mathias/hyperguild
1
0
Fork 0
Code Issues 7 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(watcher,api): use extract.Text() for file reading — fixes PDF ingestion

Browse Source
This commit is contained in:
Mathias Bergqvist
2026-04-23 16:01:36 +02:00
parent 53e46781b1
commit 2975eadc87
2 changed files with 11 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
ingestion/internal/api/handler.go
View File

@@ -11,6 +11,7 @@ import (
"strings"
"time"
"github.com/mathiasbq/hyperguild/ingestion/internal/extract"
"github.com/mathiasbq/hyperguild/ingestion/internal/pipeline"
"github.com/mathiasbq/hyperguild/ingestion/internal/search"
)
@@ -214,16 +215,16 @@ func (h *Handler) IngestPath(w http.ResponseWriter, r *http.Request) {
if !supportedExtensions[ext] {
return nil
}
content, readErr := os.ReadFile(path)
content, readErr := extract.Text(path)
if readErr != nil {
allWarnings = append(allWarnings, fmt.Sprintf("read %s: %v", path, readErr))
allWarnings = append(allWarnings, fmt.Sprintf("extract %s: %v", path, readErr))
return nil
}
source := req.Source
if source == "" {
source = filepath.Base(path)
}
result, runErr := pipeline.Run(r.Context(), h.pipeline, h.brainDir, string(content), source, req.DryRun)
result, runErr := pipeline.Run(r.Context(), h.pipeline, h.brainDir, content, source, req.DryRun)
if runErr != nil {
allWarnings = append(allWarnings, fmt.Sprintf("ingest %s: %v", path, runErr))
return nil
@@ -243,16 +244,16 @@ func (h *Handler) IngestPath(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, fmt.Sprintf("unsupported file extension: %s", ext))
return
}
content, readErr := os.ReadFile(req.Path)
content, readErr := extract.Text(req.Path)
if readErr != nil {
writeError(w, http.StatusInternalServerError, fmt.Sprintf("read file: %v", readErr))
writeError(w, http.StatusInternalServerError, fmt.Sprintf("extract text: %v", readErr))
return
}
source := req.Source
if source == "" {
source = filepath.Base(req.Path)
}
result, runErr := pipeline.Run(r.Context(), h.pipeline, h.brainDir, string(content), source, req.DryRun)
result, runErr := pipeline.Run(r.Context(), h.pipeline, h.brainDir, content, source, req.DryRun)
if runErr != nil {
h.logger.Error("ingest-path failed", "path", req.Path, "err", runErr)
writeError(w, http.StatusInternalServerError, "ingest error")

7
ingestion/internal/watcher/watcher.go
View File

@@ -11,6 +11,7 @@ import (
"time"
"unicode"
"github.com/mathiasbq/hyperguild/ingestion/internal/extract"
"github.com/mathiasbq/hyperguild/ingestion/internal/pipeline"
)
@@ -88,12 +89,12 @@ func processFile(ctx context.Context, cfg Config, path, date string) error {
filename := filepath.Base(path)
source := deriveSource(filename)
content, err := os.ReadFile(path)
content, err := extract.Text(path)
if err != nil {
return fmt.Errorf("read file: %w", err)
return fmt.Errorf("extract text: %w", err)
}
_, runErr := pipeline.Run(ctx, cfg.Pipeline, cfg.BrainDir, string(content), source, false)
_, runErr := pipeline.Run(ctx, cfg.Pipeline, cfg.BrainDir, content, source, false)
if runErr != nil {
// Move to failed/.
failedDir := filepath.Join(cfg.BrainDir, "raw", "failed")