mathias/hyperguild
1
0
Fork 0
Code Issues 7 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(pipeline): wire ParseRawPages+BuildPages+CanonicalizeLinks into Run

Browse Source 
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Mathias Bergqvist
2026-04-23 19:07:33 +02:00
parent 26855f69b0
commit de35d4dbb0
3 changed files with 49 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
ingestion/internal/pipeline/pipeline.go
View File

@@ -41,9 +41,11 @@ func Run(ctx context.Context, cfg Config, brainDir, content, source string, dryR
schema = loadSchema(brainDir)
}
sourceSlug := wiki.Slug(source)
date := time.Now().UTC().Format("2006-01-02")
chunks := Chunk(content, cfg.ChunkSize)
var allPages []wiki.Page
var allRaw []RawPage
var allWarnings []string
for _, chunk := range chunks {
@@ -52,25 +54,19 @@ func Run(ctx context.Context, cfg Config, brainDir, content, source string, dryR
if err != nil {
return Result{}, fmt.Errorf("LLM call: %w", err)
}
// TODO(task4): replace with RawPage-based pipeline
rawPages, warnings := ParseRawPages(output)
for _, rp := range rawPages {
if rp.Title == "" {
allWarnings = append(allWarnings, "skipped RawPage with empty title (TODO task4)")
continue
}
allPages = append(allPages, wiki.Page{Path: rp.Type + "/" + rp.Title, Content: rp.Content})
}
raw, warnings := ParseRawPages(output)
allRaw = append(allRaw, raw...)
allWarnings = append(allWarnings, warnings...)
}
resolved := Resolve(allPages, inventory)
withRefs := injectSourceRefs(resolved, inventory, brainDir)
pages := BuildPages(allRaw, sourceSlug, date)
resolved := Resolve(pages, inventory)
canonicalized, linkWarnings := CanonicalizeLinks(resolved, inventory)
allWarnings = append(allWarnings, linkWarnings...)
withRefs := injectSourceRefs(canonicalized, inventory, brainDir)
merged := mergeAll(withRefs)
date := time.Now().UTC().Format("2006-01-02")
var written []string
for _, page := range merged {
if !dryRun {
dest := filepath.Join(brainDir, filepath.FromSlash(page.Path))