From 4f78fecd065b2429e1bb79866ea8609fc7d8dcb1 Mon Sep 17 00:00:00 2001 From: Mathias Date: Mon, 25 May 2026 18:45:20 +0200 Subject: [PATCH] feat(search): M4 tier-weighted BM25 re-rank (infra#72) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The eval set under brain/eval/qa-2026-05.md showed BM25 top-1 at 20% with 5 of the missing slugs being short focused knowledge entries that lost to long aggregate docs on raw term-frequency. Tier weighting addresses that without touching the BM25 algorithm itself. How - Result struct gains a Tier field, populated during the file walk via extractTier (frontmatter wins, path prefix as fallback — mirrors the graph.inferTierFromPath logic so the two callers stay in lockstep). - After the existing sort (and optional hybridMerge), do a final stable re-sort by float64(Score) * tierWeight(Tier). Knowledge ×1.5, note ×1.0, inbox ×0.3, unknown ×1.0. - hydrate() (vector-only hits) also fills Tier so re-ranking covers the hybrid path. Test covers the load-bearing case: a long note-tier doc with raw=10 loses to a short knowledge-tier doc with raw=8 after weighting (8×1.5=12 vs 10×1.0=10). Measurement gate is in infra#72: re-run brain/eval/score.py against the live brain after this image lands; close the issue when top-1 hit rate lifts by ≥10 absolute points. --- ingestion/internal/search/search.go | 81 ++++++++++++++++++++++++ ingestion/internal/search/search_test.go | 24 +++++++ 2 files changed, 105 insertions(+) diff --git a/ingestion/internal/search/search.go b/ingestion/internal/search/search.go index d5c289b..32d58a2 100644 --- a/ingestion/internal/search/search.go +++ b/ingestion/internal/search/search.go @@ -43,6 +43,30 @@ type Result struct { Score int `json:"score"` Wing string `json:"wing,omitempty"` Hall string `json:"hall,omitempty"` + // Tier is the DIKW classification used for retrieval weighting + // (infra#72). Read from frontmatter when present, otherwise + // inferred from the parent directory. + Tier string `json:"tier,omitempty"` +} + +// tierWeight maps the DIKW tier to a score multiplier applied right +// before the final truncation. Knowledge entries (focused lessons that +// age well) get boosted; inbox entries (raw captures, sessions, clips) +// get demoted. Empty / unknown tiers keep the original BM25 score +// (multiplier 1.0). See infra#72 for the failure mode this addresses: +// short focused entries lose to long aggregate dump-files under +// raw BM25 ranking. +func tierWeight(tier string) float64 { + switch tier { + case "knowledge": + return 1.5 + case "note": + return 1.0 + case "inbox": + return 0.3 + default: + return 1.0 + } } // QueryOptions configures a search. @@ -120,6 +144,7 @@ func QueryContext(ctx context.Context, brainDir string, opts QueryOptions) ([]Re } rel = filepath.ToSlash(rel) wing, hall := extractWingHall(string(content), rel) + tier := extractTier(string(content), rel) results = append(results, Result{ Path: rel, Title: extractTitle(string(content), d.Name()), @@ -127,6 +152,7 @@ func QueryContext(ctx context.Context, brainDir string, opts QueryOptions) ([]Re Score: score, Wing: wing, Hall: hall, + Tier: tier, }) return nil }) @@ -150,6 +176,15 @@ func QueryContext(ctx context.Context, brainDir string, opts QueryOptions) ([]Re } } + // Tier-weighted final re-rank (infra#72). Knowledge tier entries + // boost ×1.5, inbox demote ×0.3, note stays at ×1.0. Applied after + // hybridMerge so RRF ranking still drives candidate generation; + // the tier weight only re-orders the merged set. + sort.SliceStable(results, func(i, j int) bool { + return float64(results[i].Score)*tierWeight(results[i].Tier) > + float64(results[j].Score)*tierWeight(results[j].Tier) + }) + if len(results) > opts.Limit { results = results[:opts.Limit] } @@ -235,12 +270,14 @@ func hydrate(brainDir, relPath string) (Result, error) { return Result{}, err } wing, hall := extractWingHall(string(content), relPath) + tier := extractTier(string(content), relPath) return Result{ Path: relPath, Title: extractTitle(string(content), filepath.Base(relPath)), Excerpt: excerpt(string(content), 300), Wing: wing, Hall: hall, + Tier: tier, }, nil } @@ -269,6 +306,50 @@ func resolveRoots(brainDir, wing, hall string) ([]string, error) { }, nil } +// extractTier reads the DIKW tier from frontmatter first, falling back +// to the path prefix mapping (infra#72). Mirrors graph.inferTierFromPath +// so the two callers stay in lockstep — frontmatter is canonical, +// path inference is the migration-window fallback. +func extractTier(content, relPath string) string { + scanner := bufio.NewScanner(strings.NewReader(content)) + inFrontmatter := false + for scanner.Scan() { + line := scanner.Text() + if strings.TrimSpace(line) == "---" { + if !inFrontmatter { + inFrontmatter = true + continue + } + break + } + if !inFrontmatter { + continue + } + key, val, ok := strings.Cut(line, ":") + if !ok { + continue + } + if strings.TrimSpace(key) == "tier" { + return strings.Trim(strings.TrimSpace(val), `"'`) + } + } + parts := strings.Split(relPath, "/") + if len(parts) == 0 { + return "" + } + switch parts[0] { + case "inbox", "raw", "sessions", "clips": + return "inbox" + case "notes": + return "note" + case "wiki": + return "note" + case "knowledge": + return "knowledge" + } + return "" +} + // extractWingHall reads wing/hall from frontmatter first, falling back to // path segments brain/wiki///. func extractWingHall(content, relPath string) (wing, hall string) { diff --git a/ingestion/internal/search/search_test.go b/ingestion/internal/search/search_test.go index db7a7f1..e5f0670 100644 --- a/ingestion/internal/search/search_test.go +++ b/ingestion/internal/search/search_test.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "testing" "github.com/mathiasbq/hyperguild/ingestion/internal/search" @@ -130,6 +131,29 @@ func TestSearch_ReturnsMatchingPages(t *testing.T) { assert.Contains(t, results[0].Excerpt, "Retry") } +func TestSearch_TierWeightingReordersResults(t *testing.T) { + dir := t.TempDir() + // A long note-tier dump mentions the keyword many times (high raw + // BM25 score); a short knowledge entry mentions it three times. + // Raw BM25 prefers the dump; tier weighting (knowledge ×1.5 vs + // note ×1.0) flips the order if the score gap is within reach. + // note raw = 5 × 2 terms = 10 hits, weight 1.0 → 10 + // knowledge raw = 4 × 2 terms = 8 hits, weight 1.5 → 12 (overtakes) + noteBody := "---\ntier: note\n---\n" + strings.Repeat("scram trap. ", 5) + knowledgeBody := "---\ntier: knowledge\n---\n" + strings.Repeat("scram trap. ", 4) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "wiki", "sources"), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "knowledge"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "wiki", "sources", "dump.md"), []byte(noteBody), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "knowledge", "trap.md"), []byte(knowledgeBody), 0o644)) + + results, err := search.Query(dir, search.QueryOptions{Query: "scram trap", Limit: 5}) + require.NoError(t, err) + require.GreaterOrEqual(t, len(results), 2) + assert.Equal(t, "knowledge/trap.md", results[0].Path, "knowledge tier weight should beat note tier") + assert.Equal(t, "knowledge", results[0].Tier) + assert.Equal(t, "note", results[1].Tier) +} + func TestSearch_WingHallScoping(t *testing.T) { dir := t.TempDir() for _, p := range []struct{ rel, body string }{