feat(brain_answer): Qwen3-Reranker cross-encoder filter (opt-in)
All checks were successful
CI / Lint / Test / Vet (push) Successful in 10s
CI / Mirror to GitHub (push) Successful in 3s

Adds an opt-in cross-encoder rerank step between BM25 retrieval and LLM
synthesis. With BRAIN_RERANKER_URL set, brain_answer retrieves BM25
top-20, scores each excerpt against the query via Qwen3-Reranker on
Ollama, drops the "no" answers, and forwards up to 5 surviving sources
to the LLM. Unset, behaviour is unchanged (BM25 top-10 → LLM).

The reranker is a *filter*, not a re-ranker: Qwen3-Reranker emits a
binary yes/no token under its native chat template, and ties within the
"yes" set are broken by BM25 rank — what got retrieved first stays
ahead.

New package ingestion/internal/reranker:
- Client with URL, Model, HTTP fields.
- New(url, model) returns nil on empty url so callers can treat
  "feature disabled" as a single nil check.
- Score(ctx, query, docs) issues one /api/generate call per doc using
  the Qwen3-Reranker yes/no chat template (verbatim, because the model
  was trained on this exact wording). Parses the first non-think token.

Wiring:
- mcp.Server gains a WithReranker fluent setter to keep NewServer
  signature stable.
- brain_answer's BM25 limit jumps to 20 only when a reranker is wired,
  to give the filter something to do.
- cmd/server/main.go reads BRAIN_RERANKER_URL (+ optional
  BRAIN_RERANKER_MODEL, default dengcao/Qwen3-Reranker-0.6B:F16).

Tests cover: nil-on-empty-url, ordered yes/no scoring, request shape
(model, prompt contents, yes/no template), ambiguous response → 0,
empty doc slice, upstream-error propagation, plus an end-to-end
brain_answer integration that proves only the relevant note reaches the
LLM when noise.md is rejected.

Closes hyperguild#7.
This commit is contained in:
Mathias
2026-05-18 22:55:46 +02:00
parent 58c57412a9
commit a56a4db963
6 changed files with 346 additions and 1 deletions

View File

@@ -10,6 +10,7 @@ import (
"net/http"
"github.com/mathiasbq/hyperguild/ingestion/internal/pipeline"
"github.com/mathiasbq/hyperguild/ingestion/internal/reranker"
)
type request struct {
@@ -37,6 +38,7 @@ type Server struct {
pipeline pipeline.Config
llm pipeline.CompleteFunc
answerLLM pipeline.CompleteFunc // nil = brain_answer and brain_classify unavailable
reranker *reranker.Client // nil = no rerank, BM25 top-10 → LLM
}
// NewServer constructs a Server bound to brainDir. pipelineCfg supplies the
@@ -50,6 +52,15 @@ func NewServer(brainDir string, pipelineCfg *pipeline.Config, llm pipeline.Compl
return &Server{brainDir: brainDir, pipeline: cfg, llm: llm, answerLLM: answerLLM}
}
// WithReranker installs an opt-in cross-encoder reranker. When set,
// brain_answer retrieves a wider BM25 candidate set and prunes it to
// the relevant ones before LLM synthesis. Returns the server for
// fluent chaining.
func (s *Server) WithReranker(r *reranker.Client) *Server {
s.reranker = r
return s
}
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// MCP streamable HTTP: GET establishes the SSE stream for server-to-client events.
if r.Method == http.MethodGet {

View File

@@ -6,9 +6,35 @@ import (
"fmt"
"strings"
"github.com/mathiasbq/hyperguild/ingestion/internal/reranker"
"github.com/mathiasbq/hyperguild/ingestion/internal/search"
)
// rerankResults scores each candidate's excerpt against the query and
// returns up to top results whose score is positive, preserving the
// caller's input order (BM25 rank) within the kept set. The reranker is
// a filter: ties are broken by BM25, not by the reranker's binary score.
func rerankResults(ctx context.Context, rr *reranker.Client, query string, results []search.Result, top int) ([]search.Result, error) {
docs := make([]string, len(results))
for i, r := range results {
docs[i] = r.Excerpt
}
scores, err := rr.Score(ctx, query, docs)
if err != nil {
return nil, err
}
kept := make([]search.Result, 0, top)
for i, r := range results {
if scores[i] > 0 {
kept = append(kept, r)
}
if len(kept) == top {
break
}
}
return kept, nil
}
const (
answerSystemPrompt = `You are a knowledge assistant. Answer the question using ONLY the provided sources.
Cite source file paths inline when referencing specific content.
@@ -35,10 +61,22 @@ func (s *Server) brainAnswer(ctx context.Context, args json.RawMessage) (json.Ra
return nil, fmt.Errorf("query is required")
}
results, err := search.Query(s.brainDir, search.QueryOptions{Query: a.Query, Limit: 10})
// With reranker disabled: BM25 top-10 straight to the LLM.
// With reranker enabled: BM25 top-20 → cross-encoder filter → top-5.
bm25Limit := 10
if s.reranker != nil {
bm25Limit = 20
}
results, err := search.Query(s.brainDir, search.QueryOptions{Query: a.Query, Limit: bm25Limit})
if err != nil {
return nil, fmt.Errorf("search: %w", err)
}
if s.reranker != nil && len(results) > 0 {
results, err = rerankResults(ctx, s.reranker, a.Query, results, 5)
if err != nil {
return nil, fmt.Errorf("rerank: %w", err)
}
}
if len(results) == 0 {
return json.Marshal(map[string]any{
"answer": "No relevant content found in brain.",

View File

@@ -3,14 +3,17 @@ package mcp_test
import (
"context"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"github.com/mathiasbq/hyperguild/ingestion/internal/mcp"
"github.com/mathiasbq/hyperguild/ingestion/internal/pipeline"
"github.com/mathiasbq/hyperguild/ingestion/internal/reranker"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -46,6 +49,55 @@ func callTool(t *testing.T, ts *httptest.Server, name string, arguments map[stri
return out
}
func TestBrainAnswer_RerankerFiltersBeforeLLM(t *testing.T) {
brainDir := t.TempDir()
wikiDir := filepath.Join(brainDir, "wiki")
require.NoError(t, os.MkdirAll(wikiDir, 0o755))
// Two notes — both BM25-match the query, but only one is truly relevant.
require.NoError(t, os.WriteFile(filepath.Join(wikiDir, "good.md"), []byte(
"---\ntitle: Pass-rate Logging\n---\nPass-rate logging tracks skill invocations.",
), 0o644))
require.NoError(t, os.WriteFile(filepath.Join(wikiDir, "noise.md"), []byte(
"---\ntitle: Pass-rate Tangent\n---\nPass-rate appears here too but as a tangent.",
), 0o644))
// Fake Ollama reranker: yes only when prompt contains "tracks skill invocations".
rrSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
raw, _ := io.ReadAll(r.Body)
yes := strings.Contains(string(raw), "tracks skill invocations")
ans := "no"
if yes {
ans = "yes"
}
_ = json.NewEncoder(w).Encode(map[string]any{"response": ans, "done": true})
}))
defer rrSrv.Close()
// LLM mock captures the rendered sources so we can assert what reached it.
var sawSources string
llm := func(_ context.Context, _, user string) (string, error) {
sawSources = user
return "answer text", nil
}
srv := mcp.NewServer(brainDir, nil, nil, llm).
WithReranker(reranker.New(rrSrv.URL, "qwen3"))
ts := httptest.NewServer(srv)
defer ts.Close()
rpc := callTool(t, ts, "brain_answer", map[string]any{"query": "pass-rate logging"})
require.Nil(t, rpc["error"])
content := rpc["result"].(map[string]any)["content"].([]any)[0].(map[string]any)["text"].(string)
var result map[string]any
require.NoError(t, json.Unmarshal([]byte(content), &result))
sources := result["sources"].([]any)
require.Len(t, sources, 1, "reranker should drop noise.md")
assert.Equal(t, "wiki/good.md", sources[0])
assert.Contains(t, sawSources, "good.md")
assert.NotContains(t, sawSources, "noise.md")
}
func TestBrainAnswer_NoLLM(t *testing.T) {
srv := mcp.NewServer(t.TempDir(), nil, nil, nil)
ts := httptest.NewServer(srv)