fix(brain_answer): reranker is a filter, not a gate — fall back to BM25 when it keeps nothing
The Qwen3-Reranker is a web-search cross-encoder. Against conversational / personal-intent queries (e.g. 'what am I optimizing toward?') it scores every candidate as 'no', so brain_answer collapsed to 'No relevant content found' even though BM25 had retrieved on-topic notes (incl. the telos wing). Treat the reranker as a filter: when it keeps zero results, fall back to the BM25/vector ordering (capped to the no-reranker depth of 10). Refs brain#11. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -77,9 +77,22 @@ func (s *Server) brainAnswer(ctx context.Context, args json.RawMessage) (json.Ra
|
||||
return nil, fmt.Errorf("search: %w", err)
|
||||
}
|
||||
if s.reranker != nil && len(results) > 0 {
|
||||
results, err = rerankResults(ctx, s.reranker, a.Query, results, 5)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("rerank: %w", err)
|
||||
reranked, rerr := rerankResults(ctx, s.reranker, a.Query, results, 5)
|
||||
if rerr != nil {
|
||||
return nil, fmt.Errorf("rerank: %w", rerr)
|
||||
}
|
||||
// The reranker is a filter, not a gate. The Qwen3-Reranker is a
|
||||
// web-search cross-encoder: against a conversational / personal-
|
||||
// intent query ("what am I optimizing toward?") it scores even
|
||||
// on-topic notes as "no", which would collapse the whole answer to
|
||||
// "no relevant content" despite BM25 having retrieved relevant
|
||||
// content. When the reranker keeps nothing, fall back to the
|
||||
// BM25/vector ordering (capped to the no-reranker depth) rather
|
||||
// than returning an empty answer.
|
||||
if len(reranked) > 0 {
|
||||
results = reranked
|
||||
} else if len(results) > 10 {
|
||||
results = results[:10]
|
||||
}
|
||||
}
|
||||
if len(results) == 0 {
|
||||
|
||||
@@ -98,6 +98,42 @@ func TestBrainAnswer_RerankerFiltersBeforeLLM(t *testing.T) {
|
||||
assert.NotContains(t, sawSources, "noise.md")
|
||||
}
|
||||
|
||||
func TestBrainAnswer_RerankerKeepsNone_FallsBackToBM25(t *testing.T) {
|
||||
brainDir := brainDirWithContent(t) // test.md BM25-matches "pass-rate logging"
|
||||
|
||||
// Reranker rejects every candidate ("no" to all) — models a
|
||||
// web-search cross-encoder facing a conversational / personal-intent
|
||||
// query, which is exactly when it wrongly scores on-topic notes as
|
||||
// irrelevant. The answer must still synthesize from the BM25 hits, not
|
||||
// collapse to "no relevant content".
|
||||
rrSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"response": "no", "done": true})
|
||||
}))
|
||||
defer rrSrv.Close()
|
||||
|
||||
var sawSources string
|
||||
llm := func(_ context.Context, _, user string) (string, error) {
|
||||
sawSources = user
|
||||
return "fallback answer", nil
|
||||
}
|
||||
|
||||
srv := mcp.NewServer(brainDir, nil, nil, llm).
|
||||
WithReranker(reranker.New(rrSrv.URL, "qwen3"))
|
||||
ts := httptest.NewServer(srv)
|
||||
defer ts.Close()
|
||||
|
||||
rpc := callTool(t, ts, "brain_answer", map[string]any{"query": "pass-rate logging"})
|
||||
require.Nil(t, rpc["error"])
|
||||
|
||||
content := rpc["result"].(map[string]any)["content"].([]any)[0].(map[string]any)["text"].(string)
|
||||
var result map[string]any
|
||||
require.NoError(t, json.Unmarshal([]byte(content), &result))
|
||||
|
||||
assert.Equal(t, "fallback answer", result["answer"])
|
||||
assert.NotEmpty(t, result["sources"], "reranker keeping nothing must fall back to BM25, not empty")
|
||||
assert.Contains(t, sawSources, "test.md")
|
||||
}
|
||||
|
||||
func TestBrainAnswer_NoLLM(t *testing.T) {
|
||||
srv := mcp.NewServer(t.TempDir(), nil, nil, nil)
|
||||
ts := httptest.NewServer(srv)
|
||||
|
||||
Reference in New Issue
Block a user