hyperguild/ingestion/internal/mcp/tools_answer_test.go

package mcp_test

import (
	"context"
	"encoding/json"
	"io"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"github.com/mathiasbq/hyperguild/ingestion/internal/mcp"
	"github.com/mathiasbq/hyperguild/ingestion/internal/pipeline"
	"github.com/mathiasbq/hyperguild/ingestion/internal/reranker"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func mockAnswerLLM(response string) pipeline.CompleteFunc {
	return func(_ context.Context, _, _ string) (string, error) {
		return response, nil
	}
}

func brainDirWithContent(t *testing.T) string {
	t.Helper()
	dir := t.TempDir()
	wikiDir := filepath.Join(dir, "wiki")
	require.NoError(t, os.MkdirAll(wikiDir, 0o755))
	require.NoError(t, os.WriteFile(filepath.Join(wikiDir, "test.md"), []byte(
		"---\ntitle: Pass-rate Logging\ntype: spec\n---\n\nPass-rate logging tracks skill invocations.",
	), 0o644))
	return dir
}

func callTool(t *testing.T, ts *httptest.Server, name string, arguments map[string]any) map[string]any {
	t.Helper()
	req := map[string]any{
		"jsonrpc": "2.0", "id": 1, "method": "tools/call",
		"params": map[string]any{"name": name, "arguments": arguments},
	}
	resp, err := http.Post(ts.URL, "application/json", body(t, req))
	require.NoError(t, err)
	defer resp.Body.Close() //nolint:errcheck
	var out map[string]any
	require.NoError(t, json.NewDecoder(resp.Body).Decode(&out))
	return out
}

func TestBrainAnswer_RerankerFiltersBeforeLLM(t *testing.T) {
	brainDir := t.TempDir()
	wikiDir := filepath.Join(brainDir, "wiki")
	require.NoError(t, os.MkdirAll(wikiDir, 0o755))
	// Two notes — both BM25-match the query, but only one is truly relevant.
	require.NoError(t, os.WriteFile(filepath.Join(wikiDir, "good.md"), []byte(
		"---\ntitle: Pass-rate Logging\n---\nPass-rate logging tracks skill invocations.",
	), 0o644))
	require.NoError(t, os.WriteFile(filepath.Join(wikiDir, "noise.md"), []byte(
		"---\ntitle: Pass-rate Tangent\n---\nPass-rate appears here too but as a tangent.",
	), 0o644))

	// Fake Ollama reranker: yes only when prompt contains "tracks skill invocations".
	rrSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		raw, _ := io.ReadAll(r.Body)
		yes := strings.Contains(string(raw), "tracks skill invocations")
		ans := "no"
		if yes {
			ans = "yes"
		}
		_ = json.NewEncoder(w).Encode(map[string]any{"response": ans, "done": true})
	}))
	defer rrSrv.Close()

	// LLM mock captures the rendered sources so we can assert what reached it.
	var sawSources string
	llm := func(_ context.Context, _, user string) (string, error) {
		sawSources = user
		return "answer text", nil
	}

	srv := mcp.NewServer(brainDir, nil, nil, llm).
		WithReranker(reranker.New(rrSrv.URL, "qwen3"))
	ts := httptest.NewServer(srv)
	defer ts.Close()

	rpc := callTool(t, ts, "brain_answer", map[string]any{"query": "pass-rate logging"})
	require.Nil(t, rpc["error"])

	content := rpc["result"].(map[string]any)["content"].([]any)[0].(map[string]any)["text"].(string)
	var result map[string]any
	require.NoError(t, json.Unmarshal([]byte(content), &result))
	sources := result["sources"].([]any)
	require.Len(t, sources, 1, "reranker should drop noise.md")
	assert.Equal(t, "wiki/good.md", sources[0])
	assert.Contains(t, sawSources, "good.md")
	assert.NotContains(t, sawSources, "noise.md")
}

func TestBrainAnswer_RerankerKeepsNone_FallsBackToBM25(t *testing.T) {
	brainDir := brainDirWithContent(t) // test.md BM25-matches "pass-rate logging"

	// Reranker rejects every candidate ("no" to all) — models a
	// web-search cross-encoder facing a conversational / personal-intent
	// query, which is exactly when it wrongly scores on-topic notes as
	// irrelevant. The answer must still synthesize from the BM25 hits, not
	// collapse to "no relevant content".
	rrSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
		_ = json.NewEncoder(w).Encode(map[string]any{"response": "no", "done": true})
	}))
	defer rrSrv.Close()

	var sawSources string
	llm := func(_ context.Context, _, user string) (string, error) {
		sawSources = user
		return "fallback answer", nil
	}

	srv := mcp.NewServer(brainDir, nil, nil, llm).
		WithReranker(reranker.New(rrSrv.URL, "qwen3"))
	ts := httptest.NewServer(srv)
	defer ts.Close()

	rpc := callTool(t, ts, "brain_answer", map[string]any{"query": "pass-rate logging"})
	require.Nil(t, rpc["error"])

	content := rpc["result"].(map[string]any)["content"].([]any)[0].(map[string]any)["text"].(string)
	var result map[string]any
	require.NoError(t, json.Unmarshal([]byte(content), &result))

	assert.Equal(t, "fallback answer", result["answer"])
	assert.NotEmpty(t, result["sources"], "reranker keeping nothing must fall back to BM25, not empty")
	assert.Contains(t, sawSources, "test.md")
}

func TestBrainAnswer_NoLLM(t *testing.T) {
	srv := mcp.NewServer(t.TempDir(), nil, nil, nil)
	ts := httptest.NewServer(srv)
	defer ts.Close()

	rpc := callTool(t, ts, "brain_answer", map[string]any{"query": "test"})
	assert.NotNil(t, rpc["error"], "expected error when answerLLM is nil")
}

func TestBrainAnswer_Synthesizes(t *testing.T) {
	brainDir := brainDirWithContent(t)
	srv := mcp.NewServer(brainDir, nil, nil, mockAnswerLLM("Pass-rate logging is described in spec."))
	ts := httptest.NewServer(srv)
	defer ts.Close()

	rpc := callTool(t, ts, "brain_answer", map[string]any{"query": "pass-rate logging"})
	require.Nil(t, rpc["error"])

	content := rpc["result"].(map[string]any)["content"].([]any)[0].(map[string]any)["text"].(string)
	var result map[string]any
	require.NoError(t, json.Unmarshal([]byte(content), &result))
	assert.Equal(t, "Pass-rate logging is described in spec.", result["answer"])
	assert.NotEmpty(t, result["sources"])
}

func TestBrainClassify_ReturnsJSON(t *testing.T) {
	llmResp := `{"type":"spec","title":"My Spec","tags":["go","mcp"]}`
	srv := mcp.NewServer(t.TempDir(), nil, nil, mockAnswerLLM(llmResp))
	ts := httptest.NewServer(srv)
	defer ts.Close()

	rpc := callTool(t, ts, "brain_classify", map[string]any{"text": "# My Spec\n\nThis is a Go MCP spec."})
	require.Nil(t, rpc["error"])

	content := rpc["result"].(map[string]any)["content"].([]any)[0].(map[string]any)["text"].(string)
	var result map[string]any
	require.NoError(t, json.Unmarshal([]byte(content), &result))
	assert.Equal(t, "spec", result["type"])
	assert.Equal(t, "My Spec", result["title"])
}

func TestBrainClassify_StripsFences(t *testing.T) {
	llmResp := "```json\n{\"type\":\"note\",\"title\":\"T\",\"tags\":[]}\n```"
	srv := mcp.NewServer(t.TempDir(), nil, nil, mockAnswerLLM(llmResp))
	ts := httptest.NewServer(srv)
	defer ts.Close()

	rpc := callTool(t, ts, "brain_classify", map[string]any{"text": "some text"})
	require.Nil(t, rpc["error"])

	content := rpc["result"].(map[string]any)["content"].([]any)[0].(map[string]any)["text"].(string)
	var result map[string]any
	require.NoError(t, json.Unmarshal([]byte(content), &result))
	assert.Equal(t, "note", result["type"])
}