hyperguild/ingestion/internal/pipeline/pipeline_test.go

// ingestion/internal/pipeline/pipeline_test.go
package pipeline

import (
	"context"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"github.com/mathiasbq/hyperguild/ingestion/internal/llm"
)

func TestRun_WritesPages(t *testing.T) {
	brainDir := t.TempDir()
	for _, sub := range []string{"wiki/concepts", "wiki/entities", "wiki/sources"} {
		require.NoError(t, os.MkdirAll(filepath.Join(brainDir, sub), 0o755))
	}

	llmResponse := mustJSON([]RawPage{
		{
			Title:   "Test Article",
			Type:    "source",
			Subtype: "article",
			Domain:  "software-engineering",
			Content: "## Summary\n\nA test article.\n\n## Key Claims\n\n- It tests things.\n\n## Concepts Introduced or Reinforced\n\n[[Testing]]\n\n## Entities Mentioned\n\n## Open Questions Raised\n",
		},
		{
			Title:   "Testing",
			Type:    "concept",
			Domain:  "software-engineering",
			Content: "## Definition\n\nThe practice of verifying software.\n\n## Why It Matters\n\nCatches bugs.\n\n## Related Concepts\n\n## Related Entities\n\n## Sources\n\n## Evolving Notes\n",
		},
	})

	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		_ = json.NewEncoder(w).Encode(map[string]any{
			"choices": []map[string]any{
				{"message": map[string]any{"role": "assistant", "content": llmResponse}},
			},
		})
	}))
	defer srv.Close()

	cfg := Config{
		Complete:  llm.New(srv.URL, "", "test-model", 30*time.Second).Complete,
		ChunkSize: 0,
	}

	result, err := Run(context.Background(), cfg, brainDir, "An article about testing.", "test-article", false)
	require.NoError(t, err)
	assert.Len(t, result.Pages, 2)

	_, err = os.Stat(filepath.Join(brainDir, "wiki", "sources", "test-article.md"))
	require.NoError(t, err)
	_, err = os.Stat(filepath.Join(brainDir, "wiki", "concepts", "testing.md"))
	require.NoError(t, err)
	_, err = os.Stat(filepath.Join(brainDir, "wiki", "index.md"))
	require.NoError(t, err)
	_, err = os.Stat(filepath.Join(brainDir, "log.md"))
	require.NoError(t, err)
}

func TestRun_DryRunDoesNotWrite(t *testing.T) {
	brainDir := t.TempDir()
	for _, sub := range []string{"wiki/concepts", "wiki/entities", "wiki/sources"} {
		require.NoError(t, os.MkdirAll(filepath.Join(brainDir, sub), 0o755))
	}

	llmResponse := mustJSON([]RawPage{{
		Title:   "Foo",
		Type:    "source",
		Subtype: "article",
		Content: "## Summary\n\nFoo.\n",
	}})

	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		_ = json.NewEncoder(w).Encode(map[string]any{
			"choices": []map[string]any{{"message": map[string]any{"content": llmResponse}}},
		})
	}))
	defer srv.Close()

	cfg := Config{Complete: llm.New(srv.URL, "", "m", 30*time.Second).Complete}
	result, err := Run(context.Background(), cfg, brainDir, "foo content", "foo", true)
	require.NoError(t, err)
	assert.Len(t, result.Pages, 1)

	_, err = os.Stat(filepath.Join(brainDir, "wiki", "sources", "foo.md"))
	assert.True(t, os.IsNotExist(err))
}

func TestRun_MergesDuplicatePaths(t *testing.T) {
	brainDir := t.TempDir()
	for _, sub := range []string{"wiki/concepts", "wiki/entities", "wiki/sources"} {
		require.NoError(t, os.MkdirAll(filepath.Join(brainDir, sub), 0o755))
	}

	// LLM returns same title twice (simulates multi-chunk duplicate)
	llmResponse := mustJSON([]RawPage{
		{Title: "Foo", Type: "concept", Content: "## Definition\n\nFirst.\n\n## Related Concepts\n\n[[Bar]]\n"},
		{Title: "Foo", Type: "concept", Content: "## Definition\n\nSecond.\n\n## Related Concepts\n\n[[Baz]]\n"},
	})

	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		_ = json.NewEncoder(w).Encode(map[string]any{
			"choices": []map[string]any{{"message": map[string]any{"content": llmResponse}}},
		})
	}))
	defer srv.Close()

	cfg := Config{Complete: llm.New(srv.URL, "", "m", 30*time.Second).Complete}
	result, err := Run(context.Background(), cfg, brainDir, "content", "foo", false)
	require.NoError(t, err)
	assert.Len(t, result.Pages, 1) // deduplicated

	content, err := os.ReadFile(filepath.Join(brainDir, "wiki", "concepts", "foo.md"))
	require.NoError(t, err)
	// keep-first for Definition, union for Related Concepts
	assert.Contains(t, string(content), "First.")
	// Bar and Baz unknown in empty inventory → left as plain [[links]]
	assert.Contains(t, string(content), "[[Bar]]")
	assert.Contains(t, string(content), "[[Baz]]")
}

func TestRun_ThreadsSourceAuthorPublishedFromContentFrontmatter(t *testing.T) {
	brainDir := t.TempDir()
	for _, sub := range []string{"wiki/concepts", "wiki/entities", "wiki/sources"} {
		require.NoError(t, os.MkdirAll(filepath.Join(brainDir, sub), 0o755))
	}

	llmResponse := mustJSON([]RawPage{{
		Title:   "Ornith",
		Type:    "source",
		Subtype: "article",
		Content: "## Summary\n\nAn agentic coding model.\n",
	}})

	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		_ = json.NewEncoder(w).Encode(map[string]any{
			"choices": []map[string]any{{"message": map[string]any{"content": llmResponse}}},
		})
	}))
	defer srv.Close()

	cfg := Config{Complete: llm.New(srv.URL, "", "m", 30*time.Second).Complete}
	rawContent := "---\nsource: https://example.com/ornith\nauthor: Jane Doe\npublished: 2026-07-20\n---\n\nAn agentic coding model that runs on your laptop.\n"

	result, err := Run(context.Background(), cfg, brainDir, rawContent, "ornith", false)
	require.NoError(t, err)
	require.Len(t, result.Pages, 1)

	content, err := os.ReadFile(filepath.Join(brainDir, "wiki", "sources", "ornith.md"))
	require.NoError(t, err)
	assert.Contains(t, string(content), "source: 'https://example.com/ornith'")
	assert.Contains(t, string(content), "author: 'Jane Doe'")
	assert.Contains(t, string(content), "published: '2026-07-20'")
}

func mustJSON(v any) string {
	b, err := json.Marshal(v)
	if err != nil {
		panic(err)
	}
	return string(b)
}

// --- B1: compile-on-ingest — revise the existing page, never clobber it -----

func TestRunRaw_MergesIntoAnExistingPageInsteadOfOverwritingIt(t *testing.T) {
	// Before this, buildAndWrite did an unconditional os.WriteFile: re-ingesting
	// a source replaced its page wholesale, discarding anything accumulated
	// since — including human edits. hyperguild#43 recorded the symptom as
	// "re-ingest overwrites wholesale".
	dir := t.TempDir()
	page := filepath.Join(dir, "wiki", "sources", "shape-up.md")
	require.NoError(t, os.MkdirAll(filepath.Dir(page), 0o755))
	require.NoError(t, os.WriteFile(page, []byte(`---
title: 'Shape Up'
---

## Summary

A book about shaping product work.

## My Take

Read this before the next planning cycle.
`), 0o644))

	_, err := RunRaw(context.Background(), dir, "shape-up", []RawPage{{
		Title: "Shape Up", Type: "source", Subtype: "book", Domain: "product-strategy",
		Content: "## Summary\n\nA book about shaping product work.\n\n## Key Claims\n\n- Appetite beats estimation.\n",
	}}, false)
	require.NoError(t, err)

	got, err := os.ReadFile(page)
	require.NoError(t, err)
	s := string(got)

	assert.Contains(t, s, "## My Take", "a section the extraction never saw must survive the merge")
	assert.Contains(t, s, "Read this before the next planning cycle.")
	assert.Contains(t, s, "Appetite beats estimation.", "newly extracted material must still land")
}

func TestRunRaw_ReIngestingUnchangedContentIsByteIdentical(t *testing.T) {
	// Compile-on-ingest runs on every ingest, so "nothing changed" must produce
	// no diff at all. Otherwise every re-ingest dirties the tree, brain-sync
	// commits it, and the git history fills with noise that hides real edits.
	dir := t.TempDir()
	raw := []RawPage{{
		Title: "Shape Up", Type: "source", Subtype: "book", Domain: "product-strategy",
		Content: "## Summary\n\nA book about shaping product work.\n\n## Open Questions\n\n- Does appetite scale?\n",
	}}

	_, err := RunRaw(context.Background(), dir, "shape-up", raw, false)
	require.NoError(t, err)
	first, err := os.ReadFile(filepath.Join(dir, "wiki", "sources", "shape-up.md"))
	require.NoError(t, err)

	_, err = RunRaw(context.Background(), dir, "shape-up", raw, false)
	require.NoError(t, err)
	second, err := os.ReadFile(filepath.Join(dir, "wiki", "sources", "shape-up.md"))
	require.NoError(t, err)

	assert.Equal(t, string(first), string(second),
		"re-ingesting unchanged content must leave the page byte-identical")
}

// --- #105: source drift — an unchanged re-ingest must not re-extract --------

func TestRun_SkipsExtractionWhenTheSourceIsUnchanged(t *testing.T) {
	// #103 made a re-ingest harmless (byte-identical merge). It is still not
	// free: the full LLM extraction runs over every chunk to produce a result
	// that merges to nothing. This is the cost half.
	dir := t.TempDir()
	var calls int
	cfg := Config{Complete: func(_ context.Context, _, _ string) (string, error) {
		calls++
		return `[{"title":"Shape Up","type":"source","subtype":"book","domain":"product","content":"## Summary\n\nA book.\n"}]`, nil
	}}

	content := "---\nsource: https://example.com/shape-up\n---\n\nThe body of the article.\n"

	first, err := Run(context.Background(), cfg, dir, content, "shape-up", false)
	require.NoError(t, err)
	require.Equal(t, 1, calls, "first ingest must extract")
	assert.False(t, first.Skipped)

	second, err := Run(context.Background(), cfg, dir, content, "shape-up", false)
	require.NoError(t, err)
	assert.Equal(t, 1, calls, "unchanged content must not be extracted again")
	assert.True(t, second.Skipped, "the skip must be reported, not silent")
}

func TestRun_ReExtractsWhenTheSourceBodyChanged(t *testing.T) {
	dir := t.TempDir()
	var calls int
	cfg := Config{Complete: func(_ context.Context, _, _ string) (string, error) {
		calls++
		return `[{"title":"Shape Up","type":"source","subtype":"book","domain":"product","content":"## Summary\n\nA book.\n"}]`, nil
	}}

	_, err := Run(context.Background(), cfg, dir, "---\nsource: x\n---\n\nversion one\n", "shape-up", false)
	require.NoError(t, err)
	_, err = Run(context.Background(), cfg, dir, "---\nsource: x\n---\n\nversion two — the source was edited\n", "shape-up", false)
	require.NoError(t, err)

	assert.Equal(t, 2, calls, "changed content must be re-extracted")
}

// --- #106: page-aware extraction — the model sees the page it is revising --

func TestRun_ExistingSourcePageIsIncludedInTheExtractionPrompt(t *testing.T) {
	// #103 made the merge asymmetric (on-disk wins); #106 is the other half —
	// the extraction itself gets to see what's already there instead of
	// producing a page blind and letting the merge reconcile it afterwards.
	// This only proves the existing page reaches the prompt; the byte-identical
	// guarantee for an echoed-back revision is a separate test below.
	dir := t.TempDir()
	for _, sub := range []string{"wiki/concepts", "wiki/entities", "wiki/sources"} {
		require.NoError(t, os.MkdirAll(filepath.Join(dir, sub), 0o755))
	}

	seeded := "---\ntitle: 'Shape Up'\n---\n\n## Summary\n\nA book about shaping product work.\n\n## Key Claims\n\n- Appetite beats estimation.\n"
	page := filepath.Join(dir, "wiki", "sources", "shape-up.md")
	require.NoError(t, os.WriteFile(page, []byte(seeded), 0o644))

	var gotPrompt string
	cfg := Config{Complete: func(_ context.Context, _, user string) (string, error) {
		gotPrompt = user
		return mustJSON([]RawPage{{
			Title: "Shape Up", Type: "source", Subtype: "book",
			Content: "## Summary\n\nA book about shaping product work.\n",
		}}), nil
	}}

	// Deliberately does not mention "Appetite beats estimation." — that phrase
	// can only reach the prompt via the existing on-disk page, not the source
	// content, so its presence proves the page was threaded through.
	content := "A short editorial update to the book's listing. No new claims here."
	_, err := Run(context.Background(), cfg, dir, content, "shape-up", false)
	require.NoError(t, err)

	assert.Contains(t, gotPrompt, "Appetite beats estimation.",
		"the existing page's content must be visible to the extraction prompt")
	assert.Contains(t, gotPrompt, "revis", "the prompt must ask for a revision, not a fresh page")
}

func TestRun_NewSourceHasNoExistingPageInThePrompt(t *testing.T) {
	// The other side of the same behavior: a source with no page on disk yet
	// must not get an "existing page" section injected into its prompt at all.
	dir := t.TempDir()
	for _, sub := range []string{"wiki/concepts", "wiki/entities", "wiki/sources"} {
		require.NoError(t, os.MkdirAll(filepath.Join(dir, sub), 0o755))
	}

	var gotPrompt string
	cfg := Config{Complete: func(_ context.Context, _, user string) (string, error) {
		gotPrompt = user
		return mustJSON([]RawPage{{
			Title: "New Thing", Type: "source", Subtype: "article",
			Content: "## Summary\n\nBrand new.\n",
		}}), nil
	}}

	_, err := Run(context.Background(), cfg, dir, "Brand new content.", "new-thing", false)
	require.NoError(t, err)

	assert.NotContains(t, gotPrompt, "Current page",
		"a source with no page on disk yet must not get a revision section in its prompt")
}

func TestRun_RevisionEchoingExistingPageStaysByteIdentical(t *testing.T) {
	// The issue's explicit risk: a model given the current page will sometimes
	// echo it back verbatim. That must not duplicate content or otherwise
	// perturb the page — the deterministic merge is still the backstop.
	dir := t.TempDir()
	for _, sub := range []string{"wiki/concepts", "wiki/entities", "wiki/sources"} {
		require.NoError(t, os.MkdirAll(filepath.Join(dir, sub), 0o755))
	}

	seeded := "---\ntitle: 'Shape Up'\n---\n\n## Summary\n\nA book about shaping product work.\n\n## Key Claims\n\n- Appetite beats estimation.\n"
	page := filepath.Join(dir, "wiki", "sources", "shape-up.md")
	require.NoError(t, os.WriteFile(page, []byte(seeded), 0o644))

	cfg := Config{Complete: func(_ context.Context, _, _ string) (string, error) {
		return mustJSON([]RawPage{{
			Title: "Shape Up", Type: "source", Subtype: "book",
			Content: "## Summary\n\nA book about shaping product work.\n\n## Key Claims\n\n- Appetite beats estimation.\n",
		}}), nil
	}}

	content := "A short editorial update to the book's listing. No new claims here."
	result, err := Run(context.Background(), cfg, dir, content, "shape-up", false)
	require.NoError(t, err)
	require.False(t, result.Skipped)

	got, err := os.ReadFile(page)
	require.NoError(t, err)

	want := "---\ntitle: 'Shape Up'\nlast_updated: " + time.Now().UTC().Format("2006-01-02") +
		"\nsource_hash: " + hashBody(content) +
		// Every page an ingest writes carries provenance (hyperguild#145);
		// no stamp in ctx here, so origin: unknown.
		"\norigin: unknown\n---\n\n" +
		"## Summary\n\nA book about shaping product work.\n\n" +
		"## Key Claims\n\n- Appetite beats estimation.\n"
	assert.Equal(t, want, string(got),
		"a revision pass that echoes the existing page back must not duplicate or alter its content")
}

func TestRun_FrontmatterOnlyChangeIsNotDrift(t *testing.T) {
	// The hash covers the body, so re-fetching a page whose metadata moved
	// (a re-crawl stamping a new date) is not treated as the source changing.
	dir := t.TempDir()
	var calls int
	cfg := Config{Complete: func(_ context.Context, _, _ string) (string, error) {
		calls++
		return `[{"title":"Shape Up","type":"source","subtype":"book","domain":"product","content":"## Summary\n\nA book.\n"}]`, nil
	}}

	_, err := Run(context.Background(), cfg, dir, "---\nsource: x\nfetched: monday\n---\n\nsame body\n", "shape-up", false)
	require.NoError(t, err)
	_, err = Run(context.Background(), cfg, dir, "---\nsource: x\nfetched: tuesday\n---\n\nsame body\n", "shape-up", false)
	require.NoError(t, err)

	assert.Equal(t, 1, calls, "only the body counts as the source")
}

func TestRunRaw_ReportsWhenTheSourceDisagreesWithWhatIsOnDisk(t *testing.T) {
	// The on-disk version still wins — that is what makes unattended ingest
	// safe. But the run must say that the source now claims something else,
	// or a changed source is indistinguishable from an unchanged one.
	dir := t.TempDir()
	page := filepath.Join(dir, "wiki", "sources", "shape-up.md")
	require.NoError(t, os.MkdirAll(filepath.Dir(page), 0o755))
	require.NoError(t, os.WriteFile(page, []byte(`---
title: 'Shape Up'
---

## Summary

Appetite is fixed, scope is variable.
`), 0o644))

	res, err := RunRaw(context.Background(), dir, "shape-up", []RawPage{{
		Title: "Shape Up", Type: "source", Subtype: "book", Domain: "product",
		Content: "## Summary\n\nAppetite is negotiable and scope is fixed.\n",
	}}, false)
	require.NoError(t, err)

	got, err := os.ReadFile(page)
	require.NoError(t, err)
	assert.Contains(t, string(got), "Appetite is fixed", "on-disk version wins")

	joined := strings.Join(res.Warnings, " ")
	assert.Contains(t, joined, "Summary", "the disagreeing section must be named")
	assert.Contains(t, joined, "shape-up", "and the page it happened on")
}