diff --git a/ingestion/internal/pipeline/pipeline.go b/ingestion/internal/pipeline/pipeline.go index ac63c52..1007729 100644 --- a/ingestion/internal/pipeline/pipeline.go +++ b/ingestion/internal/pipeline/pipeline.go @@ -62,13 +62,20 @@ func Run(ctx context.Context, cfg Config, brainDir, content, source string, dryR }}, nil } + // The source's own page path is deterministic (sourceSlug, computed above) + // even before extraction runs, unlike a concept/entity page's path, which + // only exists once the model has produced a title. So page-aware + // extraction (#106) is scoped to the source page: it's the one page every + // chunk's prompt can be given in advance. + existingPage := readSourcePage(brainDir, sourceSlug) + chunks := Chunk(content, cfg.ChunkSize) var allRaw []RawPage var allWarnings []string for _, chunk := range chunks { - userPrompt := BuildPrompt(schema, source, chunk, inventory) + userPrompt := BuildPrompt(schema, source, chunk, inventory, existingPage) output, err := cfg.Complete(ctx, systemPrompt, userPrompt) if err != nil { return Result{}, fmt.Errorf("LLM call: %w", err) @@ -168,6 +175,16 @@ func hashBody(content string) string { return hex.EncodeToString(sum[:]) } +// readSourcePage returns the full on-disk content of the source page at +// wiki/sources/.md, or "" when it doesn't exist yet. +func readSourcePage(brainDir, sourceSlug string) string { + raw, err := os.ReadFile(filepath.Join(brainDir, "wiki", "sources", sourceSlug+".md")) + if err != nil { + return "" + } + return string(raw) +} + // storedSourceHash reads source_hash from the source page a previous ingest // wrote, or "" when there is no such page or no such field. // @@ -175,11 +192,7 @@ func hashBody(content string) string { // skip costs one extraction, while a wrong skip loses an update silently — and // silence is the failure mode this repo keeps paying for. func storedSourceHash(brainDir, sourceSlug string) string { - raw, err := os.ReadFile(filepath.Join(brainDir, "wiki", "sources", sourceSlug+".md")) - if err != nil { - return "" - } - for _, line := range strings.Split(string(raw), "\n") { + for _, line := range strings.Split(readSourcePage(brainDir, sourceSlug), "\n") { if key, value, found := strings.Cut(line, ":"); found && strings.TrimSpace(key) == "source_hash" { return strings.TrimSpace(value) } diff --git a/ingestion/internal/pipeline/pipeline_test.go b/ingestion/internal/pipeline/pipeline_test.go index 208f913..5c6d048 100644 --- a/ingestion/internal/pipeline/pipeline_test.go +++ b/ingestion/internal/pipeline/pipeline_test.go @@ -277,6 +277,104 @@ func TestRun_ReExtractsWhenTheSourceBodyChanged(t *testing.T) { assert.Equal(t, 2, calls, "changed content must be re-extracted") } +// --- #106: page-aware extraction — the model sees the page it is revising -- + +func TestRun_ExistingSourcePageIsIncludedInTheExtractionPrompt(t *testing.T) { + // #103 made the merge asymmetric (on-disk wins); #106 is the other half — + // the extraction itself gets to see what's already there instead of + // producing a page blind and letting the merge reconcile it afterwards. + // This only proves the existing page reaches the prompt; the byte-identical + // guarantee for an echoed-back revision is a separate test below. + dir := t.TempDir() + for _, sub := range []string{"wiki/concepts", "wiki/entities", "wiki/sources"} { + require.NoError(t, os.MkdirAll(filepath.Join(dir, sub), 0o755)) + } + + seeded := "---\ntitle: 'Shape Up'\n---\n\n## Summary\n\nA book about shaping product work.\n\n## Key Claims\n\n- Appetite beats estimation.\n" + page := filepath.Join(dir, "wiki", "sources", "shape-up.md") + require.NoError(t, os.WriteFile(page, []byte(seeded), 0o644)) + + var gotPrompt string + cfg := Config{Complete: func(_ context.Context, _, user string) (string, error) { + gotPrompt = user + return mustJSON([]RawPage{{ + Title: "Shape Up", Type: "source", Subtype: "book", + Content: "## Summary\n\nA book about shaping product work.\n", + }}), nil + }} + + // Deliberately does not mention "Appetite beats estimation." — that phrase + // can only reach the prompt via the existing on-disk page, not the source + // content, so its presence proves the page was threaded through. + content := "A short editorial update to the book's listing. No new claims here." + _, err := Run(context.Background(), cfg, dir, content, "shape-up", false) + require.NoError(t, err) + + assert.Contains(t, gotPrompt, "Appetite beats estimation.", + "the existing page's content must be visible to the extraction prompt") + assert.Contains(t, gotPrompt, "revis", "the prompt must ask for a revision, not a fresh page") +} + +func TestRun_NewSourceHasNoExistingPageInThePrompt(t *testing.T) { + // The other side of the same behavior: a source with no page on disk yet + // must not get an "existing page" section injected into its prompt at all. + dir := t.TempDir() + for _, sub := range []string{"wiki/concepts", "wiki/entities", "wiki/sources"} { + require.NoError(t, os.MkdirAll(filepath.Join(dir, sub), 0o755)) + } + + var gotPrompt string + cfg := Config{Complete: func(_ context.Context, _, user string) (string, error) { + gotPrompt = user + return mustJSON([]RawPage{{ + Title: "New Thing", Type: "source", Subtype: "article", + Content: "## Summary\n\nBrand new.\n", + }}), nil + }} + + _, err := Run(context.Background(), cfg, dir, "Brand new content.", "new-thing", false) + require.NoError(t, err) + + assert.NotContains(t, gotPrompt, "Current page", + "a source with no page on disk yet must not get a revision section in its prompt") +} + +func TestRun_RevisionEchoingExistingPageStaysByteIdentical(t *testing.T) { + // The issue's explicit risk: a model given the current page will sometimes + // echo it back verbatim. That must not duplicate content or otherwise + // perturb the page — the deterministic merge is still the backstop. + dir := t.TempDir() + for _, sub := range []string{"wiki/concepts", "wiki/entities", "wiki/sources"} { + require.NoError(t, os.MkdirAll(filepath.Join(dir, sub), 0o755)) + } + + seeded := "---\ntitle: 'Shape Up'\n---\n\n## Summary\n\nA book about shaping product work.\n\n## Key Claims\n\n- Appetite beats estimation.\n" + page := filepath.Join(dir, "wiki", "sources", "shape-up.md") + require.NoError(t, os.WriteFile(page, []byte(seeded), 0o644)) + + cfg := Config{Complete: func(_ context.Context, _, _ string) (string, error) { + return mustJSON([]RawPage{{ + Title: "Shape Up", Type: "source", Subtype: "book", + Content: "## Summary\n\nA book about shaping product work.\n\n## Key Claims\n\n- Appetite beats estimation.\n", + }}), nil + }} + + content := "A short editorial update to the book's listing. No new claims here." + result, err := Run(context.Background(), cfg, dir, content, "shape-up", false) + require.NoError(t, err) + require.False(t, result.Skipped) + + got, err := os.ReadFile(page) + require.NoError(t, err) + + want := "---\ntitle: 'Shape Up'\nlast_updated: " + time.Now().UTC().Format("2006-01-02") + + "\nsource_hash: " + hashBody(content) + "\n---\n\n" + + "## Summary\n\nA book about shaping product work.\n\n" + + "## Key Claims\n\n- Appetite beats estimation.\n" + assert.Equal(t, want, string(got), + "a revision pass that echoes the existing page back must not duplicate or alter its content") +} + func TestRun_FrontmatterOnlyChangeIsNotDrift(t *testing.T) { // The hash covers the body, so re-fetching a page whose metadata moved // (a re-crawl stamping a new date) is not treated as the source changing. diff --git a/ingestion/internal/pipeline/prompt.go b/ingestion/internal/pipeline/prompt.go index 00b65c7..bdfa2e5 100644 --- a/ingestion/internal/pipeline/prompt.go +++ b/ingestion/internal/pipeline/prompt.go @@ -22,8 +22,18 @@ Each element must have exactly these fields: Wikilinks in content: [[Display Name]] — just the display name, no slug, no pipe separator. Only link to pages listed in the inventory or pages you are creating in this response.` -// BuildPrompt constructs the user prompt for a single chunk. -func BuildPrompt(schema, source, content string, inventory map[wiki.PageType][]wiki.Entry) string { +// BuildPrompt constructs the user prompt for a single chunk. existingPage is +// the full current content (frontmatter + body) of the source's own wiki page +// when one already exists on disk, or "" for a source being ingested for the +// first time. When non-empty, the model is asked to revise that page rather +// than write a fresh one — see hyperguild#106. The deterministic merge in +// wiki.Merge remains the backstop regardless of what the model returns: an +// on-disk section the model doesn't touch survives untouched, and a section +// it does touch only takes effect where Merge's strategy allows the fresh +// side to change anything (bullet union, append) — never for an ordinary +// section, so the model proposing a revision is never how a page gets +// overwritten. +func BuildPrompt(schema, source, content string, inventory map[wiki.PageType][]wiki.Entry, existingPage string) string { var sb strings.Builder fmt.Fprintf(&sb, "Today's date is %s.\n\n", time.Now().UTC().Format("2006-01-02")) @@ -49,6 +59,18 @@ func BuildPrompt(schema, source, content string, inventory map[wiki.PageType][]w sb.WriteString("\n") } + if existingPage != "" { + sb.WriteString("## Current page for this source — revise it, don't restart\n\n") + sb.WriteString("This source has already been ingested and has the wiki page shown below.\n") + sb.WriteString("You are revising that page, not writing a fresh one:\n") + sb.WriteString(" - Add sections or claims the source below supports that this page doesn't have yet.\n") + sb.WriteString(" - Strengthen a claim where the source gives more detail, evidence, or nuance than what's already there.\n") + sb.WriteString(" - Leave unchanged whatever the source doesn't affect — reuse the existing wording verbatim rather than paraphrasing it, so an unchanged claim is recognized as unchanged.\n") + sb.WriteString(" - A merge step keeps this page's on-disk content for any section you don't return, so it's fine to omit a section you aren't revising.\n\n") + sb.WriteString(existingPage) + sb.WriteString("\n\n") + } + sb.WriteString("## Non-negotiable rules\n\n") sb.WriteString("1. Output ONLY a valid JSON array — no prose, no fences.\n") sb.WriteString("2. Fields: title, type, subtype (if applicable), domain (if applicable), content.\n")