feat(pipeline): feed the current source page into the extraction prompt (#106) (#132)

Co-authored-by: Mathias <mthbqv@gmail.com>

ingestion/internal/pipeline/pipeline.go

@@ -62,13 +62,20 @@ func Run(ctx context.Context, cfg Config, brainDir, content, source string, dryR
 		}}, nil
 	}
 
+	// The source's own page path is deterministic (sourceSlug, computed above)
+	// even before extraction runs, unlike a concept/entity page's path, which
+	// only exists once the model has produced a title. So page-aware
+	// extraction (#106) is scoped to the source page: it's the one page every
+	// chunk's prompt can be given in advance.
+	existingPage := readSourcePage(brainDir, sourceSlug)
+
 	chunks := Chunk(content, cfg.ChunkSize)
 
 	var allRaw []RawPage
 	var allWarnings []string
 
 	for _, chunk := range chunks {
-		userPrompt := BuildPrompt(schema, source, chunk, inventory)
+		userPrompt := BuildPrompt(schema, source, chunk, inventory, existingPage)
 		output, err := cfg.Complete(ctx, systemPrompt, userPrompt)
 		if err != nil {
 			return Result{}, fmt.Errorf("LLM call: %w", err)
@@ -168,6 +175,16 @@ func hashBody(content string) string {
 	return hex.EncodeToString(sum[:])
 }
 
+// readSourcePage returns the full on-disk content of the source page at
+// wiki/sources/<sourceSlug>.md, or "" when it doesn't exist yet.
+func readSourcePage(brainDir, sourceSlug string) string {
+	raw, err := os.ReadFile(filepath.Join(brainDir, "wiki", "sources", sourceSlug+".md"))
+	if err != nil {
+		return ""
+	}
+	return string(raw)
+}
+
 // storedSourceHash reads source_hash from the source page a previous ingest
 // wrote, or "" when there is no such page or no such field.
 //
@@ -175,11 +192,7 @@ func hashBody(content string) string {
 // skip costs one extraction, while a wrong skip loses an update silently — and
 // silence is the failure mode this repo keeps paying for.
 func storedSourceHash(brainDir, sourceSlug string) string {
-	raw, err := os.ReadFile(filepath.Join(brainDir, "wiki", "sources", sourceSlug+".md"))
-	if err != nil {
-		return ""
-	}
-	for _, line := range strings.Split(string(raw), "\n") {
+	for _, line := range strings.Split(readSourcePage(brainDir, sourceSlug), "\n") {
 		if key, value, found := strings.Cut(line, ":"); found && strings.TrimSpace(key) == "source_hash" {
 			return strings.TrimSpace(value)
 		}

ingestion/internal/pipeline/pipeline_test.go

@@ -277,6 +277,104 @@ func TestRun_ReExtractsWhenTheSourceBodyChanged(t *testing.T) {
 	assert.Equal(t, 2, calls, "changed content must be re-extracted")
 }
 
+// --- #106: page-aware extraction — the model sees the page it is revising --
+
+func TestRun_ExistingSourcePageIsIncludedInTheExtractionPrompt(t *testing.T) {
+	// #103 made the merge asymmetric (on-disk wins); #106 is the other half —
+	// the extraction itself gets to see what's already there instead of
+	// producing a page blind and letting the merge reconcile it afterwards.
+	// This only proves the existing page reaches the prompt; the byte-identical
+	// guarantee for an echoed-back revision is a separate test below.
+	dir := t.TempDir()
+	for _, sub := range []string{"wiki/concepts", "wiki/entities", "wiki/sources"} {
+		require.NoError(t, os.MkdirAll(filepath.Join(dir, sub), 0o755))
+	}
+
+	seeded := "---\ntitle: 'Shape Up'\n---\n\n## Summary\n\nA book about shaping product work.\n\n## Key Claims\n\n- Appetite beats estimation.\n"
+	page := filepath.Join(dir, "wiki", "sources", "shape-up.md")
+	require.NoError(t, os.WriteFile(page, []byte(seeded), 0o644))
+
+	var gotPrompt string
+	cfg := Config{Complete: func(_ context.Context, _, user string) (string, error) {
+		gotPrompt = user
+		return mustJSON([]RawPage{{
+			Title: "Shape Up", Type: "source", Subtype: "book",
+			Content: "## Summary\n\nA book about shaping product work.\n",
+		}}), nil
+	}}
+
+	// Deliberately does not mention "Appetite beats estimation." — that phrase
+	// can only reach the prompt via the existing on-disk page, not the source
+	// content, so its presence proves the page was threaded through.
+	content := "A short editorial update to the book's listing. No new claims here."
+	_, err := Run(context.Background(), cfg, dir, content, "shape-up", false)
+	require.NoError(t, err)
+
+	assert.Contains(t, gotPrompt, "Appetite beats estimation.",
+		"the existing page's content must be visible to the extraction prompt")
+	assert.Contains(t, gotPrompt, "revis", "the prompt must ask for a revision, not a fresh page")
+}
+
+func TestRun_NewSourceHasNoExistingPageInThePrompt(t *testing.T) {
+	// The other side of the same behavior: a source with no page on disk yet
+	// must not get an "existing page" section injected into its prompt at all.
+	dir := t.TempDir()
+	for _, sub := range []string{"wiki/concepts", "wiki/entities", "wiki/sources"} {
+		require.NoError(t, os.MkdirAll(filepath.Join(dir, sub), 0o755))
+	}
+
+	var gotPrompt string
+	cfg := Config{Complete: func(_ context.Context, _, user string) (string, error) {
+		gotPrompt = user
+		return mustJSON([]RawPage{{
+			Title: "New Thing", Type: "source", Subtype: "article",
+			Content: "## Summary\n\nBrand new.\n",
+		}}), nil
+	}}
+
+	_, err := Run(context.Background(), cfg, dir, "Brand new content.", "new-thing", false)
+	require.NoError(t, err)
+
+	assert.NotContains(t, gotPrompt, "Current page",
+		"a source with no page on disk yet must not get a revision section in its prompt")
+}
+
+func TestRun_RevisionEchoingExistingPageStaysByteIdentical(t *testing.T) {
+	// The issue's explicit risk: a model given the current page will sometimes
+	// echo it back verbatim. That must not duplicate content or otherwise
+	// perturb the page — the deterministic merge is still the backstop.
+	dir := t.TempDir()
+	for _, sub := range []string{"wiki/concepts", "wiki/entities", "wiki/sources"} {
+		require.NoError(t, os.MkdirAll(filepath.Join(dir, sub), 0o755))
+	}
+
+	seeded := "---\ntitle: 'Shape Up'\n---\n\n## Summary\n\nA book about shaping product work.\n\n## Key Claims\n\n- Appetite beats estimation.\n"
+	page := filepath.Join(dir, "wiki", "sources", "shape-up.md")
+	require.NoError(t, os.WriteFile(page, []byte(seeded), 0o644))
+
+	cfg := Config{Complete: func(_ context.Context, _, _ string) (string, error) {
+		return mustJSON([]RawPage{{
+			Title: "Shape Up", Type: "source", Subtype: "book",
+			Content: "## Summary\n\nA book about shaping product work.\n\n## Key Claims\n\n- Appetite beats estimation.\n",
+		}}), nil
+	}}
+
+	content := "A short editorial update to the book's listing. No new claims here."
+	result, err := Run(context.Background(), cfg, dir, content, "shape-up", false)
+	require.NoError(t, err)
+	require.False(t, result.Skipped)
+
+	got, err := os.ReadFile(page)
+	require.NoError(t, err)
+
+	want := "---\ntitle: 'Shape Up'\nlast_updated: " + time.Now().UTC().Format("2006-01-02") +
+		"\nsource_hash: " + hashBody(content) + "\n---\n\n" +
+		"## Summary\n\nA book about shaping product work.\n\n" +
+		"## Key Claims\n\n- Appetite beats estimation.\n"
+	assert.Equal(t, want, string(got),
+		"a revision pass that echoes the existing page back must not duplicate or alter its content")
+}
+
 func TestRun_FrontmatterOnlyChangeIsNotDrift(t *testing.T) {
 	// The hash covers the body, so re-fetching a page whose metadata moved
 	// (a re-crawl stamping a new date) is not treated as the source changing.

ingestion/internal/pipeline/prompt.go

@@ -22,8 +22,18 @@ Each element must have exactly these fields:
 Wikilinks in content: [[Display Name]] — just the display name, no slug, no pipe separator.
 Only link to pages listed in the inventory or pages you are creating in this response.`
 
-// BuildPrompt constructs the user prompt for a single chunk.
-func BuildPrompt(schema, source, content string, inventory map[wiki.PageType][]wiki.Entry) string {
+// BuildPrompt constructs the user prompt for a single chunk. existingPage is
+// the full current content (frontmatter + body) of the source's own wiki page
+// when one already exists on disk, or "" for a source being ingested for the
+// first time. When non-empty, the model is asked to revise that page rather
+// than write a fresh one — see hyperguild#106. The deterministic merge in
+// wiki.Merge remains the backstop regardless of what the model returns: an
+// on-disk section the model doesn't touch survives untouched, and a section
+// it does touch only takes effect where Merge's strategy allows the fresh
+// side to change anything (bullet union, append) — never for an ordinary
+// section, so the model proposing a revision is never how a page gets
+// overwritten.
+func BuildPrompt(schema, source, content string, inventory map[wiki.PageType][]wiki.Entry, existingPage string) string {
 	var sb strings.Builder
 
 	fmt.Fprintf(&sb, "Today's date is %s.\n\n", time.Now().UTC().Format("2006-01-02"))
@@ -49,6 +59,18 @@ func BuildPrompt(schema, source, content string, inventory map[wiki.PageType][]w
 		sb.WriteString("\n")
 	}
 
+	if existingPage != "" {
+		sb.WriteString("## Current page for this source — revise it, don't restart\n\n")
+		sb.WriteString("This source has already been ingested and has the wiki page shown below.\n")
+		sb.WriteString("You are revising that page, not writing a fresh one:\n")
+		sb.WriteString("  - Add sections or claims the source below supports that this page doesn't have yet.\n")
+		sb.WriteString("  - Strengthen a claim where the source gives more detail, evidence, or nuance than what's already there.\n")
+		sb.WriteString("  - Leave unchanged whatever the source doesn't affect — reuse the existing wording verbatim rather than paraphrasing it, so an unchanged claim is recognized as unchanged.\n")
+		sb.WriteString("  - A merge step keeps this page's on-disk content for any section you don't return, so it's fine to omit a section you aren't revising.\n\n")
+		sb.WriteString(existingPage)
+		sb.WriteString("\n\n")
+	}
+
 	sb.WriteString("## Non-negotiable rules\n\n")
 	sb.WriteString("1. Output ONLY a valid JSON array — no prose, no fences.\n")
 	sb.WriteString("2. Fields: title, type, subtype (if applicable), domain (if applicable), content.\n")