feat(pipeline): feed the current source page into the extraction prompt (#106) (#132)

Co-authored-by: Mathias <mthbqv@gmail.com>

ingestion/internal/pipeline/pipeline.go

@@ -62,13 +62,20 @@ func Run(ctx context.Context, cfg Config, brainDir, content, source string, dryR
 		}}, nil
 	}
 
+	// The source's own page path is deterministic (sourceSlug, computed above)
+	// even before extraction runs, unlike a concept/entity page's path, which
+	// only exists once the model has produced a title. So page-aware
+	// extraction (#106) is scoped to the source page: it's the one page every
+	// chunk's prompt can be given in advance.
+	existingPage := readSourcePage(brainDir, sourceSlug)
+
 	chunks := Chunk(content, cfg.ChunkSize)
 
 	var allRaw []RawPage
 	var allWarnings []string
 
 	for _, chunk := range chunks {
-		userPrompt := BuildPrompt(schema, source, chunk, inventory)
+		userPrompt := BuildPrompt(schema, source, chunk, inventory, existingPage)
 		output, err := cfg.Complete(ctx, systemPrompt, userPrompt)
 		if err != nil {
 			return Result{}, fmt.Errorf("LLM call: %w", err)
@@ -168,6 +175,16 @@ func hashBody(content string) string {
 	return hex.EncodeToString(sum[:])
 }
 
+// readSourcePage returns the full on-disk content of the source page at
+// wiki/sources/<sourceSlug>.md, or "" when it doesn't exist yet.
+func readSourcePage(brainDir, sourceSlug string) string {
+	raw, err := os.ReadFile(filepath.Join(brainDir, "wiki", "sources", sourceSlug+".md"))
+	if err != nil {
+		return ""
+	}
+	return string(raw)
+}
+
 // storedSourceHash reads source_hash from the source page a previous ingest
 // wrote, or "" when there is no such page or no such field.
 //
@@ -175,11 +192,7 @@ func hashBody(content string) string {
 // skip costs one extraction, while a wrong skip loses an update silently — and
 // silence is the failure mode this repo keeps paying for.
 func storedSourceHash(brainDir, sourceSlug string) string {
-	raw, err := os.ReadFile(filepath.Join(brainDir, "wiki", "sources", sourceSlug+".md"))
-	if err != nil {
-		return ""
-	}
-	for _, line := range strings.Split(string(raw), "\n") {
+	for _, line := range strings.Split(readSourcePage(brainDir, sourceSlug), "\n") {
 		if key, value, found := strings.Cut(line, ":"); found && strings.TrimSpace(key) == "source_hash" {
 			return strings.TrimSpace(value)
 		}