feat(pipeline): feed the current source page into the extraction prompt
#103 made ingest merge into the page already on disk instead of clobbering it, but the extraction itself stayed page-blind: the model wrote every page as if fresh, and Merge reconciled the two texts afterwards by heading. It could restate a claim but never refine it, and had no way to know what already existed. BuildPrompt now accepts the source page's current on-disk content (when one exists) and asks the model to revise it explicitly: add what's missing, strengthen what the source now supports, leave the rest alone and reuse its wording verbatim so unchanged claims are recognized as unchanged. Scoped to the source page specifically — its path (wiki/sources/<sourceSlug>.md) is deterministic before extraction runs, unlike a concept/entity page's path, which only exists once the model has produced a title. The deterministic merge in wiki.Merge is unchanged and stays the backstop: an on-disk section the model doesn't touch survives untouched regardless of what the prompt asks for, so a model that ignores the instruction still cannot destroy human content. Closes hyperguild#106 (first half; heading normalization follows). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -62,13 +62,20 @@ func Run(ctx context.Context, cfg Config, brainDir, content, source string, dryR
|
||||
}}, nil
|
||||
}
|
||||
|
||||
// The source's own page path is deterministic (sourceSlug, computed above)
|
||||
// even before extraction runs, unlike a concept/entity page's path, which
|
||||
// only exists once the model has produced a title. So page-aware
|
||||
// extraction (#106) is scoped to the source page: it's the one page every
|
||||
// chunk's prompt can be given in advance.
|
||||
existingPage := readSourcePage(brainDir, sourceSlug)
|
||||
|
||||
chunks := Chunk(content, cfg.ChunkSize)
|
||||
|
||||
var allRaw []RawPage
|
||||
var allWarnings []string
|
||||
|
||||
for _, chunk := range chunks {
|
||||
userPrompt := BuildPrompt(schema, source, chunk, inventory)
|
||||
userPrompt := BuildPrompt(schema, source, chunk, inventory, existingPage)
|
||||
output, err := cfg.Complete(ctx, systemPrompt, userPrompt)
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("LLM call: %w", err)
|
||||
@@ -160,6 +167,16 @@ func hashBody(content string) string {
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// readSourcePage returns the full on-disk content of the source page at
|
||||
// wiki/sources/<sourceSlug>.md, or "" when it doesn't exist yet.
|
||||
func readSourcePage(brainDir, sourceSlug string) string {
|
||||
raw, err := os.ReadFile(filepath.Join(brainDir, "wiki", "sources", sourceSlug+".md"))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return string(raw)
|
||||
}
|
||||
|
||||
// storedSourceHash reads source_hash from the source page a previous ingest
|
||||
// wrote, or "" when there is no such page or no such field.
|
||||
//
|
||||
@@ -167,11 +184,7 @@ func hashBody(content string) string {
|
||||
// skip costs one extraction, while a wrong skip loses an update silently — and
|
||||
// silence is the failure mode this repo keeps paying for.
|
||||
func storedSourceHash(brainDir, sourceSlug string) string {
|
||||
raw, err := os.ReadFile(filepath.Join(brainDir, "wiki", "sources", sourceSlug+".md"))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
for _, line := range strings.Split(string(raw), "\n") {
|
||||
for _, line := range strings.Split(readSourcePage(brainDir, sourceSlug), "\n") {
|
||||
if key, value, found := strings.Cut(line, ":"); found && strings.TrimSpace(key) == "source_hash" {
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user