mathias
a5a328fea5
Ingest now stamps source_hash — sha256 of the ingested body, frontmatter excluded — onto the source page, and compares it before doing any work. An unchanged source returns Skipped with no LLM call at all. This is cost and signal, not correctness: #103 already made a re-ingest byte-identical, so a repeated run was harmless but not free — a full extraction over every chunk to produce a result that merges to nothing. Building this first would have put idempotency in the guard rather than in the write, which is the wrong dependency direction. Body-only hashing on purpose. A re-crawl that stamps a new fetch date has not changed the source, and treating that as drift would re-extract the whole corpus every time a crawler ran. A missing or unreadable hash always means "extract": a missed skip costs one extraction, a wrong skip loses an update silently, and silence is the failure mode this repo keeps paying for. The skip is reported (Result.Skipped plus a warning naming the hash), not silent — a run that does nothing must say so, or "the pipeline is fine" and "the pipeline stopped working" look identical (#107). Storing the hash required settling who owns which frontmatter field. #103 made the on-disk block win wholesale, which protects human-set metadata from a regenerating writer but froze fields the generator legitimately owns. wiki.Merge now refreshes an explicit generatorOwned allowlist — last_updated and source_hash — and keeps everything else on disk. An allowlist rather than a carve-out, so a field nobody has thought about stays human-owned by default. That also retires the last_updated limitation #103 shipped with. Refs #105, #103, #107, brain#21.
296 lines
11 KiB
Go
296 lines
11 KiB
Go
// ingestion/internal/pipeline/pipeline_test.go
|
|
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/mathiasbq/hyperguild/ingestion/internal/llm"
|
|
)
|
|
|
|
func TestRun_WritesPages(t *testing.T) {
|
|
brainDir := t.TempDir()
|
|
for _, sub := range []string{"wiki/concepts", "wiki/entities", "wiki/sources"} {
|
|
require.NoError(t, os.MkdirAll(filepath.Join(brainDir, sub), 0o755))
|
|
}
|
|
|
|
llmResponse := mustJSON([]RawPage{
|
|
{
|
|
Title: "Test Article",
|
|
Type: "source",
|
|
Subtype: "article",
|
|
Domain: "software-engineering",
|
|
Content: "## Summary\n\nA test article.\n\n## Key Claims\n\n- It tests things.\n\n## Concepts Introduced or Reinforced\n\n[[Testing]]\n\n## Entities Mentioned\n\n## Open Questions Raised\n",
|
|
},
|
|
{
|
|
Title: "Testing",
|
|
Type: "concept",
|
|
Domain: "software-engineering",
|
|
Content: "## Definition\n\nThe practice of verifying software.\n\n## Why It Matters\n\nCatches bugs.\n\n## Related Concepts\n\n## Related Entities\n\n## Sources\n\n## Evolving Notes\n",
|
|
},
|
|
})
|
|
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(map[string]any{
|
|
"choices": []map[string]any{
|
|
{"message": map[string]any{"role": "assistant", "content": llmResponse}},
|
|
},
|
|
})
|
|
}))
|
|
defer srv.Close()
|
|
|
|
cfg := Config{
|
|
Complete: llm.New(srv.URL, "", "test-model", 30*time.Second).Complete,
|
|
ChunkSize: 0,
|
|
}
|
|
|
|
result, err := Run(context.Background(), cfg, brainDir, "An article about testing.", "test-article", false)
|
|
require.NoError(t, err)
|
|
assert.Len(t, result.Pages, 2)
|
|
|
|
_, err = os.Stat(filepath.Join(brainDir, "wiki", "sources", "test-article.md"))
|
|
require.NoError(t, err)
|
|
_, err = os.Stat(filepath.Join(brainDir, "wiki", "concepts", "testing.md"))
|
|
require.NoError(t, err)
|
|
_, err = os.Stat(filepath.Join(brainDir, "wiki", "index.md"))
|
|
require.NoError(t, err)
|
|
_, err = os.Stat(filepath.Join(brainDir, "log.md"))
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
func TestRun_DryRunDoesNotWrite(t *testing.T) {
|
|
brainDir := t.TempDir()
|
|
for _, sub := range []string{"wiki/concepts", "wiki/entities", "wiki/sources"} {
|
|
require.NoError(t, os.MkdirAll(filepath.Join(brainDir, sub), 0o755))
|
|
}
|
|
|
|
llmResponse := mustJSON([]RawPage{{
|
|
Title: "Foo",
|
|
Type: "source",
|
|
Subtype: "article",
|
|
Content: "## Summary\n\nFoo.\n",
|
|
}})
|
|
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
_ = json.NewEncoder(w).Encode(map[string]any{
|
|
"choices": []map[string]any{{"message": map[string]any{"content": llmResponse}}},
|
|
})
|
|
}))
|
|
defer srv.Close()
|
|
|
|
cfg := Config{Complete: llm.New(srv.URL, "", "m", 30*time.Second).Complete}
|
|
result, err := Run(context.Background(), cfg, brainDir, "foo content", "foo", true)
|
|
require.NoError(t, err)
|
|
assert.Len(t, result.Pages, 1)
|
|
|
|
_, err = os.Stat(filepath.Join(brainDir, "wiki", "sources", "foo.md"))
|
|
assert.True(t, os.IsNotExist(err))
|
|
}
|
|
|
|
func TestRun_MergesDuplicatePaths(t *testing.T) {
|
|
brainDir := t.TempDir()
|
|
for _, sub := range []string{"wiki/concepts", "wiki/entities", "wiki/sources"} {
|
|
require.NoError(t, os.MkdirAll(filepath.Join(brainDir, sub), 0o755))
|
|
}
|
|
|
|
// LLM returns same title twice (simulates multi-chunk duplicate)
|
|
llmResponse := mustJSON([]RawPage{
|
|
{Title: "Foo", Type: "concept", Content: "## Definition\n\nFirst.\n\n## Related Concepts\n\n[[Bar]]\n"},
|
|
{Title: "Foo", Type: "concept", Content: "## Definition\n\nSecond.\n\n## Related Concepts\n\n[[Baz]]\n"},
|
|
})
|
|
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
_ = json.NewEncoder(w).Encode(map[string]any{
|
|
"choices": []map[string]any{{"message": map[string]any{"content": llmResponse}}},
|
|
})
|
|
}))
|
|
defer srv.Close()
|
|
|
|
cfg := Config{Complete: llm.New(srv.URL, "", "m", 30*time.Second).Complete}
|
|
result, err := Run(context.Background(), cfg, brainDir, "content", "foo", false)
|
|
require.NoError(t, err)
|
|
assert.Len(t, result.Pages, 1) // deduplicated
|
|
|
|
content, err := os.ReadFile(filepath.Join(brainDir, "wiki", "concepts", "foo.md"))
|
|
require.NoError(t, err)
|
|
// keep-first for Definition, union for Related Concepts
|
|
assert.Contains(t, string(content), "First.")
|
|
// Bar and Baz unknown in empty inventory → left as plain [[links]]
|
|
assert.Contains(t, string(content), "[[Bar]]")
|
|
assert.Contains(t, string(content), "[[Baz]]")
|
|
}
|
|
|
|
func TestRun_ThreadsSourceAuthorPublishedFromContentFrontmatter(t *testing.T) {
|
|
brainDir := t.TempDir()
|
|
for _, sub := range []string{"wiki/concepts", "wiki/entities", "wiki/sources"} {
|
|
require.NoError(t, os.MkdirAll(filepath.Join(brainDir, sub), 0o755))
|
|
}
|
|
|
|
llmResponse := mustJSON([]RawPage{{
|
|
Title: "Ornith",
|
|
Type: "source",
|
|
Subtype: "article",
|
|
Content: "## Summary\n\nAn agentic coding model.\n",
|
|
}})
|
|
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
_ = json.NewEncoder(w).Encode(map[string]any{
|
|
"choices": []map[string]any{{"message": map[string]any{"content": llmResponse}}},
|
|
})
|
|
}))
|
|
defer srv.Close()
|
|
|
|
cfg := Config{Complete: llm.New(srv.URL, "", "m", 30*time.Second).Complete}
|
|
rawContent := "---\nsource: https://example.com/ornith\nauthor: Jane Doe\npublished: 2026-07-20\n---\n\nAn agentic coding model that runs on your laptop.\n"
|
|
|
|
result, err := Run(context.Background(), cfg, brainDir, rawContent, "ornith", false)
|
|
require.NoError(t, err)
|
|
require.Len(t, result.Pages, 1)
|
|
|
|
content, err := os.ReadFile(filepath.Join(brainDir, "wiki", "sources", "ornith.md"))
|
|
require.NoError(t, err)
|
|
assert.Contains(t, string(content), "source: 'https://example.com/ornith'")
|
|
assert.Contains(t, string(content), "author: 'Jane Doe'")
|
|
assert.Contains(t, string(content), "published: '2026-07-20'")
|
|
}
|
|
|
|
func mustJSON(v any) string {
|
|
b, err := json.Marshal(v)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
return string(b)
|
|
}
|
|
|
|
// --- B1: compile-on-ingest — revise the existing page, never clobber it -----
|
|
|
|
func TestRunRaw_MergesIntoAnExistingPageInsteadOfOverwritingIt(t *testing.T) {
|
|
// Before this, buildAndWrite did an unconditional os.WriteFile: re-ingesting
|
|
// a source replaced its page wholesale, discarding anything accumulated
|
|
// since — including human edits. hyperguild#43 recorded the symptom as
|
|
// "re-ingest overwrites wholesale".
|
|
dir := t.TempDir()
|
|
page := filepath.Join(dir, "wiki", "sources", "shape-up.md")
|
|
require.NoError(t, os.MkdirAll(filepath.Dir(page), 0o755))
|
|
require.NoError(t, os.WriteFile(page, []byte(`---
|
|
title: 'Shape Up'
|
|
---
|
|
|
|
## Summary
|
|
|
|
A book about shaping product work.
|
|
|
|
## My Take
|
|
|
|
Read this before the next planning cycle.
|
|
`), 0o644))
|
|
|
|
_, err := RunRaw(dir, "shape-up", []RawPage{{
|
|
Title: "Shape Up", Type: "source", Subtype: "book", Domain: "product-strategy",
|
|
Content: "## Summary\n\nA book about shaping product work.\n\n## Key Claims\n\n- Appetite beats estimation.\n",
|
|
}}, false)
|
|
require.NoError(t, err)
|
|
|
|
got, err := os.ReadFile(page)
|
|
require.NoError(t, err)
|
|
s := string(got)
|
|
|
|
assert.Contains(t, s, "## My Take", "a section the extraction never saw must survive the merge")
|
|
assert.Contains(t, s, "Read this before the next planning cycle.")
|
|
assert.Contains(t, s, "Appetite beats estimation.", "newly extracted material must still land")
|
|
}
|
|
|
|
func TestRunRaw_ReIngestingUnchangedContentIsByteIdentical(t *testing.T) {
|
|
// Compile-on-ingest runs on every ingest, so "nothing changed" must produce
|
|
// no diff at all. Otherwise every re-ingest dirties the tree, brain-sync
|
|
// commits it, and the git history fills with noise that hides real edits.
|
|
dir := t.TempDir()
|
|
raw := []RawPage{{
|
|
Title: "Shape Up", Type: "source", Subtype: "book", Domain: "product-strategy",
|
|
Content: "## Summary\n\nA book about shaping product work.\n\n## Open Questions\n\n- Does appetite scale?\n",
|
|
}}
|
|
|
|
_, err := RunRaw(dir, "shape-up", raw, false)
|
|
require.NoError(t, err)
|
|
first, err := os.ReadFile(filepath.Join(dir, "wiki", "sources", "shape-up.md"))
|
|
require.NoError(t, err)
|
|
|
|
_, err = RunRaw(dir, "shape-up", raw, false)
|
|
require.NoError(t, err)
|
|
second, err := os.ReadFile(filepath.Join(dir, "wiki", "sources", "shape-up.md"))
|
|
require.NoError(t, err)
|
|
|
|
assert.Equal(t, string(first), string(second),
|
|
"re-ingesting unchanged content must leave the page byte-identical")
|
|
}
|
|
|
|
// --- #105: source drift — an unchanged re-ingest must not re-extract --------
|
|
|
|
func TestRun_SkipsExtractionWhenTheSourceIsUnchanged(t *testing.T) {
|
|
// #103 made a re-ingest harmless (byte-identical merge). It is still not
|
|
// free: the full LLM extraction runs over every chunk to produce a result
|
|
// that merges to nothing. This is the cost half.
|
|
dir := t.TempDir()
|
|
var calls int
|
|
cfg := Config{Complete: func(_ context.Context, _, _ string) (string, error) {
|
|
calls++
|
|
return `[{"title":"Shape Up","type":"source","subtype":"book","domain":"product","content":"## Summary\n\nA book.\n"}]`, nil
|
|
}}
|
|
|
|
content := "---\nsource: https://example.com/shape-up\n---\n\nThe body of the article.\n"
|
|
|
|
first, err := Run(context.Background(), cfg, dir, content, "shape-up", false)
|
|
require.NoError(t, err)
|
|
require.Equal(t, 1, calls, "first ingest must extract")
|
|
assert.False(t, first.Skipped)
|
|
|
|
second, err := Run(context.Background(), cfg, dir, content, "shape-up", false)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 1, calls, "unchanged content must not be extracted again")
|
|
assert.True(t, second.Skipped, "the skip must be reported, not silent")
|
|
}
|
|
|
|
func TestRun_ReExtractsWhenTheSourceBodyChanged(t *testing.T) {
|
|
dir := t.TempDir()
|
|
var calls int
|
|
cfg := Config{Complete: func(_ context.Context, _, _ string) (string, error) {
|
|
calls++
|
|
return `[{"title":"Shape Up","type":"source","subtype":"book","domain":"product","content":"## Summary\n\nA book.\n"}]`, nil
|
|
}}
|
|
|
|
_, err := Run(context.Background(), cfg, dir, "---\nsource: x\n---\n\nversion one\n", "shape-up", false)
|
|
require.NoError(t, err)
|
|
_, err = Run(context.Background(), cfg, dir, "---\nsource: x\n---\n\nversion two — the source was edited\n", "shape-up", false)
|
|
require.NoError(t, err)
|
|
|
|
assert.Equal(t, 2, calls, "changed content must be re-extracted")
|
|
}
|
|
|
|
func TestRun_FrontmatterOnlyChangeIsNotDrift(t *testing.T) {
|
|
// The hash covers the body, so re-fetching a page whose metadata moved
|
|
// (a re-crawl stamping a new date) is not treated as the source changing.
|
|
dir := t.TempDir()
|
|
var calls int
|
|
cfg := Config{Complete: func(_ context.Context, _, _ string) (string, error) {
|
|
calls++
|
|
return `[{"title":"Shape Up","type":"source","subtype":"book","domain":"product","content":"## Summary\n\nA book.\n"}]`, nil
|
|
}}
|
|
|
|
_, err := Run(context.Background(), cfg, dir, "---\nsource: x\nfetched: monday\n---\n\nsame body\n", "shape-up", false)
|
|
require.NoError(t, err)
|
|
_, err = Run(context.Background(), cfg, dir, "---\nsource: x\nfetched: tuesday\n---\n\nsame body\n", "shape-up", false)
|
|
require.NoError(t, err)
|
|
|
|
assert.Equal(t, 1, calls, "only the body counts as the source")
|
|
}
|