feat(pipeline): skip extraction when the source has not changed (#105) (#126)

Co-authored-by: Mathias <mthbqv@gmail.com>

ingestion/internal/pipeline/pipeline_test.go

@@ -233,3 +233,63 @@ func TestRunRaw_ReIngestingUnchangedContentIsByteIdentical(t *testing.T) {
 	assert.Equal(t, string(first), string(second),
 		"re-ingesting unchanged content must leave the page byte-identical")
 }
+
+// --- #105: source drift — an unchanged re-ingest must not re-extract --------
+
+func TestRun_SkipsExtractionWhenTheSourceIsUnchanged(t *testing.T) {
+	// #103 made a re-ingest harmless (byte-identical merge). It is still not
+	// free: the full LLM extraction runs over every chunk to produce a result
+	// that merges to nothing. This is the cost half.
+	dir := t.TempDir()
+	var calls int
+	cfg := Config{Complete: func(_ context.Context, _, _ string) (string, error) {
+		calls++
+		return `[{"title":"Shape Up","type":"source","subtype":"book","domain":"product","content":"## Summary\n\nA book.\n"}]`, nil
+	}}
+
+	content := "---\nsource: https://example.com/shape-up\n---\n\nThe body of the article.\n"
+
+	first, err := Run(context.Background(), cfg, dir, content, "shape-up", false)
+	require.NoError(t, err)
+	require.Equal(t, 1, calls, "first ingest must extract")
+	assert.False(t, first.Skipped)
+
+	second, err := Run(context.Background(), cfg, dir, content, "shape-up", false)
+	require.NoError(t, err)
+	assert.Equal(t, 1, calls, "unchanged content must not be extracted again")
+	assert.True(t, second.Skipped, "the skip must be reported, not silent")
+}
+
+func TestRun_ReExtractsWhenTheSourceBodyChanged(t *testing.T) {
+	dir := t.TempDir()
+	var calls int
+	cfg := Config{Complete: func(_ context.Context, _, _ string) (string, error) {
+		calls++
+		return `[{"title":"Shape Up","type":"source","subtype":"book","domain":"product","content":"## Summary\n\nA book.\n"}]`, nil
+	}}
+
+	_, err := Run(context.Background(), cfg, dir, "---\nsource: x\n---\n\nversion one\n", "shape-up", false)
+	require.NoError(t, err)
+	_, err = Run(context.Background(), cfg, dir, "---\nsource: x\n---\n\nversion two — the source was edited\n", "shape-up", false)
+	require.NoError(t, err)
+
+	assert.Equal(t, 2, calls, "changed content must be re-extracted")
+}
+
+func TestRun_FrontmatterOnlyChangeIsNotDrift(t *testing.T) {
+	// The hash covers the body, so re-fetching a page whose metadata moved
+	// (a re-crawl stamping a new date) is not treated as the source changing.
+	dir := t.TempDir()
+	var calls int
+	cfg := Config{Complete: func(_ context.Context, _, _ string) (string, error) {
+		calls++
+		return `[{"title":"Shape Up","type":"source","subtype":"book","domain":"product","content":"## Summary\n\nA book.\n"}]`, nil
+	}}
+
+	_, err := Run(context.Background(), cfg, dir, "---\nsource: x\nfetched: monday\n---\n\nsame body\n", "shape-up", false)
+	require.NoError(t, err)
+	_, err = Run(context.Background(), cfg, dir, "---\nsource: x\nfetched: tuesday\n---\n\nsame body\n", "shape-up", false)
+	require.NoError(t, err)
+
+	assert.Equal(t, 1, calls, "only the body counts as the source")
+}