From 90752f2ac983d18e59c72a545ff3c25d51d87be3 Mon Sep 17 00:00:00 2001 From: Mathias Date: Tue, 20 Oct 2026 21:00:18 +0200 Subject: [PATCH] fix(pipeline,api): write the source page last; say how to clear an unreadable note (review of #211, pass 4) The source page carries source_hash, which the drift check trusts. Written first, a batch that failed midway left a hash behind, so re-ingesting the same content was skipped and the rest of the batch was lost for good. In a wing it is now written last. An over-cap note in brain_pending says promote/reject cannot act on it and to delete or move it by hand. Mutation-checked. Co-Authored-By: Claude Sonnet 5.5 --- ingestion/internal/api/promote.go | 2 +- ingestion/internal/pipeline/pipeline.go | 7 ++++++ .../internal/pipeline/wing_layout_test.go | 22 +++++++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/ingestion/internal/api/promote.go b/ingestion/internal/api/promote.go index fe9115e..9e489db 100644 --- a/ingestion/internal/api/promote.go +++ b/ingestion/internal/api/promote.go @@ -105,7 +105,7 @@ func listNotes(root *os.Root, queue string) ([]PendingNote, error) { } out = append(out, PendingNote{ Filename: e.Name(), CreatedAt: fi.ModTime().UTC().Format(time.RFC3339), SizeBytes: fi.Size(), - Excerpt: "(cannot be read: " + readErr.Error() + ")", Queue: queue, + Excerpt: "(cannot be read: " + readErr.Error() + "; promote and reject cannot act on it: delete or move it by hand)", Queue: queue, }) continue } diff --git a/ingestion/internal/pipeline/pipeline.go b/ingestion/internal/pipeline/pipeline.go index f166a4a..654f71e 100644 --- a/ingestion/internal/pipeline/pipeline.go +++ b/ingestion/internal/pipeline/pipeline.go @@ -11,6 +11,7 @@ import ( "github.com/mathiasbq/hyperguild/ingestion/internal/brainfs" "io/fs" "path/filepath" + "sort" "strings" "time" @@ -166,6 +167,12 @@ func buildAndWrite(ctx context.Context, wing string, rawPages []RawPage, sourceS warnings = append(warnings, clashWarnings...) } + if wing != "" { + // The source page carries source_hash, which the drift check trusts: it + // goes last, so a batch that fails midway leaves nothing to hide behind. + sort.SliceStable(merged, func(i, j int) bool { return !isSourcePath(merged[i].Path) && isSourcePath(merged[j].Path) }) + } + var written []string for _, page := range merged { if !dryRun { diff --git a/ingestion/internal/pipeline/wing_layout_test.go b/ingestion/internal/pipeline/wing_layout_test.go index fe9915d..17958d0 100644 --- a/ingestion/internal/pipeline/wing_layout_test.go +++ b/ingestion/internal/pipeline/wing_layout_test.go @@ -372,3 +372,25 @@ func TestRun_ABatchTieBetweenAConceptAndAnEntityPrefersThePageAlreadyOnDisk(t *t assert.False(t, exists(dir, "wiki/entities/foo.md")) assert.Contains(t, read(t, dir, "wiki/concepts/foo.md"), "talk", "the existing concept got the new source reference") } + +// Pass 4 of #211: the source page carries the hash the drift check reads. If it +// is written FIRST and a later page fails to write, the next ingest of the same +// content is skipped and the rest of the batch is lost for good. Written LAST, +// a failed batch leaves no hash and is simply redone. +func TestRun_WithAWingAFailedBatchLeavesNoSourcePageToHideBehind(t *testing.T) { + dir := t.TempDir() + // An entity path that cannot be written: a directory where the page goes. + require.NoError(t, os.MkdirAll(filepath.Join(dir, "wiki/entities/jev.md"), 0o755)) + + _, err := Run(context.Background(), wingCfg("web-clip", sourceEntityConcept()...), dir, "same body", "talk", false) + require.Error(t, err, "the entity write fails") + assert.False(t, exists(dir, "wiki/web-clip/sources/talk.md"), + "the source page (and its hash) must not exist when the batch did not complete") + + require.NoError(t, os.RemoveAll(filepath.Join(dir, "wiki/entities/jev.md"))) + res, err := Run(context.Background(), wingCfg("web-clip", sourceEntityConcept()...), dir, "same body", "talk", false) + require.NoError(t, err) + assert.False(t, res.Skipped, "the retry is a real ingest, not a hash skip") + assert.True(t, exists(dir, "wiki/entities/jev.md")) + assert.True(t, exists(dir, "wiki/web-clip/sources/talk.md")) +}