fix(pipeline,api): write the source page last; say how to clear an unreadable note (review of #211, pass 4)
CI / Lint / Test / Vet (pull_request) Successful in 1m3s
CI / Mirror to GitHub (pull_request) Skipped

The source page carries source_hash, which the drift check trusts. Written first,
a batch that failed midway left a hash behind, so re-ingesting the same content
was skipped and the rest of the batch was lost for good. In a wing it is now
written last. An over-cap note in brain_pending says promote/reject cannot act
on it and to delete or move it by hand. Mutation-checked.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-20 21:00:18 +02:00
co-authored by Claude Sonnet 5.5
parent 27b1579b80
commit 90752f2ac9
3 changed files with 30 additions and 1 deletions
+1 -1
View File
@@ -105,7 +105,7 @@ func listNotes(root *os.Root, queue string) ([]PendingNote, error) {
} }
out = append(out, PendingNote{ out = append(out, PendingNote{
Filename: e.Name(), CreatedAt: fi.ModTime().UTC().Format(time.RFC3339), SizeBytes: fi.Size(), Filename: e.Name(), CreatedAt: fi.ModTime().UTC().Format(time.RFC3339), SizeBytes: fi.Size(),
Excerpt: "(cannot be read: " + readErr.Error() + ")", Queue: queue, Excerpt: "(cannot be read: " + readErr.Error() + "; promote and reject cannot act on it: delete or move it by hand)", Queue: queue,
}) })
continue continue
} }
+7
View File
@@ -11,6 +11,7 @@ import (
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs" "github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
"io/fs" "io/fs"
"path/filepath" "path/filepath"
"sort"
"strings" "strings"
"time" "time"
@@ -166,6 +167,12 @@ func buildAndWrite(ctx context.Context, wing string, rawPages []RawPage, sourceS
warnings = append(warnings, clashWarnings...) warnings = append(warnings, clashWarnings...)
} }
if wing != "" {
// The source page carries source_hash, which the drift check trusts: it
// goes last, so a batch that fails midway leaves nothing to hide behind.
sort.SliceStable(merged, func(i, j int) bool { return !isSourcePath(merged[i].Path) && isSourcePath(merged[j].Path) })
}
var written []string var written []string
for _, page := range merged { for _, page := range merged {
if !dryRun { if !dryRun {
@@ -372,3 +372,25 @@ func TestRun_ABatchTieBetweenAConceptAndAnEntityPrefersThePageAlreadyOnDisk(t *t
assert.False(t, exists(dir, "wiki/entities/foo.md")) assert.False(t, exists(dir, "wiki/entities/foo.md"))
assert.Contains(t, read(t, dir, "wiki/concepts/foo.md"), "talk", "the existing concept got the new source reference") assert.Contains(t, read(t, dir, "wiki/concepts/foo.md"), "talk", "the existing concept got the new source reference")
} }
// Pass 4 of #211: the source page carries the hash the drift check reads. If it
// is written FIRST and a later page fails to write, the next ingest of the same
// content is skipped and the rest of the batch is lost for good. Written LAST,
// a failed batch leaves no hash and is simply redone.
func TestRun_WithAWingAFailedBatchLeavesNoSourcePageToHideBehind(t *testing.T) {
dir := t.TempDir()
// An entity path that cannot be written: a directory where the page goes.
require.NoError(t, os.MkdirAll(filepath.Join(dir, "wiki/entities/jev.md"), 0o755))
_, err := Run(context.Background(), wingCfg("web-clip", sourceEntityConcept()...), dir, "same body", "talk", false)
require.Error(t, err, "the entity write fails")
assert.False(t, exists(dir, "wiki/web-clip/sources/talk.md"),
"the source page (and its hash) must not exist when the batch did not complete")
require.NoError(t, os.RemoveAll(filepath.Join(dir, "wiki/entities/jev.md")))
res, err := Run(context.Background(), wingCfg("web-clip", sourceEntityConcept()...), dir, "same body", "talk", false)
require.NoError(t, err)
assert.False(t, res.Skipped, "the retry is a real ingest, not a hash skip")
assert.True(t, exists(dir, "wiki/entities/jev.md"))
assert.True(t, exists(dir, "wiki/web-clip/sources/talk.md"))
}