fix(pipeline,api): write the source page last; say how to clear an unreadable note (review of #211, pass 4)
The source page carries source_hash, which the drift check trusts. Written first, a batch that failed midway left a hash behind, so re-ingesting the same content was skipped and the rest of the batch was lost for good. In a wing it is now written last. An over-cap note in brain_pending says promote/reject cannot act on it and to delete or move it by hand. Mutation-checked. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -105,7 +105,7 @@ func listNotes(root *os.Root, queue string) ([]PendingNote, error) {
|
|||||||
}
|
}
|
||||||
out = append(out, PendingNote{
|
out = append(out, PendingNote{
|
||||||
Filename: e.Name(), CreatedAt: fi.ModTime().UTC().Format(time.RFC3339), SizeBytes: fi.Size(),
|
Filename: e.Name(), CreatedAt: fi.ModTime().UTC().Format(time.RFC3339), SizeBytes: fi.Size(),
|
||||||
Excerpt: "(cannot be read: " + readErr.Error() + ")", Queue: queue,
|
Excerpt: "(cannot be read: " + readErr.Error() + "; promote and reject cannot act on it: delete or move it by hand)", Queue: queue,
|
||||||
})
|
})
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
|
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -166,6 +167,12 @@ func buildAndWrite(ctx context.Context, wing string, rawPages []RawPage, sourceS
|
|||||||
warnings = append(warnings, clashWarnings...)
|
warnings = append(warnings, clashWarnings...)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if wing != "" {
|
||||||
|
// The source page carries source_hash, which the drift check trusts: it
|
||||||
|
// goes last, so a batch that fails midway leaves nothing to hide behind.
|
||||||
|
sort.SliceStable(merged, func(i, j int) bool { return !isSourcePath(merged[i].Path) && isSourcePath(merged[j].Path) })
|
||||||
|
}
|
||||||
|
|
||||||
var written []string
|
var written []string
|
||||||
for _, page := range merged {
|
for _, page := range merged {
|
||||||
if !dryRun {
|
if !dryRun {
|
||||||
|
|||||||
@@ -372,3 +372,25 @@ func TestRun_ABatchTieBetweenAConceptAndAnEntityPrefersThePageAlreadyOnDisk(t *t
|
|||||||
assert.False(t, exists(dir, "wiki/entities/foo.md"))
|
assert.False(t, exists(dir, "wiki/entities/foo.md"))
|
||||||
assert.Contains(t, read(t, dir, "wiki/concepts/foo.md"), "talk", "the existing concept got the new source reference")
|
assert.Contains(t, read(t, dir, "wiki/concepts/foo.md"), "talk", "the existing concept got the new source reference")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Pass 4 of #211: the source page carries the hash the drift check reads. If it
|
||||||
|
// is written FIRST and a later page fails to write, the next ingest of the same
|
||||||
|
// content is skipped and the rest of the batch is lost for good. Written LAST,
|
||||||
|
// a failed batch leaves no hash and is simply redone.
|
||||||
|
func TestRun_WithAWingAFailedBatchLeavesNoSourcePageToHideBehind(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
// An entity path that cannot be written: a directory where the page goes.
|
||||||
|
require.NoError(t, os.MkdirAll(filepath.Join(dir, "wiki/entities/jev.md"), 0o755))
|
||||||
|
|
||||||
|
_, err := Run(context.Background(), wingCfg("web-clip", sourceEntityConcept()...), dir, "same body", "talk", false)
|
||||||
|
require.Error(t, err, "the entity write fails")
|
||||||
|
assert.False(t, exists(dir, "wiki/web-clip/sources/talk.md"),
|
||||||
|
"the source page (and its hash) must not exist when the batch did not complete")
|
||||||
|
|
||||||
|
require.NoError(t, os.RemoveAll(filepath.Join(dir, "wiki/entities/jev.md")))
|
||||||
|
res, err := Run(context.Background(), wingCfg("web-clip", sourceEntityConcept()...), dir, "same body", "talk", false)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.False(t, res.Skipped, "the retry is a real ingest, not a hash skip")
|
||||||
|
assert.True(t, exists(dir, "wiki/entities/jev.md"))
|
||||||
|
assert.True(t, exists(dir, "wiki/web-clip/sources/talk.md"))
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user