feat(ingestion): clips land in web-clip by rule; shared concepts/entities; approve parked notes; remove the classifier (#208, #209) #211

Merged
mathias merged 16 commits from wt/web-clip into main 2026-10-20 19:36:51 +00:00
3 changed files with 30 additions and 1 deletions
Showing only changes of commit 90752f2ac9 - Show all commits
+1 -1
View File
@@ -105,7 +105,7 @@ func listNotes(root *os.Root, queue string) ([]PendingNote, error) {
}
out = append(out, PendingNote{
Filename: e.Name(), CreatedAt: fi.ModTime().UTC().Format(time.RFC3339), SizeBytes: fi.Size(),
Excerpt: "(cannot be read: " + readErr.Error() + ")", Queue: queue,
Excerpt: "(cannot be read: " + readErr.Error() + "; promote and reject cannot act on it: delete or move it by hand)", Queue: queue,
})
continue
}
+7
View File
@@ -11,6 +11,7 @@ import (
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
"io/fs"
"path/filepath"
"sort"
"strings"
"time"
@@ -166,6 +167,12 @@ func buildAndWrite(ctx context.Context, wing string, rawPages []RawPage, sourceS
warnings = append(warnings, clashWarnings...)
}
if wing != "" {
// The source page carries source_hash, which the drift check trusts: it
// goes last, so a batch that fails midway leaves nothing to hide behind.
sort.SliceStable(merged, func(i, j int) bool { return !isSourcePath(merged[i].Path) && isSourcePath(merged[j].Path) })
}
var written []string
for _, page := range merged {
if !dryRun {
@@ -372,3 +372,25 @@ func TestRun_ABatchTieBetweenAConceptAndAnEntityPrefersThePageAlreadyOnDisk(t *t
assert.False(t, exists(dir, "wiki/entities/foo.md"))
assert.Contains(t, read(t, dir, "wiki/concepts/foo.md"), "talk", "the existing concept got the new source reference")
}
// Pass 4 of #211: the source page carries the hash the drift check reads. If it
// is written FIRST and a later page fails to write, the next ingest of the same
// content is skipped and the rest of the batch is lost for good. Written LAST,
// a failed batch leaves no hash and is simply redone.
func TestRun_WithAWingAFailedBatchLeavesNoSourcePageToHideBehind(t *testing.T) {
dir := t.TempDir()
// An entity path that cannot be written: a directory where the page goes.
require.NoError(t, os.MkdirAll(filepath.Join(dir, "wiki/entities/jev.md"), 0o755))
_, err := Run(context.Background(), wingCfg("web-clip", sourceEntityConcept()...), dir, "same body", "talk", false)
require.Error(t, err, "the entity write fails")
assert.False(t, exists(dir, "wiki/web-clip/sources/talk.md"),
"the source page (and its hash) must not exist when the batch did not complete")
require.NoError(t, os.RemoveAll(filepath.Join(dir, "wiki/entities/jev.md")))
res, err := Run(context.Background(), wingCfg("web-clip", sourceEntityConcept()...), dir, "same body", "talk", false)
require.NoError(t, err)
assert.False(t, res.Skipped, "the retry is a real ingest, not a hash skip")
assert.True(t, exists(dir, "wiki/entities/jev.md"))
assert.True(t, exists(dir, "wiki/web-clip/sources/talk.md"))
}