fix(watcher): never walk into raw/rejected/ — it was auto-publishing declined notes
Review of PR #133 found a serious bug: the file watcher's processDir walks brain/raw/ with filepath.WalkDir (recurses by default) and only skipped processed/ and failed/ by name. brain_reject moves a declined note to raw/rejected/, which the watcher had no reason to know about -- so on its next tick (30s default), it picked the note back up, ran it through the full LLM extraction pipeline, and published it into the wiki. A human's explicit "not worth promoting" decision was silently reversed by a background job 30 seconds later. Reproduced directly: flip-testing the fix back out shows the watcher's own log line, "file processed file=declined-file.md", proving this wasn't hypothetical. Fixed by adding "rejected" to the same subdirectory skip-list processed/failed already use. Checked the other two consumers of raw/ content for the same class of bug and found neither is affected: graphsync.BackfillFromBrainDir only walks wiki/ and knowledge/; search's resolveRoots only ever returns knowledge/ and wiki/. Neither touches raw/ at all, rejected or not. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -59,10 +59,16 @@ func processDir(ctx context.Context, cfg Config, date string) []error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Skip processed/ and failed/ subdirectories entirely.
|
// Skip processed/, failed/, and rejected/ subdirectories entirely.
|
||||||
|
// rejected/ (hyperguild#81, brain_reject) holds notes a human already
|
||||||
|
// looked at and declined — walking into it would run the extraction
|
||||||
|
// pipeline over a note and publish it to the wiki, the exact opposite
|
||||||
|
// of the human's decision. The whole reason RejectNote moves a note
|
||||||
|
// there instead of leaving it in raw/ was to keep it out of every
|
||||||
|
// path that treats raw/ as "awaiting processing" — this is that path.
|
||||||
if d.IsDir() {
|
if d.IsDir() {
|
||||||
name := d.Name()
|
name := d.Name()
|
||||||
if name == "processed" || name == "failed" {
|
if name == "processed" || name == "failed" || name == "rejected" {
|
||||||
return filepath.SkipDir
|
return filepath.SkipDir
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -176,15 +176,20 @@ func TestDeriveSource(t *testing.T) {
|
|||||||
func TestProcessDir_SkipsSubdirs(t *testing.T) {
|
func TestProcessDir_SkipsSubdirs(t *testing.T) {
|
||||||
brainDir := setupBrainDir(t)
|
brainDir := setupBrainDir(t)
|
||||||
|
|
||||||
// Create processed/ and failed/ subdirs with files inside.
|
// Create processed/, failed/, and rejected/ subdirs with files inside.
|
||||||
for _, sub := range []string{"processed/2026-04-22", "failed"} {
|
for _, sub := range []string{"processed/2026-04-22", "failed", "rejected"} {
|
||||||
require.NoError(t, os.MkdirAll(filepath.Join(brainDir, "raw", sub), 0o755))
|
require.NoError(t, os.MkdirAll(filepath.Join(brainDir, "raw", sub), 0o755))
|
||||||
}
|
}
|
||||||
|
|
||||||
processedFile := filepath.Join(brainDir, "raw", "processed", "2026-04-22", "old-file.md")
|
processedFile := filepath.Join(brainDir, "raw", "processed", "2026-04-22", "old-file.md")
|
||||||
failedFile := filepath.Join(brainDir, "raw", "failed", "broken-file.md")
|
failedFile := filepath.Join(brainDir, "raw", "failed", "broken-file.md")
|
||||||
|
// hyperguild#81/#133: a human already declined this note via brain_reject.
|
||||||
|
// If the watcher walks into rejected/, it re-ingests and publishes it to
|
||||||
|
// the wiki — the exact opposite of the human's decision.
|
||||||
|
rejectedFile := filepath.Join(brainDir, "raw", "rejected", "declined-file.md")
|
||||||
require.NoError(t, os.WriteFile(processedFile, []byte("old"), 0o644))
|
require.NoError(t, os.WriteFile(processedFile, []byte("old"), 0o644))
|
||||||
require.NoError(t, os.WriteFile(failedFile, []byte("broken"), 0o644))
|
require.NoError(t, os.WriteFile(failedFile, []byte("broken"), 0o644))
|
||||||
|
require.NoError(t, os.WriteFile(rejectedFile, []byte("---\nrejected_at: 2026-09-23T00:00:00Z\n---\ndeclined"), 0o644))
|
||||||
|
|
||||||
// Also place a valid file in raw/ root that should be processed.
|
// Also place a valid file in raw/ root that should be processed.
|
||||||
validFile := filepath.Join(brainDir, "raw", "valid.md")
|
validFile := filepath.Join(brainDir, "raw", "valid.md")
|
||||||
@@ -223,11 +228,13 @@ func TestProcessDir_SkipsSubdirs(t *testing.T) {
|
|||||||
// Complete should have been called exactly once (for valid.md, not for files in subdirs).
|
// Complete should have been called exactly once (for valid.md, not for files in subdirs).
|
||||||
assert.Len(t, processedSources, 1, "only the file in raw/ root should be processed")
|
assert.Len(t, processedSources, 1, "only the file in raw/ root should be processed")
|
||||||
|
|
||||||
// Files in processed/ and failed/ must remain untouched.
|
// Files in processed/, failed/, and rejected/ must remain untouched.
|
||||||
_, err := os.Stat(processedFile)
|
_, err := os.Stat(processedFile)
|
||||||
assert.NoError(t, err, "processed subdir file should be untouched")
|
assert.NoError(t, err, "processed subdir file should be untouched")
|
||||||
_, err = os.Stat(failedFile)
|
_, err = os.Stat(failedFile)
|
||||||
assert.NoError(t, err, "failed subdir file should be untouched")
|
assert.NoError(t, err, "failed subdir file should be untouched")
|
||||||
|
_, err = os.Stat(rejectedFile)
|
||||||
|
assert.NoError(t, err, "rejected subdir file should be untouched, never re-ingested")
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestProcessDir_SkipsTunnelCandidateFiles guards against hyperguild#88:
|
// TestProcessDir_SkipsTunnelCandidateFiles guards against hyperguild#88:
|
||||||
|
|||||||
Reference in New Issue
Block a user