fix(watcher): never re-ingest AutoTunnel's own tunnel-candidates log (#88)
Root cause was neither of the two mechanisms the issue named (CanonicalizeLinks over-canonicalizing, or a schema/prompt bias) — CanonicalizeLinks correctly resolved [[Mission]] because wiki/concepts/mission.md genuinely exists; the extraction prompt has no "mission" bias either. The real bug: watcher.processDir walks brain/raw/ and feeds every .md/.txt/.pdf file to pipeline.Run as source material to extract, with no exclusion for tunnel-candidates-*.md — AutoTunnel's own fuzzy-match human-review queue (logFuzzyCandidates writes it into that same raw/ directory). The watcher's next poll re-ingested the raw candidate log itself, and the LLM naturally surfaced "mission" as a wikilink because the log literally lists `(term: "mission")` entries in its content — a generic-word title match DetectTunnels correctly flagged as fuzzy (not auto-tunneled) but which still leaked downstream once treated as real source material. api.ListPending already excludes tunnel-candidates-* when listing raw/ for promotion; processDir gets the same exclusion. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Roq1ajWKR5f1hG5Df9wC6A
This commit is contained in:
@@ -74,6 +74,13 @@ func processDir(ctx context.Context, cfg Config, date string) []error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// AutoTunnel's own fuzzy-match human-review queue, not source
|
||||
// material to extract (hyperguild#88) — same exclusion as
|
||||
// api.ListPending already applies when listing raw/ for promotion.
|
||||
if strings.HasPrefix(d.Name(), "tunnel-candidates-") {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Skip files that have already been processed or permanently failed.
|
||||
if _, err := os.Stat(path + ".processed"); err == nil {
|
||||
return nil
|
||||
|
||||
@@ -229,3 +229,49 @@ func TestProcessDir_SkipsSubdirs(t *testing.T) {
|
||||
_, err = os.Stat(failedFile)
|
||||
assert.NoError(t, err, "failed subdir file should be untouched")
|
||||
}
|
||||
|
||||
// TestProcessDir_SkipsTunnelCandidateFiles guards against hyperguild#88:
|
||||
// AutoTunnel's own human-review queue (brain/raw/tunnel-candidates-*.md)
|
||||
// must never be re-ingested as if it were external source material — doing
|
||||
// so fed the raw "(term: X)" log entries back through the LLM extraction
|
||||
// pipeline, which then legitimately (from its own perspective) surfaced
|
||||
// [[X]] as a wikilink for any term that happened to match a real page
|
||||
// title, however generic (e.g. "mission").
|
||||
func TestProcessDir_SkipsTunnelCandidateFiles(t *testing.T) {
|
||||
brainDir := setupBrainDir(t)
|
||||
|
||||
tunnelFile := filepath.Join(brainDir, "raw", "tunnel-candidates-2026-07-19.md")
|
||||
require.NoError(t, os.WriteFile(tunnelFile, []byte(
|
||||
"# Tunnel candidates 2026-07-19\n\n- `wiki/homelab/decisions/foo.md` ↔ `wiki/telos/decisions/mission.md` (term: \"mission\")\n",
|
||||
), 0o644))
|
||||
|
||||
var completeCalls int
|
||||
completeFn := func(ctx context.Context, system, user string) (string, error) {
|
||||
completeCalls++
|
||||
raw := pipeline.RawPage{Title: "Should not be written", Type: "source", Subtype: "article", Content: "## Summary\n\nx.\n"}
|
||||
b, _ := json.Marshal([]pipeline.RawPage{raw})
|
||||
return string(b), nil
|
||||
}
|
||||
|
||||
cfg := Config{
|
||||
BrainDir: brainDir,
|
||||
Interval: time.Hour, // not used; we call processDir directly
|
||||
Pipeline: pipeline.Config{
|
||||
Complete: completeFn,
|
||||
ChunkSize: 0,
|
||||
Schema: "# Schema\nThree page types.",
|
||||
},
|
||||
}
|
||||
|
||||
date := time.Now().UTC().Format("2006-01-02")
|
||||
errs := processDir(context.Background(), cfg, date)
|
||||
assert.Empty(t, errs)
|
||||
|
||||
assert.Zero(t, completeCalls, "tunnel-candidates file must never reach the LLM extraction pipeline")
|
||||
|
||||
// File must be left alone in raw/ — not moved to processed/, no marker written.
|
||||
_, err := os.Stat(tunnelFile + ".processed")
|
||||
assert.True(t, os.IsNotExist(err), "tunnel-candidates file should not get a .processed marker")
|
||||
_, err = os.Stat(filepath.Join(brainDir, "raw", "processed", date, "tunnel-candidates-2026-07-19.md"))
|
||||
assert.True(t, os.IsNotExist(err), "tunnel-candidates file should not be copied to processed/")
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user