From 6e5a9cfce3a9e06b544d774ab17d4d2964225a6c Mon Sep 17 00:00:00 2001 From: Mathias Date: Wed, 16 Sep 2026 23:53:42 +0200 Subject: [PATCH] fix(brain): unwrap wikilinks that name no entry at the write boundary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every brain write accepted [[targets]] that resolve to nothing — skill names, repo names, agent-local memory-file slugs — and each one became a permanent [high] broken-link finding. The brain-audit digest grew 19 -> 63 high findings between 2026-08-26 and 2026-09-16; 53 of the 63 were exactly this, and none are fixable by brain-gardener's BrokenLinkFixer, which only repoints edit-distance-1 renames. There is no target to repoint to. StripUnresolvedLinks resolves each link against knowledge/ + wiki/ using brain-gardener's own resolver semantics (bare slug or wing/hall/slug path form, code spans excluded) and unwraps the misses to plain text, reporting them as Ref.Unlinked / the MCP `unlinked` field. Unwrap rather than reject: a refused write on claudewatcher's unattended ingest path is a lost learning, and the author's words are worth keeping even when the link is not. An empty corpus is a no-op, so a misconfigured brainDir cannot strip every link in a note. Co-Authored-By: Claude Opus 5 --- ingestion/internal/brain/linkcheck.go | 157 ++++++++++++++++++++ ingestion/internal/brain/linkcheck_test.go | 93 ++++++++++++ ingestion/internal/brainstore/store.go | 16 +- ingestion/internal/brainstore/store_test.go | 42 ++++++ ingestion/internal/capture/ports.go | 4 + ingestion/internal/mcp/handlers.go | 25 +++- 6 files changed, 332 insertions(+), 5 deletions(-) create mode 100644 ingestion/internal/brain/linkcheck.go create mode 100644 ingestion/internal/brain/linkcheck_test.go diff --git a/ingestion/internal/brain/linkcheck.go b/ingestion/internal/brain/linkcheck.go new file mode 100644 index 0000000..d81d08b --- /dev/null +++ b/ingestion/internal/brain/linkcheck.go @@ -0,0 +1,157 @@ +package brain + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strings" +) + +// wikilinkRE matches [[target]] and [[target|Display Text]]. +var wikilinkRE = regexp.MustCompile(`\[\[([^\]|]+)(?:\|([^\]]*))?\]\]`) + +// fencedCodeRE and inlineCodeRE mark the regions where a [[...]] is a +// syntax example rather than a link. Same exclusion brain-gardener's audit +// applies on the read side, so the write gate and the audit agree on what +// counts as a link at all. +var ( + fencedCodeRE = regexp.MustCompile("(?s)```.*?```") + inlineCodeRE = regexp.MustCompile("`[^`\n]*`") +) + +// StripUnresolvedLinks unwraps every [[wikilink]] in content whose target +// resolves to no entry under knowledge/ or wiki/, leaving the link's display +// text (or its target) as plain prose. It returns the rewritten content and +// the sorted, de-duplicated targets it unwrapped. +// +// WHY UNWRAP RATHER THAN REJECT THE WRITE +// Agents write [[...]] around anything that feels related — a skill name, a +// repo, an agent-local memory file — and none of those are brain entries. +// Every one of those becomes a permanent [high] broken-link finding that no +// deterministic Fixer can repair, because there is no target to repoint to: +// the brain-audit digest grew from 19 to 63 high findings between 2026-08-26 +// and 2026-09-16, and 53 of the 63 were exactly this. Rejecting the write +// instead would push the failure onto claudewatcher's unattended ingest path, +// where a refused write is a lost learning. Unwrapping keeps the author's +// words, keeps the write, and keeps the graph honest. +// +// An empty corpus (no knowledge/ or wiki/ yet, or a misconfigured brainDir) +// returns content unchanged: nothing resolving is a bootstrap state, not +// evidence that every target was invented. +func StripUnresolvedLinks(brainDir, content string) (string, []string, error) { + index, err := entryIndex(brainDir) + if err != nil { + return "", nil, err + } + if len(index) == 0 { + return content, nil, nil + } + + code := codeSpans(content) + stripped := map[string]struct{}{} + + var b strings.Builder + last := 0 + for _, m := range wikilinkRE.FindAllStringSubmatchIndex(content, -1) { + start, end := m[0], m[1] + if overlaps(start, end, code) { + continue + } + target := strings.TrimSpace(content[m[2]:m[3]]) + if _, ok := index[linkKey(target)]; ok { + continue + } + display := target + if m[4] >= 0 { + if d := strings.TrimSpace(content[m[4]:m[5]]); d != "" { + display = d + } + } + b.WriteString(content[last:start]) + b.WriteString(display) + last = end + stripped[target] = struct{}{} + } + if last == 0 { + return content, nil, nil + } + b.WriteString(content[last:]) + + targets := make([]string, 0, len(stripped)) + for t := range stripped { + targets = append(targets, t) + } + sort.Strings(targets) + return b.String(), targets, nil +} + +// linkedTrees are the two trees that hold linkable entries — the same pair +// package notes admits under ScopeNotes. Duplicated rather than imported +// because that list is unexported there and this is a read-only mirror. +var linkedTrees = []string{"knowledge", "wiki"} + +// entryIndex maps every form a wikilink may name an entry by — its bare slug +// and its path form (wiki///.md -> wing/hall/slug) — to a +// present marker. Mirrors brain-gardener's resolver so an accepted link is +// one the audit will also resolve. +func entryIndex(brainDir string) (map[string]struct{}, error) { + index := map[string]struct{}{} + for _, dir := range linkedTrees { + root := filepath.Join(brainDir, dir) + if _, err := os.Stat(root); os.IsNotExist(err) { + continue + } else if err != nil { + return nil, fmt.Errorf("stat %s: %w", dir, err) + } + err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".md") { + return nil + } + rel, err := filepath.Rel(brainDir, path) + if err != nil { + return err + } + rel = filepath.ToSlash(rel) + index[strings.TrimSuffix(filepath.Base(rel), ".md")] = struct{}{} + index[strings.TrimPrefix(strings.TrimSuffix(rel, ".md"), "wiki/")] = struct{}{} + return nil + }) + if err != nil { + return nil, fmt.Errorf("walk %s: %w", dir, err) + } + } + return index, nil +} + +// linkKey normalises a link target for index lookup: an exact path form is +// tried as written, and anything else falls back to its final segment. +func linkKey(target string) string { + return strings.TrimSuffix(target, ".md") +} + +// codeSpans returns the byte ranges of fenced blocks and inline code spans. +func codeSpans(content string) [][2]int { + var spans [][2]int + for _, m := range fencedCodeRE.FindAllStringIndex(content, -1) { + spans = append(spans, [2]int{m[0], m[1]}) + } + for _, m := range inlineCodeRE.FindAllStringIndex(content, -1) { + spans = append(spans, [2]int{m[0], m[1]}) + } + return spans +} + +// overlaps reports whether [start,end) intersects any span. +func overlaps(start, end int, spans [][2]int) bool { + for _, s := range spans { + if start < s[1] && end > s[0] { + return true + } + } + return false +} diff --git a/ingestion/internal/brain/linkcheck_test.go b/ingestion/internal/brain/linkcheck_test.go new file mode 100644 index 0000000..dd0cd34 --- /dev/null +++ b/ingestion/internal/brain/linkcheck_test.go @@ -0,0 +1,93 @@ +package brain_test + +import ( + "testing" + + "github.com/mathiasbq/hyperguild/ingestion/internal/brain" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// seedCorpus writes the two entries every linkcheck case resolves against. +func seedCorpus(t *testing.T, dir string) { + t.Helper() + seedNote(t, dir, "wiki/homelab/facts/real-entry.md", + "---\nwing: homelab\nhall: facts\ntitle: 'Real entry'\n---\n\nbody.\n") + seedNote(t, dir, "knowledge/legacy-entry.md", + "---\ntitle: 'Legacy entry'\n---\n\nbody.\n") +} + +func TestStripUnresolvedLinks(t *testing.T) { + tests := []struct { + name string + body string + want string + stripped []string + }{ + { + name: "bare slug that resolves is kept", + body: "See [[real-entry]] for the rule.", + want: "See [[real-entry]] for the rule.", + }, + { + name: "path-style target that resolves is kept", + body: "See [[homelab/facts/real-entry]].", + want: "See [[homelab/facts/real-entry]].", + }, + { + name: "knowledge/ entry resolves by bare slug", + body: "See [[legacy-entry]].", + want: "See [[legacy-entry]].", + }, + { + name: "unresolvable target is unwrapped to plain text", + body: "Related: [[secret-hygiene]].", + want: "Related: secret-hygiene.", + stripped: []string{"secret-hygiene"}, + }, + { + name: "unresolvable aliased target keeps its display text", + body: "Related: [[feedback_check_host_not_assume|check the host]].", + want: "Related: check the host.", + stripped: []string{"feedback_check_host_not_assume"}, + }, + { + name: "link inside a fenced code block is left alone", + body: "Syntax:\n\n```\n[[wing/hall/slug]]\n```\n", + want: "Syntax:\n\n```\n[[wing/hall/slug]]\n```\n", + }, + { + name: "link inside an inline code span is left alone", + body: "Write `[[wing/hall/slug]]` to link.", + want: "Write `[[wing/hall/slug]]` to link.", + }, + { + name: "every unresolvable target is reported once, sorted", + body: "[[zeta-gap]] and [[alpha-gap]] and [[zeta-gap]] again.", + want: "zeta-gap and alpha-gap and zeta-gap again.", + stripped: []string{"alpha-gap", "zeta-gap"}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + seedCorpus(t, dir) + + got, stripped, err := brain.StripUnresolvedLinks(dir, tc.body) + require.NoError(t, err) + assert.Equal(t, tc.want, got) + assert.Equal(t, tc.stripped, stripped) + }) + } +} + +func TestStripUnresolvedLinks_MissingBrainDirIsNotAGate(t *testing.T) { + // A brain dir with no knowledge//wiki/ tree yet must not turn every link + // into an unresolved one — an empty corpus is a bootstrap state, not + // evidence that the author invented the target. + got, stripped, err := brain.StripUnresolvedLinks(t.TempDir(), "See [[real-entry]].") + require.NoError(t, err) + assert.Equal(t, "See [[real-entry]].", got) + assert.Nil(t, stripped) +} diff --git a/ingestion/internal/brainstore/store.go b/ingestion/internal/brainstore/store.go index 3666217..f2dcc0e 100644 --- a/ingestion/internal/brainstore/store.go +++ b/ingestion/internal/brainstore/store.go @@ -42,6 +42,12 @@ func (s *Store) WithGraph(g graphsync.Store) *Store { // Write creates a brain note and returns its read-after-write handle. func (s *Store) Write(ctx context.Context, n capture.Note) (capture.Ref, error) { + content, unlinked, err := brain.StripUnresolvedLinks(s.brainDir, n.Content) + if err != nil { + return capture.Ref{}, err + } + n.Content = content + relPath, err := api.WriteNote(s.brainDir, api.WriteNoteOptions{ Content: n.Content, Filename: n.Filename, @@ -57,13 +63,19 @@ func (s *Store) Write(ctx context.Context, n capture.Note) (capture.Ref, error) s.indexInGraph(ctx, "brain_write", relPath) _, _, hash, _ := api.ReadNote(s.brainDir, relPath) - return capture.Ref{ID: relPath, Path: relPath, ContentHash: hash}, nil + return capture.Ref{ID: relPath, Path: relPath, ContentHash: hash, Unlinked: unlinked}, nil } // Update supersedes an existing note in place. slug may be a bare slug // (resolved against n.Wing/n.Hall) or a full brain-relative path (when it // contains a slash). It never creates — a missing target is an error. func (s *Store) Update(ctx context.Context, slug string, n capture.Note) (capture.Ref, error) { + content, unlinked, err := brain.StripUnresolvedLinks(s.brainDir, n.Content) + if err != nil { + return capture.Ref{}, err + } + n.Content = content + opts := api.UpdateNoteOptions{Content: n.Content, Reason: n.Reason} if strings.Contains(slug, "/") { opts.Path = slug @@ -80,7 +92,7 @@ func (s *Store) Update(ctx context.Context, slug string, n capture.Note) (captur } s.indexInGraph(ctx, "brain_update", relPath) - return capture.Ref{ID: relPath, Path: relPath, ContentHash: hash, Superseded: true}, nil + return capture.Ref{ID: relPath, Path: relPath, ContentHash: hash, Superseded: true, Unlinked: unlinked}, nil } // Get fetches a note by id/path — the read-after-write confirmation diff --git a/ingestion/internal/brainstore/store_test.go b/ingestion/internal/brainstore/store_test.go index e2a444b..34a348f 100644 --- a/ingestion/internal/brainstore/store_test.go +++ b/ingestion/internal/brainstore/store_test.go @@ -83,3 +83,45 @@ func TestStoreGetRoundTripsHash(t *testing.T) { assert.Equal(t, "a", note.Frontmatter["wing"]) assert.Contains(t, note.Body, "# Body") } + +func TestStoreWriteUnwrapsUnresolvedWikilinks(t *testing.T) { + dir := t.TempDir() + s := brainstore.New(dir) + _, err := s.Write(context.Background(), capture.Note{ + Content: "# Target\n\nbody\n", Filename: "target", Wing: "a", Hall: "facts", + }) + require.NoError(t, err) + + ref, err := s.Write(context.Background(), capture.Note{ + Content: "See [[target]] and [[secret-hygiene]].\n", + Filename: "src", Wing: "a", Hall: "facts", + }) + require.NoError(t, err) + assert.Equal(t, []string{"secret-hygiene"}, ref.Unlinked) + + got, err := os.ReadFile(filepath.Join(dir, "wiki/a/facts/src.md")) + require.NoError(t, err) + assert.Contains(t, string(got), "[[target]]") + assert.Contains(t, string(got), "and secret-hygiene.") + assert.NotContains(t, string(got), "[[secret-hygiene]]") +} + +func TestStoreUpdateUnwrapsUnresolvedWikilinks(t *testing.T) { + dir := t.TempDir() + s := brainstore.New(dir) + _, err := s.Write(context.Background(), capture.Note{ + Content: "old\n", Filename: "n", Wing: "a", Hall: "facts", + }) + require.NoError(t, err) + + ref, err := s.Update(context.Background(), "n", capture.Note{ + Content: "Related: [[cad-babysit]].\n", Wing: "a", Hall: "facts", Reason: "r", + }) + require.NoError(t, err) + assert.Equal(t, []string{"cad-babysit"}, ref.Unlinked) + + got, err := os.ReadFile(filepath.Join(dir, "wiki/a/facts/n.md")) + require.NoError(t, err) + assert.Contains(t, string(got), "Related: cad-babysit.") + assert.NotContains(t, string(got), "[[cad-babysit]]") +} diff --git a/ingestion/internal/capture/ports.go b/ingestion/internal/capture/ports.go index 92595fb..f850e87 100644 --- a/ingestion/internal/capture/ports.go +++ b/ingestion/internal/capture/ports.go @@ -15,6 +15,10 @@ type Ref struct { Path string ContentHash string Superseded bool + // Unlinked names the [[wikilink]] targets the store unwrapped to plain + // text because they resolve to no entry. Reported rather than rejected: + // see brain.StripUnresolvedLinks for why the write still lands. + Unlinked []string } // StoredNote is a brain note fetched by Get: the read-after-write diff --git a/ingestion/internal/mcp/handlers.go b/ingestion/internal/mcp/handlers.go index deb6f8b..1aa963d 100644 --- a/ingestion/internal/mcp/handlers.go +++ b/ingestion/internal/mcp/handlers.go @@ -233,6 +233,15 @@ type brainWriteArgs struct { Hall string `json:"hall,omitempty"` } +// unlinkedNote explains the `unlinked` field to the caller that caused it. +// The write landed; what is gone is the [[...]] around a target that names no +// brain entry. Said here rather than only in a doc comment because the caller +// is a model that will otherwise repeat it on the next write. +const unlinkedNote = "these [[wikilink]] targets resolve to no brain entry, " + + "so they were unwrapped to plain text. Link only entries that exist " + + "(a filename under knowledge/ or wiki/); write the target entry first if " + + "you want the link, and keep skills, repos and agent-local memory files as prose." + func (s *Server) brainWrite(ctx context.Context, args json.RawMessage) (json.RawMessage, error) { var a brainWriteArgs if err := json.Unmarshal(args, &a); err != nil { @@ -253,7 +262,12 @@ func (s *Server) brainWrite(ctx context.Context, args json.RawMessage) (json.Raw if err != nil { return nil, err } - return json.Marshal(map[string]string{"id": ref.ID, "path": ref.Path, "content_hash": ref.ContentHash}) + out := map[string]any{"id": ref.ID, "path": ref.Path, "content_hash": ref.ContentHash} + if len(ref.Unlinked) > 0 { + out["unlinked"] = ref.Unlinked + out["unlinked_note"] = unlinkedNote + } + return json.Marshal(out) } type brainUpdateArgs struct { @@ -297,9 +311,14 @@ func (s *Server) brainUpdate(ctx context.Context, args json.RawMessage) (json.Ra if err != nil { return nil, err } - return json.Marshal(map[string]any{ + out := map[string]any{ "id": ref.ID, "path": ref.Path, "content_hash": ref.ContentHash, "superseded": ref.Superseded, - }) + } + if len(ref.Unlinked) > 0 { + out["unlinked"] = ref.Unlinked + out["unlinked_note"] = unlinkedNote + } + return json.Marshal(out) } type brainGetArgs struct {