fix(graph,wiki): read the brain through brainfs (#157, slice 4)
CI / Lint / Test / Vet (pull_request) Successful in 48s
CI / Mirror to GitHub (pull_request) Skipped

graphsync.IndexDoc/BackfillFromBrainDir and wiki.LoadInventory/
RebuildIndex read notes by path (os.ReadFile, os.Open, os.ReadDir +
open). A symlinked note put the target's title, aliases and wikilinks
into the knowledge graph, the wiki inventory (and the LLM prompts built
from it) and wiki/index.md; a FIFO hung each read.

- graphsync: IndexDoc and the backfill read through brainfs; the
  backfill walks with WalkRegular, skips a missing root and aborts on
  any other walk error, including a symlinked root.
- wiki: LoadInventory lists type dirs with the new
  brainfs.ReadDirRegular (every component a real dir, regular files
  only) and reads frontmatter through the root; a symlinked type dir is
  an error. RebuildIndex reads first sentences through the root.
- brainfstest's outside file now carries the sentinel in frontmatter
  title and aliases, a heading and a wikilink, so extractors that ignore
  the body are tested too.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-04 22:17:44 +02:00
co-authored by Claude Opus 5.5
parent 57d0fb0fae
commit c7a6429bae
7 changed files with 241 additions and 42 deletions
+11 -4
View File
@@ -7,6 +7,7 @@ import (
"path/filepath"
"strings"
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
"github.com/mathiasbq/hyperguild/ingestion/internal/textutil"
)
@@ -16,6 +17,11 @@ func RebuildIndex(brainDir, date string) error {
if err != nil {
return fmt.Errorf("load inventory: %w", err)
}
bfs, err := brainfs.Open(brainDir)
if err != nil {
return fmt.Errorf("open brain: %w", err)
}
defer func() { _ = bfs.Close() }()
total := len(inv[PageTypeConcept]) + len(inv[PageTypeEntity]) + len(inv[PageTypeSource])
var sb strings.Builder
@@ -34,7 +40,7 @@ func RebuildIndex(brainDir, date string) error {
label := strings.ToUpper(string(pt)[:1]) + string(pt)[1:]
fmt.Fprintf(&sb, "## %s\n\n", label)
for _, e := range entries {
summary := pageFirstSentence(brainDir, e)
summary := pageFirstSentence(bfs, e)
if summary != "" {
fmt.Fprintf(&sb, "- [[%s|%s]] — %s\n", e.Slug, e.Title, summary)
} else {
@@ -48,9 +54,10 @@ func RebuildIndex(brainDir, date string) error {
return os.WriteFile(dest, []byte(sb.String()), 0o644)
}
func pageFirstSentence(brainDir string, e Entry) string {
path := filepath.Join(brainDir, "wiki", string(e.Type), e.Slug+".md")
content, err := os.ReadFile(path)
// pageFirstSentence reads the page through the brain root, regular files
// only (hyperguild#157).
func pageFirstSentence(bfs *brainfs.Brain, e Entry) string {
content, err := bfs.ReadRegular("wiki/" + string(e.Type) + "/" + e.Slug + ".md")
if err != nil {
return ""
}
+23 -15
View File
@@ -3,36 +3,45 @@ package wiki
import (
"bufio"
"bytes"
"errors"
"fmt"
"os"
"path/filepath"
"io/fs"
"strings"
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
)
// LoadInventory walks brain/wiki/ and returns all pages grouped by type.
// Missing subdirectories are silently skipped.
// Missing subdirectories are silently skipped. Pages are listed and read
// through the brain root, regular files only (hyperguild#157); a type
// directory that is a symlink is an error.
func LoadInventory(brainDir string) (map[PageType][]Entry, error) {
result := map[PageType][]Entry{
PageTypeConcept: {},
PageTypeEntity: {},
PageTypeSource: {},
}
bfs, err := brainfs.Open(brainDir)
if err != nil {
return nil, fmt.Errorf("open brain: %w", err)
}
defer func() { _ = bfs.Close() }()
for pt := range result {
dir := filepath.Join(brainDir, "wiki", string(pt))
entries, err := os.ReadDir(dir)
if os.IsNotExist(err) {
dir := "wiki/" + string(pt)
names, err := bfs.ReadDirRegular(dir)
if errors.Is(err, fs.ErrNotExist) {
continue
}
if err != nil {
return nil, fmt.Errorf("read dir %s: %w", dir, err)
}
for _, e := range entries {
if e.IsDir() || !strings.HasSuffix(e.Name(), ".md") {
for _, name := range names {
if !strings.HasSuffix(name, ".md") {
continue
}
slug := strings.TrimSuffix(e.Name(), ".md")
path := filepath.Join(dir, e.Name())
title, aliases := readFrontmatter(path, slug)
slug := strings.TrimSuffix(name, ".md")
title, aliases := readFrontmatter(bfs, dir+"/"+name, slug)
result[pt] = append(result[pt], Entry{Slug: slug, Title: title, Aliases: aliases, Type: pt})
}
}
@@ -41,15 +50,14 @@ func LoadInventory(brainDir string) (map[PageType][]Entry, error) {
// readFrontmatter extracts title and aliases from YAML frontmatter.
// Falls back to slug for title and empty aliases on any error.
func readFrontmatter(path, fallbackSlug string) (title string, aliases []string) {
func readFrontmatter(bfs *brainfs.Brain, rel, fallbackSlug string) (title string, aliases []string) {
title = fallbackSlug
f, err := os.Open(path)
content, err := bfs.ReadRegular(rel)
if err != nil {
return
}
defer f.Close() //nolint:errcheck
scanner := bufio.NewScanner(f)
scanner := bufio.NewScanner(bytes.NewReader(content))
inFM := false
inAliases := false
for scanner.Scan() {
+71
View File
@@ -0,0 +1,71 @@
package wiki
import (
"fmt"
"os"
"path/filepath"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs/brainfstest"
)
// hyperguild#157 slice 4: LoadInventory listed wiki/<type>/ with os.ReadDir
// and opened each .md by path, so a symlinked page put the target's title and
// aliases into the inventory (and the LLM prompts built from it), and a FIFO
// hung it. RebuildIndex read first sentences the same way.
func TestLoadInventoryRefusesSymlinksAndSurvivesSwapRaces(t *testing.T) {
brain := t.TempDir()
dir := filepath.Join(brain, "wiki", "concepts")
require.NoError(t, os.MkdirAll(dir, 0o755))
require.NoError(t, os.WriteFile(filepath.Join(dir, "ok.md"), []byte("---\ntitle: zqinside\n---\n"), 0o644))
require.NoError(t, os.Symlink(brainfstest.Outside(t), filepath.Join(dir, "leak.md")))
inv, err := LoadInventory(brain)
require.NoError(t, err)
got := fmt.Sprintf("%+v", inv)
assert.Contains(t, got, "zqinside", "positive control")
assert.NotContains(t, got, brainfstest.Sentinel)
assert.NotContains(t, got, "leak", "a symlinked page is not listed")
p := filepath.Join(dir, "r.md")
read := func() string {
inv, _ := LoadInventory(brain)
return fmt.Sprintf("%+v", inv)
}
brainfstest.AssertNoLeak(t, p, "---\ntitle: zqinside\n---\n", 1500*time.Millisecond, read)
brainfstest.AssertNoHang(t, p, "---\ntitle: zqinside\n---\n", 1500*time.Millisecond, 6*time.Second, func() { _ = read() })
}
func TestLoadInventorySkipsASymlinkedTypeDir(t *testing.T) {
brain := t.TempDir()
require.NoError(t, os.MkdirAll(filepath.Join(brain, "wiki"), 0o755))
require.NoError(t, os.MkdirAll(filepath.Join(brain, "raw"), 0o755))
require.NoError(t, os.WriteFile(filepath.Join(brain, "raw", "r.md"), []byte("---\ntitle: zqrawonly\n---\n"), 0o644))
require.NoError(t, os.Symlink(filepath.Join("..", "raw"), filepath.Join(brain, "wiki", "concepts")))
inv, _ := LoadInventory(brain)
assert.NotContains(t, fmt.Sprintf("%+v", inv), "zqrawonly")
}
func TestRebuildIndexDoesNotReadAPageSwappedForASymlink(t *testing.T) {
brain := t.TempDir()
dir := filepath.Join(brain, "wiki", "concepts")
require.NoError(t, os.MkdirAll(dir, 0o755))
p := filepath.Join(dir, "r.md")
require.NoError(t, os.WriteFile(p, []byte("---\ntitle: R\n---\nzqinside sentence.\n"), 0o644))
require.NoError(t, RebuildIndex(brain, "2026-10-04"))
idx, err := os.ReadFile(filepath.Join(brain, "wiki", "index.md"))
require.NoError(t, err)
require.Contains(t, string(idx), "zqinside", "positive control: the summary is read")
read := func() string {
_ = RebuildIndex(brain, "2026-10-04")
b, _ := os.ReadFile(filepath.Join(brain, "wiki", "index.md"))
return string(b)
}
brainfstest.AssertNoLeak(t, p, "---\ntitle: R\n---\nzqinside sentence.\n", 1500*time.Millisecond, read)
brainfstest.AssertNoHang(t, p, "---\ntitle: R\n---\nzqinside sentence.\n", 1500*time.Millisecond, 6*time.Second, func() { _ = read() })
}