fix(graph,wiki): read the brain through brainfs (#157, slice 4)
graphsync.IndexDoc/BackfillFromBrainDir and wiki.LoadInventory/ RebuildIndex read notes by path (os.ReadFile, os.Open, os.ReadDir + open). A symlinked note put the target's title, aliases and wikilinks into the knowledge graph, the wiki inventory (and the LLM prompts built from it) and wiki/index.md; a FIFO hung each read. - graphsync: IndexDoc and the backfill read through brainfs; the backfill walks with WalkRegular, skips a missing root and aborts on any other walk error, including a symlinked root. - wiki: LoadInventory lists type dirs with the new brainfs.ReadDirRegular (every component a real dir, regular files only) and reads frontmatter through the root; a symlinked type dir is an error. RebuildIndex reads first sentences through the root. - brainfstest's outside file now carries the sentinel in frontmatter title and aliases, a heading and a wikilink, so extractors that ignore the body are tested too. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -7,6 +7,7 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/textutil"
|
||||
)
|
||||
|
||||
@@ -16,6 +17,11 @@ func RebuildIndex(brainDir, date string) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("load inventory: %w", err)
|
||||
}
|
||||
bfs, err := brainfs.Open(brainDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open brain: %w", err)
|
||||
}
|
||||
defer func() { _ = bfs.Close() }()
|
||||
|
||||
total := len(inv[PageTypeConcept]) + len(inv[PageTypeEntity]) + len(inv[PageTypeSource])
|
||||
var sb strings.Builder
|
||||
@@ -34,7 +40,7 @@ func RebuildIndex(brainDir, date string) error {
|
||||
label := strings.ToUpper(string(pt)[:1]) + string(pt)[1:]
|
||||
fmt.Fprintf(&sb, "## %s\n\n", label)
|
||||
for _, e := range entries {
|
||||
summary := pageFirstSentence(brainDir, e)
|
||||
summary := pageFirstSentence(bfs, e)
|
||||
if summary != "" {
|
||||
fmt.Fprintf(&sb, "- [[%s|%s]] — %s\n", e.Slug, e.Title, summary)
|
||||
} else {
|
||||
@@ -48,9 +54,10 @@ func RebuildIndex(brainDir, date string) error {
|
||||
return os.WriteFile(dest, []byte(sb.String()), 0o644)
|
||||
}
|
||||
|
||||
func pageFirstSentence(brainDir string, e Entry) string {
|
||||
path := filepath.Join(brainDir, "wiki", string(e.Type), e.Slug+".md")
|
||||
content, err := os.ReadFile(path)
|
||||
// pageFirstSentence reads the page through the brain root, regular files
|
||||
// only (hyperguild#157).
|
||||
func pageFirstSentence(bfs *brainfs.Brain, e Entry) string {
|
||||
content, err := bfs.ReadRegular("wiki/" + string(e.Type) + "/" + e.Slug + ".md")
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
@@ -3,36 +3,45 @@ package wiki
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"io/fs"
|
||||
"strings"
|
||||
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
|
||||
)
|
||||
|
||||
// LoadInventory walks brain/wiki/ and returns all pages grouped by type.
|
||||
// Missing subdirectories are silently skipped.
|
||||
// Missing subdirectories are silently skipped. Pages are listed and read
|
||||
// through the brain root, regular files only (hyperguild#157); a type
|
||||
// directory that is a symlink is an error.
|
||||
func LoadInventory(brainDir string) (map[PageType][]Entry, error) {
|
||||
result := map[PageType][]Entry{
|
||||
PageTypeConcept: {},
|
||||
PageTypeEntity: {},
|
||||
PageTypeSource: {},
|
||||
}
|
||||
bfs, err := brainfs.Open(brainDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open brain: %w", err)
|
||||
}
|
||||
defer func() { _ = bfs.Close() }()
|
||||
for pt := range result {
|
||||
dir := filepath.Join(brainDir, "wiki", string(pt))
|
||||
entries, err := os.ReadDir(dir)
|
||||
if os.IsNotExist(err) {
|
||||
dir := "wiki/" + string(pt)
|
||||
names, err := bfs.ReadDirRegular(dir)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read dir %s: %w", dir, err)
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.IsDir() || !strings.HasSuffix(e.Name(), ".md") {
|
||||
for _, name := range names {
|
||||
if !strings.HasSuffix(name, ".md") {
|
||||
continue
|
||||
}
|
||||
slug := strings.TrimSuffix(e.Name(), ".md")
|
||||
path := filepath.Join(dir, e.Name())
|
||||
title, aliases := readFrontmatter(path, slug)
|
||||
slug := strings.TrimSuffix(name, ".md")
|
||||
title, aliases := readFrontmatter(bfs, dir+"/"+name, slug)
|
||||
result[pt] = append(result[pt], Entry{Slug: slug, Title: title, Aliases: aliases, Type: pt})
|
||||
}
|
||||
}
|
||||
@@ -41,15 +50,14 @@ func LoadInventory(brainDir string) (map[PageType][]Entry, error) {
|
||||
|
||||
// readFrontmatter extracts title and aliases from YAML frontmatter.
|
||||
// Falls back to slug for title and empty aliases on any error.
|
||||
func readFrontmatter(path, fallbackSlug string) (title string, aliases []string) {
|
||||
func readFrontmatter(bfs *brainfs.Brain, rel, fallbackSlug string) (title string, aliases []string) {
|
||||
title = fallbackSlug
|
||||
f, err := os.Open(path)
|
||||
content, err := bfs.ReadRegular(rel)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer f.Close() //nolint:errcheck
|
||||
|
||||
scanner := bufio.NewScanner(f)
|
||||
scanner := bufio.NewScanner(bytes.NewReader(content))
|
||||
inFM := false
|
||||
inAliases := false
|
||||
for scanner.Scan() {
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
package wiki
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs/brainfstest"
|
||||
)
|
||||
|
||||
// hyperguild#157 slice 4: LoadInventory listed wiki/<type>/ with os.ReadDir
|
||||
// and opened each .md by path, so a symlinked page put the target's title and
|
||||
// aliases into the inventory (and the LLM prompts built from it), and a FIFO
|
||||
// hung it. RebuildIndex read first sentences the same way.
|
||||
func TestLoadInventoryRefusesSymlinksAndSurvivesSwapRaces(t *testing.T) {
|
||||
brain := t.TempDir()
|
||||
dir := filepath.Join(brain, "wiki", "concepts")
|
||||
require.NoError(t, os.MkdirAll(dir, 0o755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "ok.md"), []byte("---\ntitle: zqinside\n---\n"), 0o644))
|
||||
require.NoError(t, os.Symlink(brainfstest.Outside(t), filepath.Join(dir, "leak.md")))
|
||||
|
||||
inv, err := LoadInventory(brain)
|
||||
require.NoError(t, err)
|
||||
got := fmt.Sprintf("%+v", inv)
|
||||
assert.Contains(t, got, "zqinside", "positive control")
|
||||
assert.NotContains(t, got, brainfstest.Sentinel)
|
||||
assert.NotContains(t, got, "leak", "a symlinked page is not listed")
|
||||
|
||||
p := filepath.Join(dir, "r.md")
|
||||
read := func() string {
|
||||
inv, _ := LoadInventory(brain)
|
||||
return fmt.Sprintf("%+v", inv)
|
||||
}
|
||||
brainfstest.AssertNoLeak(t, p, "---\ntitle: zqinside\n---\n", 1500*time.Millisecond, read)
|
||||
brainfstest.AssertNoHang(t, p, "---\ntitle: zqinside\n---\n", 1500*time.Millisecond, 6*time.Second, func() { _ = read() })
|
||||
}
|
||||
|
||||
func TestLoadInventorySkipsASymlinkedTypeDir(t *testing.T) {
|
||||
brain := t.TempDir()
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(brain, "wiki"), 0o755))
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(brain, "raw"), 0o755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(brain, "raw", "r.md"), []byte("---\ntitle: zqrawonly\n---\n"), 0o644))
|
||||
require.NoError(t, os.Symlink(filepath.Join("..", "raw"), filepath.Join(brain, "wiki", "concepts")))
|
||||
inv, _ := LoadInventory(brain)
|
||||
assert.NotContains(t, fmt.Sprintf("%+v", inv), "zqrawonly")
|
||||
}
|
||||
|
||||
func TestRebuildIndexDoesNotReadAPageSwappedForASymlink(t *testing.T) {
|
||||
brain := t.TempDir()
|
||||
dir := filepath.Join(brain, "wiki", "concepts")
|
||||
require.NoError(t, os.MkdirAll(dir, 0o755))
|
||||
p := filepath.Join(dir, "r.md")
|
||||
require.NoError(t, os.WriteFile(p, []byte("---\ntitle: R\n---\nzqinside sentence.\n"), 0o644))
|
||||
require.NoError(t, RebuildIndex(brain, "2026-10-04"))
|
||||
idx, err := os.ReadFile(filepath.Join(brain, "wiki", "index.md"))
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, string(idx), "zqinside", "positive control: the summary is read")
|
||||
|
||||
read := func() string {
|
||||
_ = RebuildIndex(brain, "2026-10-04")
|
||||
b, _ := os.ReadFile(filepath.Join(brain, "wiki", "index.md"))
|
||||
return string(b)
|
||||
}
|
||||
brainfstest.AssertNoLeak(t, p, "---\ntitle: R\n---\nzqinside sentence.\n", 1500*time.Millisecond, read)
|
||||
brainfstest.AssertNoHang(t, p, "---\ntitle: R\n---\nzqinside sentence.\n", 1500*time.Millisecond, 6*time.Second, func() { _ = read() })
|
||||
}
|
||||
Reference in New Issue
Block a user