Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1c9c218dbf | ||
|
|
da020d1e6c |
@@ -0,0 +1,125 @@
|
|||||||
|
package brain_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"syscall"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"github.com/mathiasbq/hyperguild/ingestion/internal/brain"
|
||||||
|
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs/brainfstest"
|
||||||
|
)
|
||||||
|
|
||||||
|
// hyperguild#174 / #157 slice 5b: DetectTunnels walked wiki/ and opened
|
||||||
|
// every note with a blocking os.Open, so a FIFO named *.md anywhere under
|
||||||
|
// wiki/ hung every brain_write and brain_update (wikiUpkeep -> AutoTunnel),
|
||||||
|
// and a symlinked note's title became a tunnel candidate.
|
||||||
|
func TestDetectTunnelsReturnsPromptlyWithAFIFOUnderWikiWithAndWithoutAWriter(t *testing.T) {
|
||||||
|
for _, withWriter := range []bool{false, true} {
|
||||||
|
t.Run(fmt.Sprintf("writer=%v", withWriter), func(t *testing.T) {
|
||||||
|
b := t.TempDir()
|
||||||
|
facts := filepath.Join(b, "wiki", "w", "facts")
|
||||||
|
require.NoError(t, os.MkdirAll(facts, 0o755))
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(facts, "ok.md"), []byte("---\ntitle: Zqinside Phrase\n---\n"), 0o644))
|
||||||
|
fifo := filepath.Join(facts, "f.md")
|
||||||
|
require.NoError(t, syscall.Mkfifo(fifo, 0o644))
|
||||||
|
if withWriter {
|
||||||
|
// A writer held open: a reader that opens the FIFO gets no
|
||||||
|
// EOF and parks, unless the FIFO is never opened.
|
||||||
|
w, err := os.OpenFile(fifo, os.O_RDWR, 0)
|
||||||
|
require.NoError(t, err)
|
||||||
|
t.Cleanup(func() { _ = w.Close() })
|
||||||
|
}
|
||||||
|
done := make(chan []brain.TunnelCandidate, 1)
|
||||||
|
go func() {
|
||||||
|
cs, _ := brain.DetectTunnels(b, "text mentioning zqinside phrase here")
|
||||||
|
done <- cs
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case cs := <-done:
|
||||||
|
assert.Contains(t, fmt.Sprintf("%+v", cs), "Zqinside Phrase", "positive control")
|
||||||
|
case <-time.After(3 * time.Second):
|
||||||
|
if r, err := os.OpenFile(fifo, os.O_WRONLY|syscall.O_NONBLOCK, 0); err == nil {
|
||||||
|
_ = r.Close()
|
||||||
|
}
|
||||||
|
t.Fatal("DetectTunnels blocked on a FIFO")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDetectTunnelsRefusesSymlinkedNotesAndSurvivesSwapRaces(t *testing.T) {
|
||||||
|
b := t.TempDir()
|
||||||
|
facts := filepath.Join(b, "wiki", "w", "facts")
|
||||||
|
require.NoError(t, os.MkdirAll(facts, 0o755))
|
||||||
|
// The static note's title differs from the raced note's, so the
|
||||||
|
// harness's "inside was read" floor can only be met by reading r.md.
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(facts, "ok.md"), []byte("---\ntitle: Zqstatic Words\n---\n"), 0o644))
|
||||||
|
require.NoError(t, os.Symlink(brainfstest.Outside(t), filepath.Join(facts, "out.md")))
|
||||||
|
inBrain := brainfstest.WriteSentinelAt(t, filepath.Join(b, "raw", "s.md"))
|
||||||
|
require.NoError(t, os.Symlink(filepath.Join("..", "..", "..", "raw", "s.md"), filepath.Join(facts, "in.md")))
|
||||||
|
content := "zqstatic words, zqinside phrase and " + brainfstest.Sentinel
|
||||||
|
|
||||||
|
cs, err := brain.DetectTunnels(b, content)
|
||||||
|
require.NoError(t, err)
|
||||||
|
got := fmt.Sprintf("%+v", cs)
|
||||||
|
assert.Contains(t, got, "Zqstatic Words", "positive control")
|
||||||
|
assert.NotContains(t, got, brainfstest.Sentinel)
|
||||||
|
|
||||||
|
p := filepath.Join(facts, "r.md")
|
||||||
|
read := func() string {
|
||||||
|
cs, _ := brain.DetectTunnels(b, content)
|
||||||
|
return fmt.Sprintf("%+v", cs)
|
||||||
|
}
|
||||||
|
brainfstest.AssertNoLeak(t, p, "---\ntitle: Zqinside Phrase\n---\n", 1500*time.Millisecond, read)
|
||||||
|
brainfstest.AssertNoLeakFrom(t, p, "---\ntitle: Zqinside Phrase\n---\n", inBrain, 1500*time.Millisecond, read)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A wiki/ that is not a real directory yields no candidates and no error:
|
||||||
|
// AutoTunnel runs inside brain_write.
|
||||||
|
func TestDetectTunnelsSkipsAWikiThatIsNotARealDirectory(t *testing.T) {
|
||||||
|
b := t.TempDir()
|
||||||
|
require.NoError(t, os.MkdirAll(filepath.Join(b, "raw", "w", "facts"), 0o755))
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(b, "raw", "w", "facts", "r.md"), []byte("---\ntitle: Zqrawonly Phrase\n---\n"), 0o644))
|
||||||
|
require.NoError(t, os.Symlink("raw", filepath.Join(b, "wiki")))
|
||||||
|
cs, err := brain.DetectTunnels(b, "zqrawonly phrase")
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Empty(t, cs)
|
||||||
|
}
|
||||||
|
|
||||||
|
// An unreadable directory under wiki/ is a walk error and DetectTunnels
|
||||||
|
// reports it, as filepath.WalkDir did; it is not silently dropped.
|
||||||
|
func TestDetectTunnelsReportsAnUnreadableDirectoryUnderWiki(t *testing.T) {
|
||||||
|
if os.Geteuid() == 0 {
|
||||||
|
t.Skip("root ignores directory permissions")
|
||||||
|
}
|
||||||
|
b := t.TempDir()
|
||||||
|
require.NoError(t, os.MkdirAll(filepath.Join(b, "wiki", "w", "facts"), 0o755))
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(b, "wiki", "w", "facts", "ok.md"), []byte("---\ntitle: Zqinside Phrase\n---\n"), 0o644))
|
||||||
|
locked := filepath.Join(b, "wiki", "other", "locked")
|
||||||
|
require.NoError(t, os.MkdirAll(locked, 0o755))
|
||||||
|
require.NoError(t, os.Chmod(locked, 0o000))
|
||||||
|
t.Cleanup(func() { _ = os.Chmod(locked, 0o755) })
|
||||||
|
_, err := brain.DetectTunnels(b, "zqinside phrase")
|
||||||
|
assert.ErrorIs(t, err, os.ErrPermission)
|
||||||
|
}
|
||||||
|
|
||||||
|
// _index.md files are generated, not notes: their title never becomes a
|
||||||
|
// candidate.
|
||||||
|
func TestDetectTunnelsSkipsIndexFiles(t *testing.T) {
|
||||||
|
b := t.TempDir()
|
||||||
|
w := filepath.Join(b, "wiki", "w")
|
||||||
|
require.NoError(t, os.MkdirAll(filepath.Join(w, "facts"), 0o755))
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(w, "_index.md"), []byte("---\ntitle: Zqindex Words\n---\n"), 0o644))
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(w, "facts", "ok.md"), []byte("---\ntitle: Zqinside Phrase\n---\n"), 0o644))
|
||||||
|
cs, err := brain.DetectTunnels(b, "zqindex words and zqinside phrase")
|
||||||
|
require.NoError(t, err)
|
||||||
|
got := fmt.Sprintf("%+v", cs)
|
||||||
|
assert.Contains(t, got, "Zqinside Phrase", "positive control")
|
||||||
|
assert.NotContains(t, got, "Zqindex")
|
||||||
|
}
|
||||||
@@ -116,23 +116,6 @@ func collectWingEntries(bfs *brainfs.Brain, wingRel string) ([]noteEntry, error)
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// readTitleAndCreated reads YAML frontmatter for title + created_at; falls
|
|
||||||
// back to slug and file mtime when absent.
|
|
||||||
func readTitleAndCreated(path, slug string) (string, string) {
|
|
||||||
f, err := os.Open(path)
|
|
||||||
if err != nil {
|
|
||||||
return slug, ""
|
|
||||||
}
|
|
||||||
defer func() { _ = f.Close() }()
|
|
||||||
title, created := parseTitleAndCreated(f)
|
|
||||||
if created == "" {
|
|
||||||
if info, err := os.Stat(path); err == nil {
|
|
||||||
created = info.ModTime().UTC().Format("2006-01-02")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return titleOrSlug(title, slug), created
|
|
||||||
}
|
|
||||||
|
|
||||||
// readTitleAndCreatedIn is readTitleAndCreated through the brain root,
|
// readTitleAndCreatedIn is readTitleAndCreated through the brain root,
|
||||||
// regular files only (hyperguild#157).
|
// regular files only (hyperguild#157).
|
||||||
func readTitleAndCreatedIn(bfs *brainfs.Brain, rel, slug string) (string, string) {
|
func readTitleAndCreatedIn(bfs *brainfs.Brain, rel, slug string) (string, string) {
|
||||||
|
|||||||
@@ -2,7 +2,9 @@ package brain
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bufio"
|
"bufio"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
"path"
|
"path"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -39,24 +41,29 @@ type TunnelCandidate struct {
|
|||||||
// A note's title is read from YAML frontmatter `title:`; failing that,
|
// A note's title is read from YAML frontmatter `title:`; failing that,
|
||||||
// the filename slug (sans `.md`, hyphens → spaces) is used.
|
// the filename slug (sans `.md`, hyphens → spaces) is used.
|
||||||
func DetectTunnels(brainDir, content string) ([]TunnelCandidate, error) {
|
func DetectTunnels(brainDir, content string) ([]TunnelCandidate, error) {
|
||||||
wikiDir := filepath.Join(brainDir, "wiki")
|
// Walked and read through the brain root, regular files only
|
||||||
if _, err := os.Stat(wikiDir); os.IsNotExist(err) {
|
// (hyperguild#174, #157): this runs inside every brain_write, so a FIFO
|
||||||
|
// must not block it and a symlinked note's title must not become a
|
||||||
|
// candidate.
|
||||||
|
bfs, err := brainfs.Open(brainDir)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("open brain: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = bfs.Close() }()
|
||||||
|
if _, err := bfs.Lstat("wiki"); errors.Is(err, fs.ErrNotExist) {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
} else if err != nil {
|
|
||||||
return nil, fmt.Errorf("stat wiki: %w", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
lowerContent := strings.ToLower(content)
|
lowerContent := strings.ToLower(content)
|
||||||
|
|
||||||
var out []TunnelCandidate
|
var out []TunnelCandidate
|
||||||
err := filepath.WalkDir(wikiDir, func(path string, d os.DirEntry, err error) error {
|
var walkErr error
|
||||||
if err != nil {
|
err = bfs.WalkRegular("wiki", func(rel string) error {
|
||||||
return err
|
name := path.Base(rel)
|
||||||
}
|
if !strings.HasSuffix(name, ".md") || name == "_index.md" {
|
||||||
if d.IsDir() || !strings.HasSuffix(path, ".md") || d.Name() == "_index.md" {
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
title, _ := readTitleAndCreated(path, strings.TrimSuffix(d.Name(), ".md"))
|
title, _ := readTitleAndCreatedIn(bfs, rel, strings.TrimSuffix(name, ".md"))
|
||||||
title = strings.TrimSpace(title)
|
title = strings.TrimSpace(title)
|
||||||
if title == "" {
|
if title == "" {
|
||||||
return nil
|
return nil
|
||||||
@@ -78,17 +85,25 @@ func DetectTunnels(brainDir, content string) ([]TunnelCandidate, error) {
|
|||||||
if idx == -1 {
|
if idx == -1 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
rel, err := filepath.Rel(brainDir, path)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
out = append(out, TunnelCandidate{
|
out = append(out, TunnelCandidate{
|
||||||
TargetPath: filepath.ToSlash(rel),
|
TargetPath: rel,
|
||||||
MatchedTerm: title,
|
MatchedTerm: title,
|
||||||
Exact: isWholeWord(haystack, idx, len(needle)),
|
Exact: isWholeWord(haystack, idx, len(needle)),
|
||||||
})
|
})
|
||||||
return nil
|
return nil
|
||||||
|
}, func(rel string, err error) {
|
||||||
|
// A wiki/ that is not a real directory yields no candidates, as
|
||||||
|
// filepath.WalkDir did; other walk errors still abort.
|
||||||
|
if rel == "wiki" && errors.Is(err, brainfs.ErrNotRegular) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if walkErr == nil {
|
||||||
|
walkErr = fmt.Errorf("%s: %w", rel, err)
|
||||||
|
}
|
||||||
})
|
})
|
||||||
|
if err == nil {
|
||||||
|
err = walkErr
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,10 +16,9 @@
|
|||||||
package classification
|
package classification
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
|
"os"
|
||||||
"io/fs"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v3"
|
||||||
@@ -111,20 +110,9 @@ type rawConfig struct {
|
|||||||
func Load(brainDir string) (*Config, error) {
|
func Load(brainDir string) (*Config, error) {
|
||||||
cfg := &Config{wings: map[string]Level{}, repos: map[string]Level{}}
|
cfg := &Config{wings: map[string]Level{}, repos: map[string]Level{}}
|
||||||
|
|
||||||
// Read through the brain root, regular files only (hyperguild#157): a
|
data, err := os.ReadFile(filepath.Join(brainDir, "classification.yaml"))
|
||||||
// symlinked or special file is not trusted as the config, which is a hard
|
|
||||||
// error like a malformed one.
|
|
||||||
bfs, err := brainfs.Open(brainDir)
|
|
||||||
if errors.Is(err, fs.ErrNotExist) {
|
|
||||||
return cfg, nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("open brain: %w", err)
|
if os.IsNotExist(err) {
|
||||||
}
|
|
||||||
defer func() { _ = bfs.Close() }()
|
|
||||||
data, err := bfs.ReadRegular("classification.yaml")
|
|
||||||
if err != nil {
|
|
||||||
if errors.Is(err, fs.ErrNotExist) {
|
|
||||||
return cfg, nil
|
return cfg, nil
|
||||||
}
|
}
|
||||||
return nil, fmt.Errorf("read classification.yaml: %w", err)
|
return nil, fmt.Errorf("read classification.yaml: %w", err)
|
||||||
|
|||||||
@@ -1,46 +0,0 @@
|
|||||||
package classification
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"syscall"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
// hyperguild#157 slice 6: Load read classification.yaml with os.ReadFile. It
|
|
||||||
// sets the I1 sovereignty levels; a symlinked file must not be trusted as the
|
|
||||||
// config (a hard error, as for a malformed file), and a FIFO must not hang
|
|
||||||
// startup.
|
|
||||||
func TestLoadRefusesASymlinkedOrFIFOClassificationFile(t *testing.T) {
|
|
||||||
ok := t.TempDir()
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(ok, "classification.yaml"), []byte("wings:\n w: confidential\n"), 0o644))
|
|
||||||
_, err := Load(ok)
|
|
||||||
require.NoError(t, err, "positive control")
|
|
||||||
|
|
||||||
linked := t.TempDir()
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(linked, "real.yaml"), []byte("wings:\n w: public\n"), 0o644))
|
|
||||||
require.NoError(t, os.Symlink("real.yaml", filepath.Join(linked, "classification.yaml")))
|
|
||||||
_, err = Load(linked)
|
|
||||||
assert.Error(t, err, "an in-brain symlink is not trusted as the config")
|
|
||||||
|
|
||||||
fifo := t.TempDir()
|
|
||||||
require.NoError(t, syscall.Mkfifo(filepath.Join(fifo, "classification.yaml"), 0o644))
|
|
||||||
w, err := os.OpenFile(filepath.Join(fifo, "classification.yaml"), os.O_RDWR, 0)
|
|
||||||
require.NoError(t, err)
|
|
||||||
defer func() { _ = w.Close() }()
|
|
||||||
done := make(chan error, 1)
|
|
||||||
go func() { _, err := Load(fifo); done <- err }()
|
|
||||||
select {
|
|
||||||
case err := <-done:
|
|
||||||
assert.Error(t, err)
|
|
||||||
case <-time.After(3 * time.Second):
|
|
||||||
t.Fatal("Load hung on a FIFO")
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = Load(t.TempDir())
|
|
||||||
assert.NoError(t, err, "an absent file is still not an error")
|
|
||||||
}
|
|
||||||
@@ -3,10 +3,7 @@ package pipeline
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
|
|
||||||
"io/fs"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -26,16 +23,10 @@ func BackfillRefs(ctx context.Context, brainDir string) (int, error) {
|
|||||||
return 0, fmt.Errorf("load inventory: %w", err)
|
return 0, fmt.Errorf("load inventory: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Listed and read through the brain root, regular files only
|
sourcesDir := filepath.Join(brainDir, "wiki", "sources")
|
||||||
// (hyperguild#157).
|
entries, err := os.ReadDir(sourcesDir)
|
||||||
bfs, err := brainfs.Open(brainDir)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, fmt.Errorf("open brain: %w", err)
|
if os.IsNotExist(err) {
|
||||||
}
|
|
||||||
defer func() { _ = bfs.Close() }()
|
|
||||||
names, err := bfs.ReadDirRegular("wiki/sources")
|
|
||||||
if err != nil {
|
|
||||||
if errors.Is(err, fs.ErrNotExist) {
|
|
||||||
return 0, nil
|
return 0, nil
|
||||||
}
|
}
|
||||||
return 0, fmt.Errorf("read sources dir: %w", err)
|
return 0, fmt.Errorf("read sources dir: %w", err)
|
||||||
@@ -46,20 +37,20 @@ func BackfillRefs(ctx context.Context, brainDir string) (int, error) {
|
|||||||
// get both refs merged before a single write.
|
// get both refs merged before a single write.
|
||||||
pending := make(map[string]wiki.Page)
|
pending := make(map[string]wiki.Page)
|
||||||
|
|
||||||
for _, name := range names {
|
for _, e := range entries {
|
||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
return 0, ctx.Err()
|
return 0, ctx.Err()
|
||||||
}
|
}
|
||||||
if !strings.HasSuffix(name, ".md") {
|
if e.IsDir() || !strings.HasSuffix(e.Name(), ".md") {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
b, err := bfs.ReadRegular("wiki/sources/" + name)
|
b, err := os.ReadFile(filepath.Join(sourcesDir, e.Name()))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
sourceContent := string(b)
|
sourceContent := string(b)
|
||||||
sourceSlug := strings.TrimSuffix(name, ".md")
|
sourceSlug := strings.TrimSuffix(e.Name(), ".md")
|
||||||
sourceTitle := extractTitle(sourceContent)
|
sourceTitle := extractTitle(sourceContent)
|
||||||
if sourceTitle == "" {
|
if sourceTitle == "" {
|
||||||
sourceTitle = sourceSlug
|
sourceTitle = sourceSlug
|
||||||
@@ -79,7 +70,7 @@ func BackfillRefs(ctx context.Context, brainDir string) (int, error) {
|
|||||||
// Start from already-accumulated version if we've seen this page.
|
// Start from already-accumulated version if we've seen this page.
|
||||||
page, seen := pending[relPath]
|
page, seen := pending[relPath]
|
||||||
if !seen {
|
if !seen {
|
||||||
raw, err := bfs.ReadRegular(relPath)
|
raw, err := os.ReadFile(filepath.Join(brainDir, filepath.FromSlash(relPath)))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,10 +5,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
|
|
||||||
"io/fs"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -132,7 +129,7 @@ func buildAndWrite(ctx context.Context, rawPages []RawPage, sourceSlug, date, br
|
|||||||
if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil {
|
if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil {
|
||||||
return Result{}, fmt.Errorf("mkdir for %s: %w", page.Path, err)
|
return Result{}, fmt.Errorf("mkdir for %s: %w", page.Path, err)
|
||||||
}
|
}
|
||||||
content, conflicts, err := reviseOrCreate(brainDir, page)
|
content, conflicts, err := reviseOrCreate(dest, page)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Result{}, err
|
return Result{}, err
|
||||||
}
|
}
|
||||||
@@ -191,27 +188,14 @@ func hashBody(content string) string {
|
|||||||
|
|
||||||
// readSourcePage returns the full on-disk content of the source page at
|
// readSourcePage returns the full on-disk content of the source page at
|
||||||
// wiki/sources/<sourceSlug>.md, or "" when it doesn't exist yet.
|
// wiki/sources/<sourceSlug>.md, or "" when it doesn't exist yet.
|
||||||
// It reads through the brain root, regular files only (hyperguild#157): the
|
|
||||||
// content goes into the LLM prompt.
|
|
||||||
func readSourcePage(brainDir, sourceSlug string) string {
|
func readSourcePage(brainDir, sourceSlug string) string {
|
||||||
raw, err := readBrainFile(brainDir, "wiki/sources/"+sourceSlug+".md")
|
raw, err := os.ReadFile(filepath.Join(brainDir, "wiki", "sources", sourceSlug+".md"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
return string(raw)
|
return string(raw)
|
||||||
}
|
}
|
||||||
|
|
||||||
// readBrainFile reads one brain-relative file through the brain root,
|
|
||||||
// regular files only (hyperguild#157).
|
|
||||||
func readBrainFile(brainDir, rel string) ([]byte, error) {
|
|
||||||
bfs, err := brainfs.Open(brainDir)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer func() { _ = bfs.Close() }()
|
|
||||||
return bfs.ReadRegular(rel)
|
|
||||||
}
|
|
||||||
|
|
||||||
// storedSourceHash reads source_hash from the source page a previous ingest
|
// storedSourceHash reads source_hash from the source page a previous ingest
|
||||||
// wrote, or "" when there is no such page or no such field.
|
// wrote, or "" when there is no such page or no such field.
|
||||||
//
|
//
|
||||||
@@ -246,10 +230,10 @@ func storedSourceHash(brainDir, sourceSlug string) string {
|
|||||||
// its first-ingest date on a revised page. Left deliberately — the alternative
|
// its first-ingest date on a revised page. Left deliberately — the alternative
|
||||||
// is this function editing frontmatter it does not own, and no lint reads that
|
// is this function editing frontmatter it does not own, and no lint reads that
|
||||||
// field today.
|
// field today.
|
||||||
func reviseOrCreate(brainDir string, page wiki.Page) (string, []string, error) {
|
func reviseOrCreate(dest string, page wiki.Page) (string, []string, error) {
|
||||||
existing, err := readBrainFile(brainDir, page.Path)
|
existing, err := os.ReadFile(dest)
|
||||||
switch {
|
switch {
|
||||||
case errors.Is(err, fs.ErrNotExist):
|
case os.IsNotExist(err):
|
||||||
return page.Content, nil, nil
|
return page.Content, nil, nil
|
||||||
case err != nil:
|
case err != nil:
|
||||||
return "", nil, fmt.Errorf("read existing %s: %w", page.Path, err)
|
return "", nil, fmt.Errorf("read existing %s: %w", page.Path, err)
|
||||||
@@ -283,7 +267,7 @@ See brain/schema.md for the full schema.
|
|||||||
`
|
`
|
||||||
|
|
||||||
func loadSchema(brainDir string) string {
|
func loadSchema(brainDir string) string {
|
||||||
b, err := readBrainFile(brainDir, "schema.md")
|
b, err := os.ReadFile(filepath.Join(brainDir, "schema.md"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return defaultSchema
|
return defaultSchema
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
package pipeline
|
package pipeline
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -52,7 +53,8 @@ func injectSourceRefs(pages []wiki.Page, inventory map[wiki.PageType][]wiki.Entr
|
|||||||
if !ok {
|
if !ok {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
b, err := readBrainFile(brainDir, "wiki/"+string(pt)+"/"+slug+".md")
|
diskPath := filepath.Join(brainDir, "wiki", string(pt), slug+".md")
|
||||||
|
b, err := os.ReadFile(diskPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,167 +0,0 @@
|
|||||||
package pipeline
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"syscall"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
|
|
||||||
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs/brainfstest"
|
|
||||||
"github.com/mathiasbq/hyperguild/ingestion/internal/wiki"
|
|
||||||
)
|
|
||||||
|
|
||||||
// hyperguild#157 slice 6: the pipeline read the source page, schema.md, the
|
|
||||||
// pages it revises and the pages it adds source refs to with os.ReadFile, so
|
|
||||||
// a symlinked file's content reached the LLM prompt and the written pages,
|
|
||||||
// and a FIFO hung the ingest.
|
|
||||||
|
|
||||||
func mkdirs(t *testing.T, brain string, dirs ...string) {
|
|
||||||
t.Helper()
|
|
||||||
for _, d := range dirs {
|
|
||||||
require.NoError(t, os.MkdirAll(filepath.Join(brain, filepath.FromSlash(d)), 0o755))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReadSourcePageAndSchemaRefuseSymlinksAndSurviveSwapRaces(t *testing.T) {
|
|
||||||
brain := t.TempDir()
|
|
||||||
mkdirs(t, brain, "wiki/sources")
|
|
||||||
inBrain := brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "s.md"))
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "sources", "ok.md"), []byte("zqinside source"), 0o644))
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(brain, "schema.md"), []byte("zqinside schema"), 0o644))
|
|
||||||
assert.Equal(t, "zqinside source", readSourcePage(brain, "ok"), "positive control")
|
|
||||||
assert.Equal(t, "zqinside schema", loadSchema(brain), "positive control")
|
|
||||||
|
|
||||||
require.NoError(t, os.Symlink(brainfstest.Outside(t), filepath.Join(brain, "wiki", "sources", "out.md")))
|
|
||||||
require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "s.md"), filepath.Join(brain, "wiki", "sources", "in.md")))
|
|
||||||
assert.NotContains(t, readSourcePage(brain, "out"), brainfstest.Sentinel)
|
|
||||||
assert.NotContains(t, readSourcePage(brain, "in"), brainfstest.Sentinel)
|
|
||||||
assert.NotContains(t, storedSourceHash(brain, "in"), brainfstest.Sentinel)
|
|
||||||
|
|
||||||
src := filepath.Join(brain, "wiki", "sources", "r.md")
|
|
||||||
read := func() string { return readSourcePage(brain, "r") }
|
|
||||||
brainfstest.AssertNoLeak(t, src, "zqinside", 1500*time.Millisecond, read)
|
|
||||||
brainfstest.AssertNoLeakFrom(t, src, "zqinside", inBrain, 1500*time.Millisecond, read)
|
|
||||||
brainfstest.AssertNoHang(t, src, "zqinside", 1500*time.Millisecond, 6*time.Second, func() { _ = read() })
|
|
||||||
|
|
||||||
schema := filepath.Join(brain, "schema.md")
|
|
||||||
rs := func() string { return loadSchema(brain) }
|
|
||||||
brainfstest.AssertNoLeakFrom(t, schema, "zqinside schema", inBrain, 1500*time.Millisecond, rs)
|
|
||||||
brainfstest.AssertNoHang(t, schema, "zqinside schema", 1500*time.Millisecond, 6*time.Second, func() { _ = rs() })
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReviseOrCreateRefusesASymlinkedExistingPage(t *testing.T) {
|
|
||||||
brain := t.TempDir()
|
|
||||||
mkdirs(t, brain, "wiki/concepts")
|
|
||||||
brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "s.md"))
|
|
||||||
require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "s.md"), filepath.Join(brain, "wiki", "concepts", "c.md")))
|
|
||||||
page := wiki.Page{Path: "wiki/concepts/c.md", Content: "---\ntitle: C\n---\nnew\n"}
|
|
||||||
out, _, err := reviseOrCreate(brain, page)
|
|
||||||
assert.Error(t, err)
|
|
||||||
assert.NotContains(t, out, brainfstest.Sentinel)
|
|
||||||
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "concepts", "d.md"), []byte("---\ntitle: D\n---\nzqold\n"), 0o644))
|
|
||||||
out, _, err = reviseOrCreate(brain, wiki.Page{Path: "wiki/concepts/d.md", Content: "---\ntitle: D\n---\nnew\n"})
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Contains(t, out, "zqold", "positive control: the existing page is read")
|
|
||||||
out, _, err = reviseOrCreate(brain, wiki.Page{Path: "wiki/concepts/new.md", Content: "fresh"})
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, "fresh", out, "positive control: a missing page is created")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestInjectSourceRefsDoesNotReadASymlinkedPage(t *testing.T) {
|
|
||||||
brain := t.TempDir()
|
|
||||||
mkdirs(t, brain, "wiki/concepts")
|
|
||||||
brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "s.md"))
|
|
||||||
require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "s.md"), filepath.Join(brain, "wiki", "concepts", "c.md")))
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "concepts", "d.md"), []byte("zqinside d"), 0o644))
|
|
||||||
inv := map[wiki.PageType][]wiki.Entry{wiki.PageTypeConcept: {
|
|
||||||
{Slug: "c", Type: wiki.PageTypeConcept}, {Slug: "d", Type: wiki.PageTypeConcept},
|
|
||||||
}}
|
|
||||||
src := wiki.Page{Path: "wiki/sources/src.md", Content: "---\ntitle: Src\n---\nsee [[c|C]] and [[d|D]]\n"}
|
|
||||||
out := injectSourceRefs([]wiki.Page{src}, inv, brain)
|
|
||||||
got := fmt.Sprintf("%+v", out)
|
|
||||||
assert.Contains(t, got, "zqinside d", "positive control")
|
|
||||||
assert.NotContains(t, got, brainfstest.Sentinel)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBackfillRefsSkipsSymlinkedSourcesAndPages(t *testing.T) {
|
|
||||||
brain := t.TempDir()
|
|
||||||
mkdirs(t, brain, "wiki/sources", "wiki/concepts", "raw")
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "concepts", "real.md"), []byte("---\ntitle: R\n---\nbody\n"), 0o644))
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "sources", "ok.md"), []byte("---\ntitle: zqinside\n---\n[[real|R]]\n"), 0o644))
|
|
||||||
// A source elsewhere in the brain whose title is the sentinel and which
|
|
||||||
// links the real concept with the [[slug|title]] form the pipeline follows.
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(brain, "raw", "src.md"),
|
|
||||||
[]byte("---\ntitle: "+brainfstest.Sentinel+"\n---\n[[real|R]]\n"), 0o644))
|
|
||||||
require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "src.md"), filepath.Join(brain, "wiki", "sources", "in.md")))
|
|
||||||
// A concept page linked by the regular source that is itself a symlink.
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "sources", "ok2.md"), []byte("---\ntitle: Two\n---\n[[lnk|L]]\n"), 0o644))
|
|
||||||
brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "page.md"))
|
|
||||||
require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "page.md"), filepath.Join(brain, "wiki", "concepts", "lnk.md")))
|
|
||||||
|
|
||||||
_, err := BackfillRefs(context.Background(), brain)
|
|
||||||
require.NoError(t, err)
|
|
||||||
b, _ := os.ReadFile(filepath.Join(brain, "wiki", "concepts", "real.md"))
|
|
||||||
assert.Contains(t, string(b), "zqinside", "positive control: the regular source's ref is added")
|
|
||||||
assert.NotContains(t, string(b), brainfstest.Sentinel, "a symlinked source's title was written as a ref")
|
|
||||||
raw, _ := os.ReadFile(filepath.Join(brain, "raw", "page.md"))
|
|
||||||
assert.NotContains(t, string(raw), "[[ok2", "the symlinked concept page's target was rewritten")
|
|
||||||
}
|
|
||||||
|
|
||||||
// A source swapped between a regular file and a symlink to an in-brain source
|
|
||||||
// whose title is the sentinel: the read refuses the swapped-in link, so the
|
|
||||||
// sentinel never lands in the concept page as a ref.
|
|
||||||
func TestBackfillRefsDoesNotWriteARefFromASourceSwappedInBehindASymlink(t *testing.T) {
|
|
||||||
brain := t.TempDir()
|
|
||||||
mkdirs(t, brain, "wiki/sources", "wiki/concepts", "raw")
|
|
||||||
realPage := filepath.Join(brain, "wiki", "concepts", "real.md")
|
|
||||||
target := filepath.Join(brain, "raw", "src.md")
|
|
||||||
require.NoError(t, os.WriteFile(target, []byte("---\ntitle: "+brainfstest.Sentinel+"\n---\n[[real|R]]\n"), 0o644))
|
|
||||||
src := filepath.Join(brain, "wiki", "sources", "r.md")
|
|
||||||
inside := "---\ntitle: zqinside\n---\n[[real|R]]\n"
|
|
||||||
read := func() string {
|
|
||||||
_ = os.WriteFile(realPage, []byte("---\ntitle: R\n---\nbody\n"), 0o644)
|
|
||||||
_, _ = BackfillRefs(context.Background(), brain)
|
|
||||||
b, _ := os.ReadFile(realPage)
|
|
||||||
return string(b)
|
|
||||||
}
|
|
||||||
require.NoError(t, os.WriteFile(src, []byte(inside), 0o644))
|
|
||||||
require.Contains(t, read(), "zqinside", "positive control")
|
|
||||||
brainfstest.AssertNoLeakFrom(t, src, inside, target, 1500*time.Millisecond, read)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A concept page that is regular when the inventory lists it, then swapped
|
|
||||||
// for a symlink to an in-brain file before BackfillRefs reads it: only the
|
|
||||||
// page read's own check refuses it (a static symlink never gets this far,
|
|
||||||
// because LoadInventory does not list it).
|
|
||||||
func TestBackfillRefsDoesNotCopyAPageSwappedInBehindASymlink(t *testing.T) {
|
|
||||||
brain := t.TempDir()
|
|
||||||
mkdirs(t, brain, "wiki/sources", "wiki/concepts", "raw")
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(brain, "wiki", "sources", "s.md"), []byte("---\ntitle: S\n---\n[[c|C]]\n"), 0o644))
|
|
||||||
page := filepath.Join(brain, "wiki", "concepts", "c.md")
|
|
||||||
target := filepath.Join(brain, "raw", "secret.md")
|
|
||||||
inside := "---\ntitle: C\n---\nzqinside\n"
|
|
||||||
read := func() string {
|
|
||||||
// The write path still follows a swapped-in link (#158) and can
|
|
||||||
// overwrite the target, so restore it each round.
|
|
||||||
_ = os.WriteFile(target, []byte("---\ntitle: X\n---\n"+brainfstest.Sentinel+"\n"), 0o644)
|
|
||||||
_, _ = BackfillRefs(context.Background(), brain)
|
|
||||||
f, err := os.OpenFile(page, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0)
|
|
||||||
if err != nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
defer func() { _ = f.Close() }()
|
|
||||||
b, _ := io.ReadAll(f)
|
|
||||||
return string(b)
|
|
||||||
}
|
|
||||||
require.NoError(t, os.WriteFile(page, []byte(inside), 0o644))
|
|
||||||
require.Contains(t, read(), "[[s|S]]", "positive control: the regular page gets the source ref")
|
|
||||||
brainfstest.AssertNoLeakFrom(t, page, inside, target, 3*time.Second, read)
|
|
||||||
}
|
|
||||||
Reference in New Issue
Block a user