Compare commits

..
Author SHA1 Message Date
mathiasandClaude Opus 5.5 4ba9d20b4e fix(ingestion): tripwire alerts survive outages; drop only permanent rejections
CI / Lint / Test / Vet (pull_request) Successful in 41s
CI / Mirror to GitHub (pull_request) Skipped
PR #160 round 2: dropping a message after 5 failures could not tell a
poison message from an ntfy outage. A 15-minute outage dropped the only
incident with nothing reported, and drops were not counted (74 of 300
removals were neither delivered nor counted).

- audit.NtfyNotifier returns *RejectedError for non-2xx; Permanent() is
  true for 4xx other than 429.
- The tripwire queue drops a message only on a permanent rejection and
  counts it; every other failure is an outage and is retried, however
  long. Each flush tries every queued message, so an undeliverable one
  cannot block the rest.
- Overflow and drops go into one counter, sent as a summary retried
  every tick and never dropped. Every removal is delivered or counted.

Tests (red first): an outage of ~100 ticks still delivers the only
incident; delivered + counted equals the number of removals; a
permanently rejected message is dropped and counted. The short-outage
test is renamed to say it is short. The Watch doc no longer claims
'loses none' unconditionally.

Refs #157

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-03 18:30:46 +02:00
mathiasandClaude Opus 5.5 01de163452 fix(ingestion): bound the tripwire's alerts; test removal guards; deflake race tests
CI / Lint / Test / Vet (pull_request) Successful in 47s
CI / Mirror to GitHub (pull_request) Skipped
PR #160 review:
- The planter chooses the paths, and one alert listed them all, while the
  queue retried oldest-first forever: a message ntfy refuses (too large)
  blocked every later alert (2000 links: 0 alerts delivered), and an
  outage queued one message per removal, unbounded. Alerts now name at
  most 10 paths and count the rest; the queue holds at most 5, further
  alerts become one suppression summary; a message failing 5 times is
  dropped so it cannot block the rest.
- removeSymlinks' Lstat re-check had no test: a direct test passes a
  regular file, a path through an in-brain dir link, a ../ escape and a
  missing path, and asserts all fail with nothing deleted. A Watch test
  covers dir and in-brain targets. root.Remove (not RemoveAll) is
  commented: it refuses a dir swapped in after the Lstat.
- Every queued alert is delivered after an outage (truncating the queue
  after the first send is now red).
- hyperguild#162: race tests did not wait for their swapper goroutines
  before t.TempDir cleanup ("directory not empty" in CI run 8912). Each
  swapper is now stopped and awaited in a t.Cleanup.

Fixes #162
Refs #157

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-03 18:24:13 +02:00
mathiasandClaude Opus 5.5 504feb372f feat(ingestion): the runtime symlink tripwire removes what it finds
CI / Lint / Test / Vet (pull_request) Failing after 33s
CI / Mirror to GitHub (pull_request) Skipped
Decision (#157): the tripwire only alerted, so a symlink planted in the
Syncthing-shared brain dir stayed readable until a human removed it.
Watch now unlinks every symlink it finds through an os.Root on the brain
dir (re-checking it is still a symlink; the target is never touched),
and the default scan interval drops from 5 min to 30 s, so a planted
link is readable for at most one interval.

Each removal is an alert naming what was removed; a persistent finding
(a link it could not remove, an unreadable dir, a scan error) alerts
once per distinct state. Alerts queue until Notify succeeds, so an ntfy
outage delays them without losing them, even though the next scan of
the cleaned brain is clean.

The startup check still refuses to start on any symlink.

Refs #157

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-03 18:16:43 +02:00
33 changed files with 188 additions and 2445 deletions
-25
View File
@@ -1,25 +0,0 @@
package main
import (
"context"
"log/slog"
chassisauth "git.d-ma.be/mathias/mcp-chassis/auth"
"github.com/mathiasbq/hyperguild/ingestion/internal/startupretry"
)
// buildJWTValidator runs OIDC discovery under policy, so a 502 from an auth
// upstream that is not serving yet on a cold boot is retried instead of
// exiting (#83). It honours ctx between attempts.
func buildJWTValidator(
ctx context.Context, logger *slog.Logger, issuer, audience string, policy startupretry.Policy,
) (*chassisauth.JWTValidator, error) {
return startupretry.Do(ctx, policy, func(ctx context.Context) (*chassisauth.JWTValidator, error) {
v, err := chassisauth.NewJWTValidator(ctx, issuer, audience)
if err != nil {
logger.Warn("oidc discovery not ready, retrying", "err", err)
}
return v, err
})
}
@@ -1,86 +0,0 @@
package main
import (
"context"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mathiasbq/hyperguild/ingestion/internal/startupretry"
)
// fakeIssuer answers OIDC discovery with 502 for the first fail requests, as
// ingress does while dex is not yet serving on a cold boot (#83), then with a
// real discovery document and an empty JWKS.
func fakeIssuer(t *testing.T, fail int32) (*httptest.Server, *atomic.Int32) {
t.Helper()
var discoveries atomic.Int32
mux := http.NewServeMux()
var srv *httptest.Server
mux.HandleFunc("/.well-known/openid-configuration", func(w http.ResponseWriter, _ *http.Request) {
if discoveries.Add(1) <= fail {
w.WriteHeader(http.StatusBadGateway)
return
}
w.Header().Set("Content-Type", "application/json")
_, _ = io.WriteString(w, `{"issuer":"`+srv.URL+`","jwks_uri":"`+srv.URL+`/keys"}`)
})
mux.HandleFunc("/keys", func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = io.WriteString(w, `{"keys":[]}`)
})
srv = httptest.NewServer(mux)
t.Cleanup(srv.Close)
return srv, &discoveries
}
var fastPolicy = startupretry.Policy{Attempts: 8, Base: time.Millisecond, Max: 5 * time.Millisecond}
func TestBuildJWTValidatorRetriesDiscovery502sThenSucceeds(t *testing.T) {
srv, discoveries := fakeIssuer(t, 3)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
v, err := buildJWTValidator(ctx, slog.New(slog.DiscardHandler), srv.URL, "brain", fastPolicy)
require.NoError(t, err)
assert.NotNil(t, v)
assert.Equal(t, int32(4), discoveries.Load(), "three 502s, then the successful discovery")
}
func TestBuildJWTValidatorGivesUpAfterTheBudgetWithTheLastError(t *testing.T) {
srv, discoveries := fakeIssuer(t, 1000)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
_, err := buildJWTValidator(ctx, slog.New(slog.DiscardHandler), srv.URL, "brain", fastPolicy)
require.Error(t, err)
assert.Contains(t, err.Error(), "status 502")
assert.Equal(t, int32(8), discoveries.Load(), "bounded: exactly the attempt budget")
}
func TestBuildJWTValidatorStopsRetryingWhenTheContextIsCancelled(t *testing.T) {
srv, _ := fakeIssuer(t, 1000)
ctx, cancel := context.WithCancel(context.Background())
slow := startupretry.Policy{Attempts: 8, Base: time.Hour, Max: time.Hour}
done := make(chan error, 1)
go func() {
_, err := buildJWTValidator(ctx, slog.New(slog.DiscardHandler), srv.URL, "brain", slow)
done <- err
}()
time.Sleep(50 * time.Millisecond)
cancel()
select {
case err := <-done:
assert.ErrorIs(t, err, context.Canceled)
case <-time.After(3 * time.Second):
t.Fatal("did not return after the context was cancelled")
}
}
+9 -3
View File
@@ -381,9 +381,15 @@ func main() {
// //
// The budget stays bounded: retrying forever would turn a misconfiguration // The budget stays bounded: retrying forever would turn a misconfiguration
// into a process that looks alive and never serves. // into a process that looks alive and never serves.
jwtValidator, err := buildJWTValidator(ctx, logger, jwtValidator, err := startupretry.Do(ctx,
os.Getenv("DEX_ISSUER_URL"), os.Getenv("MCP_AUDIENCE"), startupretry.Policy{Attempts: 8, Base: time.Second, Max: 30 * time.Second},
startupretry.Policy{Attempts: 8, Base: time.Second, Max: 30 * time.Second}) func(ctx context.Context) (*chassisauth.JWTValidator, error) {
v, err := chassisauth.NewJWTValidator(ctx, os.Getenv("DEX_ISSUER_URL"), os.Getenv("MCP_AUDIENCE"))
if err != nil {
logger.Warn("oidc discovery not ready, retrying", "err", err)
}
return v, err
})
if err != nil { if err != nil {
logger.Error("build jwt validator", "err", err) logger.Error("build jwt validator", "err", err)
os.Exit(1) os.Exit(1)
+6 -6
View File
@@ -34,9 +34,9 @@ require (
github.com/rogpeppe/go-internal v1.15.0 // indirect github.com/rogpeppe/go-internal v1.15.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/pflag v1.0.5 // indirect
github.com/x448/float16 v0.8.4 // indirect github.com/x448/float16 v0.8.4 // indirect
golang.org/x/net v0.55.0 // indirect golang.org/x/net v0.26.0 // indirect
golang.org/x/oauth2 v0.21.0 // indirect golang.org/x/oauth2 v0.21.0 // indirect
golang.org/x/term v0.43.0 // indirect golang.org/x/term v0.28.0 // indirect
golang.org/x/time v0.3.0 // indirect golang.org/x/time v0.3.0 // indirect
google.golang.org/protobuf v1.34.2 // indirect google.golang.org/protobuf v1.34.2 // indirect
gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
@@ -66,9 +66,9 @@ require (
github.com/lestrrat-go/option v1.0.1 // indirect github.com/lestrrat-go/option v1.0.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/segmentio/asm v1.2.0 // indirect github.com/segmentio/asm v1.2.0 // indirect
golang.org/x/crypto v0.51.0 // indirect golang.org/x/crypto v0.32.0 // indirect
golang.org/x/sync v0.21.0 // indirect golang.org/x/sync v0.17.0 // indirect
golang.org/x/sys v0.45.0 // indirect golang.org/x/sys v0.31.0 // indirect
golang.org/x/text v0.39.0 // indirect golang.org/x/text v0.29.0 // indirect
gopkg.in/yaml.v3 v3.0.1 gopkg.in/yaml.v3 v3.0.1
) )
+14 -14
View File
@@ -117,42 +117,42 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc=
golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs=
golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg=
golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg=
golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1,43 +0,0 @@
package api_test
import (
"os"
"path/filepath"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mathiasbq/hyperguild/ingestion/internal/api"
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs/brainfstest"
)
// hyperguild#157 slice 2: ReadNote (brain_get, read-after-write) read by
// path; it now reads through brainfs.
func TestReadNoteRefusesSymlinksAndSurvivesSwapRaces(t *testing.T) {
brain := t.TempDir()
facts := filepath.Join(brain, "wiki", "w", "facts")
require.NoError(t, os.MkdirAll(facts, 0o755))
require.NoError(t, os.Symlink(brainfstest.Outside(t), filepath.Join(facts, "leak.md")))
_, body, _, err := api.ReadNote(brain, "wiki/w/facts/leak.md")
assert.Error(t, err)
assert.NotContains(t, body, brainfstest.Sentinel)
p := filepath.Join(facts, "r.md")
require.NoError(t, os.WriteFile(p, []byte("---\nx: y\n---\ninside"), 0o644))
fm, body, _, err := api.ReadNote(brain, "wiki/w/facts/r.md")
require.NoError(t, err)
assert.Equal(t, "y", fm["x"])
assert.Contains(t, body, "inside")
read := func() string {
fm, body, _, _ := api.ReadNote(brain, "wiki/w/facts/r.md")
s := body
for k, v := range fm {
s += k + v
}
return s
}
brainfstest.AssertNoLeak(t, p, "---\nx: y\n---\ninside", 1500*time.Millisecond, read)
brainfstest.AssertNoHang(t, p, "---\nx: y\n---\ninside", 1500*time.Millisecond, 6*time.Second, func() { _ = read() })
}
+6 -25
View File
@@ -3,18 +3,14 @@ package api
import ( import (
"crypto/sha256" "crypto/sha256"
"encoding/hex" "encoding/hex"
"errors"
"fmt" "fmt"
"io/fs"
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
"time" "time"
"github.com/mathiasbq/hyperguild/ingestion/internal/brain" "github.com/mathiasbq/hyperguild/ingestion/internal/brain"
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
"github.com/mathiasbq/hyperguild/ingestion/internal/claim" "github.com/mathiasbq/hyperguild/ingestion/internal/claim"
"github.com/mathiasbq/hyperguild/ingestion/internal/notes"
) )
// ContentHash returns the lowercase hex sha256 of b. It is the note's // ContentHash returns the lowercase hex sha256 of b. It is the note's
@@ -94,23 +90,14 @@ func UpdateNote(brainDir string, opts UpdateNoteOptions) (relPath, contentHash,
rel = filepath.ToSlash(rel) rel = filepath.ToSlash(rel)
} }
// The path is caller-supplied (brain_update): resolve it as brain_edit rel, abs, err := resolveWithin(brainDir, rel)
// does, knowledge/ and wiki/ notes only, and read and rewrite it through
// the brain root, never through a symlink (hyperguild#169).
t, err := notes.Resolve(brainDir, rel, notes.ScopeNotes)
if err != nil { if err != nil {
return "", "", "", err return "", "", "", err
} }
rel = t.Rel
bfs, err := brainfs.Open(brainDir)
if err != nil {
return "", "", "", fmt.Errorf("open brain: %w", err)
}
defer func() { _ = bfs.Close() }()
prior, err := bfs.ReadRegular(rel) prior, err := os.ReadFile(abs)
if err != nil { if err != nil {
if errors.Is(err, fs.ErrNotExist) { if os.IsNotExist(err) {
return "", "", "", fmt.Errorf("note %q does not exist: use brain_write to create", rel) return "", "", "", fmt.Errorf("note %q does not exist: use brain_write to create", rel)
} }
return "", "", "", fmt.Errorf("read target: %w", err) return "", "", "", fmt.Errorf("read target: %w", err)
@@ -176,7 +163,7 @@ func UpdateNote(brainDir string, opts UpdateNoteOptions) (relPath, contentHash,
return "", "", "", err return "", "", "", err
} }
out := []byte(stamped) out := []byte(stamped)
if err := bfs.WriteRegular(rel, out); err != nil { if err := os.WriteFile(abs, out, 0o644); err != nil {
return "", "", "", fmt.Errorf("write: %w", err) return "", "", "", fmt.Errorf("write: %w", err)
} }
return rel, ContentHash(out), priorHash, nil return rel, ContentHash(out), priorHash, nil
@@ -187,17 +174,11 @@ func UpdateNote(brainDir string, opts UpdateNoteOptions) (relPath, contentHash,
// write primitive behind brain_get: the hash it returns equals the hash // write primitive behind brain_get: the hash it returns equals the hash
// brain_write / brain_update returned for the same bytes. // brain_write / brain_update returned for the same bytes.
func ReadNote(brainDir, relPath string) (fm map[string]string, body, contentHash string, err error) { func ReadNote(brainDir, relPath string) (fm map[string]string, body, contentHash string, err error) {
rel, _, err := resolveWithin(brainDir, relPath) _, abs, err := resolveWithin(brainDir, relPath)
if err != nil { if err != nil {
return nil, "", "", err return nil, "", "", err
} }
// Through the brain root, regular files only (hyperguild#157). raw, err := os.ReadFile(abs)
bfs, err := brainfs.Open(brainDir)
if err != nil {
return nil, "", "", fmt.Errorf("read note: %w", err)
}
defer func() { _ = bfs.Close() }()
raw, err := bfs.ReadRegular(rel)
if err != nil { if err != nil {
if os.IsNotExist(err) { if os.IsNotExist(err) {
return nil, "", "", fmt.Errorf("note %q does not exist", relPath) return nil, "", "", fmt.Errorf("note %q does not exist", relPath)
-112
View File
@@ -1,112 +0,0 @@
package api_test
import (
"os"
"path/filepath"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mathiasbq/hyperguild/ingestion/internal/api"
)
// brain_update passed the caller's path through resolveWithin, a lexical
// containment check only: any file in the brain could be superseded
// (classification.yaml, raw/, sessions/), and a final symlink was written
// through. It now resolves like brain_edit: knowledge/ and wiki/ notes only,
// read and rewritten through the brain root.
func TestUpdateNoteRefusesPathsOutsideKnowledgeAndWikiNotes(t *testing.T) {
brain := t.TempDir()
files := map[string]string{
"classification.yaml": "levels: {}\n",
"raw/r.md": "raw\n",
"sessions/s.jsonl": "{}\n",
"wiki/w/facts/ok.md": "---\ntitle: ok\n---\nold\n",
"knowledge/k.md": "---\ntitle: k\n---\nold\n",
"wiki/w/facts/x.txt": "txt\n",
}
for rel, body := range files {
p := filepath.Join(brain, filepath.FromSlash(rel))
require.NoError(t, os.MkdirAll(filepath.Dir(p), 0o755))
require.NoError(t, os.WriteFile(p, []byte(body), 0o644))
}
for _, rel := range []string{"classification.yaml", "raw/r.md", "sessions/s.jsonl", "wiki/w/facts/x.txt", "wiki/w/../../raw/r.md"} {
_, _, _, err := api.UpdateNote(brain, api.UpdateNoteOptions{Path: rel, Content: "replaced"})
assert.Error(t, err, rel)
}
for rel, body := range files {
if rel == "wiki/w/facts/ok.md" || rel == "knowledge/k.md" {
continue
}
got, err := os.ReadFile(filepath.Join(brain, filepath.FromSlash(rel)))
require.NoError(t, err)
assert.Equal(t, body, string(got), rel)
}
for _, rel := range []string{"wiki/w/facts/ok.md", "knowledge/k.md"} {
_, _, _, err := api.UpdateNote(brain, api.UpdateNoteOptions{Path: rel, Content: "new body"})
require.NoError(t, err, "positive control: %s", rel)
got, _ := os.ReadFile(filepath.Join(brain, filepath.FromSlash(rel)))
assert.Contains(t, string(got), "new body")
}
}
func TestUpdateNoteDoesNotWriteThroughASymlinkedNote(t *testing.T) {
brain := t.TempDir()
facts := filepath.Join(brain, "wiki", "w", "facts")
require.NoError(t, os.MkdirAll(facts, 0o755))
victim := filepath.Join(brain, "raw", "victim.md")
require.NoError(t, os.MkdirAll(filepath.Dir(victim), 0o755))
require.NoError(t, os.WriteFile(victim, []byte("untouched\n"), 0o644))
require.NoError(t, os.Symlink(filepath.Join("..", "..", "..", "raw", "victim.md"), filepath.Join(facts, "l.md")))
_, _, _, err := api.UpdateNote(brain, api.UpdateNoteOptions{Path: "wiki/w/facts/l.md", Content: "replaced"})
assert.Error(t, err)
got, _ := os.ReadFile(victim)
assert.Equal(t, "untouched\n", string(got))
}
// The note is swapped between a regular file and an in-brain symlink to a
// raw/ file while UpdateNote runs: the read and the write each refuse a
// symlink, so the raw/ file is never rewritten.
func TestUpdateNoteNeverRewritesASymlinkSwappedInForTheNote(t *testing.T) {
brain := t.TempDir()
facts := filepath.Join(brain, "wiki", "w", "facts")
require.NoError(t, os.MkdirAll(facts, 0o755))
victim := filepath.Join(brain, "raw", "victim.md")
require.NoError(t, os.MkdirAll(filepath.Dir(victim), 0o755))
require.NoError(t, os.WriteFile(victim, []byte("untouched\n"), 0o644))
note := filepath.Join(facts, "n.md")
body := "---\ntitle: n\n---\nold\n"
require.NoError(t, os.WriteFile(note, []byte(body), 0o644))
var stop atomic.Bool
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
tmp := note + ".swap"
for i := 0; !stop.Load(); i++ {
_ = os.Remove(tmp)
if i%2 == 0 {
_ = os.WriteFile(tmp, []byte(body), 0o644)
} else {
_ = os.Symlink(filepath.Join("..", "..", "..", "raw", "victim.md"), tmp)
}
_ = os.Rename(tmp, note)
}
}()
ok := 0
for deadline := time.Now().Add(1500 * time.Millisecond); time.Now().Before(deadline); {
if _, _, _, err := api.UpdateNote(brain, api.UpdateNoteOptions{Path: "wiki/w/facts/n.md", Content: "new"}); err == nil {
ok++
}
}
stop.Store(true)
wg.Wait()
got, _ := os.ReadFile(victim)
assert.Equal(t, "untouched\n", string(got))
assert.Positive(t, ok, "positive control: some updates reach the regular note")
}
+37 -67
View File
@@ -2,18 +2,12 @@ package brain
import ( import (
"bufio" "bufio"
"bytes"
"errors"
"fmt" "fmt"
"io"
"io/fs"
"os" "os"
"path/filepath" "path/filepath"
"sort" "sort"
"strings" "strings"
"time" "time"
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
) )
// noteEntry is one row in a Wing _index.md. // noteEntry is one row in a Wing _index.md.
@@ -33,18 +27,13 @@ func BuildWingIndex(brainDir, wing string) error {
return fmt.Errorf("invalid wing %q", wing) return fmt.Errorf("invalid wing %q", wing)
} }
wingDir := filepath.Join(brainDir, "wiki", w) wingDir := filepath.Join(brainDir, "wiki", w)
// Listed and read through the brain root (hyperguild#157): a wing or if _, err := os.Stat(wingDir); os.IsNotExist(err) {
// hall that is a symlink is not followed, and notes are regular files.
bfs, err := brainfs.Open(brainDir)
if err != nil {
return fmt.Errorf("open brain: %w", err)
}
defer func() { _ = bfs.Close() }()
if _, err := bfs.Lstat("wiki/" + w); errors.Is(err, fs.ErrNotExist) {
return nil return nil
} else if err != nil {
return fmt.Errorf("stat wing: %w", err)
} }
entries, err := collectWingEntries(bfs, "wiki/"+w) entries, err := collectWingEntries(wingDir)
if err != nil { if err != nil {
return err return err
} }
@@ -69,47 +58,51 @@ func BuildWingIndex(brainDir, wing string) error {
// BuildAllWingIndexes regenerates _index.md for every wing under brain/wiki/. // BuildAllWingIndexes regenerates _index.md for every wing under brain/wiki/.
func BuildAllWingIndexes(brainDir string) error { func BuildAllWingIndexes(brainDir string) error {
bfs, err := brainfs.Open(brainDir) wikiDir := filepath.Join(brainDir, "wiki")
if err != nil { ents, err := os.ReadDir(wikiDir)
return fmt.Errorf("open brain: %w", err) if os.IsNotExist(err) {
}
wings, err := bfs.ReadDirDirs("wiki")
_ = bfs.Close()
if errors.Is(err, fs.ErrNotExist) {
return nil return nil
} }
if err != nil { if err != nil {
return fmt.Errorf("read wiki: %w", err) return fmt.Errorf("read wiki: %w", err)
} }
for _, w := range wings { for _, e := range ents {
if err := BuildWingIndex(brainDir, w); err != nil { if !e.IsDir() {
return fmt.Errorf("index %s: %w", w, err) continue
}
if err := BuildWingIndex(brainDir, e.Name()); err != nil {
return fmt.Errorf("index %s: %w", e.Name(), err)
} }
} }
return nil return nil
} }
func collectWingEntries(bfs *brainfs.Brain, wingRel string) ([]noteEntry, error) { func collectWingEntries(wingDir string) ([]noteEntry, error) {
var out []noteEntry var out []noteEntry
halls, err := bfs.ReadDirDirs(wingRel) ents, err := os.ReadDir(wingDir)
if err != nil { if err != nil {
return nil, fmt.Errorf("read wing: %w", err) return nil, fmt.Errorf("read wing: %w", err)
} }
for _, hall := range halls { for _, hallEnt := range ents {
if !hallEnt.IsDir() {
continue
}
hall := hallEnt.Name()
if !IsValidHall(hall) { if !IsValidHall(hall) {
continue continue
} }
hallRel := wingRel + "/" + hall hallDir := filepath.Join(wingDir, hall)
names, err := bfs.ReadDirRegular(hallRel) notes, err := os.ReadDir(hallDir)
if err != nil { if err != nil {
return nil, fmt.Errorf("read hall %s: %w", hall, err) return nil, fmt.Errorf("read hall %s: %w", hall, err)
} }
for _, name := range names { for _, n := range notes {
if !strings.HasSuffix(name, ".md") || name == "_index.md" { if n.IsDir() || !strings.HasSuffix(n.Name(), ".md") || n.Name() == "_index.md" {
continue continue
} }
slug := strings.TrimSuffix(name, ".md") slug := strings.TrimSuffix(n.Name(), ".md")
title, created := readTitleAndCreatedIn(bfs, hallRel+"/"+name, slug) full := filepath.Join(hallDir, n.Name())
title, created := readTitleAndCreated(full, slug)
out = append(out, noteEntry{Hall: hall, Slug: slug, Title: title, Created: created}) out = append(out, noteEntry{Hall: hall, Slug: slug, Title: title, Created: created})
} }
} }
@@ -124,40 +117,9 @@ func readTitleAndCreated(path, slug string) (string, string) {
return slug, "" return slug, ""
} }
defer func() { _ = f.Close() }() defer func() { _ = f.Close() }()
title, created := parseTitleAndCreated(f)
if created == "" {
if info, err := os.Stat(path); err == nil {
created = info.ModTime().UTC().Format("2006-01-02")
}
}
return titleOrSlug(title, slug), created
}
// readTitleAndCreatedIn is readTitleAndCreated through the brain root, title, created := "", ""
// regular files only (hyperguild#157). scanner := bufio.NewScanner(f)
func readTitleAndCreatedIn(bfs *brainfs.Brain, rel, slug string) (string, string) {
content, err := bfs.ReadRegular(rel)
if err != nil {
return slug, ""
}
title, created := parseTitleAndCreated(bytes.NewReader(content))
if created == "" {
if info, err := bfs.Lstat(rel); err == nil {
created = info.ModTime().UTC().Format("2006-01-02")
}
}
return titleOrSlug(title, slug), created
}
func titleOrSlug(title, slug string) string {
if title == "" {
return strings.ReplaceAll(slug, "-", " ")
}
return title
}
func parseTitleAndCreated(r io.Reader) (title, created string) {
scanner := bufio.NewScanner(r)
inFrontmatter := false inFrontmatter := false
for scanner.Scan() { for scanner.Scan() {
line := scanner.Text() line := scanner.Text()
@@ -187,5 +149,13 @@ func parseTitleAndCreated(r io.Reader) (title, created string) {
} }
} }
} }
if title == "" {
title = strings.ReplaceAll(slug, "-", " ")
}
if created == "" {
if info, err := os.Stat(path); err == nil {
created = info.ModTime().UTC().Format("2006-01-02")
}
}
return title, created return title, created
} }
-132
View File
@@ -1,132 +0,0 @@
package brain_test
import (
"os"
"path/filepath"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mathiasbq/hyperguild/ingestion/internal/brain"
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs/brainfstest"
)
func readIndex(t *testing.T, brainDir, wing string) string {
t.Helper()
b, _ := os.ReadFile(filepath.Join(brainDir, "wiki", wing, "_index.md"))
return string(b)
}
// hyperguild#157 slice 5: BuildWingIndex listed wiki/<wing>/<hall>/ with
// os.ReadDir and opened each note by path, so a symlinked note put the
// target's title into _index.md, and a FIFO hung the index build.
func TestBuildWingIndexRefusesSymlinkedNotesAndSurvivesSwapRaces(t *testing.T) {
b := t.TempDir()
facts := filepath.Join(b, "wiki", "w", "facts")
require.NoError(t, os.MkdirAll(facts, 0o755))
require.NoError(t, os.WriteFile(filepath.Join(facts, "ok.md"), []byte("---\ntitle: zqinside\n---\n"), 0o644))
require.NoError(t, os.Symlink(brainfstest.Outside(t), filepath.Join(facts, "leak.md")))
inBrain := brainfstest.WriteSentinelAt(t, filepath.Join(b, "raw", "s.md"))
require.NoError(t, os.Symlink(filepath.Join("..", "..", "..", "raw", "s.md"), filepath.Join(facts, "inlink.md")))
require.NoError(t, brain.BuildWingIndex(b, "w"))
idx := readIndex(t, b, "w")
assert.Contains(t, idx, "zqinside", "positive control")
assert.NotContains(t, idx, brainfstest.Sentinel)
assert.NotContains(t, idx, "leak")
assert.NotContains(t, idx, "inlink")
p := filepath.Join(facts, "r.md")
read := func() string {
_ = brain.BuildWingIndex(b, "w")
return readIndex(t, b, "w")
}
brainfstest.AssertNoLeak(t, p, "---\ntitle: zqinside\n---\n", 1500*time.Millisecond, read)
brainfstest.AssertNoLeakFrom(t, p, "---\ntitle: zqinside\n---\n", inBrain, 1500*time.Millisecond, read)
brainfstest.AssertNoHang(t, p, "---\ntitle: zqinside\n---\n", 1500*time.Millisecond, 6*time.Second, func() { _ = read() })
}
// A symlinked hall is skipped while a real hall next to it is indexed; a
// symlinked wing is refused and nothing is written into its target.
func TestBuildWingIndexSkipsASymlinkedHallAndRefusesASymlinkedWing(t *testing.T) {
b := t.TempDir()
require.NoError(t, os.MkdirAll(filepath.Join(b, "raw", "facts"), 0o755))
require.NoError(t, os.WriteFile(filepath.Join(b, "raw", "facts", "r.md"), []byte("---\ntitle: zqrawonly\n---\n"), 0o644))
require.NoError(t, os.MkdirAll(filepath.Join(b, "wiki", "w", "decisions"), 0o755))
require.NoError(t, os.WriteFile(filepath.Join(b, "wiki", "w", "decisions", "d.md"), []byte("---\ntitle: zqrealhall\n---\n"), 0o644))
require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "facts"), filepath.Join(b, "wiki", "w", "facts")))
require.NoError(t, os.Symlink(filepath.Join("..", "raw"), filepath.Join(b, "wiki", "alias")))
require.NoError(t, brain.BuildWingIndex(b, "w"))
idx, err := os.ReadFile(filepath.Join(b, "wiki", "w", "_index.md"))
require.NoError(t, err, "the wing index is written")
assert.Contains(t, string(idx), "zqrealhall", "the real hall is indexed")
assert.NotContains(t, string(idx), "zqrawonly", "the symlinked hall is not")
assert.Error(t, brain.BuildWingIndex(b, "alias"))
_, err = os.Stat(filepath.Join(b, "raw", "_index.md"))
assert.True(t, os.IsNotExist(err), "an index was written into the symlinked wing's target")
}
// BuildAllWingIndexes indexes the real wings and skips a symlinked one.
func TestBuildAllWingIndexesSkipsASymlinkedWing(t *testing.T) {
b := t.TempDir()
require.NoError(t, os.MkdirAll(filepath.Join(b, "wiki", "w", "facts"), 0o755))
require.NoError(t, os.WriteFile(filepath.Join(b, "wiki", "w", "facts", "n.md"), []byte("---\ntitle: zqrealwing\n---\n"), 0o644))
require.NoError(t, os.MkdirAll(filepath.Join(b, "raw", "facts"), 0o755))
require.NoError(t, os.Symlink(filepath.Join("..", "raw"), filepath.Join(b, "wiki", "alias")))
require.NoError(t, brain.BuildAllWingIndexes(b))
idx, err := os.ReadFile(filepath.Join(b, "wiki", "w", "_index.md"))
require.NoError(t, err)
assert.Contains(t, string(idx), "zqrealwing")
_, err = os.Stat(filepath.Join(b, "raw", "_index.md"))
assert.True(t, os.IsNotExist(err))
}
// A link to a symlinked note does not resolve: only regular files count as
// entries, so StripUnresolvedLinks unwraps it.
func TestStripUnresolvedLinksDoesNotCountASymlinkedNoteAsAnEntry(t *testing.T) {
b := t.TempDir()
facts := filepath.Join(b, "wiki", "w", "facts")
require.NoError(t, os.MkdirAll(facts, 0o755))
require.NoError(t, os.WriteFile(filepath.Join(facts, "real.md"), []byte("x"), 0o644))
brainfstest.WriteSentinelAt(t, filepath.Join(b, "raw", "s.md"))
require.NoError(t, os.Symlink(filepath.Join("..", "..", "..", "raw", "s.md"), filepath.Join(facts, "ghost.md")))
out, unlinked, err := brain.StripUnresolvedLinks(b, "see [[real]] and [[ghost]]")
require.NoError(t, err)
assert.Contains(t, out, "[[real]]", "positive control")
assert.NotContains(t, out, "[[ghost]]")
assert.Contains(t, unlinked, "ghost")
}
// A linked tree (knowledge/ or wiki/) that is not a real directory is
// skipped, not an error: StripUnresolvedLinks runs before every brain_write,
// and an error there loses the learning.
func TestStripUnresolvedLinksSkipsALinkedTreeThatIsNotARealDirectory(t *testing.T) {
for name, mk := range map[string]func(t *testing.T, b string){
"symlink to an in-brain dir": func(t *testing.T, b string) {
require.NoError(t, os.MkdirAll(filepath.Join(b, "raw", "k"), 0o755))
require.NoError(t, os.Symlink(filepath.Join("raw", "k"), filepath.Join(b, "knowledge")))
},
"dangling symlink": func(t *testing.T, b string) {
require.NoError(t, os.Symlink("nowhere", filepath.Join(b, "knowledge")))
},
"regular file": func(t *testing.T, b string) {
require.NoError(t, os.WriteFile(filepath.Join(b, "knowledge"), []byte("x"), 0o644))
},
} {
t.Run(name, func(t *testing.T) {
b := t.TempDir()
require.NoError(t, os.MkdirAll(filepath.Join(b, "wiki", "w", "facts"), 0o755))
require.NoError(t, os.WriteFile(filepath.Join(b, "wiki", "w", "facts", "real.md"), []byte("x"), 0o644))
mk(t, b)
out, _, err := brain.StripUnresolvedLinks(b, "learning [[real]] and [[gone]]")
require.NoError(t, err)
assert.Contains(t, out, "[[real]]", "links into wiki/ still resolve")
assert.NotContains(t, out, "[[gone]]")
})
}
}
+17 -32
View File
@@ -1,16 +1,12 @@
package brain package brain
import ( import (
"errors"
"fmt" "fmt"
"io/fs" "os"
"log/slog" "path/filepath"
"path"
"regexp" "regexp"
"sort" "sort"
"strings" "strings"
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
) )
// wikilinkRE matches [[target]] and [[target|Display Text]]. // wikilinkRE matches [[target]] and [[target|Display Text]].
@@ -102,40 +98,29 @@ var linkedTrees = []string{"knowledge", "wiki"}
// one the audit will also resolve. // one the audit will also resolve.
func entryIndex(brainDir string) (map[string]struct{}, error) { func entryIndex(brainDir string) (map[string]struct{}, error) {
index := map[string]struct{}{} index := map[string]struct{}{}
// Walked through the brain root (hyperguild#157): only regular files
// count as entries, so a symlinked .md name cannot make a link resolve.
bfs, err := brainfs.Open(brainDir)
if err != nil {
return nil, fmt.Errorf("open brain: %w", err)
}
defer func() { _ = bfs.Close() }()
for _, dir := range linkedTrees { for _, dir := range linkedTrees {
if _, err := bfs.Lstat(dir); errors.Is(err, fs.ErrNotExist) { root := filepath.Join(brainDir, dir)
if _, err := os.Stat(root); os.IsNotExist(err) {
continue continue
} else if err != nil {
return nil, fmt.Errorf("stat %s: %w", dir, err)
} }
var walkErr error err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
err := bfs.WalkRegular(dir, func(rel string) error { if err != nil {
if !strings.HasSuffix(rel, ".md") { return err
}
if d.IsDir() || !strings.HasSuffix(path, ".md") {
return nil return nil
} }
index[strings.TrimSuffix(path.Base(rel), ".md")] = struct{}{} rel, err := filepath.Rel(brainDir, path)
if err != nil {
return err
}
rel = filepath.ToSlash(rel)
index[strings.TrimSuffix(filepath.Base(rel), ".md")] = struct{}{}
index[strings.TrimPrefix(strings.TrimSuffix(rel, ".md"), "wiki/")] = struct{}{} index[strings.TrimPrefix(strings.TrimSuffix(rel, ".md"), "wiki/")] = struct{}{}
return nil return nil
}, func(rel string, err error) {
// A linked tree that is not a real directory (a symlink, a
// file) is skipped, as filepath.WalkDir did: this runs before
// every brain_write, and an error here loses the learning.
if rel == dir && errors.Is(err, brainfs.ErrNotRegular) {
slog.Warn("link check: skipping a linked tree that is not a real directory", "dir", dir)
return
}
if walkErr == nil {
walkErr = fmt.Errorf("%s: %w", rel, err)
}
}) })
if err == nil {
err = walkErr
}
if err != nil { if err != nil {
return nil, fmt.Errorf("walk %s: %w", dir, err) return nil, fmt.Errorf("walk %s: %w", dir, err)
} }
+20 -54
View File
@@ -4,12 +4,9 @@ import (
"bufio" "bufio"
"fmt" "fmt"
"os" "os"
"path"
"path/filepath" "path/filepath"
"strings" "strings"
"time" "time"
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
) )
// seeAlsoHeader is the markdown heading used to group cross-wing links. // seeAlsoHeader is the markdown heading used to group cross-wing links.
@@ -200,12 +197,6 @@ func fileExists(p string) bool {
// section headers. Rejects same-wing pairs (a tunnel is by definition // section headers. Rejects same-wing pairs (a tunnel is by definition
// cross-wing) and missing notes. // cross-wing) and missing notes.
func WriteTunnel(brainDir, sourcePath, targetPath string) error { func WriteTunnel(brainDir, sourcePath, targetPath string) error {
if err := checkNotePath(sourcePath); err != nil {
return fmt.Errorf("source: %w", err)
}
if err := checkNotePath(targetPath); err != nil {
return fmt.Errorf("target: %w", err)
}
srcWing, err := wingOf(sourcePath) srcWing, err := wingOf(sourcePath)
if err != nil { if err != nil {
return fmt.Errorf("source: %w", err) return fmt.Errorf("source: %w", err)
@@ -218,50 +209,20 @@ func WriteTunnel(brainDir, sourcePath, targetPath string) error {
return fmt.Errorf("tunnel must cross wings; got both in %q", srcWing) return fmt.Errorf("tunnel must cross wings; got both in %q", srcWing)
} }
// Both notes are read and rewritten through the brain root, regular srcFull := filepath.Join(brainDir, filepath.FromSlash(sourcePath))
// files only: brain_tunnel passes caller-supplied paths, and the brain is tgtFull := filepath.Join(brainDir, filepath.FromSlash(targetPath))
// a Syncthing folder that can hold symlinks. if _, err := os.Stat(srcFull); err != nil {
bfs, err := brainfs.Open(brainDir)
if err != nil {
return fmt.Errorf("open brain: %w", err)
}
defer func() { _ = bfs.Close() }()
srcContent, err := bfs.ReadRegular(sourcePath)
if err != nil {
return fmt.Errorf("source note: %w", err) return fmt.Errorf("source note: %w", err)
} }
tgtContent, err := bfs.ReadRegular(targetPath) if _, err := os.Stat(tgtFull); err != nil {
if err != nil {
return fmt.Errorf("target note: %w", err) return fmt.Errorf("target note: %w", err)
} }
if out, changed, err := withSeeAlso(srcContent, wikilinkOf(targetPath)); err != nil { if err := appendSeeAlso(srcFull, wikilinkOf(targetPath)); err != nil {
return fmt.Errorf("update source: %w", err) return fmt.Errorf("update source: %w", err)
} else if changed {
if err := bfs.WriteRegular(sourcePath, out); err != nil {
return fmt.Errorf("update source: %w", err)
}
} }
if out, changed, err := withSeeAlso(tgtContent, wikilinkOf(sourcePath)); err != nil { if err := appendSeeAlso(tgtFull, wikilinkOf(sourcePath)); err != nil {
return fmt.Errorf("update target: %w", err) return fmt.Errorf("update target: %w", err)
} else if changed {
if err := bfs.WriteRegular(targetPath, out); err != nil {
return fmt.Errorf("update target: %w", err)
}
}
return nil
}
// checkNotePath accepts only a clean, local wiki/<wing>/<hall>/.../<slug>.md
// path: no "." or ".." segments, no empty segments, no leading slash.
func checkNotePath(p string) error {
if p == "" || path.Clean(p) != p || !filepath.IsLocal(p) || !strings.HasSuffix(p, ".md") {
return fmt.Errorf("not a clean wiki note path: %q", p)
}
for _, part := range strings.Split(p, "/") {
if part == "" || part == "." || part == ".." {
return fmt.Errorf("not a clean wiki note path: %q", p)
}
} }
return nil return nil
} }
@@ -287,13 +248,16 @@ func wingOf(relPath string) (string, error) {
return parts[1], nil return parts[1], nil
} }
// withSeeAlso returns content with `- [[link]]` under its See also section, // appendSeeAlso inserts `- [[link]]` under the file's See also section,
// creating the section if absent. changed is false when the link is already // creating the section if absent. No-op when the link is already present.
// present. func appendSeeAlso(filePath, link string) error {
func withSeeAlso(content []byte, link string) (out []byte, changed bool, err error) { content, err := os.ReadFile(filePath)
if err != nil {
return err
}
wikilink := "[[" + link + "]]" wikilink := "[[" + link + "]]"
if strings.Contains(string(content), wikilink) { if strings.Contains(string(content), wikilink) {
return content, false, nil return nil
} }
bullet := "- " + wikilink bullet := "- " + wikilink
@@ -302,7 +266,8 @@ func withSeeAlso(content []byte, link string) (out []byte, changed bool, err err
// No section yet — append a fresh one. Always emit a trailing // No section yet — append a fresh one. Always emit a trailing
// newline so subsequent appends don't merge into the previous line. // newline so subsequent appends don't merge into the previous line.
trimmed := strings.TrimRight(string(content), "\n") trimmed := strings.TrimRight(string(content), "\n")
return []byte(trimmed + "\n\n" + seeAlsoHeader + "\n\n" + bullet + "\n"), true, nil out := trimmed + "\n\n" + seeAlsoHeader + "\n\n" + bullet + "\n"
return os.WriteFile(filePath, []byte(out), 0o644)
} }
// Section exists — splice the bullet in just before the next `## ` // Section exists — splice the bullet in just before the next `## `
@@ -328,11 +293,12 @@ func withSeeAlso(content []byte, link string) (out []byte, changed bool, err err
b.WriteByte('\n') b.WriteByte('\n')
} }
if err := scanner.Err(); err != nil { if err := scanner.Err(); err != nil {
return nil, false, err return err
} }
if !inserted { if !inserted {
// section was the last thing in the file — just append bullet // section was the last thing in the file — just append bullet
return []byte(strings.TrimRight(b.String(), "\n") + "\n" + bullet + "\n"), true, nil out := strings.TrimRight(b.String(), "\n") + "\n" + bullet + "\n"
return os.WriteFile(filePath, []byte(out), 0o644)
} }
return []byte(b.String()), true, nil return os.WriteFile(filePath, []byte(b.String()), 0o644)
} }
@@ -1,141 +0,0 @@
package brain_test
import (
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mathiasbq/hyperguild/ingestion/internal/brain"
)
// brain_tunnel passes caller-supplied source and target paths to WriteTunnel,
// which checked only "wiki/<wing>/..." and then joined them onto the brain
// dir: ".." segments resolved outside the brain and appendSeeAlso rewrote the
// file there.
func TestWriteTunnelRefusesATargetThatTraversesOutOfTheBrain(t *testing.T) {
brain0 := t.TempDir()
src := filepath.Join(brain0, "wiki", "a", "facts", "s.md")
require.NoError(t, os.MkdirAll(filepath.Dir(src), 0o755))
require.NoError(t, os.WriteFile(src, []byte("---\ntitle: S\n---\nbody\n"), 0o644))
victim := filepath.Join(t.TempDir(), "victim.md")
require.NoError(t, os.WriteFile(victim, []byte("untouched\n"), 0o644))
up := strings.Repeat("../", strings.Count(filepath.Clean(brain0), "/")+3)
target := "wiki/b/x/" + up + strings.TrimPrefix(victim, "/")
err := brain.WriteTunnel(brain0, "wiki/a/facts/s.md", target)
assert.Error(t, err)
got, rerr := os.ReadFile(victim)
require.NoError(t, rerr)
assert.Equal(t, "untouched\n", string(got), "a file outside the brain was modified")
}
func TestWriteTunnelRefusesUncleanOrNonWikiPaths(t *testing.T) {
brain0 := t.TempDir()
for _, p := range []string{"wiki/a/facts/s.md", "wiki/b/facts/t.md", "wiki/b/facts/t.txt", "raw/r.md"} {
require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Join(brain0, p)), 0o755))
require.NoError(t, os.WriteFile(filepath.Join(brain0, p), []byte("x\n"), 0o644))
}
for _, target := range []string{
"wiki/b/facts/../../../raw/r.md",
"wiki/b/./facts/t.md",
"wiki/b//facts/t.md",
"/wiki/b/facts/t.md",
"wiki/b/facts/t.txt",
"wiki/b/facts/",
} {
assert.Error(t, brain.WriteTunnel(brain0, "wiki/a/facts/s.md", target), target)
}
for _, p := range []string{"raw/r.md", "wiki/b/facts/t.txt"} {
got, err := os.ReadFile(filepath.Join(brain0, p))
require.NoError(t, err)
assert.Equal(t, "x\n", string(got), p)
}
require.NoError(t, brain.WriteTunnel(brain0, "wiki/a/facts/s.md", "wiki/b/facts/t.md"), "positive control")
got, err := os.ReadFile(filepath.Join(brain0, "wiki", "b", "facts", "t.md"))
require.NoError(t, err)
assert.Contains(t, string(got), "[[a/facts/s]]")
}
func TestWriteTunnelDoesNotWriteThroughASymlinkedNote(t *testing.T) {
brain0 := t.TempDir()
src := filepath.Join(brain0, "wiki", "a", "facts", "s.md")
require.NoError(t, os.MkdirAll(filepath.Dir(src), 0o755))
require.NoError(t, os.WriteFile(src, []byte("x\n"), 0o644))
tdir := filepath.Join(brain0, "wiki", "b", "facts")
require.NoError(t, os.MkdirAll(tdir, 0o755))
victim := filepath.Join(t.TempDir(), "victim.md")
require.NoError(t, os.WriteFile(victim, []byte("untouched\n"), 0o644))
require.NoError(t, os.Symlink(victim, filepath.Join(tdir, "t.md")))
assert.Error(t, brain.WriteTunnel(brain0, "wiki/a/facts/s.md", "wiki/b/facts/t.md"))
got, err := os.ReadFile(victim)
require.NoError(t, err)
assert.Equal(t, "untouched\n", string(got))
// The same with a link to a file elsewhere in the brain, which os.Root
// would follow.
inVictim := filepath.Join(brain0, "raw", "v.md")
require.NoError(t, os.MkdirAll(filepath.Dir(inVictim), 0o755))
require.NoError(t, os.WriteFile(inVictim, []byte("untouched\n"), 0o644))
require.NoError(t, os.Symlink(filepath.Join("..", "..", "..", "raw", "v.md"), filepath.Join(tdir, "u.md")))
assert.Error(t, brain.WriteTunnel(brain0, "wiki/a/facts/s.md", "wiki/b/facts/u.md"))
got, err = os.ReadFile(inVictim)
require.NoError(t, err)
assert.Equal(t, "untouched\n", string(got))
got, err = os.ReadFile(src)
require.NoError(t, err)
assert.Equal(t, "x\n", string(got), "the source is not touched when the target is refused")
}
// The target is swapped between a regular note and a symlink to an in-brain
// file outside wiki/ while WriteTunnel runs: the read and the write each
// refuse a symlink, so the other file is never rewritten.
func TestWriteTunnelNeverRewritesASymlinkSwappedInForTheTarget(t *testing.T) {
brain0 := t.TempDir()
src := filepath.Join(brain0, "wiki", "a", "facts", "s.md")
require.NoError(t, os.MkdirAll(filepath.Dir(src), 0o755))
require.NoError(t, os.WriteFile(src, []byte("x\n"), 0o644))
tdir := filepath.Join(brain0, "wiki", "b", "facts")
require.NoError(t, os.MkdirAll(tdir, 0o755))
victim := filepath.Join(brain0, "raw", "victim.md")
require.NoError(t, os.MkdirAll(filepath.Dir(victim), 0o755))
require.NoError(t, os.WriteFile(victim, []byte("untouched\n"), 0o644))
tgt := filepath.Join(tdir, "t.md")
require.NoError(t, os.WriteFile(tgt, []byte("t\n"), 0o644))
var stop atomic.Bool
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
tmp := tgt + ".swap"
for i := 0; !stop.Load(); i++ {
_ = os.Remove(tmp)
if i%2 == 0 {
_ = os.WriteFile(tmp, []byte("t\n"), 0o644)
} else {
_ = os.Symlink(filepath.Join("..", "..", "..", "raw", "victim.md"), tmp)
}
_ = os.Rename(tmp, tgt)
}
}()
ok := 0
for deadline := time.Now().Add(1500 * time.Millisecond); time.Now().Before(deadline); {
if brain.WriteTunnel(brain0, "wiki/a/facts/s.md", "wiki/b/facts/t.md") == nil {
ok++
}
_ = os.WriteFile(src, []byte("x\n"), 0o644)
}
stop.Store(true)
wg.Wait()
got, err := os.ReadFile(victim)
require.NoError(t, err)
assert.Equal(t, "untouched\n", string(got), "the swapped-in symlink's target was rewritten")
assert.Positive(t, ok, "positive control: some tunnels were written to the regular target")
}
-289
View File
@@ -1,289 +0,0 @@
// Package brainfs reads the brain directory through an os.Root (hyperguild#157).
// The brain dir is a Syncthing sendreceive folder and Syncthing syncs
// symlinks; a reader that walks and reads by path follows a planted symlink
// out of the brain, and blocks forever on a planted FIFO. Readers that go
// through a Brain only ever read regular files inside the brain: os.Root
// refuses to leave the root even if an entry is swapped mid-read, and every
// open is non-blocking with the type checked on the opened handle.
package brainfs
import (
"errors"
"fmt"
"io"
"io/fs"
"os"
"path"
"path/filepath"
"strings"
"syscall"
)
// ErrNotRegular is returned for an entry that is not a regular file.
var ErrNotRegular = errors.New("not a regular file")
// ErrTooLarge is returned by ReadRegularMax for a file over the limit.
var ErrTooLarge = errors.New("file too large")
// Brain is an open brain directory.
type Brain struct {
root *os.Root
dir string
}
// Open opens brainDir (which may itself be a mount symlink) as a root.
func Open(brainDir string) (*Brain, error) {
r, err := os.OpenRoot(brainDir)
if err != nil {
return nil, fmt.Errorf("open brain root: %w", err)
}
return &Brain{root: r, dir: brainDir}, nil
}
// Close releases the root.
func (b *Brain) Close() error { return b.root.Close() }
// Rel converts a path under the brain dir to a root-relative path.
func (b *Brain) Rel(path string) (string, error) {
rel, err := filepath.Rel(b.dir, path)
if err != nil {
return "", err
}
if rel == ".." || len(rel) > 2 && rel[:3] == ".."+string(filepath.Separator) {
return "", fmt.Errorf("%s is outside the brain dir", path)
}
return filepath.ToSlash(rel), nil
}
// Lstat stats a brain-relative path through the root without following a
// final symlink.
func (b *Brain) Lstat(rel string) (fs.FileInfo, error) {
return b.root.Lstat(filepath.ToSlash(rel))
}
// realDir returns ErrNotRegular unless every component of rel is a real
// directory. A swap after the check is not caught (accepted residual,
// hyperguild#157).
func (b *Brain) realDir(rel string) error {
if rel == "." {
return nil
}
prefix := ""
for _, part := range strings.Split(rel, "/") {
prefix = path.Join(prefix, part)
fi, err := b.root.Lstat(prefix)
if err != nil {
return err
}
if !fi.IsDir() {
return fmt.Errorf("%s: walk root component %s is not a directory: %w", rel, prefix, ErrNotRegular)
}
}
return nil
}
// ReadDirRegular lists the regular files directly in a brain-relative
// directory, by name. Every component of relDir must be a real directory
// (ErrNotRegular otherwise); symlinks, FIFOs, devices and subdirectories in
// it are left out.
func (b *Brain) ReadDirRegular(relDir string) ([]string, error) {
relDir = filepath.ToSlash(relDir)
if err := b.realDir(relDir); err != nil {
return nil, err
}
ents, err := fs.ReadDir(b.root.FS(), relDir)
if err != nil {
return nil, err
}
var names []string
for _, e := range ents {
if e.Type().IsRegular() {
names = append(names, e.Name())
}
}
return names, nil
}
// ReadDirDirs lists the real subdirectories directly in a brain-relative
// directory, by name. Every component of relDir must be a real directory
// (ErrNotRegular otherwise); symlinks to directories are left out.
func (b *Brain) ReadDirDirs(relDir string) ([]string, error) {
relDir = filepath.ToSlash(relDir)
if err := b.realDir(relDir); err != nil {
return nil, err
}
ents, err := fs.ReadDir(b.root.FS(), relDir)
if err != nil {
return nil, err
}
var names []string
for _, e := range ents {
if e.Type().IsDir() {
names = append(names, e.Name())
}
}
return names, nil
}
// ReadRegular reads a brain-relative path, refusing anything that is not a
// regular file, both before the open and on the opened handle.
func (b *Brain) ReadRegular(rel string) ([]byte, error) {
fi, err := b.root.Lstat(rel)
if err != nil {
return nil, err
}
if !fi.Mode().IsRegular() {
return nil, fmt.Errorf("%s: %w", rel, ErrNotRegular)
}
return b.readOpenedMax(rel, -1, fi)
}
// readOpened opens rel non-blocking and reads it only if the OPENED handle is
// a regular file. This is the check that matters under a race: an entry
// swapped for a FIFO after the Lstat, with a writer holding it open, would
// otherwise park io.ReadAll forever (PR #161 review: removing it hung search
// after 16 iterations).
//
// Accepted residual, as in api/rawroot.go: a HARD link to an outside file is
// a regular file and is read. Syncthing syncs contents, not links, so a peer
// cannot create one; fs.protected_hardlinks=1 and the pod's secrets are on a
// different filesystem from the brain volume, so a local caller cannot link
// them in. Refusing Nlink > 1 is the next step if either changes.
func (b *Brain) readOpened(rel string) ([]byte, error) { return b.readOpenedMax(rel, -1, nil) }
// ReadRegularMax is ReadRegular with a size limit, checked on the opened
// handle before reading (max < 0 means no limit).
func (b *Brain) ReadRegularMax(rel string, max int64) ([]byte, error) {
fi, err := b.root.Lstat(rel)
if err != nil {
return nil, err
}
if !fi.Mode().IsRegular() {
return nil, fmt.Errorf("%s: %w", rel, ErrNotRegular)
}
return b.readOpenedMax(rel, max, fi)
}
// readOpenedMax opens rel and reads it if the opened handle is a regular file
// and, when lfi is set, the same file the caller Lstat-ed: os.Root follows an
// in-root final symlink even with O_NOFOLLOW, so a symlink to another brain
// file swapped in between the Lstat and the open is refused only here.
func (b *Brain) readOpenedMax(rel string, max int64, lfi fs.FileInfo) ([]byte, error) {
f, err := b.root.OpenFile(rel, os.O_RDONLY|syscall.O_NONBLOCK, 0)
if err != nil {
return nil, err
}
defer func() { _ = f.Close() }()
hfi, err := f.Stat()
if err != nil {
return nil, err
}
if !hfi.Mode().IsRegular() {
return nil, fmt.Errorf("%s: %w", rel, ErrNotRegular)
}
if lfi != nil && !os.SameFile(lfi, hfi) {
return nil, fmt.Errorf("%s: changed while opening: %w", rel, ErrNotRegular)
}
if max >= 0 && hfi.Size() > max {
return nil, fmt.Errorf("%s is %d bytes (limit %d): %w", rel, hfi.Size(), max, ErrTooLarge)
}
var r io.Reader = f
if wrapAfterStat != nil {
r = wrapAfterStat(r)
}
if max < 0 {
return io.ReadAll(r)
}
// The handle's size can grow after the check; bound the read itself.
data, err := io.ReadAll(io.LimitReader(r, max+1))
if err != nil {
return nil, err
}
if int64(len(data)) > max {
return nil, fmt.Errorf("%s grew past %d bytes while read: %w", rel, max, ErrTooLarge)
}
return data, nil
}
// wrapAfterStat is a test seam: it runs between the size check and the read,
// and wraps the reader the read consumes. A package global: brainfs tests
// must not use t.Parallel.
var wrapAfterStat func(io.Reader) io.Reader
// WalkRegular walks a brain-relative directory through the root and calls fn
// for each regular file. Directories are descended; symlinks, FIFOs and
// devices found by the walk are skipped. A walk root with any component that
// is not a real directory goes to onErr as ErrNotRegular and nothing is
// yielded. An entry error is passed to onErr and the walk continues; a nil
// onErr ignores it.
//
// Accepted residual (hyperguild#157): a directory swapped for a symlink
// between the listing and the read is followed. os.Root still keeps the read
// inside the brain, but not inside relDir.
func (b *Brain) WalkRegular(relDir string, fn func(rel string) error, onErr func(rel string, err error)) error {
relDir = filepath.ToSlash(relDir)
// fs.WalkDir stats its root with a follow, and os.Root allows a symlink
// that stays inside the brain: knowledge -> . would alias the whole
// brain. Require every component of the root to be a real directory
// (Lstat checks only the last one).
err := b.realDir(relDir)
if err != nil {
if onErr != nil {
onErr(relDir, err)
}
return nil
}
return fs.WalkDir(b.root.FS(), relDir, func(p string, d fs.DirEntry, err error) error {
if err != nil {
if onErr != nil {
onErr(p, err)
}
if d != nil && d.IsDir() {
return fs.SkipDir
}
return nil
}
if d.IsDir() || !d.Type().IsRegular() {
return nil
}
return fn(p)
})
}
// WriteRegular replaces the contents of an existing regular file at a
// brain-relative path. It never creates a file and never writes through a
// final symlink: os.Root.OpenFile follows an in-root final symlink even with
// O_NOFOLLOW (verified on go1.27), so the path is Lstat-ed through the root
// first and the opened handle must be that same file (os.SameFile), which
// also catches a swap between the two. O_NONBLOCK keeps a FIFO from blocking
// the open. Intermediate in-brain symlinks are still followed (os.Root keeps
// the write inside the brain; hyperguild#167).
func (b *Brain) WriteRegular(rel string, data []byte) error {
rel = filepath.ToSlash(rel)
lfi, err := b.root.Lstat(rel)
if err != nil {
return err
}
if !lfi.Mode().IsRegular() {
return fmt.Errorf("%s: %w", rel, ErrNotRegular)
}
f, err := b.root.OpenFile(rel, os.O_WRONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0)
if err != nil {
return err
}
defer func() { _ = f.Close() }()
fi, err := f.Stat()
if err != nil {
return err
}
if !fi.Mode().IsRegular() || !os.SameFile(lfi, fi) {
return fmt.Errorf("%s: changed while opening: %w", rel, ErrNotRegular)
}
if err := f.Truncate(0); err != nil {
return err
}
if _, err := f.Write(data); err != nil {
return err
}
return f.Close()
}
-328
View File
@@ -1,328 +0,0 @@
package brainfs
import (
"errors"
"io"
"io/fs"
"os"
"path/filepath"
"sync"
"sync/atomic"
"syscall"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// The handle check is what stops a FIFO swapped in after the Lstat. A FIFO
// with no writer reads EOF at once, so a race test could not see the check
// missing; here a writer holds the FIFO open, as any local process can, and
// readOpened must refuse it promptly (PR #161 review).
func TestReadOpenedRefusesAFIFOHeldOpenByAWriter(t *testing.T) {
dir := t.TempDir()
fifo := filepath.Join(dir, "p.md")
require.NoError(t, syscall.Mkfifo(fifo, 0o644))
w, err := os.OpenFile(fifo, os.O_RDWR, 0)
require.NoError(t, err)
defer func() { _ = w.Close() }()
b, err := Open(dir)
require.NoError(t, err)
defer func() { _ = b.Close() }()
done := make(chan error, 1)
go func() { _, err := b.readOpened("p.md"); done <- err }()
select {
case err := <-done:
assert.True(t, errors.Is(err, ErrNotRegular), "got %v", err)
case <-time.After(2 * time.Second):
t.Fatal("readOpened blocked on a FIFO with a writer")
}
}
func TestReadRegularRefusesSymlinksAndReadsRegularFiles(t *testing.T) {
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "ok.md"), []byte("ok"), 0o644))
outside := filepath.Join(t.TempDir(), "s.md")
require.NoError(t, os.WriteFile(outside, []byte("secret"), 0o644))
require.NoError(t, os.Symlink(outside, filepath.Join(dir, "link.md")))
b, err := Open(dir)
require.NoError(t, err)
defer func() { _ = b.Close() }()
got, err := b.ReadRegular("ok.md")
require.NoError(t, err)
assert.Equal(t, "ok", string(got))
_, err = b.ReadRegular("link.md")
assert.True(t, errors.Is(err, ErrNotRegular), "got %v", err)
_, err = b.ReadRegular("../" + filepath.Base(outside))
assert.Error(t, err)
}
func TestRelAcceptsPathsInsideAndRefusesEscapes(t *testing.T) {
dir := t.TempDir()
b, err := Open(dir)
require.NoError(t, err)
defer func() { _ = b.Close() }()
for in, want := range map[string]string{
filepath.Join(dir, "wiki", "x.md"): "wiki/x.md",
filepath.Join(dir, "..foo", "x"): "..foo/x",
dir: ".",
} {
got, err := b.Rel(in)
require.NoError(t, err, in)
assert.Equal(t, want, got, in)
}
for _, bad := range []string{filepath.Join(dir, "..", "x"), dir + "-evil/x", "/etc/passwd", filepath.Join(dir, "wiki", "..", "..", "x")} {
_, err := b.Rel(bad)
assert.Error(t, err, bad)
}
}
func TestWalkRegularYieldsOnlyRegularFiles(t *testing.T) {
dir := t.TempDir()
require.NoError(t, os.MkdirAll(filepath.Join(dir, "a"), 0o755))
require.NoError(t, os.WriteFile(filepath.Join(dir, "a", "n.md"), []byte("n"), 0o644))
require.NoError(t, os.Symlink(t.TempDir(), filepath.Join(dir, "a", "dirlink")))
require.NoError(t, os.Symlink(filepath.Join(dir, "a", "n.md"), filepath.Join(dir, "a", "filelink.md")))
require.NoError(t, syscall.Mkfifo(filepath.Join(dir, "a", "p.md"), 0o644))
b, err := Open(dir)
require.NoError(t, err)
defer func() { _ = b.Close() }()
var got []string
require.NoError(t, b.WalkRegular(".", func(rel string) error { got = append(got, rel); return nil }, nil))
assert.Equal(t, []string{"a/n.md"}, got)
}
func TestReadRegularMaxRefusesAFileOverTheLimit(t *testing.T) {
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "big.md"), make([]byte, 100), 0o644))
b, err := Open(dir)
require.NoError(t, err)
defer func() { _ = b.Close() }()
_, err = b.ReadRegularMax("big.md", 99)
assert.True(t, errors.Is(err, ErrTooLarge), "got %v", err)
got, err := b.ReadRegularMax("big.md", 100)
require.NoError(t, err)
assert.Len(t, got, 100)
}
func TestReadRegularMaxRefusesAndStopsReadingAFileThatGrowsPastTheLimit(t *testing.T) {
dir := t.TempDir()
p := filepath.Join(dir, "grow.md")
require.NoError(t, os.WriteFile(p, make([]byte, 10), 0o644))
consumed := &countingReader{}
wrapAfterStat = func(r io.Reader) io.Reader {
f, err := os.OpenFile(p, os.O_WRONLY|os.O_APPEND, 0)
require.NoError(t, err)
_, err = f.Write(make([]byte, 1000))
require.NoError(t, err)
require.NoError(t, f.Close())
consumed.r = r
return consumed
}
t.Cleanup(func() { wrapAfterStat = nil })
b, err := Open(dir)
require.NoError(t, err)
defer func() { _ = b.Close() }()
got, err := b.ReadRegularMax("grow.md", 100)
assert.True(t, errors.Is(err, ErrTooLarge), "got %v", err)
assert.Nil(t, got)
assert.LessOrEqual(t, consumed.n, 101, "read past max+1 bytes")
}
type countingReader struct {
r io.Reader
n int
}
func (c *countingReader) Read(p []byte) (int, error) {
n, err := c.r.Read(p)
c.n += n
return n, err
}
func TestLstatDoesNotFollowSymlinksAndRefusesEscapes(t *testing.T) {
dir := t.TempDir()
outside := filepath.Join(t.TempDir(), "o.md")
require.NoError(t, os.WriteFile(outside, []byte("o"), 0o644))
require.NoError(t, os.Symlink(outside, filepath.Join(dir, "l.md")))
b, err := Open(dir)
require.NoError(t, err)
defer func() { _ = b.Close() }()
fi, err := b.Lstat("l.md")
require.NoError(t, err)
assert.Equal(t, fs.ModeSymlink, fi.Mode().Type())
_, err = b.Lstat("../" + filepath.Base(filepath.Dir(outside)) + "/o.md")
assert.Error(t, err)
}
func TestWalkRegularReportsASymlinkedWalkRootAndYieldsNothing(t *testing.T) {
dir := t.TempDir()
require.NoError(t, os.MkdirAll(filepath.Join(dir, "real"), 0o755))
require.NoError(t, os.WriteFile(filepath.Join(dir, "real", "n.md"), []byte("n"), 0o644))
require.NoError(t, os.Symlink("real", filepath.Join(dir, "alias")))
b, err := Open(dir)
require.NoError(t, err)
defer func() { _ = b.Close() }()
var got []string
var errs []error
require.NoError(t, b.WalkRegular("alias", func(rel string) error { got = append(got, rel); return nil },
func(_ string, err error) { errs = append(errs, err) }))
assert.Empty(t, got)
require.Len(t, errs, 1)
assert.ErrorIs(t, errs[0], ErrNotRegular)
}
func TestWalkRegularReportsASymlinkedIntermediateComponentOfTheWalkRoot(t *testing.T) {
dir := t.TempDir()
require.NoError(t, os.MkdirAll(filepath.Join(dir, "real", "sub"), 0o755))
require.NoError(t, os.WriteFile(filepath.Join(dir, "real", "sub", "n.md"), []byte("n"), 0o644))
require.NoError(t, os.Symlink("real", filepath.Join(dir, "alias")))
b, err := Open(dir)
require.NoError(t, err)
defer func() { _ = b.Close() }()
var got []string
var errs []error
require.NoError(t, b.WalkRegular("alias/sub", func(rel string) error { got = append(got, rel); return nil },
func(_ string, err error) { errs = append(errs, err) }))
assert.Empty(t, got)
require.Len(t, errs, 1)
assert.ErrorIs(t, errs[0], ErrNotRegular)
}
func TestReadDirRegularListsOnlyRegularFilesAndRefusesSymlinkedComponents(t *testing.T) {
dir := t.TempDir()
require.NoError(t, os.MkdirAll(filepath.Join(dir, "a", "b", "sub"), 0o755))
require.NoError(t, os.WriteFile(filepath.Join(dir, "a", "b", "n.md"), []byte("n"), 0o644))
require.NoError(t, os.WriteFile(filepath.Join(dir, "other.md"), []byte("o"), 0o644))
require.NoError(t, os.Symlink(filepath.Join("..", "..", "other.md"), filepath.Join(dir, "a", "b", "l.md")))
require.NoError(t, syscall.Mkfifo(filepath.Join(dir, "a", "b", "f.md"), 0o644))
require.NoError(t, os.Symlink("a", filepath.Join(dir, "alias")))
require.NoError(t, os.MkdirAll(filepath.Join(dir, "x"), 0o755))
require.NoError(t, os.Symlink(filepath.Join("..", "a"), filepath.Join(dir, "x", "mid")))
b, err := Open(dir)
require.NoError(t, err)
defer func() { _ = b.Close() }()
names, err := b.ReadDirRegular("a/b")
require.NoError(t, err)
assert.Equal(t, []string{"n.md"}, names)
for _, rel := range []string{"alias/b", "x/mid/b"} {
_, err := b.ReadDirRegular(rel)
assert.ErrorIs(t, err, ErrNotRegular, rel)
}
}
// A final component swapped for a symlink to another regular file in the
// brain between the Lstat and the open: os.Root follows in-root links, so
// only comparing the two (os.SameFile) refuses it.
func TestReadRegularNeverReturnsAnInBrainFileSwappedInBehindASymlink(t *testing.T) {
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "s.md"), []byte("zqsamefilesentinel"), 0o644))
p := filepath.Join(dir, "r.md")
require.NoError(t, os.WriteFile(p, []byte("inside"), 0o644))
b, err := Open(dir)
require.NoError(t, err)
defer func() { _ = b.Close() }()
var stop atomic.Bool
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
tmp := p + ".swap"
for i := 0; !stop.Load(); i++ {
_ = os.Remove(tmp)
if i%2 == 0 {
_ = os.WriteFile(tmp, []byte("inside"), 0o644)
} else {
_ = os.Symlink("s.md", tmp)
}
_ = os.Rename(tmp, p)
}
}()
leaks, ok := 0, 0
for deadline := time.Now().Add(1500 * time.Millisecond); time.Now().Before(deadline); {
got, err := b.ReadRegular("r.md")
if err == nil {
ok++
}
if string(got) == "zqsamefilesentinel" {
leaks++
}
got, _ = b.ReadRegularMax("r.md", 1<<20)
if string(got) == "zqsamefilesentinel" {
leaks++
}
}
stop.Store(true)
wg.Wait()
assert.Zero(t, leaks)
assert.Positive(t, ok, "positive control: the regular file is read")
}
func TestWriteRegularRefusesFinalSymlinksFIFOsEscapesAndMissingFilesAndRewritesRegularFiles(t *testing.T) {
dir := t.TempDir()
outside := filepath.Join(t.TempDir(), "o.md")
require.NoError(t, os.WriteFile(outside, []byte("untouched"), 0o644))
require.NoError(t, os.Symlink(outside, filepath.Join(dir, "l.md")))
require.NoError(t, syscall.Mkfifo(filepath.Join(dir, "f.md"), 0o644))
require.NoError(t, os.WriteFile(filepath.Join(dir, "n.md"), []byte("old content"), 0o644))
b, err := Open(dir)
require.NoError(t, err)
defer func() { _ = b.Close() }()
assert.Error(t, b.WriteRegular("l.md", []byte("x")))
got, _ := os.ReadFile(outside)
assert.Equal(t, "untouched", string(got))
// A symlink to another file inside the brain: os.Root would allow it,
// only O_NOFOLLOW refuses it.
require.NoError(t, os.WriteFile(filepath.Join(dir, "in.md"), []byte("inside"), 0o644))
require.NoError(t, os.Symlink("in.md", filepath.Join(dir, "il.md")))
assert.Error(t, b.WriteRegular("il.md", []byte("x")))
got, _ = os.ReadFile(filepath.Join(dir, "in.md"))
assert.Equal(t, "inside", string(got))
// A reader held open, so a non-blocking write open would succeed. The
// Lstat type check refuses the FIFO first; the handle check and
// O_NONBLOCK matter only if it is swapped in after the Lstat.
r, err := os.OpenFile(filepath.Join(dir, "f.md"), os.O_RDONLY|syscall.O_NONBLOCK, 0)
require.NoError(t, err)
defer func() { _ = r.Close() }()
done := make(chan error, 1)
go func() { done <- b.WriteRegular("f.md", []byte("x")) }()
select {
case err := <-done:
assert.Error(t, err)
case <-time.After(3 * time.Second):
t.Fatal("WriteRegular hung on a FIFO")
}
assert.Error(t, b.WriteRegular("../"+filepath.Base(filepath.Dir(outside))+"/o.md", []byte("x")))
assert.Error(t, b.WriteRegular("missing.md", []byte("x")), "does not create files")
require.NoError(t, b.WriteRegular("n.md", []byte("new")))
got, _ = os.ReadFile(filepath.Join(dir, "n.md"))
assert.Equal(t, "new", string(got), "truncated and rewritten")
}
func TestReadDirDirsListsOnlyRealSubdirsAndRefusesSymlinkedComponents(t *testing.T) {
dir := t.TempDir()
require.NoError(t, os.MkdirAll(filepath.Join(dir, "a", "real"), 0o755))
require.NoError(t, os.MkdirAll(filepath.Join(dir, "other"), 0o755))
require.NoError(t, os.Symlink(filepath.Join("..", "other"), filepath.Join(dir, "a", "linked")))
require.NoError(t, os.WriteFile(filepath.Join(dir, "a", "f.md"), []byte("f"), 0o644))
require.NoError(t, os.Symlink("a", filepath.Join(dir, "alias")))
b, err := Open(dir)
require.NoError(t, err)
defer func() { _ = b.Close() }()
names, err := b.ReadDirDirs("a")
require.NoError(t, err)
assert.Equal(t, []string{"real"}, names)
_, err = b.ReadDirDirs("alias")
assert.ErrorIs(t, err, ErrNotRegular)
}
@@ -1,171 +0,0 @@
// Package brainfstest is the shared harness for hyperguild#157's swap-race
// tests: it keeps swapping a regular note, a symlink to an outside file, or a
// FIFO (with a writer held open) into a path while a reader under test loops.
// It observes two things: no sentinel byte from the outside file reaches the
// reader's output, and the reader returns within a timeout. It does not cover
// hard links, other paths into the reader, or races it does not generate.
package brainfstest
import (
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"syscall"
"testing"
"time"
)
// Sentinel marks the outside file's contents.
const Sentinel = "zqbrainfssentinelonefiftyseven"
// outsideContent carries Sentinel in every field a reader might extract: the
// frontmatter title and aliases, a heading, a wikilink and the body.
const outsideContent = "---\ntitle: " + Sentinel + "\naliases:\n - " + Sentinel + "\n---\n# " + Sentinel +
"\n\n" + Sentinel + " [[" + Sentinel + "]]\n"
// Outside writes a file outside any brain dir containing Sentinel.
func Outside(t testing.TB) string {
t.Helper()
p := filepath.Join(t.TempDir(), "secret.md")
if err := os.WriteFile(p, []byte(outsideContent), 0o644); err != nil {
t.Fatal(err)
}
return p
}
var swapSeq atomic.Int64
// StartSwap renames, in a loop, a regular note (content inside) and either
// a symlink to outside or a FIFO into path, until the test ends. FIFOs get a
// writer held open until the swapper exits, so a reader that skips the
// handle check parks forever instead of reading EOF. The swapper is stopped
// and awaited before t.TempDir cleanup. It returns the number of symlinks
// or FIFOs actually renamed into path so far, so a caller can tell a race
// that ran from one whose every swap failed (hyperguild#177).
func StartSwap(t testing.TB, path, inside, outside string, fifo bool) *atomic.Int64 {
t.Helper()
var swaps atomic.Int64
var stop atomic.Bool
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
// A name of its own: swappers on one path share a test, and one
// opening another's FIFO for write would block forever.
tmp := fmt.Sprintf("%s.swap%d", path, swapSeq.Add(1))
// A relative link: os.Root refuses every absolute link on its own, so
// an absolute in-brain target would test nothing.
link := outside
if rel, err := filepath.Rel(filepath.Dir(path), outside); err == nil {
link = rel
}
var writers []*os.File
defer func() {
for _, w := range writers {
_ = w.Close()
}
}()
for i := 0; !stop.Load(); i++ {
_ = os.Remove(tmp)
var made error
switch {
case i%2 == 0:
_ = os.WriteFile(tmp, []byte(inside), 0o644)
case fifo:
if made = syscall.Mkfifo(tmp, 0o644); made == nil {
if w, err := os.OpenFile(tmp, os.O_RDWR, 0); err == nil {
writers = append(writers, w)
}
}
default:
made = os.Symlink(link, tmp)
}
if err := os.Rename(tmp, path); err == nil && made == nil && i%2 == 1 {
swaps.Add(1)
}
}
}()
t.Cleanup(func() { stop.Store(true); wg.Wait() })
return &swaps
}
// WriteSentinelAt writes a file carrying Sentinel at p (creating its
// directory), for an in-brain symlink target.
func WriteSentinelAt(t *testing.T, p string) string {
t.Helper()
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(p, []byte(outsideContent), 0o644); err != nil {
t.Fatal(err)
}
return p
}
// AssertNoLeak loops read for d while a symlink swapper runs on path, and
// fails if any output contains Sentinel. The symlink points outside the
// brain; os.Root refuses that on its own, so pair it with AssertNoLeakFrom
// and an in-brain target.
func AssertNoLeak(t *testing.T, path, inside string, d time.Duration, read func() string) {
t.Helper()
AssertNoLeakFrom(t, path, inside, Outside(t), d, read)
}
// InsideMarker must appear in the inside note's content and in read's
// output whenever read saw that note: the floor below counts such reads.
const InsideMarker = "inside"
// AssertNoLeakFrom is AssertNoLeak with the symlink pointing at target, e.g.
// a file elsewhere in the brain written with WriteSentinelAt.
//
// Beyond "no Sentinel in any output" it sets two floors, because without
// them the assertion passes when nothing was tested (hyperguild#177): at
// least one symlink must have been swapped into path, and at least one read
// must have returned InsideMarker (a reader that always errors returns
// nothing, and so never leaks).
func AssertNoLeakFrom(t testing.TB, path, inside, target string, d time.Duration, read func() string) {
t.Helper()
if !strings.Contains(inside, InsideMarker) {
t.Fatalf("inside content must contain %q", InsideMarker)
}
swaps := StartSwap(t, path, inside, target, false)
leaks, iters, insideReads := 0, 0, 0
for deadline := time.Now().Add(d); time.Now().Before(deadline); iters++ {
out := read()
leaks += strings.Count(out, Sentinel)
if strings.Contains(out, InsideMarker) {
insideReads++
}
}
if leaks > 0 {
t.Fatalf("outside file leaked %d times in %d iterations", leaks, iters)
}
if n := swaps.Load(); n == 0 {
t.Fatalf("no symlink was swapped into %s in %d iterations: the race did not run", path, iters)
}
if insideReads == 0 {
t.Fatalf("no read in %d iterations returned the inside note (%q): the reader read nothing", iters, InsideMarker)
}
}
// AssertNoHang loops read for d while a FIFO swapper (held writers) runs on
// path, and fails if a read does not return within timeout.
func AssertNoHang(t *testing.T, path, inside string, d, timeout time.Duration, read func()) {
t.Helper()
StartSwap(t, path, inside, Outside(t), true)
done := make(chan struct{})
go func() {
for deadline := time.Now().Add(d); time.Now().Before(deadline); {
read()
}
close(done)
}()
select {
case <-done:
case <-time.After(timeout):
t.Fatal("reader hung on a swapped-in FIFO with a writer held open")
}
}
@@ -1,79 +0,0 @@
package brainfstest
import (
"fmt"
"os"
"path/filepath"
"runtime"
"syscall"
"testing"
"time"
)
// recordingTB stands in for the test under AssertNoLeakFrom so a fatal
// there can be observed instead of failing this test. Fatalf ends the
// calling goroutine the way testing.T does.
type recordingTB struct {
testing.TB
failed bool
msg string
}
func (r *recordingTB) Fatalf(format string, args ...any) {
r.failed, r.msg = true, fmt.Sprintf(format, args...)
runtime.Goexit()
}
func runAssert(t *testing.T, path, target string, read func() string) *recordingTB {
t.Helper()
r := &recordingTB{TB: t}
done := make(chan struct{})
go func() {
defer close(done)
AssertNoLeakFrom(r, path, "inside", target, 200*time.Millisecond, read)
}()
<-done
return r
}
// hyperguild#177: with every swap failing, no leak can occur, so the leak
// assertion passed without testing anything.
func TestAssertNoLeakFromFailsWhenNoSymlinkSwapSucceeds(t *testing.T) {
path := filepath.Join(t.TempDir(), "missing-dir", "r.md")
r := runAssert(t, path, Outside(t), func() string { return "inside" })
if !r.failed {
t.Fatal("AssertNoLeakFrom passed with zero successful symlink swaps")
}
}
// hyperguild#177: a reader that always errors returns nothing, so it never
// leaks either.
func TestAssertNoLeakFromFailsWhenNoReadReturnsTheInsideNote(t *testing.T) {
path := filepath.Join(t.TempDir(), "r.md")
r := runAssert(t, path, Outside(t), func() string { return "" })
if !r.failed {
t.Fatal("AssertNoLeakFrom passed with zero reads returning the inside note")
}
}
// Positive control: a reader that refuses symlinks and reads regular files
// passes, so the floors do not fail a working reader.
func TestAssertNoLeakFromPassesAReaderThatRefusesSymlinks(t *testing.T) {
path := filepath.Join(t.TempDir(), "r.md")
read := func() string {
f, err := os.OpenFile(path, os.O_RDONLY|noFollow, 0)
if err != nil {
return ""
}
defer func() { _ = f.Close() }()
b := make([]byte, 64)
n, _ := f.Read(b)
return string(b[:n])
}
r := runAssert(t, path, Outside(t), read)
if r.failed {
t.Fatalf("a symlink-refusing reader failed: %s", r.msg)
}
}
const noFollow = syscall.O_NOFOLLOW
+21 -43
View File
@@ -12,14 +12,10 @@ package graphsync
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
"io/fs" "os"
"path"
"path/filepath" "path/filepath"
"strings"
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
"github.com/mathiasbq/hyperguild/ingestion/internal/graph" "github.com/mathiasbq/hyperguild/ingestion/internal/graph"
"github.com/mathiasbq/hyperguild/ingestion/internal/graphstore" "github.com/mathiasbq/hyperguild/ingestion/internal/graphstore"
) )
@@ -50,25 +46,8 @@ func IndexDoc(ctx context.Context, store Store, brainDir, relPath string) error
if relPath == "" { if relPath == "" {
return nil return nil
} }
// Callers pass caller-supplied paths (brain_tunnel, brain_update): index abs := filepath.Join(brainDir, filepath.FromSlash(relPath))
// only a clean path under wiki/ or knowledge/. content, err := os.ReadFile(abs)
if path.Clean(relPath) != relPath || !filepath.IsLocal(relPath) ||
(!strings.HasPrefix(relPath, "wiki/") && !strings.HasPrefix(relPath, "knowledge/")) {
return fmt.Errorf("index %q: not a clean wiki/ or knowledge/ path", relPath)
}
bfs, err := brainfs.Open(brainDir)
if err != nil {
return fmt.Errorf("open brain: %w", err)
}
defer func() { _ = bfs.Close() }()
return indexDoc(ctx, store, bfs, relPath)
}
// indexDoc reads relPath through the brain root, regular files only
// (hyperguild#157): the brain is a Syncthing folder, and os.ReadFile
// followed a synced symlink into the graph and blocked on a FIFO.
func indexDoc(ctx context.Context, store Store, bfs *brainfs.Brain, relPath string) error {
content, err := bfs.ReadRegular(relPath)
if err != nil { if err != nil {
return fmt.Errorf("read %q: %w", relPath, err) return fmt.Errorf("read %q: %w", relPath, err)
} }
@@ -98,34 +77,33 @@ func BackfillFromBrainDir(ctx context.Context, store Store, brainDir string) (in
if store == nil { if store == nil {
return 0, nil return 0, nil
} }
bfs, err := brainfs.Open(brainDir) roots := []string{"wiki", "knowledge"}
if err != nil { for _, root := range roots {
return 0, fmt.Errorf("open brain: %w", err) base := filepath.Join(brainDir, root)
} if _, err := os.Stat(base); os.IsNotExist(err) {
defer func() { _ = bfs.Close() }()
for _, root := range []string{"wiki", "knowledge"} {
if _, err := bfs.Lstat(root); errors.Is(err, fs.ErrNotExist) {
continue continue
} }
// Any walk error, including a root that is a symlink, aborts. err := filepath.WalkDir(base, func(path string, d os.DirEntry, walkErr error) error {
var walkErr error if walkErr != nil {
err := bfs.WalkRegular(root, func(rel string) error { return walkErr
if path.Ext(rel) != ".md" { }
if d.IsDir() {
return nil return nil
} }
if err := indexDoc(ctx, store, bfs, rel); err != nil { if filepath.Ext(path) != ".md" {
return nil
}
rel, relErr := filepath.Rel(brainDir, path)
if relErr != nil {
return fmt.Errorf("rel %q: %w", path, relErr)
}
rel = filepath.ToSlash(rel)
if err := IndexDoc(ctx, store, brainDir, rel); err != nil {
return fmt.Errorf("index %q: %w", rel, err) return fmt.Errorf("index %q: %w", rel, err)
} }
indexed++ indexed++
return nil return nil
}, func(rel string, err error) {
if walkErr == nil {
walkErr = fmt.Errorf("%s: %w", rel, err)
}
}) })
if err == nil {
err = walkErr
}
if err != nil { if err != nil {
return indexed, fmt.Errorf("walk %s: %w", root, err) return indexed, fmt.Errorf("walk %s: %w", root, err)
} }
@@ -1,92 +0,0 @@
package graphsync
import (
"context"
"fmt"
"os"
"path/filepath"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs/brainfstest"
)
func (f *fakeStore) dump() string {
f.mu.Lock()
defer f.mu.Unlock()
return fmt.Sprintf("%+v %+v", f.upserts, f.replaces)
}
// hyperguild#157 slice 4: IndexDoc and BackfillFromBrainDir read notes with
// os.ReadFile, so a symlinked note put the target's title and wikilinks into
// the graph, and a FIFO hung the read.
func TestIndexDocRefusesSymlinksAndSurvivesSwapRaces(t *testing.T) {
brain := t.TempDir()
facts := filepath.Join(brain, "wiki", "w", "facts")
require.NoError(t, os.MkdirAll(facts, 0o755))
require.NoError(t, os.WriteFile(filepath.Join(facts, "ok.md"), []byte("---\ntitle: zqinside\n---\nbody [[other]]\n"), 0o644))
require.NoError(t, os.Symlink(brainfstest.Outside(t), filepath.Join(facts, "leak.md")))
store := newFakeStore()
require.NoError(t, IndexDoc(context.Background(), store, brain, "wiki/w/facts/ok.md"))
assert.Contains(t, store.dump(), "zqinside", "positive control: a regular note is indexed")
assert.Error(t, IndexDoc(context.Background(), store, brain, "wiki/w/facts/leak.md"))
// A link to a file elsewhere in the brain: os.Root allows it, only the
// regular-file check refuses it.
inBrain := brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "s.md"))
require.NoError(t, os.Symlink(filepath.Join("..", "..", "..", "raw", "s.md"), filepath.Join(facts, "inlink.md")))
assert.Error(t, IndexDoc(context.Background(), store, brain, "wiki/w/facts/inlink.md"))
assert.NotContains(t, store.dump(), brainfstest.Sentinel)
p := filepath.Join(facts, "r.md")
read := func() string {
s := newFakeStore()
_ = IndexDoc(context.Background(), s, brain, "wiki/w/facts/r.md")
return s.dump()
}
brainfstest.AssertNoLeak(t, p, "---\ntitle: zqinside\n---\n", 1500*time.Millisecond, read)
brainfstest.AssertNoLeakFrom(t, p, "---\ntitle: zqinside\n---\n", inBrain, 1500*time.Millisecond, read)
brainfstest.AssertNoHang(t, p, "---\ntitle: zqinside\n---\n", 1500*time.Millisecond, 6*time.Second, func() { _ = read() })
}
func TestBackfillSkipsSymlinkedNotesAndAbortsOnASymlinkedRoot(t *testing.T) {
brain := t.TempDir()
facts := filepath.Join(brain, "wiki", "w", "facts")
require.NoError(t, os.MkdirAll(facts, 0o755))
require.NoError(t, os.WriteFile(filepath.Join(facts, "ok.md"), []byte("---\ntitle: zqinside\n---\n"), 0o644))
require.NoError(t, os.Symlink(brainfstest.Outside(t), filepath.Join(facts, "leak.md")))
require.NoError(t, os.MkdirAll(filepath.Join(brain, "raw"), 0o755))
require.NoError(t, os.WriteFile(filepath.Join(brain, "raw", "r.md"), []byte("---\ntitle: zqrawonly\n---\n"), 0o644))
require.NoError(t, os.Symlink(filepath.Join("..", "..", "..", "raw", "r.md"), filepath.Join(facts, "inlink.md")))
require.NoError(t, os.Symlink(".", filepath.Join(brain, "knowledge")))
store := newFakeStore()
_, err := BackfillFromBrainDir(context.Background(), store, brain)
assert.Error(t, err, "a symlinked root aborts the backfill")
assert.NotContains(t, store.dump(), brainfstest.Sentinel)
assert.NotContains(t, store.dump(), "zqrawonly")
require.NoError(t, os.Remove(filepath.Join(brain, "knowledge")))
store = newFakeStore()
n, err := BackfillFromBrainDir(context.Background(), store, brain)
require.NoError(t, err)
assert.Equal(t, 1, n, "only the regular note is indexed")
assert.Contains(t, store.dump(), "zqinside")
assert.NotContains(t, store.dump(), brainfstest.Sentinel)
}
// IndexDoc gets caller-supplied paths (brain_tunnel, brain_update): only a
// clean path under wiki/ or knowledge/ is indexed.
func TestIndexDocRefusesPathsOutsideWikiAndKnowledgeAndUncleanPaths(t *testing.T) {
brain := t.TempDir()
brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "secret.md"))
require.NoError(t, os.MkdirAll(filepath.Join(brain, "wiki", "w"), 0o755))
for _, p := range []string{"raw/secret.md", "wiki/w/../../raw/secret.md", "./raw/secret.md", "/raw/secret.md"} {
store := newFakeStore()
assert.Error(t, IndexDoc(context.Background(), store, brain, p), p)
assert.NotContains(t, store.dump(), brainfstest.Sentinel, p)
}
}
@@ -1,34 +0,0 @@
package mcp
import (
"os"
"path/filepath"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs/brainfstest"
)
// hyperguild#157 slice 2: readExcerpt (brain_context) read by path, so a
// symlink in the Syncthing-shared brain put an outside file in context, and
// a FIFO hung it. It now reads through brainfs.
func TestReadExcerptRefusesSymlinksAndSurvivesSwapRaces(t *testing.T) {
brain := t.TempDir()
facts := filepath.Join(brain, "wiki", "w", "facts")
require.NoError(t, os.MkdirAll(facts, 0o755))
require.NoError(t, os.Symlink(brainfstest.Outside(t), filepath.Join(facts, "leak.md")))
assert.NotContains(t, readExcerpt(brain, "wiki/w/facts/leak.md", 500), brainfstest.Sentinel)
p := filepath.Join(facts, "r.md")
require.NoError(t, os.WriteFile(p, []byte("---\nx: y\n---\ninside"), 0o644))
assert.Equal(t, "inside", readExcerpt(brain, "wiki/w/facts/r.md", 500))
brainfstest.AssertNoLeak(t, p, "---\nx: y\n---\ninside", 1500*time.Millisecond, func() string {
return readExcerpt(brain, "wiki/w/facts/r.md", 500)
})
brainfstest.AssertNoHang(t, p, "---\nx: y\n---\ninside", 1500*time.Millisecond, 6*time.Second, func() {
_ = readExcerpt(brain, "wiki/w/facts/r.md", 500)
})
}
+3 -8
View File
@@ -4,7 +4,7 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"fmt" "fmt"
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs" "os"
"path/filepath" "path/filepath"
"sort" "sort"
"strings" "strings"
@@ -181,13 +181,8 @@ func readExcerpt(brainDir, relPath string, maxLen int) string {
if relPath == "" { if relPath == "" {
return "" return ""
} }
// Through the brain root, regular files only (hyperguild#157). full := filepath.Join(brainDir, filepath.FromSlash(relPath))
bfs, err := brainfs.Open(brainDir) content, err := os.ReadFile(full)
if err != nil {
return ""
}
defer func() { _ = bfs.Close() }()
content, err := bfs.ReadRegular(filepath.ToSlash(relPath))
if err != nil { if err != nil {
return "" return ""
} }
+5 -10
View File
@@ -5,7 +5,6 @@ import (
"encoding/hex" "encoding/hex"
"errors" "errors"
"fmt" "fmt"
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
@@ -72,18 +71,14 @@ func ReadScoped(brainDir, path string, scope Scope) ([]byte, error) {
} }
func readContained(t Target) ([]byte, error) { func readContained(t Target) ([]byte, error) {
// Through the brain root, regular files only, size checked on the opened fi, err := os.Stat(t.Abs)
// handle (hyperguild#157): Resolve's Lstat could be raced by a swap, and
// os.ReadFile blocked on a FIFO.
bfs, err := brainfs.Open(t.Brain)
if err != nil { if err != nil {
return nil, fmt.Errorf("notes: %w", err) return nil, fmt.Errorf("notes: stat %s: %w", t.Rel, err)
} }
defer func() { _ = bfs.Close() }() if fi.Size() > maxNoteBytes {
b, err := bfs.ReadRegularMax(t.Rel, maxNoteBytes) return nil, fmt.Errorf("%w: %s is %d bytes (limit %d)", ErrTooLarge, t.Rel, fi.Size(), maxNoteBytes)
if errors.Is(err, brainfs.ErrTooLarge) {
return nil, fmt.Errorf("%w: %s (limit %d)", ErrTooLarge, t.Rel, maxNoteBytes)
} }
b, err := os.ReadFile(t.Abs)
if err != nil { if err != nil {
return nil, fmt.Errorf("notes: read %s: %w", t.Rel, err) return nil, fmt.Errorf("notes: read %s: %w", t.Rel, err)
} }
+3 -4
View File
@@ -58,9 +58,8 @@ var (
// Target is a resolved, contained path. // Target is a resolved, contained path.
type Target struct { type Target struct {
Rel string // brain-relative, cleaned Rel string // brain-relative, cleaned
Abs string // absolute, symlink-checked Abs string // absolute, symlink-checked
Brain string // the brain root Rel is relative to
} }
// Resolve canonicalises path, then checks it against the scope's allowlist, // Resolve canonicalises path, then checks it against the scope's allowlist,
@@ -110,7 +109,7 @@ func Resolve(brainDir, path string, scope Scope) (Target, error) {
return Target{}, err return Target{}, err
} }
return Target{Rel: clean, Abs: abs, Brain: brainDir}, nil return Target{Rel: clean, Abs: abs}, nil
} }
// allow enforces the scope's allowlist. The two scopes are DISJOINT, not // allow enforces the scope's allowlist. The two scopes are DISJOINT, not
@@ -1,40 +0,0 @@
package notes
import (
"os"
"path/filepath"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs/brainfstest"
)
// hyperguild#157 slice 2: ReadScoped (brain_read) Lstat-checked the final
// component, then reopened by name with os.ReadFile: a swap after the check
// was followed, and a FIFO blocked the read. It now reads through brainfs.
func TestReadScopedRefusesSymlinksAndSurvivesSwapRaces(t *testing.T) {
brain := t.TempDir()
facts := filepath.Join(brain, "wiki", "w", "facts")
require.NoError(t, os.MkdirAll(facts, 0o755))
p := filepath.Join(facts, "r.md")
read := func() string {
b, _ := ReadScoped(brain, "wiki/w/facts/r.md", ScopeNotes)
return string(b)
}
require.NoError(t, os.WriteFile(p, []byte("inside"), 0o644))
assert.Equal(t, "inside", read())
brainfstest.AssertNoLeak(t, p, "inside", 1500*time.Millisecond, read)
brainfstest.AssertNoHang(t, p, "inside", 1500*time.Millisecond, 6*time.Second, func() { _ = read() })
}
func TestReadScopedRefusesANoteOverMaxNoteBytes(t *testing.T) {
brain := t.TempDir()
facts := filepath.Join(brain, "wiki", "w", "facts")
require.NoError(t, os.MkdirAll(facts, 0o755))
require.NoError(t, os.WriteFile(filepath.Join(facts, "big.md"), make([]byte, maxNoteBytes+1), 0o644))
_, err := ReadScoped(brain, "wiki/w/facts/big.md", ScopeNotes)
assert.ErrorIs(t, err, ErrTooLarge)
}
-191
View File
@@ -1,191 +0,0 @@
package search
import (
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"syscall"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// hyperguild#157 option 1: search walked and read the brain by path, so a
// symlink anywhere in the Syncthing-shared brain dir (wiki/<w>/<h>/leak.md ->
// /any/file) put the target's contents in brain_query results (reproduced in
// the PR #156 review). Search now walks and reads through an os.Root on the
// brain dir and only reads regular files.
const sentinel157 = "zqsentinelonefiftyseven"
func searchBrain(t *testing.T) (brain, facts, outside string) {
t.Helper()
brain = t.TempDir()
facts = filepath.Join(brain, "wiki", "w", "facts")
require.NoError(t, os.MkdirAll(facts, 0o755))
require.NoError(t, os.MkdirAll(filepath.Join(brain, "knowledge"), 0o755))
outside = filepath.Join(t.TempDir(), "secret.md")
require.NoError(t, os.WriteFile(outside, []byte("# secret\n\n"+sentinel157), 0o644))
return brain, facts, outside
}
func leaked(rs []Result) int {
n := 0
for _, r := range rs {
n += strings.Count(r.Excerpt+r.Title, sentinel157)
}
return n
}
func TestQueryDoesNotFollowASymlinkInTheWiki(t *testing.T) {
brain, facts, outside := searchBrain(t)
require.NoError(t, os.Symlink(outside, filepath.Join(facts, "leak.md")))
require.NoError(t, os.WriteFile(filepath.Join(facts, "real.md"), []byte("# real\n\nzqrealnote"), 0o644))
rs, err := Query(brain, QueryOptions{Query: sentinel157, Limit: 10})
require.NoError(t, err)
assert.Zero(t, leaked(rs))
rs, err = Query(brain, QueryOptions{Query: "zqrealnote", Limit: 10})
require.NoError(t, err)
assert.Len(t, rs, 1, "regular notes are still found")
}
func TestQueryDoesNotFollowASymlinkedDirectory(t *testing.T) {
brain, _, outside := searchBrain(t)
require.NoError(t, os.Symlink(filepath.Dir(outside), filepath.Join(brain, "wiki", "linked")))
rs, err := Query(brain, QueryOptions{Query: sentinel157, Limit: 10})
require.NoError(t, err)
assert.Zero(t, leaked(rs))
}
// A search root that is a symlink staying inside the brain (wiki/<w> -> ..)
// was followed by fs.WalkDir, aliasing raw/ and root files into results
// (PR #166 review). The root is now Lstat-checked and skipped.
func TestQueryDoesNotFollowASymlinkedWingRootInsideTheBrain(t *testing.T) {
brain, _, _ := searchBrain(t)
require.NoError(t, os.MkdirAll(filepath.Join(brain, "raw"), 0o755))
require.NoError(t, os.WriteFile(filepath.Join(brain, "raw", "r.md"), []byte("# r\n\nzqrawonly"), 0o644))
require.NoError(t, os.Symlink("..", filepath.Join(brain, "wiki", "alias")))
rs, err := Query(brain, QueryOptions{Query: "zqrawonly", Wing: "alias", Limit: 10})
require.NoError(t, err)
assert.Empty(t, rs)
}
// With a hall the search root is wiki/<wing>/<hall>; a symlinked <wing> with
// a real <hall> directory under its target was followed (PR #166 round 2).
func TestQueryDoesNotFollowASymlinkedWingUnderAWingAndHallRoot(t *testing.T) {
brain, _, _ := searchBrain(t)
require.NoError(t, os.MkdirAll(filepath.Join(brain, "raw", "facts"), 0o755))
require.NoError(t, os.WriteFile(filepath.Join(brain, "raw", "facts", "r.md"), []byte("# r\n\nzqrawhall"), 0o644))
require.NoError(t, os.Symlink(filepath.Join("..", "raw"), filepath.Join(brain, "wiki", "alias")))
rs, err := Query(brain, QueryOptions{Query: "zqrawhall", Wing: "alias", Hall: "facts", Limit: 10})
require.NoError(t, err)
assert.Empty(t, rs)
}
// swapInto keeps renaming a regular note, and either a symlink to outside or
// a FIFO, into path. Each FIFO has a writer held open (as any local process
// can): a FIFO with no writer reads EOF at once, which would let a reader
// that skips the handle check pass (PR #161 round 2: with a held writer, a
// bypass of readOpened hung Query after 5 iterations).
// startSwap runs swapInto until the test ends and waits for it before
// t.TempDir's cleanup (hyperguild#162).
// It returns the count of symlinks or FIFOs actually renamed into path
// (hyperguild#177: a swapper whose every swap fails tests nothing).
func startSwap(t *testing.T, path, outside string, fifo bool) *atomic.Int64 {
t.Helper()
var stop atomic.Bool
var swaps atomic.Int64
var wg sync.WaitGroup
wg.Add(1)
go func() { defer wg.Done(); swapInto(path, outside, fifo, &stop, &swaps) }()
t.Cleanup(func() { stop.Store(true); wg.Wait() })
return &swaps
}
func swapInto(path, outside string, fifo bool, stop *atomic.Bool, swaps *atomic.Int64) {
tmp := path + ".swap"
// Writers stay open until the swapper exits, so a reader parked on a
// FIFO stays parked (closing one would hand the reader EOF and hide the
// hang). Past the fd limit, new FIFOs simply have no writer.
var writers []*os.File
defer func() {
for _, w := range writers {
_ = w.Close()
}
}()
for i := 0; !stop.Load(); i++ {
_ = os.Remove(tmp)
var made error
switch {
case i%2 == 0:
_ = os.WriteFile(tmp, []byte("# r\n\nzqinside "+"zqracenote"), 0o644)
case fifo:
if made = syscall.Mkfifo(tmp, 0o644); made == nil {
if w, err := os.OpenFile(tmp, os.O_RDWR, 0); err == nil {
writers = append(writers, w)
}
}
default:
made = os.Symlink(outside, tmp)
}
if err := os.Rename(tmp, path); err == nil && made == nil && i%2 == 1 {
swaps.Add(1)
}
}
}
func TestQueryLeaksNothingUnderASymlinkSwapRace(t *testing.T) {
brain, facts, outside := searchBrain(t)
swaps := startSwap(t, filepath.Join(facts, "r.md"), outside, false)
leaks, iters, insideReads := 0, 0, 0
for deadline := time.Now().Add(1500 * time.Millisecond); time.Now().Before(deadline); iters++ {
rs, err := Query(brain, QueryOptions{Query: sentinel157, Limit: 10})
require.NoError(t, err)
leaks += leaked(rs)
// Positive control (hyperguild#177): a Query that always errors
// returns nothing and so never leaks.
rs, err = Query(brain, QueryOptions{Query: "zqracenote", Limit: 10})
require.NoError(t, err)
if len(rs) > 0 {
insideReads++
}
}
if leaks > 0 {
t.Fatalf("outside file leaked %d times in %d iterations", leaks, iters)
}
require.Positive(t, swaps.Load(), "no symlink was swapped in: the race did not run")
require.Positive(t, insideReads, "no Query returned the inside note")
}
func TestQueryDoesNotHangUnderAFIFOSwapRace(t *testing.T) {
brain, facts, outside := searchBrain(t)
startSwap(t, filepath.Join(facts, "r.md"), outside, true)
done := make(chan struct{})
go func() {
for deadline := time.Now().Add(1500 * time.Millisecond); time.Now().Before(deadline); {
_, _ = Query(brain, QueryOptions{Query: "zqracenote", Limit: 10})
}
close(done)
}()
select {
case <-done:
case <-time.After(6 * time.Second):
if f, err := os.OpenFile(filepath.Join(facts, "r.md"), os.O_WRONLY|syscall.O_NONBLOCK, 0); err == nil {
_ = f.Close()
}
t.Fatal("search hung on a swapped-in FIFO")
}
}
// hydrate (vector-only hits) reads by relative path; it must refuse a symlink.
func TestHydrateRefusesASymlink(t *testing.T) {
brain, facts, outside := searchBrain(t)
require.NoError(t, os.Symlink(outside, filepath.Join(facts, "leak.md")))
r, err := hydrate(brain, "wiki/w/facts/leak.md")
assert.Error(t, err)
assert.NotContains(t, r.Excerpt, sentinel157)
}
+10 -28
View File
@@ -5,7 +5,6 @@ import (
"bufio" "bufio"
"context" "context"
"fmt" "fmt"
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
sharedfm "github.com/mathiasbq/hyperguild/ingestion/internal/frontmatter" sharedfm "github.com/mathiasbq/hyperguild/ingestion/internal/frontmatter"
"log/slog" "log/slog"
"os" "os"
@@ -120,30 +119,20 @@ func QueryContext(ctx context.Context, brainDir string, opts QueryOptions) ([]Re
return nil, err return nil, err
} }
// Walk and read through an os.Root on the brain dir, regular files only
// (hyperguild#157): a symlink planted in the Syncthing-shared brain must
// not put an outside file in results, nor a FIFO hang the query.
bfs, err := brainfs.Open(brainDir)
if err != nil {
return nil, fmt.Errorf("search: %w", err)
}
defer func() { _ = bfs.Close() }()
var results []Result var results []Result
for _, dir := range roots { for _, dir := range roots {
if _, statErr := os.Stat(dir); os.IsNotExist(statErr) { if _, statErr := os.Stat(dir); os.IsNotExist(statErr) {
continue continue
} }
relDir, relErr := bfs.Rel(dir) err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
if relErr != nil { if err != nil {
return nil, fmt.Errorf("search: %w", relErr) slog.Warn("search: skipping path", "path", path, "err", err)
}
err := bfs.WalkRegular(relDir, func(relPath string) error {
if !strings.HasSuffix(relPath, ".md") {
return nil return nil
} }
path := filepath.Join(brainDir, filepath.FromSlash(relPath)) if d.IsDir() || !strings.HasSuffix(path, ".md") {
content, err := bfs.ReadRegular(relPath) return nil
}
content, err := os.ReadFile(path)
if err != nil { if err != nil {
slog.Warn("search: skipping unreadable file", "path", path, "err", err) slog.Warn("search: skipping unreadable file", "path", path, "err", err)
return nil return nil
@@ -166,7 +155,7 @@ func QueryContext(ctx context.Context, brainDir string, opts QueryOptions) ([]Re
origin, writtenBy := extractProvenance(string(content)) origin, writtenBy := extractProvenance(string(content))
results = append(results, Result{ results = append(results, Result{
Path: rel, Path: rel,
Title: extractTitle(string(content), filepath.Base(relPath)), Title: extractTitle(string(content), d.Name()),
Excerpt: excerpt(string(content), 300), Excerpt: excerpt(string(content), 300),
Score: score, Score: score,
Wing: wing, Wing: wing,
@@ -177,8 +166,6 @@ func QueryContext(ctx context.Context, brainDir string, opts QueryOptions) ([]Re
WrittenBy: writtenBy, WrittenBy: writtenBy,
}) })
return nil return nil
}, func(p string, err error) {
slog.Warn("search: skipping path", "path", p, "err", err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@@ -288,13 +275,8 @@ func pathInScope(relPath, wing, hall string) bool {
// excerpt, wing, and hall populated. Used for paths that surface only // excerpt, wing, and hall populated. Used for paths that surface only
// via vector search. // via vector search.
func hydrate(brainDir, relPath string) (Result, error) { func hydrate(brainDir, relPath string) (Result, error) {
// Through the brain root, regular files only (hyperguild#157). full := filepath.Join(brainDir, filepath.FromSlash(relPath))
bfs, err := brainfs.Open(brainDir) content, err := os.ReadFile(full)
if err != nil {
return Result{}, err
}
defer func() { _ = bfs.Close() }()
content, err := bfs.ReadRegular(filepath.ToSlash(relPath))
if err != nil { if err != nil {
return Result{}, err return Result{}, err
} }
-92
View File
@@ -1,92 +0,0 @@
package session
import (
"errors"
"os"
"path/filepath"
"syscall"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// session_log passed the caller's session_id straight into
// filepath.Join(sessionsDir, id+".jsonl") and opened it O_CREATE|O_APPEND:
// path segments in the id created or appended files anywhere the pod could
// write. The id is now a plain name, and the open does not follow a final
// symlink.
func TestAppendRefusesSessionIDsThatAreNotPlainNames(t *testing.T) {
root := t.TempDir()
dir := filepath.Join(root, "sessions")
for _, id := range []string{"../x", "a/b", "/abs", ".", "..", ".hidden", "", "a\\b", "a\x00b"} {
assert.Error(t, Append(dir, id, Entry{SessionID: id}), "%q", id)
}
ents, _ := os.ReadDir(root)
for _, e := range ents {
assert.Equal(t, "sessions", e.Name(), "nothing created beside the sessions dir")
}
for _, id := range []string{"1557b2e1-bf0d-5c4f-bbe3-0754d087e356", "_routing", "2026-05-05-infra.plan", "0"} {
require.NoError(t, Append(dir, id, Entry{SessionID: id}), "positive control: %q", id)
}
}
func TestAppendDoesNotWriteThroughASymlinkedLog(t *testing.T) {
root := t.TempDir()
dir := filepath.Join(root, "sessions")
require.NoError(t, os.MkdirAll(dir, 0o755))
victim := filepath.Join(root, "victim.md")
require.NoError(t, os.WriteFile(victim, []byte("untouched\n"), 0o644))
require.NoError(t, os.Symlink(filepath.Join("..", "victim.md"), filepath.Join(dir, "s.jsonl")))
assert.Error(t, Append(dir, "s", Entry{SessionID: "s"}))
got, _ := os.ReadFile(victim)
assert.Equal(t, "untouched\n", string(got))
}
func TestReadRefusesSessionIDsThatAreNotPlainNames(t *testing.T) {
root := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(root, "x.jsonl"), []byte(`{"session_id":"x"}`+"\n"), 0o644))
_, err := Read(filepath.Join(root, "sessions"), "../x")
assert.Error(t, err)
}
// A name that starts plainly and then climbs out with ".." is exactly what the
// ban on "/" in the id's tail stops: filepath.Join cleans "a/../../x" to a
// path beside the sessions dir without needing sessions/a to exist. The
// assertion is on the refusal itself (ErrInvalidID), not on any error.
func TestAppendRefusesAnIDThatClimbsOutAfterAPlainPrefix(t *testing.T) {
root := t.TempDir()
dir := filepath.Join(root, "sessions")
require.NoError(t, os.MkdirAll(dir, 0o755))
for _, id := range []string{"a/../../escape", "a/../../../escape", "a/../b"} {
err := Append(dir, id, Entry{SessionID: id})
assert.ErrorIs(t, err, ErrInvalidID, "%q", id)
}
_, err := os.Lstat(filepath.Join(root, "escape.jsonl"))
assert.True(t, errors.Is(err, os.ErrNotExist), "nothing created beside the sessions dir")
_, err = os.Lstat(filepath.Join(dir, "b.jsonl"))
assert.True(t, errors.Is(err, os.ErrNotExist), "no in-dir alias created either")
}
// Append does no Lstat type check, so O_NONBLOCK is the only thing that keeps
// a FIFO named <id>.jsonl with no reader from blocking session_log forever.
func TestAppendReturnsPromptlyOnAFIFOLogWithNoReader(t *testing.T) {
dir := filepath.Join(t.TempDir(), "sessions")
require.NoError(t, os.MkdirAll(dir, 0o755))
fifo := filepath.Join(dir, "s.jsonl")
require.NoError(t, syscall.Mkfifo(fifo, 0o644))
done := make(chan error, 1)
go func() { done <- Append(dir, "s", Entry{SessionID: "s"}) }()
select {
case err := <-done:
assert.Error(t, err, "a FIFO with no reader is refused, not written")
case <-time.After(3 * time.Second):
// Unblock the stuck open so the goroutine and TempDir cleanup finish.
if r, err := os.OpenFile(fifo, os.O_RDONLY|syscall.O_NONBLOCK, 0); err == nil {
defer r.Close() //nolint:errcheck
}
<-done
t.Fatal("Append blocked on a FIFO log with no reader")
}
}
+1 -25
View File
@@ -9,8 +9,6 @@ import (
"io/fs" "io/fs"
"os" "os"
"path/filepath" "path/filepath"
"regexp"
"syscall"
"time" "time"
) )
@@ -44,32 +42,13 @@ type Attempt struct {
RunnerOutput string `json:"runner_output,omitempty"` RunnerOutput string `json:"runner_output,omitempty"`
} }
// idPattern is a plain file name: session_log takes the id from the caller,
// and a path segment in it created or appended files anywhere the pod could
// write (hyperguild#169).
var idPattern = regexp.MustCompile(`^[A-Za-z0-9_][A-Za-z0-9._-]{0,127}$`)
// ErrInvalidID is returned when a session id is not a plain file name.
var ErrInvalidID = errors.New("session id is not a plain name")
func checkID(sessionID string) error {
if !idPattern.MatchString(sessionID) {
return fmt.Errorf("%w: %q", ErrInvalidID, sessionID)
}
return nil
}
// Append writes entry as a single JSON line to sessionsDir/{sessionID}.jsonl. // Append writes entry as a single JSON line to sessionsDir/{sessionID}.jsonl.
// The open does not follow a final symlink.
func Append(sessionsDir, sessionID string, entry Entry) error { func Append(sessionsDir, sessionID string, entry Entry) error {
if err := checkID(sessionID); err != nil {
return err
}
if err := os.MkdirAll(sessionsDir, 0o755); err != nil { if err := os.MkdirAll(sessionsDir, 0o755); err != nil {
return fmt.Errorf("create sessions dir: %w", err) return fmt.Errorf("create sessions dir: %w", err)
} }
path := filepath.Join(sessionsDir, sessionID+".jsonl") path := filepath.Join(sessionsDir, sessionID+".jsonl")
f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0o644) f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o644)
if err != nil { if err != nil {
return fmt.Errorf("open session log: %w", err) return fmt.Errorf("open session log: %w", err)
} }
@@ -91,9 +70,6 @@ func Append(sessionsDir, sessionID string, entry Entry) error {
// Read returns all entries for sessionID. Returns empty slice if no log exists. // Read returns all entries for sessionID. Returns empty slice if no log exists.
func Read(sessionsDir, sessionID string) ([]Entry, error) { func Read(sessionsDir, sessionID string) ([]Entry, error) {
if err := checkID(sessionID); err != nil {
return nil, err
}
path := filepath.Join(sessionsDir, sessionID+".jsonl") path := filepath.Join(sessionsDir, sessionID+".jsonl")
f, err := os.Open(path) f, err := os.Open(path)
if errors.Is(err, fs.ErrNotExist) { if errors.Is(err, fs.ErrNotExist) {
+17 -32
View File
@@ -2,15 +2,12 @@ package vectorstore
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
"io/fs"
"log/slog" "log/slog"
"path" "os"
"path/filepath"
"strings" "strings"
"time" "time"
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
) )
// Embedder produces dense vectors. The embed package's Client satisfies // Embedder produces dense vectors. The embed package's Client satisfies
@@ -93,38 +90,31 @@ func Sync(ctx context.Context, brainDir string, store Store, embedder Embedder)
} }
seenParents := make(map[string]struct{}) seenParents := make(map[string]struct{})
// Through the brain root, regular files only (hyperguild#157): the brain
// is a Syncthing folder, and os.ReadFile followed a synced symlink into
// the embeddings and blocked on a FIFO.
bfs, err := brainfs.Open(brainDir)
if err != nil {
return res, fmt.Errorf("open brain: %w", err)
}
defer func() { _ = bfs.Close() }()
for _, sub := range scanDirs { for _, sub := range scanDirs {
if _, err := bfs.Lstat(sub); errors.Is(err, fs.ErrNotExist) { root := filepath.Join(brainDir, sub)
if _, err := os.Stat(root); os.IsNotExist(err) {
continue continue
} }
// Any other walk error, including a scan root that is a symlink,
// aborts the run below. err = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
var walkErr error if err != nil {
onErr := func(rel string, err error) { return err
if walkErr == nil {
walkErr = fmt.Errorf("%s: %w", rel, err)
} }
} if d.IsDir() || !strings.HasSuffix(path, ".md") || d.Name() == "_index.md" {
err = bfs.WalkRegular(sub, func(relSlash string) error {
if !strings.HasSuffix(relSlash, ".md") || path.Base(relSlash) == "_index.md" {
return nil return nil
} }
rel, err := filepath.Rel(brainDir, path)
if err != nil {
return err
}
relSlash := filepath.ToSlash(rel)
seenParents[relSlash] = struct{}{} seenParents[relSlash] = struct{}{}
if ps, ok := parents[relSlash]; ok { if ps, ok := parents[relSlash]; ok {
// File already has chunks in the store. Re-embed only when // File already has chunks in the store. Re-embed only when
// the file has been edited since the oldest chunk was // the file has been edited since the oldest chunk was
// written. Tolerate clock skew with a sub-second grace. // written. Tolerate clock skew with a sub-second grace.
info, statErr := bfs.Lstat(relSlash) info, statErr := d.Info()
if statErr != nil { if statErr != nil {
res.Errors = append(res.Errors, fmt.Errorf("stat %s: %w", relSlash, statErr)) res.Errors = append(res.Errors, fmt.Errorf("stat %s: %w", relSlash, statErr))
return nil return nil
@@ -142,7 +132,7 @@ func Sync(ctx context.Context, brainDir string, store Store, embedder Embedder)
} }
} }
content, readErr := bfs.ReadRegular(relSlash) content, readErr := os.ReadFile(path)
if readErr != nil { if readErr != nil {
res.Errors = append(res.Errors, fmt.Errorf("read %s: %w", relSlash, readErr)) res.Errors = append(res.Errors, fmt.Errorf("read %s: %w", relSlash, readErr))
return nil return nil
@@ -161,12 +151,7 @@ func Sync(ctx context.Context, brainDir string, store Store, embedder Embedder)
res.Added++ res.Added++
} }
return nil return nil
}, onErr) })
if err == nil {
err = walkErr
}
// Abort before the delete pass: an incomplete walk would leave
// seenParents short and delete chunks of files that still exist.
if err != nil { if err != nil {
return res, fmt.Errorf("walk %s: %w", sub, err) return res, fmt.Errorf("walk %s: %w", sub, err)
} }
@@ -1,126 +0,0 @@
package vectorstore_test
import (
"context"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs/brainfstest"
"github.com/mathiasbq/hyperguild/ingestion/internal/vectorstore"
)
// recordingEmbedder keeps every text it was asked to embed.
type recordingEmbedder struct {
mu sync.Mutex
texts []string
}
func (e *recordingEmbedder) Embed(_ context.Context, text string) ([]float32, error) {
e.mu.Lock()
defer e.mu.Unlock()
e.texts = append(e.texts, text)
return []float32{1}, nil
}
func (e *recordingEmbedder) all() string {
e.mu.Lock()
defer e.mu.Unlock()
return strings.Join(e.texts, "\n")
}
// hyperguild#157 slice 3: Sync walked wiki/ and knowledge/ and read each .md
// with os.ReadFile, so a symlinked note had its target embedded and served by
// semantic search, and a FIFO stalled the sync.
func TestSyncRefusesSymlinkedNotesAndSurvivesSwapRaces(t *testing.T) {
brain := t.TempDir()
writeNote(t, brain, "wiki/w/facts/ok.md", "inside note")
require.NoError(t, os.Symlink(brainfstest.Outside(t), filepath.Join(brain, "wiki", "w", "facts", "leak.md")))
emb := &recordingEmbedder{}
store := &stubStore{}
_, err := vectorstore.Sync(context.Background(), brain, store, emb)
require.NoError(t, err)
assert.Contains(t, emb.all(), "inside note", "positive control: the regular note is embedded")
assert.NotContains(t, emb.all(), brainfstest.Sentinel)
for p := range store.upserts {
assert.NotContains(t, p, "leak.md")
}
p := filepath.Join(brain, "knowledge", "r.md")
require.NoError(t, os.MkdirAll(filepath.Dir(p), 0o755))
sync := func() string {
e := &recordingEmbedder{}
_, _ = vectorstore.Sync(context.Background(), brain, &stubStore{}, e)
return e.all()
}
brainfstest.AssertNoLeak(t, p, "inside", 1500*time.Millisecond, sync)
brainfstest.AssertNoHang(t, p, "inside", 1500*time.Millisecond, 6*time.Second, func() { _ = sync() })
}
// Chunks embedded from a symlinked note before this fix are deleted: the
// note is no longer seen, so the delete pass drops its rows.
func TestSyncDeletesChunksOfASymlinkedNote(t *testing.T) {
brain := t.TempDir()
writeNote(t, brain, "wiki/w/facts/ok.md", "inside note")
require.NoError(t, os.Symlink(brainfstest.Outside(t), filepath.Join(brain, "wiki", "w", "facts", "leak.md")))
store := &stubStore{known: map[string]time.Time{
"wiki/w/facts/leak.md#0001": {},
"wiki/w/facts/ok.md#0001": {},
}}
_, err := vectorstore.Sync(context.Background(), brain, store, &recordingEmbedder{})
require.NoError(t, err)
assert.Equal(t, []string{"wiki/w/facts/leak.md#0001"}, store.deletes)
}
// A walk error aborts Sync before the delete pass, so chunks of files the
// walk could not see are kept.
func TestSyncAbortsBeforeDeletingWhenASubdirIsUnreadable(t *testing.T) {
if os.Geteuid() == 0 {
t.Skip("root ignores directory permissions")
}
brain := t.TempDir()
writeNote(t, brain, "wiki/w/facts/ok.md", "inside note")
locked := filepath.Join(brain, "wiki", "w")
require.NoError(t, os.Chmod(locked, 0o000))
t.Cleanup(func() { _ = os.Chmod(locked, 0o755) })
store := &stubStore{known: map[string]time.Time{"wiki/w/facts/ok.md#0001": {}}}
_, err := vectorstore.Sync(context.Background(), brain, store, &recordingEmbedder{})
assert.Error(t, err)
assert.Empty(t, store.deletes)
}
// A scan root that is a symlink inside the brain (knowledge -> .) was
// followed by fs.WalkDir, aliasing raw/, sessions/ and root files into the
// index. Sync now aborts on it, before embedding or deleting anything.
func TestSyncAbortsOnASymlinkedScanRoot(t *testing.T) {
brain := t.TempDir()
writeNote(t, brain, "wiki/w/a.md", "wiki note")
writeNote(t, brain, "sessions/s1.md", "session note")
require.NoError(t, os.Symlink(".", filepath.Join(brain, "knowledge")))
emb := &recordingEmbedder{}
store := &stubStore{known: map[string]time.Time{"wiki/w/a.md#0001": {}}}
_, err := vectorstore.Sync(context.Background(), brain, store, emb)
assert.Error(t, err)
assert.NotContains(t, emb.all(), "session note")
for p := range store.upserts {
assert.False(t, strings.HasPrefix(p, "knowledge/"), "upserted %s", p)
}
assert.Empty(t, store.deletes)
}
// A missing scan root is skipped; the other root still syncs.
func TestSyncSkipsAMissingScanRoot(t *testing.T) {
brain := t.TempDir()
writeNote(t, brain, "wiki/w/a.md", "wiki note")
emb := &recordingEmbedder{}
_, err := vectorstore.Sync(context.Background(), brain, &stubStore{}, emb)
require.NoError(t, err)
assert.Contains(t, emb.all(), "wiki note")
}
+4 -11
View File
@@ -7,7 +7,6 @@ import (
"path/filepath" "path/filepath"
"strings" "strings"
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
"github.com/mathiasbq/hyperguild/ingestion/internal/textutil" "github.com/mathiasbq/hyperguild/ingestion/internal/textutil"
) )
@@ -17,11 +16,6 @@ func RebuildIndex(brainDir, date string) error {
if err != nil { if err != nil {
return fmt.Errorf("load inventory: %w", err) return fmt.Errorf("load inventory: %w", err)
} }
bfs, err := brainfs.Open(brainDir)
if err != nil {
return fmt.Errorf("open brain: %w", err)
}
defer func() { _ = bfs.Close() }()
total := len(inv[PageTypeConcept]) + len(inv[PageTypeEntity]) + len(inv[PageTypeSource]) total := len(inv[PageTypeConcept]) + len(inv[PageTypeEntity]) + len(inv[PageTypeSource])
var sb strings.Builder var sb strings.Builder
@@ -40,7 +34,7 @@ func RebuildIndex(brainDir, date string) error {
label := strings.ToUpper(string(pt)[:1]) + string(pt)[1:] label := strings.ToUpper(string(pt)[:1]) + string(pt)[1:]
fmt.Fprintf(&sb, "## %s\n\n", label) fmt.Fprintf(&sb, "## %s\n\n", label)
for _, e := range entries { for _, e := range entries {
summary := pageFirstSentence(bfs, e) summary := pageFirstSentence(brainDir, e)
if summary != "" { if summary != "" {
fmt.Fprintf(&sb, "- [[%s|%s]] — %s\n", e.Slug, e.Title, summary) fmt.Fprintf(&sb, "- [[%s|%s]] — %s\n", e.Slug, e.Title, summary)
} else { } else {
@@ -54,10 +48,9 @@ func RebuildIndex(brainDir, date string) error {
return os.WriteFile(dest, []byte(sb.String()), 0o644) return os.WriteFile(dest, []byte(sb.String()), 0o644)
} }
// pageFirstSentence reads the page through the brain root, regular files func pageFirstSentence(brainDir string, e Entry) string {
// only (hyperguild#157). path := filepath.Join(brainDir, "wiki", string(e.Type), e.Slug+".md")
func pageFirstSentence(bfs *brainfs.Brain, e Entry) string { content, err := os.ReadFile(path)
content, err := bfs.ReadRegular("wiki/" + string(e.Type) + "/" + e.Slug + ".md")
if err != nil { if err != nil {
return "" return ""
} }
+15 -23
View File
@@ -3,45 +3,36 @@ package wiki
import ( import (
"bufio" "bufio"
"bytes"
"errors"
"fmt" "fmt"
"io/fs" "os"
"path/filepath"
"strings" "strings"
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
) )
// LoadInventory walks brain/wiki/ and returns all pages grouped by type. // LoadInventory walks brain/wiki/ and returns all pages grouped by type.
// Missing subdirectories are silently skipped. Pages are listed and read // Missing subdirectories are silently skipped.
// through the brain root, regular files only (hyperguild#157); a type
// directory that is a symlink is an error.
func LoadInventory(brainDir string) (map[PageType][]Entry, error) { func LoadInventory(brainDir string) (map[PageType][]Entry, error) {
result := map[PageType][]Entry{ result := map[PageType][]Entry{
PageTypeConcept: {}, PageTypeConcept: {},
PageTypeEntity: {}, PageTypeEntity: {},
PageTypeSource: {}, PageTypeSource: {},
} }
bfs, err := brainfs.Open(brainDir)
if err != nil {
return nil, fmt.Errorf("open brain: %w", err)
}
defer func() { _ = bfs.Close() }()
for pt := range result { for pt := range result {
dir := "wiki/" + string(pt) dir := filepath.Join(brainDir, "wiki", string(pt))
names, err := bfs.ReadDirRegular(dir) entries, err := os.ReadDir(dir)
if errors.Is(err, fs.ErrNotExist) { if os.IsNotExist(err) {
continue continue
} }
if err != nil { if err != nil {
return nil, fmt.Errorf("read dir %s: %w", dir, err) return nil, fmt.Errorf("read dir %s: %w", dir, err)
} }
for _, name := range names { for _, e := range entries {
if !strings.HasSuffix(name, ".md") { if e.IsDir() || !strings.HasSuffix(e.Name(), ".md") {
continue continue
} }
slug := strings.TrimSuffix(name, ".md") slug := strings.TrimSuffix(e.Name(), ".md")
title, aliases := readFrontmatter(bfs, dir+"/"+name, slug) path := filepath.Join(dir, e.Name())
title, aliases := readFrontmatter(path, slug)
result[pt] = append(result[pt], Entry{Slug: slug, Title: title, Aliases: aliases, Type: pt}) result[pt] = append(result[pt], Entry{Slug: slug, Title: title, Aliases: aliases, Type: pt})
} }
} }
@@ -50,14 +41,15 @@ func LoadInventory(brainDir string) (map[PageType][]Entry, error) {
// readFrontmatter extracts title and aliases from YAML frontmatter. // readFrontmatter extracts title and aliases from YAML frontmatter.
// Falls back to slug for title and empty aliases on any error. // Falls back to slug for title and empty aliases on any error.
func readFrontmatter(bfs *brainfs.Brain, rel, fallbackSlug string) (title string, aliases []string) { func readFrontmatter(path, fallbackSlug string) (title string, aliases []string) {
title = fallbackSlug title = fallbackSlug
content, err := bfs.ReadRegular(rel) f, err := os.Open(path)
if err != nil { if err != nil {
return return
} }
defer f.Close() //nolint:errcheck
scanner := bufio.NewScanner(bytes.NewReader(content)) scanner := bufio.NewScanner(f)
inFM := false inFM := false
inAliases := false inAliases := false
for scanner.Scan() { for scanner.Scan() {
-79
View File
@@ -1,79 +0,0 @@
package wiki
import (
"fmt"
"os"
"path/filepath"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs"
"github.com/mathiasbq/hyperguild/ingestion/internal/brainfs/brainfstest"
)
// hyperguild#157 slice 4: LoadInventory listed wiki/<type>/ with os.ReadDir
// and opened each .md by path, so a symlinked page put the target's title and
// aliases into the inventory (and the LLM prompts built from it), and a FIFO
// hung it. RebuildIndex read first sentences the same way.
func TestLoadInventoryRefusesSymlinksAndSurvivesSwapRaces(t *testing.T) {
brain := t.TempDir()
dir := filepath.Join(brain, "wiki", "concepts")
require.NoError(t, os.MkdirAll(dir, 0o755))
require.NoError(t, os.WriteFile(filepath.Join(dir, "ok.md"), []byte("---\ntitle: zqinside\n---\n"), 0o644))
require.NoError(t, os.Symlink(brainfstest.Outside(t), filepath.Join(dir, "leak.md")))
inBrain := brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "s.md"))
require.NoError(t, os.Symlink(filepath.Join("..", "..", "raw", "s.md"), filepath.Join(dir, "inlink.md")))
inv, err := LoadInventory(brain)
require.NoError(t, err)
got := fmt.Sprintf("%+v", inv)
assert.Contains(t, got, "zqinside", "positive control")
assert.NotContains(t, got, brainfstest.Sentinel)
assert.NotContains(t, got, "leak", "a symlinked page is not listed")
assert.NotContains(t, got, "inlink", "a page linked to another brain file is not listed")
p := filepath.Join(dir, "r.md")
read := func() string {
inv, _ := LoadInventory(brain)
return fmt.Sprintf("%+v", inv)
}
brainfstest.AssertNoLeak(t, p, "---\ntitle: zqinside\n---\n", 1500*time.Millisecond, read)
brainfstest.AssertNoLeakFrom(t, p, "---\ntitle: zqinside\n---\n", inBrain, 1500*time.Millisecond, read)
brainfstest.AssertNoHang(t, p, "---\ntitle: zqinside\n---\n", 1500*time.Millisecond, 6*time.Second, func() { _ = read() })
}
func TestLoadInventoryReturnsErrNotRegularForASymlinkedTypeDir(t *testing.T) {
brain := t.TempDir()
require.NoError(t, os.MkdirAll(filepath.Join(brain, "wiki"), 0o755))
require.NoError(t, os.MkdirAll(filepath.Join(brain, "raw"), 0o755))
require.NoError(t, os.WriteFile(filepath.Join(brain, "raw", "r.md"), []byte("---\ntitle: zqrawonly\n---\n"), 0o644))
require.NoError(t, os.Symlink(filepath.Join("..", "raw"), filepath.Join(brain, "wiki", "concepts")))
inv, err := LoadInventory(brain)
require.ErrorIs(t, err, brainfs.ErrNotRegular)
assert.NotContains(t, fmt.Sprintf("%+v", inv), "zqrawonly")
}
func TestRebuildIndexDoesNotReadAPageSwappedForASymlink(t *testing.T) {
brain := t.TempDir()
dir := filepath.Join(brain, "wiki", "concepts")
require.NoError(t, os.MkdirAll(dir, 0o755))
p := filepath.Join(dir, "r.md")
require.NoError(t, os.WriteFile(p, []byte("---\ntitle: R\n---\nzqinside sentence.\n"), 0o644))
require.NoError(t, RebuildIndex(brain, "2026-10-04"))
idx, err := os.ReadFile(filepath.Join(brain, "wiki", "index.md"))
require.NoError(t, err)
require.Contains(t, string(idx), "zqinside", "positive control: the summary is read")
read := func() string {
_ = RebuildIndex(brain, "2026-10-04")
b, _ := os.ReadFile(filepath.Join(brain, "wiki", "index.md"))
return string(b)
}
brainfstest.AssertNoLeak(t, p, "---\ntitle: R\n---\nzqinside sentence.\n", 1500*time.Millisecond, read)
inBrain := brainfstest.WriteSentinelAt(t, filepath.Join(brain, "raw", "s.md"))
brainfstest.AssertNoLeakFrom(t, p, "---\ntitle: R\n---\nzqinside sentence.\n", inBrain, 1500*time.Millisecond, read)
brainfstest.AssertNoHang(t, p, "---\ntitle: R\n---\nzqinside sentence.\n", 1500*time.Millisecond, 6*time.Second, func() { _ = read() })
}