diff --git a/.context/mcp.json b/.context/mcp.json index 45b7f61..c6fd6a8 100644 --- a/.context/mcp.json +++ b/.context/mcp.json @@ -7,6 +7,29 @@ "supervisor": { "url": "http://localhost:3200/mcp", "description": "Hyperguild SDO — skill workers (tdd, retrospective), brain tools (brain_query, brain_write), session logging, tier detection" + }, + "brain": { + "type": "http", + "url": "https://koala.tail30472e.ts.net:8443/mcp", + "headers": { + "Authorization": "Bearer ${BRAIN_MCP_TOKEN}" + } + }, + "gitea": { + "_comment": "gitea-mcp authenticates by PASS-THROUGH: the bearer is a Gitea PAT it validates against Gitea and then acts as. GITEA_MCP_TOKEN -- which this entry used until 2026-09-08 -- 401s against it, so every harness synced from this file had a dead gitea server and no signal saying so. Probed 2026-09-08: GITEA_MCP_TOKEN 401, DMABE_GITEA_API_TOKEN 200, on both the public and the Serve URL.", + "type": "http", + "url": "https://koala.tail30472e.ts.net:8446/mcp", + "headers": { + "Authorization": "Bearer ${DMABE_GITEA_API_TOKEN}" + } + }, + "infra": { + "_comment": "The bearer is belt-and-braces: this endpoint answers 200 with or without it, since Serve already limits reach to the tailnet. Sending it anyway means the day the server starts enforcing is not the day every harness breaks.", + "type": "http", + "url": "https://koala.tail30472e.ts.net:8444/mcp", + "headers": { + "Authorization": "Bearer ${INFRA_MCP_TOKEN}" + } } } } diff --git a/ingestion/cmd/server/main.go b/ingestion/cmd/server/main.go index 7e15ae4..e512ab2 100644 --- a/ingestion/cmd/server/main.go +++ b/ingestion/cmd/server/main.go @@ -396,15 +396,39 @@ func main() { } host := envOr("CLAUDE_INGEST_HOST", systemHostname()) interval := time.Duration(envInt("CLAUDE_INGEST_INTERVAL", 60)) * time.Second + + // Project directories excluded from ingestion entirely, as globs + // against the project directory name — e.g. + // "*cobalt-dingo*,*claude-worktrees-bridge-cse*". + // + // This is a veto list, deliberately named as one. It complements the + // client-name scrubber rather than replacing it: the scrubber drops + // individual TURNS whose content matches, while this never opens a + // project's files at all. Sessions working on live company financials + // are excluded here because their content is confidential in ways no + // pattern reliably recognises, and their learnings are captured + // deliberately through the session-close ritual instead. + // + // Caveat worth knowing: a project directory is derived from the + // session's WORKING DIRECTORY, not from the repositories it touches. + // A session run from ~/dev that happens to read a skipped repo will + // NOT be excluded by this. The scrubber remains the control for + // content; this is a control for whole workspaces. + skipProjects := splitList(os.Getenv("CLAUDE_INGEST_SKIP_PROJECTS")) + if len(skipProjects) > 0 { + logger.Info("claudewatcher: projects excluded from ingestion", "patterns", skipProjects) + } + sink := &claudeSink{brainDir: brainDir, logger: logger} go func() { if err := claudewatcher.Watch(ctx, claudewatcher.Config{ - SessionsDir: claudeDir, - Host: host, - Interval: interval, - Sink: sink, - Cursors: cursorStore, - Logger: logger, + SessionsDir: claudeDir, + Host: host, + Interval: interval, + Sink: sink, + Cursors: cursorStore, + Logger: logger, + SkipProjects: skipProjects, }); err != nil && err != context.Canceled { logger.Error("claudewatcher exited", "err", err) } diff --git a/ingestion/internal/claudewatcher/skip_projects_test.go b/ingestion/internal/claudewatcher/skip_projects_test.go new file mode 100644 index 0000000..40425dc --- /dev/null +++ b/ingestion/internal/claudewatcher/skip_projects_test.go @@ -0,0 +1,231 @@ +package claudewatcher + +import ( + "context" + "log/slog" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// recordingHandler captures log records so a test can assert that a file was +// never even attempted — the absence of a read error is the evidence. +type recordingHandler struct { + mu sync.Mutex + records []string +} + +func (h *recordingHandler) Enabled(context.Context, slog.Level) bool { return true } + +func (h *recordingHandler) Handle(_ context.Context, r slog.Record) error { + h.mu.Lock() + defer h.mu.Unlock() + var sb strings.Builder + sb.WriteString(r.Message) + r.Attrs(func(a slog.Attr) bool { + sb.WriteString(" " + a.Key + "=" + a.Value.String()) + return true + }) + h.records = append(h.records, sb.String()) + return nil +} + +func (h *recordingHandler) WithAttrs([]slog.Attr) slog.Handler { return h } +func (h *recordingHandler) WithGroup(string) slog.Handler { return h } + +func (h *recordingHandler) contains(substr string) bool { + h.mu.Lock() + defer h.mu.Unlock() + for _, r := range h.records { + if strings.Contains(r, substr) { + return true + } + } + return false +} + +func session(t *testing.T, root, project, id string) string { + t.Helper() + dir := filepath.Join(root, project) + require.NoError(t, os.MkdirAll(dir, 0o755)) + return writeSession(t, dir, id, []string{ + `{"type":"user","sessionId":"` + id + `","message":"a prompt"}`, + `{"type":"assistant","sessionId":"` + id + `","message":{"content":[{"type":"text","text":"an answer"}]}}`, + }) +} + +// A named project is not ingested, and its neighbours still are. +func TestSkipProjects_skipsTheNamedProjectOnly(t *testing.T) { + tmp := t.TempDir() + session(t, tmp, "-home-mathias-dev-AGENTS-cobalt-dingo", "skipped") + session(t, tmp, "-home-mathias-dev", "kept") + + sink := &memSink{} + require.NoError(t, TickOnce(context.Background(), Config{ + SessionsDir: tmp, + Host: "koala", + Sink: sink, + SkipProjects: []string{"-home-mathias-dev-AGENTS-cobalt-dingo"}, + })) + + require.Len(t, sink.batches, 1) + assert.Equal(t, "kept", sink.batches[0].SessionID) + assert.Equal(t, "-home-mathias-dev", sink.batches[0].ProjectID) +} + +// Globs are necessary, not a convenience. A project directory is derived from +// the session's WORKING DIRECTORY, so a git worktree produces an ephemeral +// name like "-home-mathias-dev--claude-worktrees-bridge-cse-017xoC...". An +// exact-match list could never keep up with those. +func TestSkipProjects_globMatchesEphemeralWorktreeNames(t *testing.T) { + tmp := t.TempDir() + session(t, tmp, "-home-mathias-dev--claude-worktrees-bridge-cse-017xoCEcrc35", "worktree") + session(t, tmp, "-home-mathias-dev", "kept") + + sink := &memSink{} + require.NoError(t, TickOnce(context.Background(), Config{ + SessionsDir: tmp, + Host: "koala", + Sink: sink, + SkipProjects: []string{"*claude-worktrees-bridge-cse*"}, + })) + + require.Len(t, sink.batches, 1) + assert.Equal(t, "kept", sink.batches[0].SessionID) +} + +// The point of skipping at the directory walk: the file is never OPENED, so +// its content never enters this process. Proven by making it unreadable — a +// skipped file produces no read error because nothing tried to read it, while +// the same file in a non-skipped project does. +func TestSkipProjects_theFileIsNeverOpened(t *testing.T) { + tmp := t.TempDir() + path := session(t, tmp, "-home-mathias-dev-AGENTS-cobalt-dingo", "unreadable") + require.NoError(t, os.Chmod(path, 0o000)) + t.Cleanup(func() { _ = os.Chmod(path, 0o644) }) + + rec := &recordingHandler{} + sink := &memSink{} + require.NoError(t, TickOnce(context.Background(), Config{ + SessionsDir: tmp, + Host: "koala", + Sink: sink, + Logger: slog.New(rec), + SkipProjects: []string{"*cobalt-dingo*"}, + })) + + assert.Empty(t, sink.batches) + assert.False(t, rec.contains("file failed"), + "a skipped file must not even be attempted") +} + +// The control test for the one above: without the skip, the unreadable file IS +// attempted and fails. Otherwise the assertion above could pass for the wrong +// reason. +func TestSkipProjects_withoutTheSkipTheSameFileIsAttempted(t *testing.T) { + tmp := t.TempDir() + path := session(t, tmp, "-home-mathias-dev-AGENTS-cobalt-dingo", "unreadable") + require.NoError(t, os.Chmod(path, 0o000)) + t.Cleanup(func() { _ = os.Chmod(path, 0o644) }) + + rec := &recordingHandler{} + require.NoError(t, TickOnce(context.Background(), Config{ + SessionsDir: tmp, + Host: "koala", + Sink: &memSink{}, + Logger: slog.New(rec), + })) + + assert.True(t, rec.contains("file failed"), + "the unreadable file must fail when not skipped — otherwise the skip test proves nothing") +} + +// Patterns match the project directory NAME, not the whole path. A temp +// directory or a home directory component must not accidentally match and +// silently disable ingestion. +func TestSkipProjects_matchesTheProjectNameNotTheFullPath(t *testing.T) { + tmp := t.TempDir() + session(t, tmp, "-home-mathias-dev", "kept") + + sink := &memSink{} + require.NoError(t, TickOnce(context.Background(), Config{ + SessionsDir: tmp, + Host: "koala", + Sink: sink, + // tmp is somewhere under /tmp; this must not match the project name. + SkipProjects: []string{"*tmp*"}, + })) + + require.Len(t, sink.batches, 1, "a path component outside the project name must not match") +} + +// An unparseable pattern must stop the watcher rather than be ignored. A +// silently dropped skip rule means ingesting what was meant to be excluded, +// which is the failure that cannot be noticed after the fact. +func TestSkipProjects_invalidPatternRefusesToRun(t *testing.T) { + tmp := t.TempDir() + session(t, tmp, "-home-mathias-dev", "kept") + + sink := &memSink{} + err := TickOnce(context.Background(), Config{ + SessionsDir: tmp, + Host: "koala", + Sink: sink, + SkipProjects: []string{"[unclosed"}, + }) + + require.Error(t, err) + assert.Contains(t, err.Error(), "[unclosed") + assert.Empty(t, sink.batches, "nothing may be ingested while a skip rule is broken") +} + +func TestSkipProjects_emptyListChangesNothing(t *testing.T) { + tmp := t.TempDir() + session(t, tmp, "-home-mathias-dev", "kept") + + sink := &memSink{} + require.NoError(t, TickOnce(context.Background(), Config{ + SessionsDir: tmp, Host: "koala", Sink: sink, SkipProjects: nil, + })) + + require.Len(t, sink.batches, 1) +} + +// The difference between SkipDir and filtering after the walk, made +// observable: an unreadable skipped directory must not disturb anything. +// +// Without SkipDir, WalkDir reads the directory's entries, gets a permission +// error, and that error aborts the ENTIRE walk — so one skipped project with +// awkward permissions would silently stop ingesting every other project. +// Returning SkipDir means its contents are never listed at all. +// +// Added after a mutation test showed the earlier assertions could not tell the +// two implementations apart. +func TestSkipProjects_anUnreadableSkippedDirDoesNotBreakTheWalk(t *testing.T) { + tmp := t.TempDir() + session(t, tmp, "-home-mathias-dev", "kept") + + blocked := filepath.Join(tmp, "-home-mathias-dev-AGENTS-cobalt-dingo") + require.NoError(t, os.MkdirAll(blocked, 0o755)) + _ = writeSession(t, blocked, "secret", []string{ + `{"type":"user","sessionId":"secret","message":"should never be read"}`, + }) + require.NoError(t, os.Chmod(blocked, 0o000)) + t.Cleanup(func() { _ = os.Chmod(blocked, 0o755) }) + + sink := &memSink{} + require.NoError(t, TickOnce(context.Background(), Config{ + SessionsDir: tmp, + Host: "koala", + Sink: sink, + SkipProjects: []string{"*cobalt-dingo*"}, + })) + + require.Len(t, sink.batches, 1, "an unreadable skipped directory must not abort the walk") + assert.Equal(t, "kept", sink.batches[0].SessionID) +} diff --git a/ingestion/internal/claudewatcher/watcher.go b/ingestion/internal/claudewatcher/watcher.go index 26b5873..9b62018 100644 --- a/ingestion/internal/claudewatcher/watcher.go +++ b/ingestion/internal/claudewatcher/watcher.go @@ -45,6 +45,25 @@ type Config struct { Sink Sink Cursors *CursorStore Logger *slog.Logger + + // SkipProjects excludes whole project directories from ingestion, as + // filepath.Match globs tested against the project directory NAME (the + // basename, e.g. "-home-mathias-dev"), not the full path. + // + // Globs rather than exact names because a project directory is derived + // from the session's working directory, so a git worktree yields an + // ephemeral name like + // "-home-mathias-dev--claude-worktrees-bridge-cse-017xoC..." that no + // exact-match list could keep up with. + // + // Matching directories are skipped during the directory WALK, so their + // files are never opened and their content never enters this process — + // which is a stronger property than filtering it out afterwards. + // + // An unparseable pattern stops the watcher. A silently dropped skip rule + // means ingesting what was meant to be excluded, and that is not + // noticeable after the fact. + SkipProjects []string } // Watch runs the polling loop until ctx is cancelled. Returns ctx.Err() @@ -68,6 +87,9 @@ func Watch(ctx context.Context, cfg Config) error { if cfg.Logger == nil { cfg.Logger = slog.Default() } + if err := validateSkipPatterns(cfg.SkipProjects); err != nil { + return err + } cfg.Logger.Info("claudewatcher: started", "sessions_dir", cfg.SessionsDir, "host", cfg.Host, @@ -91,7 +113,7 @@ func Watch(ctx context.Context, cfg Config) error { // runTick is one polling pass. Exposed (lowercase) for tests via // TickOnce. func runTick(ctx context.Context, cfg Config) { - files, err := listSessionFiles(cfg.SessionsDir) + files, err := listSessionFiles(cfg.SessionsDir, cfg.SkipProjects, cfg.Logger) if err != nil { cfg.Logger.Warn("claudewatcher: list session files", "err", err) return @@ -119,17 +141,52 @@ func TickOnce(ctx context.Context, cfg Config) error { if cfg.Logger == nil { cfg.Logger = slog.Default() } + if err := validateSkipPatterns(cfg.SkipProjects); err != nil { + return err + } runTick(ctx, cfg) return nil } -func listSessionFiles(root string) ([]string, error) { +// validateSkipPatterns rejects a pattern filepath.Match cannot parse. +// +// Checked up front rather than per file: a bad pattern that only failed at +// match time would be reported once per session file per tick, and — worse — +// filepath.Match returns an error alongside a false match, so the directory +// would be ingested while the log complained. +func validateSkipPatterns(patterns []string) error { + for _, p := range patterns { + if _, err := filepath.Match(p, "probe"); err != nil { + return fmt.Errorf("invalid skip-project pattern %q: %w", p, err) + } + } + return nil +} + +// skipProject reports whether a project directory name matches any pattern. +func skipProject(name string, patterns []string) bool { + for _, p := range patterns { + // The error case is already excluded by validateSkipPatterns. + if ok, _ := filepath.Match(p, name); ok { + return true + } + } + return false +} + +func listSessionFiles(root string, skip []string, log *slog.Logger) ([]string, error) { var out []string err := filepath.WalkDir(root, func(path string, d os.DirEntry, walkErr error) error { if walkErr != nil { return walkErr } if d.IsDir() { + // SkipDir, so nothing inside is even listed. The files are never + // opened and their content never reaches this process. + if path != root && skipProject(d.Name(), skip) { + log.Info("claudewatcher: project skipped by configuration", "project", d.Name()) + return filepath.SkipDir + } return nil } if !strings.HasSuffix(path, ".jsonl") {