From 2d5bed1e907eb9aa991e846e20ca6a0d85d1cf74 Mon Sep 17 00:00:00 2001 From: Mathias Date: Wed, 23 Sep 2026 07:59:43 +0200 Subject: [PATCH 1/2] =?UTF-8?q?feat(brain):=20add=20brain=5Freject=20?= =?UTF-8?q?=E2=80=94=20the=20missing=20third=20option=20for=20raw/=20notes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hyperguild#81. brain_pending lists brain/raw/ notes awaiting review; brain_promote moves one into the wiki. There was no way to say "reviewed, not worth promoting" — the only options were promote it or let it sit forever. Live consequence, confirmed this session: the pending queue has grown to 11 items, the oldest 152 days old, several of which are generic web scrapes nobody is ever going to promote but nobody could clear either except an un-audited rm bypassing the tool surface entirely. RejectNote (internal/api/promote.go) mirrors PromoteNote's exact safety shape: same basename-only filename validation, same write-then-delete ordering (the source survives any write failure), same collision check (never silently overwrite an existing rejected note under the same name). Stamps rejected_at (+ reject_reason if given) into frontmatter, moves brain/raw/ -> brain/raw/rejected/. No index rebuild or auto-tunnel -- a rejected note never entered the wiki, so there's nothing to index. raw/rejected/ is a subdirectory ListPending's non-recursive os.ReadDir never walks into, so a rejected note is never re-surfaced as pending -- the human decision, once made, stays made. brain_reject wired into the MCP tool list and dispatch, mirroring brain_promote's schema shape (filename required, reason optional). Human-invoked only, same as promote -- never called automatically. Co-Authored-By: Claude Sonnet 5 --- ingestion/internal/api/promote.go | 62 +++++++++++++++++++ ingestion/internal/api/promote_test.go | 82 +++++++++++++++++++++++++ ingestion/internal/mcp/handlers.go | 28 +++++++++ ingestion/internal/mcp/handlers_test.go | 35 +++++++++++ ingestion/internal/mcp/server.go | 2 + ingestion/internal/mcp/server_test.go | 2 +- 6 files changed, 210 insertions(+), 1 deletion(-) diff --git a/ingestion/internal/api/promote.go b/ingestion/internal/api/promote.go index d34768c..8def9e0 100644 --- a/ingestion/internal/api/promote.go +++ b/ingestion/internal/api/promote.go @@ -145,6 +145,68 @@ func PromoteNote(brainDir string, opts PromoteOptions) (string, error) { return relSlash, nil } +// RejectOptions identifies a raw note to reject and why. +type RejectOptions struct { + Filename string // basename in brain/raw/ + Reason string // optional +} + +// RejectNote records a human decision not to promote a brain/raw/ note, +// without silently losing it: it stamps rejected_at (+ reject_reason when +// given) into frontmatter and moves the file to brain/raw/rejected/, using +// the same write-then-delete safety PromoteNote uses — the source survives +// any write failure — and the same basename-only filename validation. +// +// A rejected note never entered the wiki, so no index rebuild or +// auto-tunnel run: unlike PromoteNote, this is the end of the note's +// journey, not the start of a new one. It stays out of ListPending's view +// (raw/rejected/ is a subdirectory, and ListPending's os.ReadDir on raw/ +// does not recurse), so a rejected note is never re-surfaced as pending — +// the human decision, once made, stays made, the same principle behind +// this estate's "issue status is surfaced, not auto-closed, but once a +// human decides, the decision is recorded" rule applied one level earlier. +func RejectNote(brainDir string, opts RejectOptions) (string, error) { + base := filepath.Base(opts.Filename) + if base != opts.Filename || base == "." || base == ".." || strings.ContainsAny(opts.Filename, `/\`) { + return "", fmt.Errorf("invalid filename %q", opts.Filename) + } + + src := filepath.Join(brainDir, "raw", base) + raw, err := os.ReadFile(src) + if err != nil { + if os.IsNotExist(err) { + return "", fmt.Errorf("pending note %q does not exist in raw/", base) + } + return "", fmt.Errorf("read source: %w", err) + } + + dest := filepath.Join(brainDir, "raw", "rejected", base) + if _, statErr := os.Stat(dest); statErr == nil { + rel, _ := filepath.Rel(brainDir, dest) + return "", fmt.Errorf("a rejected note named %s already exists", filepath.ToSlash(rel)) + } + + fm, body := parseFrontmatter(string(raw)) + fm.set("rejected_at", time.Now().UTC().Format(time.RFC3339)) + if opts.Reason != "" { + fm.set("reject_reason", opts.Reason) + } + + if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { + return "", fmt.Errorf("create rejected dir: %w", err) + } + // Write-then-delete: the source survives any write failure. + if err := os.WriteFile(dest, []byte(fm.render()+body), 0o644); err != nil { + return "", fmt.Errorf("write rejected note: %w", err) + } + if err := os.Remove(src); err != nil { + return "", fmt.Errorf("rejected note written but source removal failed: %w", err) + } + + rel, _ := filepath.Rel(brainDir, dest) + return filepath.ToSlash(rel), nil +} + // excerpt returns the first n runes of s, trimmed, single-spaced. func excerpt(s string, n int) string { s = strings.TrimSpace(s) diff --git a/ingestion/internal/api/promote_test.go b/ingestion/internal/api/promote_test.go index ff4e428..fd9427a 100644 --- a/ingestion/internal/api/promote_test.go +++ b/ingestion/internal/api/promote_test.go @@ -119,3 +119,85 @@ func TestPromoteRebuildsWingIndex(t *testing.T) { require.NoError(t, err, "wing _index regenerated") assert.Contains(t, string(idx), "x", "promoted note appears in the index") } + +// --- RejectNote (hyperguild#81) --------------------------------------------- + +func TestRejectHappyPath(t *testing.T) { + dir := t.TempDir() + writeRaw(t, dir, "2026-04-23-fed-sentiment-scrape.md", + "---\ncreated_at: 2026-04-23T13:20:07Z\n---\n# Sentiment scrape\n\nbody.\n") + + rel, err := RejectNote(dir, RejectOptions{ + Filename: "2026-04-23-fed-sentiment-scrape.md", Reason: "generic scrape, not homelab-relevant", + }) + require.NoError(t, err) + assert.Equal(t, "raw/rejected/2026-04-23-fed-sentiment-scrape.md", rel) + + // Source removed from the pending queue. + _, statErr := os.Stat(filepath.Join(dir, "raw", "2026-04-23-fed-sentiment-scrape.md")) + assert.True(t, os.IsNotExist(statErr), "source removed from raw/ after reject") + + got, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(rel))) + require.NoError(t, err) + s := string(got) + assert.Contains(t, s, "created_at: 2026-04-23T13:20:07Z", "original created_at preserved") + assert.Contains(t, s, "rejected_at:") + assert.Contains(t, s, "reject_reason: generic scrape, not homelab-relevant") + assert.Contains(t, s, "# Sentiment scrape", "body preserved for later reference") + + // Rejected notes never entered the wiki, and are no longer pending. + pending, err := ListPending(dir) + require.NoError(t, err) + assert.Empty(t, pending) +} + +func TestRejectWithoutReason(t *testing.T) { + dir := t.TempDir() + writeRaw(t, dir, "2026-06-01-x.md", "body\n") + rel, err := RejectNote(dir, RejectOptions{Filename: "2026-06-01-x.md"}) + require.NoError(t, err) + got, _ := os.ReadFile(filepath.Join(dir, filepath.FromSlash(rel))) + assert.NotContains(t, string(got), "reject_reason:", "no reason given, none stamped") +} + +func TestRejectMissingSourceErrors(t *testing.T) { + _, err := RejectNote(t.TempDir(), RejectOptions{Filename: "ghost.md"}) + require.Error(t, err) +} + +func TestRejectRejectsTraversalFilename(t *testing.T) { + _, err := RejectNote(t.TempDir(), RejectOptions{Filename: "../escape.md"}) + require.Error(t, err) +} + +func TestRejectCollisionNoOverwrite(t *testing.T) { + dir := t.TempDir() + dest := filepath.Join(dir, "raw", "rejected", "x.md") + require.NoError(t, os.MkdirAll(filepath.Dir(dest), 0o755)) + require.NoError(t, os.WriteFile(dest, []byte("EXISTING\n"), 0o644)) + writeRaw(t, dir, "x.md", "NEW\n") + + _, err := RejectNote(dir, RejectOptions{Filename: "x.md"}) + require.Error(t, err, "collision must error, not overwrite") + + got, _ := os.ReadFile(dest) + assert.Equal(t, "EXISTING\n", string(got), "target not overwritten") + _, statErr := os.Stat(filepath.Join(dir, "raw", "x.md")) + assert.NoError(t, statErr, "source preserved on collision (atomic: no delete without write)") +} + +func TestRejectedNotesAreNotListedAsPendingAgain(t *testing.T) { + // ListPending already skips tunnel-candidates-*.md; confirm raw/rejected/ + // (a subdirectory) doesn't leak into the top-level pending listing either + // — os.ReadDir on raw/ doesn't recurse, but pin it so a future change to + // ListPending can't silently start surfacing rejected notes as pending + // again. + dir := t.TempDir() + writeRaw(t, dir, "2026-06-01-x.md", "body\n") + _, err := RejectNote(dir, RejectOptions{Filename: "2026-06-01-x.md"}) + require.NoError(t, err) + + pending, err := ListPending(dir) + require.NoError(t, err) + assert.Empty(t, pending, "a rejected note must not still appear as pending") +} diff --git a/ingestion/internal/mcp/handlers.go b/ingestion/internal/mcp/handlers.go index c60f52e..2163c94 100644 --- a/ingestion/internal/mcp/handlers.go +++ b/ingestion/internal/mcp/handlers.go @@ -96,6 +96,14 @@ func (s *Server) tools() []map[string]any { "slug": str("optional target slug; defaults to filename minus date prefix"), }), }, + { + "name": "brain_reject", + "description": "Reject a brain/raw/ note: stamps rejected_at (+ reject_reason if given) into frontmatter and moves it to brain/raw/rejected/ — never deleted outright, and never re-surfaced by brain_pending. The third option alongside promote/leave-pending; a human decision, never automatic. Returns {path}.", + "inputSchema": schema([]string{"filename"}, map[string]any{ + "filename": str("basename in brain/raw/, e.g. 2026-04-23-fed-sentiment-scrape.md"), + "reason": str("optional; why this isn't worth promoting"), + }), + }, noteToolDescriptors()[0], noteToolDescriptors()[1], { @@ -389,6 +397,26 @@ func (s *Server) brainPromote(ctx context.Context, args json.RawMessage) (json.R return json.Marshal(map[string]string{"path": relPath}) } +type brainRejectArgs struct { + Filename string `json:"filename"` + Reason string `json:"reason,omitempty"` +} + +// brainReject records a decision not to promote a raw/ note (api.RejectNote: +// frontmatter-stamp + move to raw/rejected/). No graph indexing — a rejected +// note never entered the wiki, so there's nothing to index. +func (s *Server) brainReject(_ context.Context, args json.RawMessage) (json.RawMessage, error) { + var a brainRejectArgs + if err := json.Unmarshal(args, &a); err != nil { + return nil, fmt.Errorf("parse args: %w", err) + } + relPath, err := api.RejectNote(s.brainDir, api.RejectOptions{Filename: a.Filename, Reason: a.Reason}) + if err != nil { + return nil, err + } + return json.Marshal(map[string]string{"path": relPath}) +} + // indexInGraph is a best-effort wrapper around graphsync.IndexDoc that // logs failures but never propagates them — the underlying write/ingest // has already succeeded and the graph is an augmentation, not a diff --git a/ingestion/internal/mcp/handlers_test.go b/ingestion/internal/mcp/handlers_test.go index 09b91e0..84b5114 100644 --- a/ingestion/internal/mcp/handlers_test.go +++ b/ingestion/internal/mcp/handlers_test.go @@ -396,3 +396,38 @@ func TestBrainPromoteInvalidHallErrors(t *testing.T) { _, srcErr := os.Stat(filepath.Join(raw, "x.md")) assert.NoError(t, srcErr, "source untouched on validation error") } + +func TestBrainRejectMovesToRejectedSubdir(t *testing.T) { + brainDir := t.TempDir() + raw := filepath.Join(brainDir, "raw") + require.NoError(t, os.MkdirAll(raw, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(raw, "2026-04-23-scrape.md"), + []byte("---\ncreated_at: 2026-04-23T13:20:07Z\n---\n# Scrape\n\nbody\n"), 0o644)) + srv := mcp.NewServer(brainDir, nil, nil, nil) + + resp := toolCall(t, srv, "brain_reject", map[string]any{ + "filename": "2026-04-23-scrape.md", "reason": "generic scrape, not homelab-relevant", + }) + require.Nil(t, resp["error"], "got: %v", resp["error"]) + text := resp["result"].(map[string]any)["content"].([]any)[0].(map[string]any)["text"].(string) + assert.Contains(t, text, "raw/rejected/2026-04-23-scrape.md") + + got, err := os.ReadFile(filepath.Join(brainDir, "raw/rejected/2026-04-23-scrape.md")) + require.NoError(t, err) + assert.Contains(t, string(got), "reject_reason: generic scrape, not homelab-relevant") + + _, srcErr := os.Stat(filepath.Join(raw, "2026-04-23-scrape.md")) + assert.True(t, os.IsNotExist(srcErr), "source removed from raw/") + + // A rejected note must not still show up as pending. + pendingResp := toolCall(t, srv, "brain_pending", map[string]any{}) + pendingText := pendingResp["result"].(map[string]any)["content"].([]any)[0].(map[string]any)["text"].(string) + assert.Contains(t, pendingText, `"pending":[]`) +} + +func TestBrainRejectMissingSourceErrors(t *testing.T) { + brainDir := t.TempDir() + srv := mcp.NewServer(brainDir, nil, nil, nil) + resp := toolCall(t, srv, "brain_reject", map[string]any{"filename": "ghost.md"}) + require.NotNil(t, resp["error"]) +} diff --git a/ingestion/internal/mcp/server.go b/ingestion/internal/mcp/server.go index b35c8da..3726e0b 100644 --- a/ingestion/internal/mcp/server.go +++ b/ingestion/internal/mcp/server.go @@ -275,6 +275,8 @@ func (s *Server) handleCall(ctx context.Context, name string, args json.RawMessa return s.brainPending(ctx, args) case "brain_promote": return s.brainPromote(ctx, args) + case "brain_reject": + return s.brainReject(ctx, args) case "capture": return s.brainCapture(ctx, args) case "brain_read": diff --git a/ingestion/internal/mcp/server_test.go b/ingestion/internal/mcp/server_test.go index 328ada7..4f742f4 100644 --- a/ingestion/internal/mcp/server_test.go +++ b/ingestion/internal/mcp/server_test.go @@ -58,7 +58,7 @@ func TestServerToolsList(t *testing.T) { } assert.ElementsMatch(t, []string{ "brain_query", "brain_write", "brain_update", "brain_get", - "brain_pending", "brain_promote", + "brain_pending", "brain_promote", "brain_reject", "brain_read", "brain_edit", "brain_tunnel", "brain_ingest_raw", "brain_ingest", -- 2.54.0 From 69731d92f47f7e5466f2fea50ee0edee9e2baf3e Mon Sep 17 00:00:00 2001 From: Mathias Date: Wed, 23 Sep 2026 08:04:02 +0200 Subject: [PATCH 2/2] =?UTF-8?q?fix(watcher):=20never=20walk=20into=20raw/r?= =?UTF-8?q?ejected/=20=E2=80=94=20it=20was=20auto-publishing=20declined=20?= =?UTF-8?q?notes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of PR #133 found a serious bug: the file watcher's processDir walks brain/raw/ with filepath.WalkDir (recurses by default) and only skipped processed/ and failed/ by name. brain_reject moves a declined note to raw/rejected/, which the watcher had no reason to know about -- so on its next tick (30s default), it picked the note back up, ran it through the full LLM extraction pipeline, and published it into the wiki. A human's explicit "not worth promoting" decision was silently reversed by a background job 30 seconds later. Reproduced directly: flip-testing the fix back out shows the watcher's own log line, "file processed file=declined-file.md", proving this wasn't hypothetical. Fixed by adding "rejected" to the same subdirectory skip-list processed/failed already use. Checked the other two consumers of raw/ content for the same class of bug and found neither is affected: graphsync.BackfillFromBrainDir only walks wiki/ and knowledge/; search's resolveRoots only ever returns knowledge/ and wiki/. Neither touches raw/ at all, rejected or not. Co-Authored-By: Claude Sonnet 5 --- ingestion/internal/watcher/watcher.go | 10 ++++++++-- ingestion/internal/watcher/watcher_test.go | 13 ++++++++++--- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/ingestion/internal/watcher/watcher.go b/ingestion/internal/watcher/watcher.go index d92c21f..dea92f9 100644 --- a/ingestion/internal/watcher/watcher.go +++ b/ingestion/internal/watcher/watcher.go @@ -59,10 +59,16 @@ func processDir(ctx context.Context, cfg Config, date string) []error { return nil } - // Skip processed/ and failed/ subdirectories entirely. + // Skip processed/, failed/, and rejected/ subdirectories entirely. + // rejected/ (hyperguild#81, brain_reject) holds notes a human already + // looked at and declined — walking into it would run the extraction + // pipeline over a note and publish it to the wiki, the exact opposite + // of the human's decision. The whole reason RejectNote moves a note + // there instead of leaving it in raw/ was to keep it out of every + // path that treats raw/ as "awaiting processing" — this is that path. if d.IsDir() { name := d.Name() - if name == "processed" || name == "failed" { + if name == "processed" || name == "failed" || name == "rejected" { return filepath.SkipDir } return nil diff --git a/ingestion/internal/watcher/watcher_test.go b/ingestion/internal/watcher/watcher_test.go index fe99be1..7581194 100644 --- a/ingestion/internal/watcher/watcher_test.go +++ b/ingestion/internal/watcher/watcher_test.go @@ -176,15 +176,20 @@ func TestDeriveSource(t *testing.T) { func TestProcessDir_SkipsSubdirs(t *testing.T) { brainDir := setupBrainDir(t) - // Create processed/ and failed/ subdirs with files inside. - for _, sub := range []string{"processed/2026-04-22", "failed"} { + // Create processed/, failed/, and rejected/ subdirs with files inside. + for _, sub := range []string{"processed/2026-04-22", "failed", "rejected"} { require.NoError(t, os.MkdirAll(filepath.Join(brainDir, "raw", sub), 0o755)) } processedFile := filepath.Join(brainDir, "raw", "processed", "2026-04-22", "old-file.md") failedFile := filepath.Join(brainDir, "raw", "failed", "broken-file.md") + // hyperguild#81/#133: a human already declined this note via brain_reject. + // If the watcher walks into rejected/, it re-ingests and publishes it to + // the wiki — the exact opposite of the human's decision. + rejectedFile := filepath.Join(brainDir, "raw", "rejected", "declined-file.md") require.NoError(t, os.WriteFile(processedFile, []byte("old"), 0o644)) require.NoError(t, os.WriteFile(failedFile, []byte("broken"), 0o644)) + require.NoError(t, os.WriteFile(rejectedFile, []byte("---\nrejected_at: 2026-09-23T00:00:00Z\n---\ndeclined"), 0o644)) // Also place a valid file in raw/ root that should be processed. validFile := filepath.Join(brainDir, "raw", "valid.md") @@ -223,11 +228,13 @@ func TestProcessDir_SkipsSubdirs(t *testing.T) { // Complete should have been called exactly once (for valid.md, not for files in subdirs). assert.Len(t, processedSources, 1, "only the file in raw/ root should be processed") - // Files in processed/ and failed/ must remain untouched. + // Files in processed/, failed/, and rejected/ must remain untouched. _, err := os.Stat(processedFile) assert.NoError(t, err, "processed subdir file should be untouched") _, err = os.Stat(failedFile) assert.NoError(t, err, "failed subdir file should be untouched") + _, err = os.Stat(rejectedFile) + assert.NoError(t, err, "rejected subdir file should be untouched, never re-ingested") } // TestProcessDir_SkipsTunnelCandidateFiles guards against hyperguild#88: -- 2.54.0