Merge pull request 'feat: brain_pending + brain_promote — close the raw→wiki curation loop (#38)' (#70) from feat/brain-pending-promote into main

ingestion/cmd/server/main.go

@@ -369,6 +369,8 @@ func main() {
 	mux.HandleFunc("POST /ingest-path", h.IngestPath)
 	mux.HandleFunc("POST /ingest-raw", h.IngestRaw)
 	mux.HandleFunc("POST /backfill-refs", h.BackfillRefs)
+	mux.HandleFunc("GET /pending", h.Pending)
+	mux.HandleFunc("POST /promote", h.Promote)
 	mux.HandleFunc("POST /backfill-embeddings", h.BackfillEmbeddings)
 	mux.HandleFunc("GET /pass-rate", h.PassRate)
 	jwtValidator, err := chassisauth.NewJWTValidator(ctx, os.Getenv("DEX_ISSUER_URL"), os.Getenv("MCP_AUDIENCE"))

ingestion/internal/api/handler.go

@@ -483,6 +483,40 @@ func (h *Handler) BackfillRefs(w http.ResponseWriter, r *http.Request) {
 	writeJSON(w, map[string]int{"updated": n})
 }
 
+// Pending handles GET /pending — list raw/ notes awaiting promotion.
+func (h *Handler) Pending(w http.ResponseWriter, _ *http.Request) {
+	pending, err := ListPending(h.brainDir)
+	if err != nil {
+		h.logger.Error("pending failed", "err", err)
+		writeError(w, http.StatusInternalServerError, "pending error")
+		return
+	}
+	writeJSON(w, map[string]any{"pending": pending})
+}
+
+type promoteRequest struct {
+	Filename string `json:"filename"`
+	Wing     string `json:"wing"`
+	Hall     string `json:"hall"`
+	Slug     string `json:"slug,omitempty"`
+}
+
+// Promote handles POST /promote — move a raw/ note into the wiki. A bad
+// hall / collision / missing source is a 400 (caller error), not a 500.
+func (h *Handler) Promote(w http.ResponseWriter, r *http.Request) {
+	var req promoteRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeError(w, http.StatusBadRequest, "invalid JSON")
+		return
+	}
+	rel, err := PromoteNote(h.brainDir, PromoteOptions(req))
+	if err != nil {
+		writeError(w, http.StatusBadRequest, err.Error())
+		return
+	}
+	writeJSON(w, map[string]string{"path": rel})
+}
+
 func writeJSON(w http.ResponseWriter, v any) {
 	w.Header().Set("Content-Type", "application/json")
 	json.NewEncoder(w).Encode(v) //nolint:errcheck

ingestion/internal/api/promote.go

@@ -0,0 +1,156 @@
+package api
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/mathiasbq/hyperguild/ingestion/internal/brain"
+)
+
+// PendingNote describes a raw/ note awaiting human promotion to the wiki.
+type PendingNote struct {
+	Filename  string `json:"filename"`
+	CreatedAt string `json:"created_at"`
+	SizeBytes int64  `json:"size_bytes"`
+	Excerpt   string `json:"excerpt"`
+}
+
+// datePrefix matches a leading YYYY-MM-DD- on a raw filename, stripped when
+// deriving the default promoted slug.
+var datePrefix = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}-`)
+
+// ListPending returns the notes in brain/raw/ awaiting review, oldest-first
+// (natural review order). An absent raw/ dir yields an empty slice, not an
+// error. Only .md files are listed; tunnel-candidate files are skipped.
+func ListPending(brainDir string) ([]PendingNote, error) {
+	dir := filepath.Join(brainDir, "raw")
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return []PendingNote{}, nil
+		}
+		return nil, fmt.Errorf("read raw dir: %w", err)
+	}
+
+	out := make([]PendingNote, 0, len(entries))
+	for _, e := range entries {
+		if e.IsDir() || !strings.HasSuffix(e.Name(), ".md") || strings.HasPrefix(e.Name(), "tunnel-candidates-") {
+			continue
+		}
+		info, statErr := e.Info()
+		if statErr != nil {
+			continue
+		}
+		raw, readErr := os.ReadFile(filepath.Join(dir, e.Name()))
+		if readErr != nil {
+			continue
+		}
+		fm, body := parseFrontmatter(string(raw))
+		created := fm.get("created_at")
+		if created == "" {
+			created = info.ModTime().UTC().Format(time.RFC3339)
+		}
+		out = append(out, PendingNote{
+			Filename:  e.Name(),
+			CreatedAt: created,
+			SizeBytes: info.Size(),
+			Excerpt:   excerpt(body, 200),
+		})
+	}
+	sort.SliceStable(out, func(i, j int) bool { return out[i].CreatedAt < out[j].CreatedAt })
+	return out, nil
+}
+
+// PromoteOptions identifies a raw note to promote and its wiki destination.
+type PromoteOptions struct {
+	Filename string // basename in brain/raw/
+	Wing     string
+	Hall     string
+	Slug     string // optional; defaults to Filename minus date prefix + .md
+}
+
+// PromoteNote moves a note from brain/raw/ into the structured wiki: it
+// rewrites frontmatter (sets wing/hall/promoted_at, preserves created_at and
+// any custom fields), writes to brain/wiki/<wing>/<hall>/<slug>.md, deletes
+// the source, then rebuilds the wing index and runs auto-tunnel detection.
+//
+// It is atomic from the caller's view: validation (hall, wing, slug,
+// collision) happens before any filesystem change, and the source is deleted
+// only after the destination write succeeds (write-then-delete, never move).
+// Returns the promoted note's path relative to brainDir.
+func PromoteNote(brainDir string, opts PromoteOptions) (string, error) {
+	// Validate filename (basename only — no traversal) before touching fs.
+	base := filepath.Base(opts.Filename)
+	if base != opts.Filename || base == "." || base == ".." || strings.ContainsAny(opts.Filename, `/\`) {
+		return "", fmt.Errorf("invalid filename %q", opts.Filename)
+	}
+
+	slug := opts.Slug
+	if slug == "" {
+		slug = datePrefix.ReplaceAllString(strings.TrimSuffix(base, ".md"), "")
+	}
+	// NotePath validates hall + wing + slug; do this before reading anything.
+	dest, err := brain.NotePath(brainDir, opts.Wing, opts.Hall, slug)
+	if err != nil {
+		return "", err
+	}
+
+	src := filepath.Join(brainDir, "raw", base)
+	raw, err := os.ReadFile(src)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return "", fmt.Errorf("pending note %q does not exist in raw/", base)
+		}
+		return "", fmt.Errorf("read source: %w", err)
+	}
+
+	// Collision: never silently overwrite an existing promoted note.
+	if _, statErr := os.Stat(dest); statErr == nil {
+		rel, _ := filepath.Rel(brainDir, dest)
+		return "", fmt.Errorf("target %s already exists; choose a different slug", filepath.ToSlash(rel))
+	}
+
+	fm, body := parseFrontmatter(string(raw))
+	now := time.Now().UTC().Format(time.RFC3339)
+	fm.set("wing", brain.Sanitise(opts.Wing))
+	fm.set("hall", opts.Hall)
+	if fm.get("created_at") == "" {
+		fm.set("created_at", now)
+	}
+	fm.set("promoted_at", now)
+
+	if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil {
+		return "", fmt.Errorf("create wing dir: %w", err)
+	}
+	// Write-then-delete: the source survives any write failure.
+	if err := os.WriteFile(dest, []byte(fm.render()+body), 0o644); err != nil {
+		return "", fmt.Errorf("write promoted note: %w", err)
+	}
+	if err := os.Remove(src); err != nil {
+		return "", fmt.Errorf("promoted note written but source removal failed: %w", err)
+	}
+
+	rel, _ := filepath.Rel(brainDir, dest)
+	relSlash := filepath.ToSlash(rel)
+
+	// Best-effort wiki upkeep — the note is already promoted.
+	_ = brain.BuildWingIndex(brainDir, opts.Wing)
+	_ = brain.AutoTunnel(brainDir, relSlash, body)
+
+	return relSlash, nil
+}
+
+// excerpt returns the first n runes of s, trimmed, single-spaced.
+func excerpt(s string, n int) string {
+	s = strings.TrimSpace(s)
+	r := []rune(s)
+	if len(r) > n {
+		r = r[:n]
+	}
+	return strings.TrimSpace(string(r))
+}

ingestion/internal/api/promote_test.go

@@ -0,0 +1,121 @@
+package api
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func writeRaw(t *testing.T, brainDir, name, content string) {
+	t.Helper()
+	dir := filepath.Join(brainDir, "raw")
+	require.NoError(t, os.MkdirAll(dir, 0o755))
+	require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644))
+}
+
+func TestListPendingEmptyWhenAbsent(t *testing.T) {
+	got, err := ListPending(t.TempDir())
+	require.NoError(t, err, "absent raw/ is not an error")
+	assert.Empty(t, got)
+}
+
+func TestListPendingReturnsOldestFirstWithExcerpt(t *testing.T) {
+	dir := t.TempDir()
+	writeRaw(t, dir, "2026-06-02-newer.md", "---\ncreated_at: 2026-06-02T00:00:00Z\n---\nNewer body here.\n")
+	writeRaw(t, dir, "2026-06-01-older.md", "---\ncreated_at: 2026-06-01T00:00:00Z\n---\nOlder body content.\n")
+	// non-md ignored
+	writeRaw(t, dir, "notes.txt", "ignore me")
+
+	got, err := ListPending(dir)
+	require.NoError(t, err)
+	require.Len(t, got, 2)
+	assert.Equal(t, "2026-06-01-older.md", got[0].Filename, "oldest first")
+	assert.Equal(t, "2026-06-02-newer.md", got[1].Filename)
+	assert.Contains(t, got[0].Excerpt, "Older body content")
+	assert.NotContains(t, got[0].Excerpt, "---", "excerpt is body, not frontmatter")
+	assert.Positive(t, got[0].SizeBytes)
+}
+
+func TestPromoteHappyPath(t *testing.T) {
+	dir := t.TempDir()
+	writeRaw(t, dir, "2026-06-01-lejpa-decision.md",
+		"---\ncreated_at: 2026-06-01T09:00:00Z\ncustom_field: keep-me\n---\n# LeJEPA\n\nbody.\n")
+
+	rel, err := PromoteNote(dir, PromoteOptions{
+		Filename: "2026-06-01-lejpa-decision.md", Wing: "jepa-fx", Hall: "decisions",
+	})
+	require.NoError(t, err)
+	assert.Equal(t, "wiki/jepa-fx/decisions/lejpa-decision.md", rel, "slug defaults to filename minus date prefix")
+
+	// Source deleted.
+	_, statErr := os.Stat(filepath.Join(dir, "raw", "2026-06-01-lejpa-decision.md"))
+	assert.True(t, os.IsNotExist(statErr), "source removed after promote")
+
+	got, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(rel)))
+	require.NoError(t, err)
+	s := string(got)
+	assert.Contains(t, s, "wing: jepa-fx")
+	assert.Contains(t, s, "hall: decisions")
+	assert.Contains(t, s, "created_at: 2026-06-01T09:00:00Z", "original created_at preserved")
+	assert.Contains(t, s, "promoted_at:")
+	assert.Contains(t, s, "custom_field: keep-me", "custom frontmatter preserved")
+	assert.Contains(t, s, "# LeJEPA")
+}
+
+func TestPromoteExplicitSlug(t *testing.T) {
+	dir := t.TempDir()
+	writeRaw(t, dir, "2026-06-01-x.md", "body\n")
+	rel, err := PromoteNote(dir, PromoteOptions{Filename: "2026-06-01-x.md", Wing: "a", Hall: "facts", Slug: "custom-slug"})
+	require.NoError(t, err)
+	assert.Equal(t, "wiki/a/facts/custom-slug.md", rel)
+}
+
+func TestPromoteInvalidHallErrorsBeforeTouchingFS(t *testing.T) {
+	dir := t.TempDir()
+	writeRaw(t, dir, "2026-06-01-x.md", "body\n")
+	_, err := PromoteNote(dir, PromoteOptions{Filename: "2026-06-01-x.md", Wing: "a", Hall: "garbage"})
+	require.Error(t, err)
+	// Source untouched.
+	_, statErr := os.Stat(filepath.Join(dir, "raw", "2026-06-01-x.md"))
+	assert.NoError(t, statErr, "invalid hall must not delete or move the source")
+}
+
+func TestPromoteMissingSourceErrors(t *testing.T) {
+	_, err := PromoteNote(t.TempDir(), PromoteOptions{Filename: "ghost.md", Wing: "a", Hall: "facts"})
+	require.Error(t, err)
+}
+
+func TestPromoteSlugCollisionNoOverwrite(t *testing.T) {
+	dir := t.TempDir()
+	// Pre-existing target.
+	dest := filepath.Join(dir, "wiki", "a", "facts", "x.md")
+	require.NoError(t, os.MkdirAll(filepath.Dir(dest), 0o755))
+	require.NoError(t, os.WriteFile(dest, []byte("EXISTING\n"), 0o644))
+	writeRaw(t, dir, "2026-06-01-x.md", "NEW\n")
+
+	_, err := PromoteNote(dir, PromoteOptions{Filename: "2026-06-01-x.md", Wing: "a", Hall: "facts"})
+	require.Error(t, err, "collision must error, not overwrite")
+
+	got, _ := os.ReadFile(dest)
+	assert.Equal(t, "EXISTING\n", string(got), "target not overwritten")
+	_, statErr := os.Stat(filepath.Join(dir, "raw", "2026-06-01-x.md"))
+	assert.NoError(t, statErr, "source preserved on collision (atomic: no delete without write)")
+}
+
+func TestPromoteRejectsTraversalFilename(t *testing.T) {
+	_, err := PromoteNote(t.TempDir(), PromoteOptions{Filename: "../escape.md", Wing: "a", Hall: "facts"})
+	require.Error(t, err)
+}
+
+func TestPromoteRebuildsWingIndex(t *testing.T) {
+	dir := t.TempDir()
+	writeRaw(t, dir, "2026-06-01-x.md", "---\ntitle: X Note\n---\nbody\n")
+	_, err := PromoteNote(dir, PromoteOptions{Filename: "2026-06-01-x.md", Wing: "a", Hall: "facts"})
+	require.NoError(t, err)
+	idx, err := os.ReadFile(filepath.Join(dir, "wiki", "a", "_index.md"))
+	require.NoError(t, err, "wing _index regenerated")
+	assert.Contains(t, string(idx), "x", "promoted note appears in the index")
+}

ingestion/internal/mcp/handlers.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/mathiasbq/hyperguild/ingestion/internal/api"
 	"github.com/mathiasbq/hyperguild/ingestion/internal/brain"
 	"github.com/mathiasbq/hyperguild/ingestion/internal/capture"
 	"github.com/mathiasbq/hyperguild/ingestion/internal/extract"
@@ -81,6 +82,21 @@ func (s *Server) tools() []map[string]any {
 				"path": str("brain-relative path to the note; equivalent to id"),
 			}),
 		},
+		{
+			"name":        "brain_pending",
+			"description": "List notes in brain/raw/ awaiting human promotion to the wiki, oldest-first. Returns filename, created_at, size_bytes, excerpt. The human-review queue complement to brain_promote.",
+			"inputSchema": schema([]string{}, map[string]any{}),
+		},
+		{
+			"name":        "brain_promote",
+			"description": "Promote a brain/raw/ note into brain/wiki/<wing>/<hall>/: rewrites frontmatter (sets wing/hall/promoted_at, preserves created_at + custom fields), deletes the source, rebuilds the wing index, runs auto-tunnel. Errors (without touching the fs) on invalid hall or a slug collision. Returns {path}.",
+			"inputSchema": schema([]string{"filename", "wing", "hall"}, map[string]any{
+				"filename": str("basename in brain/raw/, e.g. 2026-06-01-lejpa-decision.md"),
+				"wing":     str("target wing, e.g. jepa-fx"),
+				"hall":     enum("target hall", halls...),
+				"slug":     str("optional target slug; defaults to filename minus date prefix"),
+			}),
+		},
 		{
 			"name":        "brain_tunnel",
 			"description": "Create an explicit bidirectional [[wikilink]] between two notes in different wings. Idempotent.",
@@ -321,6 +337,40 @@ func (s *Server) brainGet(ctx context.Context, args json.RawMessage) (json.RawMe
 	})
 }
 
+// brainPending lists the raw/ review queue (oldest-first).
+func (s *Server) brainPending(_ context.Context, _ json.RawMessage) (json.RawMessage, error) {
+	pending, err := api.ListPending(s.brainDir)
+	if err != nil {
+		return nil, err
+	}
+	return json.Marshal(map[string]any{"pending": pending})
+}
+
+type brainPromoteArgs struct {
+	Filename string `json:"filename"`
+	Wing     string `json:"wing"`
+	Hall     string `json:"hall"`
+	Slug     string `json:"slug,omitempty"`
+}
+
+// brainPromote moves a raw/ note into the structured wiki (frontmatter
+// rewrite + index + auto-tunnel, all owned by api.PromoteNote) and then
+// re-indexes it into the graph. The human-facing complement to brain_write.
+func (s *Server) brainPromote(ctx context.Context, args json.RawMessage) (json.RawMessage, error) {
+	var a brainPromoteArgs
+	if err := json.Unmarshal(args, &a); err != nil {
+		return nil, fmt.Errorf("parse args: %w", err)
+	}
+	relPath, err := api.PromoteNote(s.brainDir, api.PromoteOptions{
+		Filename: a.Filename, Wing: a.Wing, Hall: a.Hall, Slug: a.Slug,
+	})
+	if err != nil {
+		return nil, err
+	}
+	s.indexInGraph(ctx, "brain_promote", relPath)
+	return json.Marshal(map[string]string{"path": relPath})
+}
+
 // indexInGraph is a best-effort wrapper around graphsync.IndexDoc that
 // logs failures but never propagates them — the underlying write/ingest
 // has already succeeded and the graph is an augmentation, not a

ingestion/internal/mcp/handlers_test.go

@@ -332,3 +332,61 @@ func TestSessionLogRequiresSessionID(t *testing.T) {
 	resp := toolCall(t, srv, "session_log", map[string]any{"skill": "tdd"})
 	require.NotNil(t, resp["error"])
 }
+
+func TestBrainPendingListsRaw(t *testing.T) {
+	brainDir := t.TempDir()
+	raw := filepath.Join(brainDir, "raw")
+	require.NoError(t, os.MkdirAll(raw, 0o755))
+	require.NoError(t, os.WriteFile(filepath.Join(raw, "2026-06-01-x.md"),
+		[]byte("---\ncreated_at: 2026-06-01T00:00:00Z\n---\npending body\n"), 0o644))
+	srv := mcp.NewServer(brainDir, nil, nil, nil)
+
+	resp := toolCall(t, srv, "brain_pending", map[string]any{})
+	require.Nil(t, resp["error"])
+	text := resp["result"].(map[string]any)["content"].([]any)[0].(map[string]any)["text"].(string)
+	assert.Contains(t, text, "2026-06-01-x.md")
+	assert.Contains(t, text, "pending body")
+}
+
+func TestBrainPendingEmpty(t *testing.T) {
+	srv := mcp.NewServer(t.TempDir(), nil, nil, nil)
+	resp := toolCall(t, srv, "brain_pending", map[string]any{})
+	require.Nil(t, resp["error"])
+	text := resp["result"].(map[string]any)["content"].([]any)[0].(map[string]any)["text"].(string)
+	assert.Contains(t, text, `"pending":[]`)
+}
+
+func TestBrainPromoteMovesToWiki(t *testing.T) {
+	brainDir := t.TempDir()
+	raw := filepath.Join(brainDir, "raw")
+	require.NoError(t, os.MkdirAll(raw, 0o755))
+	require.NoError(t, os.WriteFile(filepath.Join(raw, "2026-06-01-decision.md"),
+		[]byte("---\ncreated_at: 2026-06-01T00:00:00Z\n---\n# D\n\nbody\n"), 0o644))
+	srv := mcp.NewServer(brainDir, nil, nil, nil)
+
+	resp := toolCall(t, srv, "brain_promote", map[string]any{
+		"filename": "2026-06-01-decision.md", "wing": "jepa-fx", "hall": "decisions",
+	})
+	require.Nil(t, resp["error"], "got: %v", resp["error"])
+	text := resp["result"].(map[string]any)["content"].([]any)[0].(map[string]any)["text"].(string)
+	assert.Contains(t, text, "wiki/jepa-fx/decisions/decision.md")
+
+	_, err := os.Stat(filepath.Join(brainDir, "wiki/jepa-fx/decisions/decision.md"))
+	require.NoError(t, err)
+	_, srcErr := os.Stat(filepath.Join(raw, "2026-06-01-decision.md"))
+	assert.True(t, os.IsNotExist(srcErr), "source removed")
+}
+
+func TestBrainPromoteInvalidHallErrors(t *testing.T) {
+	brainDir := t.TempDir()
+	raw := filepath.Join(brainDir, "raw")
+	require.NoError(t, os.MkdirAll(raw, 0o755))
+	require.NoError(t, os.WriteFile(filepath.Join(raw, "x.md"), []byte("body\n"), 0o644))
+	srv := mcp.NewServer(brainDir, nil, nil, nil)
+	resp := toolCall(t, srv, "brain_promote", map[string]any{
+		"filename": "x.md", "wing": "a", "hall": "garbage",
+	})
+	require.NotNil(t, resp["error"])
+	_, srcErr := os.Stat(filepath.Join(raw, "x.md"))
+	assert.NoError(t, srcErr, "source untouched on validation error")
+}

ingestion/internal/mcp/server.go

@@ -1,8 +1,9 @@
 // Package mcp implements an MCP HTTP handler for the ingestion service.
 // Exposed tools: brain_query, brain_write, brain_update, brain_get,
-// brain_index, brain_tunnel, brain_ingest, brain_ingest_raw,
-// brain_answer, brain_classify, brain_graph, brain_context, session_log,
-// and capture (the #55 relay tool, registered only when WithCapture is set).
+// brain_pending, brain_promote, brain_index, brain_tunnel, brain_ingest,
+// brain_ingest_raw, brain_answer, brain_classify, brain_graph,
+// brain_context, session_log, and capture (the #55 relay tool, registered
+// only when WithCapture is set).
 package mcp
 
 import (
@@ -263,6 +264,10 @@ func (s *Server) handleCall(ctx context.Context, name string, args json.RawMessa
 		return s.brainUpdate(ctx, args)
 	case "brain_get":
 		return s.brainGet(ctx, args)
+	case "brain_pending":
+		return s.brainPending(ctx, args)
+	case "brain_promote":
+		return s.brainPromote(ctx, args)
 	case "capture":
 		return s.brainCapture(ctx, args)
 	case "brain_index":

ingestion/internal/mcp/server_test.go

@@ -58,6 +58,7 @@ func TestServerToolsList(t *testing.T) {
 	}
 	assert.ElementsMatch(t, []string{
 		"brain_query", "brain_write", "brain_update", "brain_get",
+		"brain_pending", "brain_promote",
 		"brain_index", "brain_tunnel",
 		"brain_ingest_raw", "brain_ingest",
 		"brain_answer", "brain_classify", "brain_graph", "brain_context",