diff --git a/ingestion/cmd/server/main.go b/ingestion/cmd/server/main.go index d3a3f18..98592a2 100644 --- a/ingestion/cmd/server/main.go +++ b/ingestion/cmd/server/main.go @@ -369,6 +369,8 @@ func main() { mux.HandleFunc("POST /ingest-path", h.IngestPath) mux.HandleFunc("POST /ingest-raw", h.IngestRaw) mux.HandleFunc("POST /backfill-refs", h.BackfillRefs) + mux.HandleFunc("GET /pending", h.Pending) + mux.HandleFunc("POST /promote", h.Promote) mux.HandleFunc("POST /backfill-embeddings", h.BackfillEmbeddings) mux.HandleFunc("GET /pass-rate", h.PassRate) jwtValidator, err := chassisauth.NewJWTValidator(ctx, os.Getenv("DEX_ISSUER_URL"), os.Getenv("MCP_AUDIENCE")) diff --git a/ingestion/internal/api/handler.go b/ingestion/internal/api/handler.go index b5d1554..04dae27 100644 --- a/ingestion/internal/api/handler.go +++ b/ingestion/internal/api/handler.go @@ -483,6 +483,40 @@ func (h *Handler) BackfillRefs(w http.ResponseWriter, r *http.Request) { writeJSON(w, map[string]int{"updated": n}) } +// Pending handles GET /pending — list raw/ notes awaiting promotion. +func (h *Handler) Pending(w http.ResponseWriter, _ *http.Request) { + pending, err := ListPending(h.brainDir) + if err != nil { + h.logger.Error("pending failed", "err", err) + writeError(w, http.StatusInternalServerError, "pending error") + return + } + writeJSON(w, map[string]any{"pending": pending}) +} + +type promoteRequest struct { + Filename string `json:"filename"` + Wing string `json:"wing"` + Hall string `json:"hall"` + Slug string `json:"slug,omitempty"` +} + +// Promote handles POST /promote — move a raw/ note into the wiki. A bad +// hall / collision / missing source is a 400 (caller error), not a 500. +func (h *Handler) Promote(w http.ResponseWriter, r *http.Request) { + var req promoteRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "invalid JSON") + return + } + rel, err := PromoteNote(h.brainDir, PromoteOptions(req)) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + writeJSON(w, map[string]string{"path": rel}) +} + func writeJSON(w http.ResponseWriter, v any) { w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(v) //nolint:errcheck diff --git a/ingestion/internal/api/promote.go b/ingestion/internal/api/promote.go new file mode 100644 index 0000000..d34768c --- /dev/null +++ b/ingestion/internal/api/promote.go @@ -0,0 +1,156 @@ +package api + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "time" + + "github.com/mathiasbq/hyperguild/ingestion/internal/brain" +) + +// PendingNote describes a raw/ note awaiting human promotion to the wiki. +type PendingNote struct { + Filename string `json:"filename"` + CreatedAt string `json:"created_at"` + SizeBytes int64 `json:"size_bytes"` + Excerpt string `json:"excerpt"` +} + +// datePrefix matches a leading YYYY-MM-DD- on a raw filename, stripped when +// deriving the default promoted slug. +var datePrefix = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}-`) + +// ListPending returns the notes in brain/raw/ awaiting review, oldest-first +// (natural review order). An absent raw/ dir yields an empty slice, not an +// error. Only .md files are listed; tunnel-candidate files are skipped. +func ListPending(brainDir string) ([]PendingNote, error) { + dir := filepath.Join(brainDir, "raw") + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return []PendingNote{}, nil + } + return nil, fmt.Errorf("read raw dir: %w", err) + } + + out := make([]PendingNote, 0, len(entries)) + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".md") || strings.HasPrefix(e.Name(), "tunnel-candidates-") { + continue + } + info, statErr := e.Info() + if statErr != nil { + continue + } + raw, readErr := os.ReadFile(filepath.Join(dir, e.Name())) + if readErr != nil { + continue + } + fm, body := parseFrontmatter(string(raw)) + created := fm.get("created_at") + if created == "" { + created = info.ModTime().UTC().Format(time.RFC3339) + } + out = append(out, PendingNote{ + Filename: e.Name(), + CreatedAt: created, + SizeBytes: info.Size(), + Excerpt: excerpt(body, 200), + }) + } + sort.SliceStable(out, func(i, j int) bool { return out[i].CreatedAt < out[j].CreatedAt }) + return out, nil +} + +// PromoteOptions identifies a raw note to promote and its wiki destination. +type PromoteOptions struct { + Filename string // basename in brain/raw/ + Wing string + Hall string + Slug string // optional; defaults to Filename minus date prefix + .md +} + +// PromoteNote moves a note from brain/raw/ into the structured wiki: it +// rewrites frontmatter (sets wing/hall/promoted_at, preserves created_at and +// any custom fields), writes to brain/wiki///.md, deletes +// the source, then rebuilds the wing index and runs auto-tunnel detection. +// +// It is atomic from the caller's view: validation (hall, wing, slug, +// collision) happens before any filesystem change, and the source is deleted +// only after the destination write succeeds (write-then-delete, never move). +// Returns the promoted note's path relative to brainDir. +func PromoteNote(brainDir string, opts PromoteOptions) (string, error) { + // Validate filename (basename only — no traversal) before touching fs. + base := filepath.Base(opts.Filename) + if base != opts.Filename || base == "." || base == ".." || strings.ContainsAny(opts.Filename, `/\`) { + return "", fmt.Errorf("invalid filename %q", opts.Filename) + } + + slug := opts.Slug + if slug == "" { + slug = datePrefix.ReplaceAllString(strings.TrimSuffix(base, ".md"), "") + } + // NotePath validates hall + wing + slug; do this before reading anything. + dest, err := brain.NotePath(brainDir, opts.Wing, opts.Hall, slug) + if err != nil { + return "", err + } + + src := filepath.Join(brainDir, "raw", base) + raw, err := os.ReadFile(src) + if err != nil { + if os.IsNotExist(err) { + return "", fmt.Errorf("pending note %q does not exist in raw/", base) + } + return "", fmt.Errorf("read source: %w", err) + } + + // Collision: never silently overwrite an existing promoted note. + if _, statErr := os.Stat(dest); statErr == nil { + rel, _ := filepath.Rel(brainDir, dest) + return "", fmt.Errorf("target %s already exists; choose a different slug", filepath.ToSlash(rel)) + } + + fm, body := parseFrontmatter(string(raw)) + now := time.Now().UTC().Format(time.RFC3339) + fm.set("wing", brain.Sanitise(opts.Wing)) + fm.set("hall", opts.Hall) + if fm.get("created_at") == "" { + fm.set("created_at", now) + } + fm.set("promoted_at", now) + + if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { + return "", fmt.Errorf("create wing dir: %w", err) + } + // Write-then-delete: the source survives any write failure. + if err := os.WriteFile(dest, []byte(fm.render()+body), 0o644); err != nil { + return "", fmt.Errorf("write promoted note: %w", err) + } + if err := os.Remove(src); err != nil { + return "", fmt.Errorf("promoted note written but source removal failed: %w", err) + } + + rel, _ := filepath.Rel(brainDir, dest) + relSlash := filepath.ToSlash(rel) + + // Best-effort wiki upkeep — the note is already promoted. + _ = brain.BuildWingIndex(brainDir, opts.Wing) + _ = brain.AutoTunnel(brainDir, relSlash, body) + + return relSlash, nil +} + +// excerpt returns the first n runes of s, trimmed, single-spaced. +func excerpt(s string, n int) string { + s = strings.TrimSpace(s) + r := []rune(s) + if len(r) > n { + r = r[:n] + } + return strings.TrimSpace(string(r)) +} diff --git a/ingestion/internal/api/promote_test.go b/ingestion/internal/api/promote_test.go new file mode 100644 index 0000000..ff4e428 --- /dev/null +++ b/ingestion/internal/api/promote_test.go @@ -0,0 +1,121 @@ +package api + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func writeRaw(t *testing.T, brainDir, name, content string) { + t.Helper() + dir := filepath.Join(brainDir, "raw") + require.NoError(t, os.MkdirAll(dir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644)) +} + +func TestListPendingEmptyWhenAbsent(t *testing.T) { + got, err := ListPending(t.TempDir()) + require.NoError(t, err, "absent raw/ is not an error") + assert.Empty(t, got) +} + +func TestListPendingReturnsOldestFirstWithExcerpt(t *testing.T) { + dir := t.TempDir() + writeRaw(t, dir, "2026-06-02-newer.md", "---\ncreated_at: 2026-06-02T00:00:00Z\n---\nNewer body here.\n") + writeRaw(t, dir, "2026-06-01-older.md", "---\ncreated_at: 2026-06-01T00:00:00Z\n---\nOlder body content.\n") + // non-md ignored + writeRaw(t, dir, "notes.txt", "ignore me") + + got, err := ListPending(dir) + require.NoError(t, err) + require.Len(t, got, 2) + assert.Equal(t, "2026-06-01-older.md", got[0].Filename, "oldest first") + assert.Equal(t, "2026-06-02-newer.md", got[1].Filename) + assert.Contains(t, got[0].Excerpt, "Older body content") + assert.NotContains(t, got[0].Excerpt, "---", "excerpt is body, not frontmatter") + assert.Positive(t, got[0].SizeBytes) +} + +func TestPromoteHappyPath(t *testing.T) { + dir := t.TempDir() + writeRaw(t, dir, "2026-06-01-lejpa-decision.md", + "---\ncreated_at: 2026-06-01T09:00:00Z\ncustom_field: keep-me\n---\n# LeJEPA\n\nbody.\n") + + rel, err := PromoteNote(dir, PromoteOptions{ + Filename: "2026-06-01-lejpa-decision.md", Wing: "jepa-fx", Hall: "decisions", + }) + require.NoError(t, err) + assert.Equal(t, "wiki/jepa-fx/decisions/lejpa-decision.md", rel, "slug defaults to filename minus date prefix") + + // Source deleted. + _, statErr := os.Stat(filepath.Join(dir, "raw", "2026-06-01-lejpa-decision.md")) + assert.True(t, os.IsNotExist(statErr), "source removed after promote") + + got, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(rel))) + require.NoError(t, err) + s := string(got) + assert.Contains(t, s, "wing: jepa-fx") + assert.Contains(t, s, "hall: decisions") + assert.Contains(t, s, "created_at: 2026-06-01T09:00:00Z", "original created_at preserved") + assert.Contains(t, s, "promoted_at:") + assert.Contains(t, s, "custom_field: keep-me", "custom frontmatter preserved") + assert.Contains(t, s, "# LeJEPA") +} + +func TestPromoteExplicitSlug(t *testing.T) { + dir := t.TempDir() + writeRaw(t, dir, "2026-06-01-x.md", "body\n") + rel, err := PromoteNote(dir, PromoteOptions{Filename: "2026-06-01-x.md", Wing: "a", Hall: "facts", Slug: "custom-slug"}) + require.NoError(t, err) + assert.Equal(t, "wiki/a/facts/custom-slug.md", rel) +} + +func TestPromoteInvalidHallErrorsBeforeTouchingFS(t *testing.T) { + dir := t.TempDir() + writeRaw(t, dir, "2026-06-01-x.md", "body\n") + _, err := PromoteNote(dir, PromoteOptions{Filename: "2026-06-01-x.md", Wing: "a", Hall: "garbage"}) + require.Error(t, err) + // Source untouched. + _, statErr := os.Stat(filepath.Join(dir, "raw", "2026-06-01-x.md")) + assert.NoError(t, statErr, "invalid hall must not delete or move the source") +} + +func TestPromoteMissingSourceErrors(t *testing.T) { + _, err := PromoteNote(t.TempDir(), PromoteOptions{Filename: "ghost.md", Wing: "a", Hall: "facts"}) + require.Error(t, err) +} + +func TestPromoteSlugCollisionNoOverwrite(t *testing.T) { + dir := t.TempDir() + // Pre-existing target. + dest := filepath.Join(dir, "wiki", "a", "facts", "x.md") + require.NoError(t, os.MkdirAll(filepath.Dir(dest), 0o755)) + require.NoError(t, os.WriteFile(dest, []byte("EXISTING\n"), 0o644)) + writeRaw(t, dir, "2026-06-01-x.md", "NEW\n") + + _, err := PromoteNote(dir, PromoteOptions{Filename: "2026-06-01-x.md", Wing: "a", Hall: "facts"}) + require.Error(t, err, "collision must error, not overwrite") + + got, _ := os.ReadFile(dest) + assert.Equal(t, "EXISTING\n", string(got), "target not overwritten") + _, statErr := os.Stat(filepath.Join(dir, "raw", "2026-06-01-x.md")) + assert.NoError(t, statErr, "source preserved on collision (atomic: no delete without write)") +} + +func TestPromoteRejectsTraversalFilename(t *testing.T) { + _, err := PromoteNote(t.TempDir(), PromoteOptions{Filename: "../escape.md", Wing: "a", Hall: "facts"}) + require.Error(t, err) +} + +func TestPromoteRebuildsWingIndex(t *testing.T) { + dir := t.TempDir() + writeRaw(t, dir, "2026-06-01-x.md", "---\ntitle: X Note\n---\nbody\n") + _, err := PromoteNote(dir, PromoteOptions{Filename: "2026-06-01-x.md", Wing: "a", Hall: "facts"}) + require.NoError(t, err) + idx, err := os.ReadFile(filepath.Join(dir, "wiki", "a", "_index.md")) + require.NoError(t, err, "wing _index regenerated") + assert.Contains(t, string(idx), "x", "promoted note appears in the index") +} diff --git a/ingestion/internal/mcp/handlers.go b/ingestion/internal/mcp/handlers.go index 8a750a1..ecf33e4 100644 --- a/ingestion/internal/mcp/handlers.go +++ b/ingestion/internal/mcp/handlers.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "github.com/mathiasbq/hyperguild/ingestion/internal/api" "github.com/mathiasbq/hyperguild/ingestion/internal/brain" "github.com/mathiasbq/hyperguild/ingestion/internal/capture" "github.com/mathiasbq/hyperguild/ingestion/internal/extract" @@ -81,6 +82,21 @@ func (s *Server) tools() []map[string]any { "path": str("brain-relative path to the note; equivalent to id"), }), }, + { + "name": "brain_pending", + "description": "List notes in brain/raw/ awaiting human promotion to the wiki, oldest-first. Returns filename, created_at, size_bytes, excerpt. The human-review queue complement to brain_promote.", + "inputSchema": schema([]string{}, map[string]any{}), + }, + { + "name": "brain_promote", + "description": "Promote a brain/raw/ note into brain/wiki///: rewrites frontmatter (sets wing/hall/promoted_at, preserves created_at + custom fields), deletes the source, rebuilds the wing index, runs auto-tunnel. Errors (without touching the fs) on invalid hall or a slug collision. Returns {path}.", + "inputSchema": schema([]string{"filename", "wing", "hall"}, map[string]any{ + "filename": str("basename in brain/raw/, e.g. 2026-06-01-lejpa-decision.md"), + "wing": str("target wing, e.g. jepa-fx"), + "hall": enum("target hall", halls...), + "slug": str("optional target slug; defaults to filename minus date prefix"), + }), + }, { "name": "brain_tunnel", "description": "Create an explicit bidirectional [[wikilink]] between two notes in different wings. Idempotent.", @@ -321,6 +337,40 @@ func (s *Server) brainGet(ctx context.Context, args json.RawMessage) (json.RawMe }) } +// brainPending lists the raw/ review queue (oldest-first). +func (s *Server) brainPending(_ context.Context, _ json.RawMessage) (json.RawMessage, error) { + pending, err := api.ListPending(s.brainDir) + if err != nil { + return nil, err + } + return json.Marshal(map[string]any{"pending": pending}) +} + +type brainPromoteArgs struct { + Filename string `json:"filename"` + Wing string `json:"wing"` + Hall string `json:"hall"` + Slug string `json:"slug,omitempty"` +} + +// brainPromote moves a raw/ note into the structured wiki (frontmatter +// rewrite + index + auto-tunnel, all owned by api.PromoteNote) and then +// re-indexes it into the graph. The human-facing complement to brain_write. +func (s *Server) brainPromote(ctx context.Context, args json.RawMessage) (json.RawMessage, error) { + var a brainPromoteArgs + if err := json.Unmarshal(args, &a); err != nil { + return nil, fmt.Errorf("parse args: %w", err) + } + relPath, err := api.PromoteNote(s.brainDir, api.PromoteOptions{ + Filename: a.Filename, Wing: a.Wing, Hall: a.Hall, Slug: a.Slug, + }) + if err != nil { + return nil, err + } + s.indexInGraph(ctx, "brain_promote", relPath) + return json.Marshal(map[string]string{"path": relPath}) +} + // indexInGraph is a best-effort wrapper around graphsync.IndexDoc that // logs failures but never propagates them — the underlying write/ingest // has already succeeded and the graph is an augmentation, not a diff --git a/ingestion/internal/mcp/handlers_test.go b/ingestion/internal/mcp/handlers_test.go index 9da1fca..7333a47 100644 --- a/ingestion/internal/mcp/handlers_test.go +++ b/ingestion/internal/mcp/handlers_test.go @@ -332,3 +332,61 @@ func TestSessionLogRequiresSessionID(t *testing.T) { resp := toolCall(t, srv, "session_log", map[string]any{"skill": "tdd"}) require.NotNil(t, resp["error"]) } + +func TestBrainPendingListsRaw(t *testing.T) { + brainDir := t.TempDir() + raw := filepath.Join(brainDir, "raw") + require.NoError(t, os.MkdirAll(raw, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(raw, "2026-06-01-x.md"), + []byte("---\ncreated_at: 2026-06-01T00:00:00Z\n---\npending body\n"), 0o644)) + srv := mcp.NewServer(brainDir, nil, nil, nil) + + resp := toolCall(t, srv, "brain_pending", map[string]any{}) + require.Nil(t, resp["error"]) + text := resp["result"].(map[string]any)["content"].([]any)[0].(map[string]any)["text"].(string) + assert.Contains(t, text, "2026-06-01-x.md") + assert.Contains(t, text, "pending body") +} + +func TestBrainPendingEmpty(t *testing.T) { + srv := mcp.NewServer(t.TempDir(), nil, nil, nil) + resp := toolCall(t, srv, "brain_pending", map[string]any{}) + require.Nil(t, resp["error"]) + text := resp["result"].(map[string]any)["content"].([]any)[0].(map[string]any)["text"].(string) + assert.Contains(t, text, `"pending":[]`) +} + +func TestBrainPromoteMovesToWiki(t *testing.T) { + brainDir := t.TempDir() + raw := filepath.Join(brainDir, "raw") + require.NoError(t, os.MkdirAll(raw, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(raw, "2026-06-01-decision.md"), + []byte("---\ncreated_at: 2026-06-01T00:00:00Z\n---\n# D\n\nbody\n"), 0o644)) + srv := mcp.NewServer(brainDir, nil, nil, nil) + + resp := toolCall(t, srv, "brain_promote", map[string]any{ + "filename": "2026-06-01-decision.md", "wing": "jepa-fx", "hall": "decisions", + }) + require.Nil(t, resp["error"], "got: %v", resp["error"]) + text := resp["result"].(map[string]any)["content"].([]any)[0].(map[string]any)["text"].(string) + assert.Contains(t, text, "wiki/jepa-fx/decisions/decision.md") + + _, err := os.Stat(filepath.Join(brainDir, "wiki/jepa-fx/decisions/decision.md")) + require.NoError(t, err) + _, srcErr := os.Stat(filepath.Join(raw, "2026-06-01-decision.md")) + assert.True(t, os.IsNotExist(srcErr), "source removed") +} + +func TestBrainPromoteInvalidHallErrors(t *testing.T) { + brainDir := t.TempDir() + raw := filepath.Join(brainDir, "raw") + require.NoError(t, os.MkdirAll(raw, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(raw, "x.md"), []byte("body\n"), 0o644)) + srv := mcp.NewServer(brainDir, nil, nil, nil) + resp := toolCall(t, srv, "brain_promote", map[string]any{ + "filename": "x.md", "wing": "a", "hall": "garbage", + }) + require.NotNil(t, resp["error"]) + _, srcErr := os.Stat(filepath.Join(raw, "x.md")) + assert.NoError(t, srcErr, "source untouched on validation error") +} diff --git a/ingestion/internal/mcp/server.go b/ingestion/internal/mcp/server.go index aa33734..fce7876 100644 --- a/ingestion/internal/mcp/server.go +++ b/ingestion/internal/mcp/server.go @@ -1,8 +1,9 @@ // Package mcp implements an MCP HTTP handler for the ingestion service. // Exposed tools: brain_query, brain_write, brain_update, brain_get, -// brain_index, brain_tunnel, brain_ingest, brain_ingest_raw, -// brain_answer, brain_classify, brain_graph, brain_context, session_log, -// and capture (the #55 relay tool, registered only when WithCapture is set). +// brain_pending, brain_promote, brain_index, brain_tunnel, brain_ingest, +// brain_ingest_raw, brain_answer, brain_classify, brain_graph, +// brain_context, session_log, and capture (the #55 relay tool, registered +// only when WithCapture is set). package mcp import ( @@ -263,6 +264,10 @@ func (s *Server) handleCall(ctx context.Context, name string, args json.RawMessa return s.brainUpdate(ctx, args) case "brain_get": return s.brainGet(ctx, args) + case "brain_pending": + return s.brainPending(ctx, args) + case "brain_promote": + return s.brainPromote(ctx, args) case "capture": return s.brainCapture(ctx, args) case "brain_index": diff --git a/ingestion/internal/mcp/server_test.go b/ingestion/internal/mcp/server_test.go index 06f4bfb..da0aa15 100644 --- a/ingestion/internal/mcp/server_test.go +++ b/ingestion/internal/mcp/server_test.go @@ -58,6 +58,7 @@ func TestServerToolsList(t *testing.T) { } assert.ElementsMatch(t, []string{ "brain_query", "brain_write", "brain_update", "brain_get", + "brain_pending", "brain_promote", "brain_index", "brain_tunnel", "brain_ingest_raw", "brain_ingest", "brain_answer", "brain_classify", "brain_graph", "brain_context",