// ingestion/internal/api/handler.go package api import ( "encoding/json" "fmt" "log/slog" "net/http" "os" "path/filepath" "strings" "time" "github.com/mathiasbq/hyperguild/ingestion/internal/brain" "github.com/mathiasbq/hyperguild/ingestion/internal/brainfs" "github.com/mathiasbq/hyperguild/ingestion/internal/claim" "github.com/mathiasbq/hyperguild/ingestion/internal/pipeline" "github.com/mathiasbq/hyperguild/ingestion/internal/search" "github.com/mathiasbq/hyperguild/ingestion/internal/vectorstore" ) // Handler serves the ingestion HTTP API. type Handler struct { brainDir string logger *slog.Logger pipeline pipeline.Config embedStore vectorstore.Store embedClient vectorstore.Embedder } // NewHandler constructs a Handler. brainDir is the absolute path to brain/. func NewHandler(brainDir string, logger *slog.Logger, pipelineCfg pipeline.Config) *Handler { if logger == nil { logger = slog.Default() } return &Handler{brainDir: brainDir, logger: logger, pipeline: pipelineCfg} } // WithEmbedSync wires the optional vector store + embedder used by the // POST /backfill-embeddings endpoint. Calling with either nil is a no-op. func (h *Handler) WithEmbedSync(store vectorstore.Store, embedder vectorstore.Embedder) *Handler { h.embedStore = store h.embedClient = embedder return h } type queryRequest struct { Query string `json:"query"` Limit int `json:"limit,omitempty"` Wing string `json:"wing,omitempty"` Hall string `json:"hall,omitempty"` } type writeRequest struct { Content string `json:"content"` Filename string `json:"filename,omitempty"` Wing string `json:"wing,omitempty"` Hall string `json:"hall,omitempty"` SourceType string `json:"source_type,omitempty"` // "external" opts a hall=facts entry out of the internal default } type ingestRequest struct { Content string `json:"content"` Source string `json:"source"` DryRun bool `json:"dry_run"` } type ingestPathRequest struct { Path string `json:"path"` Source string `json:"source"` DryRun bool `json:"dry_run"` } type ingestResponse struct { Pages []string `json:"pages"` Warnings []string `json:"warnings"` } // Query handles POST /query — full-text search across the brain wiki. func (h *Handler) Query(w http.ResponseWriter, r *http.Request) { var req queryRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { writeError(w, http.StatusBadRequest, "invalid JSON") return } if strings.TrimSpace(req.Query) == "" { writeError(w, http.StatusBadRequest, "query is required") return } if req.Limit == 0 { req.Limit = 5 } results, err := search.Query(h.brainDir, search.QueryOptions{ Query: req.Query, Limit: req.Limit, Wing: req.Wing, Hall: req.Hall, }) if err != nil { h.logger.Error("query failed", "err", err) writeError(w, http.StatusInternalServerError, "search error") return } writeJSON(w, map[string]any{"results": results}) } // WriteNoteOptions configures how a brain note is written. // // Wing and Hall are both required: the note routes into the structured // wiki at brain/wiki///.md and gets wing/hall/created_at // injected into its YAML frontmatter. There is no other valid destination // (the legacy brain/knowledge/ fallback was removed — ADR-0012 / #79). type WriteNoteOptions struct { Content string Filename string Wing string Hall string SourceType string // "internal" marks a first-party observation (e.g. claudewatcher) that needs no external citation // Origin/WrittenBy are server-owned provenance (hyperguild#145): any // origin/written_by in Content's own frontmatter is discarded, so a // caller cannot claim a more trusted origin than it authenticated as. Origin string WrittenBy string } // WriteNote writes a markdown note into the brain. Returns the path // relative to brainDir (forward-slashed). Filename traversal is rejected. func WriteNote(brainDir string, opts WriteNoteOptions) (string, error) { if opts.Content == "" { return "", fmt.Errorf("content is required") } if opts.Wing == "" || opts.Hall == "" { return "", fmt.Errorf("wing and hall are required") } // In the primitive, not in a caller: POST /write reaches here directly, // without the BrainStore, and a gate one door wide is not a gate. // BrainStore still calls the strip itself to report Ref.Unlinked; the // second pass is a no-op on already-clean content. clean, _, err := brain.StripUnresolvedLinks(brainDir, opts.Content) if err != nil { return "", err } opts.Content = clean return writeHallNote(brainDir, opts) } // writeHallNote routes a note into brain/wiki/// and injects // wing/hall/created_at frontmatter. func writeHallNote(brainDir string, opts WriteNoteOptions) (string, error) { slug := opts.Filename if slug == "" { slug = time.Now().UTC().Format("2006-01-02-150405") + "-auto" } dest, err := brain.NotePath(brainDir, opts.Wing, opts.Hall, slug) if err != nil { return "", err } existingFields, body := splitFrontmatter(opts.Content) existingByKey := make(map[string]frontmatterField, len(existingFields)) for _, f := range existingFields { existingByKey[f.key] = f } // Validate the claim-property envelope BEFORE creating the directory, so a // rejected write leaves nothing behind — not even an empty hall dir // (brain#21). Fail closed: a value outside the enum is worse than an absent // one, because a lint skips what it cannot parse and the note then reads as // classified while nothing acts on it. if err := claim.ValidateEnvelope(scalarValues(existingFields)); err != nil { return "", fmt.Errorf("claim envelope: %w", err) } emitted := make(map[string]bool, 6) var fm strings.Builder fm.WriteString("---\n") fmt.Fprintf(&fm, "wing: %s\n", brain.Sanitise(opts.Wing)) fmt.Fprintf(&fm, "hall: %s\n", opts.Hall) fmt.Fprintf(&fm, "created_at: %s\n", time.Now().UTC().Format(time.RFC3339)) emitted["wing"], emitted["hall"], emitted["created_at"] = true, true, true // writeField merges one key: opts.Content's own value (if the note already // carries this field in its own frontmatter) always wins over the fallback, // so promotion/extraction-step metadata survives verbatim instead of being // shadowed by a second, stacked frontmatter block (#86). writeField := func(key, fallback string) { emitted[key] = true if f, ok := existingByKey[key]; ok { for _, line := range f.lines { fm.WriteString(line) fm.WriteString("\n") } return } if fallback != "" { fmt.Fprintf(&fm, "%s: %s\n", key, fallback) } } sourceType := opts.SourceType if sourceType == "" && opts.Hall == "facts" { // Most hall=facts entries are first-party (an eval/benchmark the // writer ran itself), not external claims — default to internal and // require an explicit source_type: external opt-out for the rare // citation-needing entry (brain-gardener#7). sourceType = "internal" } writeField("source_type", sourceType) for _, f := range existingFields { if emitted[f.key] { continue } for _, line := range f.lines { fm.WriteString(line) fm.WriteString("\n") } } fm.WriteString("---\n") final, err := StampProvenance(fm.String()+body, opts.Origin, opts.WrittenBy) if err != nil { return "", err } // The directory is created only once the note is known to be writable: // a refused write leaves nothing behind, not even an empty hall dir // (brain#21; provenance refusal joins the envelope refusal here). // Created through the brain root, never through a symlinked wing or // hall directory (hyperguild#188). rel, err := filepath.Rel(brainDir, dest) if err != nil { return "", fmt.Errorf("note path: %w", err) } rel = filepath.ToSlash(rel) bfs, err := brainfs.Open(brainDir) if err != nil { return "", fmt.Errorf("open brain: %w", err) } defer func() { _ = bfs.Close() }() if err := bfs.PutRegular(rel, []byte(final)); err != nil { return "", fmt.Errorf("write: %w", err) } return rel, nil } // scalarValues flattens frontmatter fields to key -> scalar value for // validation. Multi-line values (lists, nested blocks) yield the text after // the colon on the first line, which is empty for a list — correct here, // since every claim-property field is a scalar enum. func scalarValues(fields []frontmatterField) map[string]string { out := make(map[string]string, len(fields)) for _, f := range fields { if len(f.lines) == 0 { continue } _, value, found := strings.Cut(f.lines[0], ":") if !found { continue } out[f.key] = strings.TrimSpace(value) } return out } // frontmatterField is one top-level YAML key from a frontmatter block, // along with its raw line and any indented continuation lines (e.g. a // bulleted list value spanning multiple lines). type frontmatterField struct { key string lines []string } // splitFrontmatter splits a leading "---\n...\n---\n" YAML block out of // content, returning its top-level fields in original order and the // remaining body. If content has no leading frontmatter block, fields is // nil and body is content unchanged. func splitFrontmatter(content string) (fields []frontmatterField, body string) { if !strings.HasPrefix(content, "---\n") { return nil, content } lines := strings.Split(content, "\n") i := 1 var cur *frontmatterField for ; i < len(lines); i++ { line := lines[i] if strings.TrimSpace(line) == "---" { i++ break } if line != "" && !strings.HasPrefix(line, " ") && !strings.HasPrefix(line, "\t") { if cur != nil { fields = append(fields, *cur) } key, _, _ := strings.Cut(line, ":") cur = &frontmatterField{key: strings.TrimSpace(key), lines: []string{line}} } else if cur != nil { cur.lines = append(cur.lines, line) } } if cur != nil { fields = append(fields, *cur) } body = strings.Join(lines[i:], "\n") return fields, body } // Write handles POST /write — write a note into brain/wiki///. func (h *Handler) Write(w http.ResponseWriter, r *http.Request) { var req writeRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { writeError(w, http.StatusBadRequest, "invalid JSON") return } // Field by field, not a struct conversion: provenance is server-owned and // this door carries no principal, so it stamps origin: unknown // (hyperguild#145). A JSON body must never be able to set it. relPath, err := WriteNote(h.brainDir, WriteNoteOptions{ Content: req.Content, Filename: req.Filename, Wing: req.Wing, Hall: req.Hall, SourceType: req.SourceType, }) if err != nil { h.logger.Error("write failed", "err", err) writeError(w, http.StatusBadRequest, err.Error()) return } // WriteNote errors out above unless both are set, so this always runs — // no legacy fallback path left to guard against. if err := brain.BuildWingIndex(h.brainDir, req.Wing); err != nil { h.logger.Warn("auto-index failed", "wing", req.Wing, "err", err) } writeJSON(w, map[string]string{"path": relPath}) } // BackfillEmbeddings handles POST /backfill-embeddings — synchronously // embeds every note under brain/wiki/ that's not yet in the vector // store, and deletes rows for files no longer on disk. func (h *Handler) BackfillEmbeddings(w http.ResponseWriter, r *http.Request) { if h.embedStore == nil || h.embedClient == nil { writeError(w, http.StatusServiceUnavailable, "embeddings not configured (set BRAIN_PG_DSN and BRAIN_EMBED_URL)") return } res, err := vectorstore.Sync(r.Context(), h.brainDir, h.embedStore, h.embedClient) if err != nil { h.logger.Error("backfill failed", "err", err) writeError(w, http.StatusInternalServerError, "backfill error") return } errStrs := make([]string, 0, len(res.Errors)) for _, e := range res.Errors { errStrs = append(errStrs, e.Error()) } writeJSON(w, map[string]any{ "added": res.Added, "deleted": res.Deleted, "errors": errStrs, }) } type indexRequest struct { Wing string `json:"wing,omitempty"` } // Index handles POST /index — regenerate the _index.md MOC for one wing // (when "wing" is set) or for every wing (when omitted). func (h *Handler) Index(w http.ResponseWriter, r *http.Request) { var req indexRequest if r.ContentLength > 0 { if err := json.NewDecoder(r.Body).Decode(&req); err != nil { writeError(w, http.StatusBadRequest, "invalid JSON") return } } if req.Wing == "" { if err := brain.BuildAllWingIndexes(h.brainDir); err != nil { h.logger.Error("index all failed", "err", err) writeError(w, http.StatusInternalServerError, "index error") return } writeJSON(w, map[string]any{"status": "ok", "scope": "all"}) return } if err := brain.BuildWingIndex(h.brainDir, req.Wing); err != nil { h.logger.Error("index failed", "wing", req.Wing, "err", err) writeError(w, http.StatusBadRequest, err.Error()) return } writeJSON(w, map[string]any{"status": "ok", "scope": req.Wing}) } // Ingest handles POST /ingest — run the pipeline on provided content. func (h *Handler) Ingest(w http.ResponseWriter, r *http.Request) { var req ingestRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { writeError(w, http.StatusBadRequest, "invalid JSON") return } if strings.TrimSpace(req.Content) == "" { writeError(w, http.StatusBadRequest, "content is required") return } if strings.TrimSpace(req.Source) == "" { writeError(w, http.StatusBadRequest, "source is required") return } result, err := pipeline.Run(r.Context(), h.pipeline, h.brainDir, req.Content, req.Source, req.DryRun) if err != nil { h.logger.Error("ingest failed", "source", req.Source, "err", err) writeError(w, http.StatusInternalServerError, "ingest error") return } pages := result.Pages if pages == nil { pages = []string{} } warnings := result.Warnings if warnings == nil { warnings = []string{} } writeJSON(w, ingestResponse{Pages: pages, Warnings: warnings}) } // isSupportedExtension reports whether IngestPath will process ext. // .docx/.xlsx/.pptx/.png/.jpg/.jpeg require docmark (ADR-0013) and are only // supported when DOCMARK_URL is configured — checked per-call (not cached at // package init) so it reflects the environment at request time. func isSupportedExtension(ext string) bool { switch ext { case ".md", ".txt", ".pdf": return true case ".docx", ".xlsx", ".pptx", ".png", ".jpg", ".jpeg": return os.Getenv("DOCMARK_URL") != "" default: return false } } // IngestPath handles POST /ingest-path — ingest a file or directory. func (h *Handler) IngestPath(w http.ResponseWriter, r *http.Request) { var req ingestPathRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { writeError(w, http.StatusBadRequest, "invalid JSON") return } if strings.TrimSpace(req.Path) == "" { writeError(w, http.StatusBadRequest, "path is required") return } resolved, err := ResolveIngestPath(h.brainDir, req.Path) if err != nil { writeError(w, http.StatusBadRequest, err.Error()) return } req.Path = resolved info, err := os.Stat(req.Path) if err != nil { writeError(w, http.StatusBadRequest, fmt.Sprintf("path not accessible: %v", err)) return } var allPages []string var allWarnings []string if info.IsDir() { err = filepath.WalkDir(req.Path, func(path string, d os.DirEntry, walkErr error) error { if walkErr != nil { return walkErr } if d.IsDir() { return nil } ext := strings.ToLower(filepath.Ext(path)) if !isSupportedExtension(ext) { return nil } // Each walked file is re-resolved: a symlinked file inside raw/ // would otherwise lead extract.Text out of the inbox (#149). if _, rerr := ResolveIngestPath(h.brainDir, path); rerr != nil { allWarnings = append(allWarnings, fmt.Sprintf("skipped %s: %v", path, rerr)) return nil } content, readErr := ReadIngestFile(h.brainDir, path) if readErr != nil { allWarnings = append(allWarnings, fmt.Sprintf("extract %s: %v", path, readErr)) return nil } source := req.Source if source == "" { source = filepath.Base(path) } result, runErr := pipeline.Run(r.Context(), h.pipeline, h.brainDir, content, source, req.DryRun) if runErr != nil { allWarnings = append(allWarnings, fmt.Sprintf("ingest %s: %v", path, runErr)) return nil } allPages = append(allPages, result.Pages...) allWarnings = append(allWarnings, result.Warnings...) return nil }) if err != nil { h.logger.Error("walk dir failed", "path", req.Path, "err", err) writeError(w, http.StatusInternalServerError, fmt.Sprintf("walk error: %v", err)) return } } else { ext := strings.ToLower(filepath.Ext(req.Path)) if !isSupportedExtension(ext) { writeError(w, http.StatusBadRequest, fmt.Sprintf("unsupported file extension: %s", ext)) return } content, readErr := ReadIngestFile(h.brainDir, req.Path) if readErr != nil { writeError(w, http.StatusInternalServerError, fmt.Sprintf("extract text: %v", readErr)) return } source := req.Source if source == "" { source = filepath.Base(req.Path) } result, runErr := pipeline.Run(r.Context(), h.pipeline, h.brainDir, content, source, req.DryRun) if runErr != nil { h.logger.Error("ingest-path failed", "path", req.Path, "err", runErr) writeError(w, http.StatusInternalServerError, "ingest error") return } allPages = result.Pages allWarnings = result.Warnings } if allPages == nil { allPages = []string{} } if allWarnings == nil { allWarnings = []string{} } writeJSON(w, ingestResponse{Pages: allPages, Warnings: allWarnings}) } type ingestRawRequest struct { Source string `json:"source"` Pages []pipeline.RawPage `json:"pages"` DryRun bool `json:"dry_run"` } // IngestRaw handles POST /ingest-raw — run the pipeline on pre-parsed RawPages, // skipping the LLM extraction step. Use when the caller has already produced // structured page data (e.g. from a more capable model or manual curation). func (h *Handler) IngestRaw(w http.ResponseWriter, r *http.Request) { var req ingestRawRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { writeError(w, http.StatusBadRequest, "invalid JSON") return } if strings.TrimSpace(req.Source) == "" { writeError(w, http.StatusBadRequest, "source is required") return } if len(req.Pages) == 0 { writeError(w, http.StatusBadRequest, "pages is required and must be non-empty") return } // No principal on this door: pages are stamped origin: unknown (#145). result, err := pipeline.RunRaw(r.Context(), h.brainDir, req.Source, req.Pages, req.DryRun) if err != nil { h.logger.Error("ingest-raw failed", "source", req.Source, "err", err) writeError(w, http.StatusInternalServerError, "ingest error") return } pages := result.Pages if pages == nil { pages = []string{} } warnings := result.Warnings if warnings == nil { warnings = []string{} } writeJSON(w, ingestResponse{Pages: pages, Warnings: warnings}) } // BackfillRefs handles POST /backfill-refs — injects source back-references // into all concept and entity pages based on existing wiki/sources/ pages. func (h *Handler) BackfillRefs(w http.ResponseWriter, r *http.Request) { n, err := pipeline.BackfillRefs(r.Context(), h.brainDir) if err != nil { h.logger.Error("backfill-refs failed", "err", err) writeError(w, http.StatusInternalServerError, "backfill error") return } writeJSON(w, map[string]int{"updated": n}) } // Pending handles GET /pending — list raw/ notes awaiting promotion. func (h *Handler) Pending(w http.ResponseWriter, _ *http.Request) { pending, err := ListPending(h.brainDir) if err != nil { h.logger.Error("pending failed", "err", err) writeError(w, http.StatusInternalServerError, "pending error") return } writeJSON(w, map[string]any{"pending": pending}) } type promoteRequest struct { Filename string `json:"filename"` Wing string `json:"wing"` Hall string `json:"hall"` Slug string `json:"slug,omitempty"` } // Promote handles POST /promote — move a raw/ note into the wiki. A bad // hall / collision / missing source is a 400 (caller error), not a 500. func (h *Handler) Promote(w http.ResponseWriter, r *http.Request) { var req promoteRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { writeError(w, http.StatusBadRequest, "invalid JSON") return } // Field by field: this door carries no principal, so the promoted note is // stamped origin: unknown and the body cannot set provenance (#145). rel, err := PromoteNote(h.brainDir, PromoteOptions{ Filename: req.Filename, Wing: req.Wing, Hall: req.Hall, Slug: req.Slug, }) if err != nil { writeError(w, http.StatusBadRequest, err.Error()) return } writeJSON(w, map[string]string{"path": rel}) } func writeJSON(w http.ResponseWriter, v any) { w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(v) //nolint:errcheck } func writeError(w http.ResponseWriter, code int, msg string) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(code) json.NewEncoder(w).Encode(map[string]string{"error": msg}) //nolint:errcheck }