hyperguild/ingestion/internal/api/handler.go

// ingestion/internal/api/handler.go
package api

import (
	"encoding/json"
	"fmt"
	"log/slog"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/mathiasbq/hyperguild/ingestion/internal/brain"
	"github.com/mathiasbq/hyperguild/ingestion/internal/claim"
	"github.com/mathiasbq/hyperguild/ingestion/internal/extract"
	"github.com/mathiasbq/hyperguild/ingestion/internal/pipeline"
	"github.com/mathiasbq/hyperguild/ingestion/internal/search"
	"github.com/mathiasbq/hyperguild/ingestion/internal/vectorstore"
)

// Handler serves the ingestion HTTP API.
type Handler struct {
	brainDir    string
	logger      *slog.Logger
	pipeline    pipeline.Config
	embedStore  vectorstore.Store
	embedClient vectorstore.Embedder
}

// NewHandler constructs a Handler. brainDir is the absolute path to brain/.
func NewHandler(brainDir string, logger *slog.Logger, pipelineCfg pipeline.Config) *Handler {
	if logger == nil {
		logger = slog.Default()
	}
	return &Handler{brainDir: brainDir, logger: logger, pipeline: pipelineCfg}
}

// WithEmbedSync wires the optional vector store + embedder used by the
// POST /backfill-embeddings endpoint. Calling with either nil is a no-op.
func (h *Handler) WithEmbedSync(store vectorstore.Store, embedder vectorstore.Embedder) *Handler {
	h.embedStore = store
	h.embedClient = embedder
	return h
}

type queryRequest struct {
	Query string `json:"query"`
	Limit int    `json:"limit,omitempty"`
	Wing  string `json:"wing,omitempty"`
	Hall  string `json:"hall,omitempty"`
}

type writeRequest struct {
	Content    string `json:"content"`
	Filename   string `json:"filename,omitempty"`
	Wing       string `json:"wing,omitempty"`
	Hall       string `json:"hall,omitempty"`
	SourceType string `json:"source_type,omitempty"` // "external" opts a hall=facts entry out of the internal default
}

type ingestRequest struct {
	Content string `json:"content"`
	Source  string `json:"source"`
	DryRun  bool   `json:"dry_run"`
}

type ingestPathRequest struct {
	Path   string `json:"path"`
	Source string `json:"source"`
	DryRun bool   `json:"dry_run"`
}

type ingestResponse struct {
	Pages    []string `json:"pages"`
	Warnings []string `json:"warnings"`
}

// Query handles POST /query — full-text search across the brain wiki.
func (h *Handler) Query(w http.ResponseWriter, r *http.Request) {
	var req queryRequest
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		writeError(w, http.StatusBadRequest, "invalid JSON")
		return
	}
	if strings.TrimSpace(req.Query) == "" {
		writeError(w, http.StatusBadRequest, "query is required")
		return
	}
	if req.Limit == 0 {
		req.Limit = 5
	}

	results, err := search.Query(h.brainDir, search.QueryOptions{
		Query: req.Query,
		Limit: req.Limit,
		Wing:  req.Wing,
		Hall:  req.Hall,
	})
	if err != nil {
		h.logger.Error("query failed", "err", err)
		writeError(w, http.StatusInternalServerError, "search error")
		return
	}

	writeJSON(w, map[string]any{"results": results})
}

// WriteNoteOptions configures how a brain note is written.
//
// Wing and Hall are both required: the note routes into the structured
// wiki at brain/wiki/<wing>/<hall>/<slug>.md and gets wing/hall/created_at
// injected into its YAML frontmatter. There is no other valid destination
// (the legacy brain/knowledge/ fallback was removed — ADR-0012 / #79).
type WriteNoteOptions struct {
	Content    string
	Filename   string
	Wing       string
	Hall       string
	SourceType string // "internal" marks a first-party observation (e.g. claudewatcher) that needs no external citation
	// Origin/WrittenBy are server-owned provenance (hyperguild#145): any
	// origin/written_by in Content's own frontmatter is discarded, so a
	// caller cannot claim a more trusted origin than it authenticated as.
	Origin    string
	WrittenBy string
}

// WriteNote writes a markdown note into the brain. Returns the path
// relative to brainDir (forward-slashed). Filename traversal is rejected.
func WriteNote(brainDir string, opts WriteNoteOptions) (string, error) {
	if opts.Content == "" {
		return "", fmt.Errorf("content is required")
	}
	if opts.Wing == "" || opts.Hall == "" {
		return "", fmt.Errorf("wing and hall are required")
	}

	// In the primitive, not in a caller: POST /write reaches here directly,
	// without the BrainStore, and a gate one door wide is not a gate.
	// BrainStore still calls the strip itself to report Ref.Unlinked; the
	// second pass is a no-op on already-clean content.
	clean, _, err := brain.StripUnresolvedLinks(brainDir, opts.Content)
	if err != nil {
		return "", err
	}
	opts.Content = clean

	return writeHallNote(brainDir, opts)
}

// writeHallNote routes a note into brain/wiki/<wing>/<hall>/ and injects
// wing/hall/created_at frontmatter.
func writeHallNote(brainDir string, opts WriteNoteOptions) (string, error) {
	slug := opts.Filename
	if slug == "" {
		slug = time.Now().UTC().Format("2006-01-02-150405") + "-auto"
	}
	dest, err := brain.NotePath(brainDir, opts.Wing, opts.Hall, slug)
	if err != nil {
		return "", err
	}
	existingFields, body := splitFrontmatter(opts.Content)
	existingByKey := make(map[string]frontmatterField, len(existingFields))
	for _, f := range existingFields {
		existingByKey[f.key] = f
	}

	// Validate the claim-property envelope BEFORE creating the directory, so a
	// rejected write leaves nothing behind — not even an empty hall dir
	// (brain#21). Fail closed: a value outside the enum is worse than an absent
	// one, because a lint skips what it cannot parse and the note then reads as
	// classified while nothing acts on it.
	if err := claim.ValidateEnvelope(scalarValues(existingFields)); err != nil {
		return "", fmt.Errorf("claim envelope: %w", err)
	}

	if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil {
		return "", fmt.Errorf("create hall dir: %w", err)
	}
	emitted := make(map[string]bool, 6)

	var fm strings.Builder
	fm.WriteString("---\n")
	fmt.Fprintf(&fm, "wing: %s\n", brain.Sanitise(opts.Wing))
	fmt.Fprintf(&fm, "hall: %s\n", opts.Hall)
	fmt.Fprintf(&fm, "created_at: %s\n", time.Now().UTC().Format(time.RFC3339))
	emitted["wing"], emitted["hall"], emitted["created_at"] = true, true, true

	// writeField merges one key: opts.Content's own value (if the note already
	// carries this field in its own frontmatter) always wins over the fallback,
	// so promotion/extraction-step metadata survives verbatim instead of being
	// shadowed by a second, stacked frontmatter block (#86).
	writeField := func(key, fallback string) {
		emitted[key] = true
		if f, ok := existingByKey[key]; ok {
			for _, line := range f.lines {
				fm.WriteString(line)
				fm.WriteString("\n")
			}
			return
		}
		if fallback != "" {
			fmt.Fprintf(&fm, "%s: %s\n", key, fallback)
		}
	}
	sourceType := opts.SourceType
	if sourceType == "" && opts.Hall == "facts" {
		// Most hall=facts entries are first-party (an eval/benchmark the
		// writer ran itself), not external claims — default to internal and
		// require an explicit source_type: external opt-out for the rare
		// citation-needing entry (brain-gardener#7).
		sourceType = "internal"
	}
	writeField("source_type", sourceType)

	for _, f := range existingFields {
		if emitted[f.key] {
			continue
		}
		for _, line := range f.lines {
			fm.WriteString(line)
			fm.WriteString("\n")
		}
	}
	fm.WriteString("---\n")

	final, err := StampProvenance(fm.String()+body, opts.Origin, opts.WrittenBy)
	if err != nil {
		return "", err
	}
	if err := os.WriteFile(dest, []byte(final), 0o644); err != nil {
		return "", fmt.Errorf("write: %w", err)
	}
	rel, _ := filepath.Rel(brainDir, dest)
	return filepath.ToSlash(rel), nil
}

// scalarValues flattens frontmatter fields to key -> scalar value for
// validation. Multi-line values (lists, nested blocks) yield the text after
// the colon on the first line, which is empty for a list — correct here,
// since every claim-property field is a scalar enum.
func scalarValues(fields []frontmatterField) map[string]string {
	out := make(map[string]string, len(fields))
	for _, f := range fields {
		if len(f.lines) == 0 {
			continue
		}
		_, value, found := strings.Cut(f.lines[0], ":")
		if !found {
			continue
		}
		out[f.key] = strings.TrimSpace(value)
	}
	return out
}

// frontmatterField is one top-level YAML key from a frontmatter block,
// along with its raw line and any indented continuation lines (e.g. a
// bulleted list value spanning multiple lines).
type frontmatterField struct {
	key   string
	lines []string
}

// splitFrontmatter splits a leading "---\n...\n---\n" YAML block out of
// content, returning its top-level fields in original order and the
// remaining body. If content has no leading frontmatter block, fields is
// nil and body is content unchanged.
func splitFrontmatter(content string) (fields []frontmatterField, body string) {
	if !strings.HasPrefix(content, "---\n") {
		return nil, content
	}

	lines := strings.Split(content, "\n")
	i := 1
	var cur *frontmatterField
	for ; i < len(lines); i++ {
		line := lines[i]
		if strings.TrimSpace(line) == "---" {
			i++
			break
		}
		if line != "" && !strings.HasPrefix(line, " ") && !strings.HasPrefix(line, "\t") {
			if cur != nil {
				fields = append(fields, *cur)
			}
			key, _, _ := strings.Cut(line, ":")
			cur = &frontmatterField{key: strings.TrimSpace(key), lines: []string{line}}
		} else if cur != nil {
			cur.lines = append(cur.lines, line)
		}
	}
	if cur != nil {
		fields = append(fields, *cur)
	}
	body = strings.Join(lines[i:], "\n")
	return fields, body
}

// Write handles POST /write — write a note into brain/wiki/<wing>/<hall>/.
func (h *Handler) Write(w http.ResponseWriter, r *http.Request) {
	var req writeRequest
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		writeError(w, http.StatusBadRequest, "invalid JSON")
		return
	}
	// Field by field, not a struct conversion: provenance is server-owned and
	// this door carries no principal, so it stamps origin: unknown
	// (hyperguild#145). A JSON body must never be able to set it.
	relPath, err := WriteNote(h.brainDir, WriteNoteOptions{
		Content:    req.Content,
		Filename:   req.Filename,
		Wing:       req.Wing,
		Hall:       req.Hall,
		SourceType: req.SourceType,
	})
	if err != nil {
		h.logger.Error("write failed", "err", err)
		writeError(w, http.StatusBadRequest, err.Error())
		return
	}
	// WriteNote errors out above unless both are set, so this always runs —
	// no legacy fallback path left to guard against.
	if err := brain.BuildWingIndex(h.brainDir, req.Wing); err != nil {
		h.logger.Warn("auto-index failed", "wing", req.Wing, "err", err)
	}
	writeJSON(w, map[string]string{"path": relPath})
}

// BackfillEmbeddings handles POST /backfill-embeddings — synchronously
// embeds every note under brain/wiki/ that's not yet in the vector
// store, and deletes rows for files no longer on disk.
func (h *Handler) BackfillEmbeddings(w http.ResponseWriter, r *http.Request) {
	if h.embedStore == nil || h.embedClient == nil {
		writeError(w, http.StatusServiceUnavailable,
			"embeddings not configured (set BRAIN_PG_DSN and BRAIN_EMBED_URL)")
		return
	}
	res, err := vectorstore.Sync(r.Context(), h.brainDir, h.embedStore, h.embedClient)
	if err != nil {
		h.logger.Error("backfill failed", "err", err)
		writeError(w, http.StatusInternalServerError, "backfill error")
		return
	}
	errStrs := make([]string, 0, len(res.Errors))
	for _, e := range res.Errors {
		errStrs = append(errStrs, e.Error())
	}
	writeJSON(w, map[string]any{
		"added":   res.Added,
		"deleted": res.Deleted,
		"errors":  errStrs,
	})
}

type indexRequest struct {
	Wing string `json:"wing,omitempty"`
}

// Index handles POST /index — regenerate the _index.md MOC for one wing
// (when "wing" is set) or for every wing (when omitted).
func (h *Handler) Index(w http.ResponseWriter, r *http.Request) {
	var req indexRequest
	if r.ContentLength > 0 {
		if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
			writeError(w, http.StatusBadRequest, "invalid JSON")
			return
		}
	}
	if req.Wing == "" {
		if err := brain.BuildAllWingIndexes(h.brainDir); err != nil {
			h.logger.Error("index all failed", "err", err)
			writeError(w, http.StatusInternalServerError, "index error")
			return
		}
		writeJSON(w, map[string]any{"status": "ok", "scope": "all"})
		return
	}
	if err := brain.BuildWingIndex(h.brainDir, req.Wing); err != nil {
		h.logger.Error("index failed", "wing", req.Wing, "err", err)
		writeError(w, http.StatusBadRequest, err.Error())
		return
	}
	writeJSON(w, map[string]any{"status": "ok", "scope": req.Wing})
}

// Ingest handles POST /ingest — run the pipeline on provided content.
func (h *Handler) Ingest(w http.ResponseWriter, r *http.Request) {
	var req ingestRequest
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		writeError(w, http.StatusBadRequest, "invalid JSON")
		return
	}
	if strings.TrimSpace(req.Content) == "" {
		writeError(w, http.StatusBadRequest, "content is required")
		return
	}
	if strings.TrimSpace(req.Source) == "" {
		writeError(w, http.StatusBadRequest, "source is required")
		return
	}

	result, err := pipeline.Run(r.Context(), h.pipeline, h.brainDir, req.Content, req.Source, req.DryRun)
	if err != nil {
		h.logger.Error("ingest failed", "source", req.Source, "err", err)
		writeError(w, http.StatusInternalServerError, "ingest error")
		return
	}

	pages := result.Pages
	if pages == nil {
		pages = []string{}
	}
	warnings := result.Warnings
	if warnings == nil {
		warnings = []string{}
	}
	writeJSON(w, ingestResponse{Pages: pages, Warnings: warnings})
}

// isSupportedExtension reports whether IngestPath will process ext.
// .docx/.xlsx/.pptx/.png/.jpg/.jpeg require docmark (ADR-0013) and are only
// supported when DOCMARK_URL is configured — checked per-call (not cached at
// package init) so it reflects the environment at request time.
func isSupportedExtension(ext string) bool {
	switch ext {
	case ".md", ".txt", ".pdf":
		return true
	case ".docx", ".xlsx", ".pptx", ".png", ".jpg", ".jpeg":
		return os.Getenv("DOCMARK_URL") != ""
	default:
		return false
	}
}

// IngestPath handles POST /ingest-path — ingest a file or directory.
func (h *Handler) IngestPath(w http.ResponseWriter, r *http.Request) {
	var req ingestPathRequest
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		writeError(w, http.StatusBadRequest, "invalid JSON")
		return
	}
	if strings.TrimSpace(req.Path) == "" {
		writeError(w, http.StatusBadRequest, "path is required")
		return
	}

	info, err := os.Stat(req.Path)
	if err != nil {
		writeError(w, http.StatusBadRequest, fmt.Sprintf("path not accessible: %v", err))
		return
	}

	var allPages []string
	var allWarnings []string

	if info.IsDir() {
		err = filepath.WalkDir(req.Path, func(path string, d os.DirEntry, walkErr error) error {
			if walkErr != nil {
				return walkErr
			}
			if d.IsDir() {
				return nil
			}
			ext := strings.ToLower(filepath.Ext(path))
			if !isSupportedExtension(ext) {
				return nil
			}
			content, readErr := extract.Text(path)
			if readErr != nil {
				allWarnings = append(allWarnings, fmt.Sprintf("extract %s: %v", path, readErr))
				return nil
			}
			source := req.Source
			if source == "" {
				source = filepath.Base(path)
			}
			result, runErr := pipeline.Run(r.Context(), h.pipeline, h.brainDir, content, source, req.DryRun)
			if runErr != nil {
				allWarnings = append(allWarnings, fmt.Sprintf("ingest %s: %v", path, runErr))
				return nil
			}
			allPages = append(allPages, result.Pages...)
			allWarnings = append(allWarnings, result.Warnings...)
			return nil
		})
		if err != nil {
			h.logger.Error("walk dir failed", "path", req.Path, "err", err)
			writeError(w, http.StatusInternalServerError, fmt.Sprintf("walk error: %v", err))
			return
		}
	} else {
		ext := strings.ToLower(filepath.Ext(req.Path))
		if !isSupportedExtension(ext) {
			writeError(w, http.StatusBadRequest, fmt.Sprintf("unsupported file extension: %s", ext))
			return
		}
		content, readErr := extract.Text(req.Path)
		if readErr != nil {
			writeError(w, http.StatusInternalServerError, fmt.Sprintf("extract text: %v", readErr))
			return
		}
		source := req.Source
		if source == "" {
			source = filepath.Base(req.Path)
		}
		result, runErr := pipeline.Run(r.Context(), h.pipeline, h.brainDir, content, source, req.DryRun)
		if runErr != nil {
			h.logger.Error("ingest-path failed", "path", req.Path, "err", runErr)
			writeError(w, http.StatusInternalServerError, "ingest error")
			return
		}
		allPages = result.Pages
		allWarnings = result.Warnings
	}

	if allPages == nil {
		allPages = []string{}
	}
	if allWarnings == nil {
		allWarnings = []string{}
	}
	writeJSON(w, ingestResponse{Pages: allPages, Warnings: allWarnings})
}

type ingestRawRequest struct {
	Source string             `json:"source"`
	Pages  []pipeline.RawPage `json:"pages"`
	DryRun bool               `json:"dry_run"`
}

// IngestRaw handles POST /ingest-raw — run the pipeline on pre-parsed RawPages,
// skipping the LLM extraction step. Use when the caller has already produced
// structured page data (e.g. from a more capable model or manual curation).
func (h *Handler) IngestRaw(w http.ResponseWriter, r *http.Request) {
	var req ingestRawRequest
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		writeError(w, http.StatusBadRequest, "invalid JSON")
		return
	}
	if strings.TrimSpace(req.Source) == "" {
		writeError(w, http.StatusBadRequest, "source is required")
		return
	}
	if len(req.Pages) == 0 {
		writeError(w, http.StatusBadRequest, "pages is required and must be non-empty")
		return
	}

	// No principal on this door: pages are stamped origin: unknown (#145).
	result, err := pipeline.RunRaw(r.Context(), h.brainDir, req.Source, req.Pages, req.DryRun)
	if err != nil {
		h.logger.Error("ingest-raw failed", "source", req.Source, "err", err)
		writeError(w, http.StatusInternalServerError, "ingest error")
		return
	}

	pages := result.Pages
	if pages == nil {
		pages = []string{}
	}
	warnings := result.Warnings
	if warnings == nil {
		warnings = []string{}
	}
	writeJSON(w, ingestResponse{Pages: pages, Warnings: warnings})
}

// BackfillRefs handles POST /backfill-refs — injects source back-references
// into all concept and entity pages based on existing wiki/sources/ pages.
func (h *Handler) BackfillRefs(w http.ResponseWriter, r *http.Request) {
	n, err := pipeline.BackfillRefs(r.Context(), h.brainDir)
	if err != nil {
		h.logger.Error("backfill-refs failed", "err", err)
		writeError(w, http.StatusInternalServerError, "backfill error")
		return
	}
	writeJSON(w, map[string]int{"updated": n})
}

// Pending handles GET /pending — list raw/ notes awaiting promotion.
func (h *Handler) Pending(w http.ResponseWriter, _ *http.Request) {
	pending, err := ListPending(h.brainDir)
	if err != nil {
		h.logger.Error("pending failed", "err", err)
		writeError(w, http.StatusInternalServerError, "pending error")
		return
	}
	writeJSON(w, map[string]any{"pending": pending})
}

type promoteRequest struct {
	Filename string `json:"filename"`
	Wing     string `json:"wing"`
	Hall     string `json:"hall"`
	Slug     string `json:"slug,omitempty"`
}

// Promote handles POST /promote — move a raw/ note into the wiki. A bad
// hall / collision / missing source is a 400 (caller error), not a 500.
func (h *Handler) Promote(w http.ResponseWriter, r *http.Request) {
	var req promoteRequest
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		writeError(w, http.StatusBadRequest, "invalid JSON")
		return
	}
	// Field by field: this door carries no principal, so the promoted note is
	// stamped origin: unknown and the body cannot set provenance (#145).
	rel, err := PromoteNote(h.brainDir, PromoteOptions{
		Filename: req.Filename, Wing: req.Wing, Hall: req.Hall, Slug: req.Slug,
	})
	if err != nil {
		writeError(w, http.StatusBadRequest, err.Error())
		return
	}
	writeJSON(w, map[string]string{"path": rel})
}

func writeJSON(w http.ResponseWriter, v any) {
	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(v) //nolint:errcheck
}

func writeError(w http.ResponseWriter, code int, msg string) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(code)
	json.NewEncoder(w).Encode(map[string]string{"error": msg}) //nolint:errcheck
}