hyperguild/ingestion/internal/api/promote.go

package api

import (
	"errors"
	"fmt"
	"io/fs"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/mathiasbq/hyperguild/ingestion/internal/brain"
)

// PendingNote describes a raw/ note awaiting human promotion to the wiki.
type PendingNote struct {
	Filename  string `json:"filename"`
	CreatedAt string `json:"created_at"`
	SizeBytes int64  `json:"size_bytes"`
	Excerpt   string `json:"excerpt"`

	// Set only for a note parked in raw/needs-review/ by the wing resolver
	// (hyperguild#205): it was not placed automatically, and the proposal says
	// what the resolver would have done and why it did not. Approve by moving
	// the note into raw/<wing>/ and the watcher ingests it there.
	Queue              string  `json:"queue"`
	ProposedWing       string  `json:"proposed_wing,omitempty"`
	ProposalReason     string  `json:"proposal_reason,omitempty"`
	ProposalConfidence float64 `json:"proposal_confidence,omitempty"`
}

// datePrefix matches a leading YYYY-MM-DD- on a raw filename, stripped when
// deriving the default promoted slug.
var datePrefix = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}-`)

// ListPending returns the notes in brain/raw/ awaiting review, oldest-first
// (natural review order). An absent raw/ dir yields an empty slice, not an
// error. Only .md files are listed; tunnel-candidate files are skipped.
func ListPending(brainDir string) ([]PendingNote, error) {
	root, _, err := OpenRawRoot(brainDir)
	if err != nil {
		if errors.Is(err, os.ErrNotExist) {
			return []PendingNote{}, nil
		}
		return nil, fmt.Errorf("read raw dir: %w", err)
	}
	defer func() { _ = root.Close() }()

	out, err := listNotes(root, QueueInbox)
	if err != nil {
		return nil, err
	}
	// raw/needs-review/ is read through its own root, so a symlinked directory
	// or note there is refused exactly like at the top level (#153).
	if sub, serr := root.OpenRoot("needs-review"); serr == nil {
		defer func() { _ = sub.Close() }()
		parked, perr := listNotes(sub, QueueNeedsReview)
		if perr != nil {
			return nil, perr
		}
		out = append(out, parked...)
	}
	sort.SliceStable(out, func(i, j int) bool { return out[i].CreatedAt < out[j].CreatedAt })
	return out, nil
}

// listNotes lists the regular .md notes directly under root. queue labels the
// notes ("" for the top of raw/) and enables reading a wing proposal.
func listNotes(root *os.Root, queue string) ([]PendingNote, error) {
	entries, err := fs.ReadDir(root.FS(), ".")
	if err != nil {
		return nil, fmt.Errorf("read raw dir: %w", err)
	}
	out := make([]PendingNote, 0, len(entries))
	for _, e := range entries {
		isMD := strings.HasSuffix(e.Name(), ".md")
		// A parked pdf or txt is listed too (the watcher parks any supported
		// file); at the top level only markdown ever was.
		parkedOther := queue == QueueNeedsReview && (strings.HasSuffix(e.Name(), ".txt") || strings.HasSuffix(e.Name(), ".pdf"))
		if !e.Type().IsRegular() || (!isMD && !parkedOther) || strings.HasPrefix(e.Name(), "tunnel-candidates-") {
			continue // directories, and symlinks whose target may be anywhere (#153)
		}
		if !isMD {
			info, ierr := root.Lstat(e.Name())
			if ierr != nil {
				continue
			}
			out = append(out, PendingNote{
				Filename: e.Name(), CreatedAt: info.ModTime().UTC().Format(time.RFC3339), SizeBytes: info.Size(),
				Excerpt: "(not markdown; the wing proposal is in log.md. Move the file into raw/<wing>/ to ingest it.)",
				Queue:   queue,
			})
			continue
		}
		raw, info, readErr := readRawNote(root, e.Name())
		if readErr != nil {
			// Not skipped: a note that cannot be read (over the cap, say) must
			// not silently vanish from the queue it is waiting in.
			fi, ierr := root.Lstat(e.Name())
			if ierr != nil {
				continue
			}
			out = append(out, PendingNote{
				Filename: e.Name(), CreatedAt: fi.ModTime().UTC().Format(time.RFC3339), SizeBytes: fi.Size(),
				Excerpt: "(cannot be read: " + readErr.Error() + ")", Queue: queue,
			})
			continue
		}
		fm, body := parseFrontmatter(string(raw))
		created := fm.get("created_at")
		if created == "" {
			created = info.ModTime().UTC().Format(time.RFC3339)
		}
		n := PendingNote{
			Filename:  e.Name(),
			CreatedAt: created,
			SizeBytes: info.Size(),
			Excerpt:   excerpt(body, 200),
			Queue:     queue,
		}
		if queue == QueueNeedsReview {
			n.ProposedWing = fm.get("proposed_wing")
			n.ProposalReason = unquoteScalar(fm.get("proposal_reason"))
			n.ProposalConfidence, _ = strconv.ParseFloat(fm.get("proposal_confidence"), 64)
		}
		out = append(out, n)
	}
	return out, nil
}

// PromoteOptions identifies a raw note to promote and its wiki destination.
type PromoteOptions struct {
	Filename string // basename in brain/raw/
	Wing     string
	Hall     string
	Slug     string // optional; defaults to Filename minus date prefix + .md
	// Queue names which listing the note is in: QueueInbox or QueueNeedsReview.
	// Empty is accepted only when the name exists in exactly one of them.
	Queue string
	// Origin/WrittenBy re-attribute the promoted note to the promoter: the raw
	// note's own frontmatter claims are not provenance (hyperguild#145).
	Origin    string
	WrittenBy string
}

// PromoteNote moves a note from brain/raw/ into the structured wiki: it
// rewrites frontmatter (sets wing/hall/promoted_at, preserves created_at and
// any custom fields), writes to brain/wiki/<wing>/<hall>/<slug>.md, deletes
// the source, then rebuilds the wing index and runs auto-tunnel detection.
//
// It is atomic from the caller's view: validation (hall, wing, slug,
// collision) happens before any filesystem change, and the source is deleted
// only after the destination write succeeds (write-then-delete, never move).
// Returns the promoted note's path relative to brainDir.
func PromoteNote(brainDir string, opts PromoteOptions) (string, error) {
	// Validate filename (basename only — no traversal) before touching fs.
	base := filepath.Base(opts.Filename)
	if base != opts.Filename || base == "." || base == ".." || strings.ContainsAny(opts.Filename, `/\`) {
		return "", fmt.Errorf("invalid filename %q", opts.Filename)
	}

	if !strings.HasSuffix(base, ".md") {
		return "", fmt.Errorf("only a markdown note can be promoted; move %q into raw/<wing>/ to ingest it", base)
	}
	slug := opts.Slug
	if slug == "" {
		slug = datePrefix.ReplaceAllString(strings.TrimSuffix(base, ".md"), "")
	}
	// The retired layout's directory names are not wings: a note promoted into
	// "sources" would land in wiki/sources/<hall>/, inside the retired dir.
	switch brain.Sanitise(opts.Wing) {
	case "sources", "concepts", "entities", "wiki":
		return "", fmt.Errorf("%q is not a wing", opts.Wing)
	}
	// NotePath validates hall + wing + slug; do this before reading anything.
	dest, err := brain.NotePath(brainDir, opts.Wing, opts.Hall, slug)
	if err != nil {
		return "", err
	}

	raw, srcRel, err := readRawSource(brainDir, base, opts.Queue)
	if err != nil {
		return "", err
	}
	src := filepath.Join(brainDir, filepath.FromSlash(srcRel))

	// Collision: never silently overwrite an existing promoted note.
	if _, statErr := os.Stat(dest); statErr == nil {
		rel, _ := filepath.Rel(brainDir, dest)
		return "", fmt.Errorf("target %s already exists; choose a different slug", filepath.ToSlash(rel))
	}

	fm, body := parseFrontmatter(string(raw))
	// The resolver's proposal described where the note was going to be placed;
	// once a human has placed it, it is noise (and `wing` is set below).
	for _, k := range []string{"proposed_wing", "proposal_confidence", "proposal_reason"} {
		fm.drop(k)
	}
	now := time.Now().UTC().Format(time.RFC3339)
	fm.set("wing", brain.Sanitise(opts.Wing))
	fm.set("hall", opts.Hall)
	if fm.get("created_at") == "" {
		fm.set("created_at", now)
	}
	fm.set("promoted_at", now)

	// Write-then-delete: the source survives any write failure. Both go
	// through the brain root (hyperguild#188).
	promoted, err := StampProvenance(fm.render()+body, opts.Origin, opts.WrittenBy)
	if err != nil {
		return "", err
	}
	// Created with O_EXCL: a note that appeared after the collision check
	// above is never overwritten.
	if err := createBrainFile(brainDir, dest, []byte(promoted)); err != nil {
		return "", fmt.Errorf("write promoted note: %w", err)
	}
	if err := removeBrainFile(brainDir, src); err != nil {
		return "", fmt.Errorf("promoted note written but source removal failed: %w", err)
	}

	rel, _ := filepath.Rel(brainDir, dest)
	relSlash := filepath.ToSlash(rel)

	// Best-effort wiki upkeep — the note is already promoted.
	_ = brain.BuildWingIndex(brainDir, opts.Wing)
	_ = brain.AutoTunnel(brainDir, relSlash, body)

	return relSlash, nil
}

// RejectOptions identifies a raw note to reject and why.
type RejectOptions struct {
	Filename string // basename in brain/raw/
	Reason   string // optional
	Queue    string // as PromoteOptions.Queue
}

// RejectNote records a human decision not to promote a brain/raw/ note,
// without silently losing it: it stamps rejected_at (+ reject_reason when
// given) into frontmatter and moves the file to brain/raw/rejected/, using
// the same write-then-delete safety PromoteNote uses — the source survives
// any write failure — and the same basename-only filename validation.
//
// A rejected note never entered the wiki, so no index rebuild or
// auto-tunnel run: unlike PromoteNote, this is the end of the note's
// journey, not the start of a new one. It stays out of ListPending's view
// (raw/rejected/ is a subdirectory, and ListPending's os.ReadDir on raw/
// does not recurse), so a rejected note is never re-surfaced as pending —
// the human decision, once made, stays made, the same principle behind
// this estate's "issue status is surfaced, not auto-closed, but once a
// human decides, the decision is recorded" rule applied one level earlier.
func RejectNote(brainDir string, opts RejectOptions) (string, error) {
	base := filepath.Base(opts.Filename)
	if base != opts.Filename || base == "." || base == ".." || strings.ContainsAny(opts.Filename, `/\`) {
		return "", fmt.Errorf("invalid filename %q", opts.Filename)
	}

	raw, srcRel, err := readRawSource(brainDir, base, opts.Queue)
	if err != nil {
		return "", err
	}
	src := filepath.Join(brainDir, filepath.FromSlash(srcRel))

	dest := filepath.Join(brainDir, "raw", "rejected", base)
	if _, statErr := os.Stat(dest); statErr == nil {
		rel, _ := filepath.Rel(brainDir, dest)
		return "", fmt.Errorf("a rejected note named %s already exists", filepath.ToSlash(rel))
	}

	content := string(raw)
	if strings.HasSuffix(base, ".md") {
		fm, body := parseFrontmatter(string(raw))
		fm.set("rejected_at", time.Now().UTC().Format(time.RFC3339))
		if opts.Reason != "" {
			fm.set("reject_reason", opts.Reason)
		}
		content = fm.render() + body
	} // a pdf or txt is moved byte for byte: prepending frontmatter would alter it

	// Write-then-delete: the source survives any write failure. Both go
	// through the brain root (hyperguild#188).
	if err := createBrainFile(brainDir, dest, []byte(content)); err != nil {
		return "", fmt.Errorf("write rejected note: %w", err)
	}
	if err := removeBrainFile(brainDir, src); err != nil {
		return "", fmt.Errorf("rejected note written but source removal failed: %w", err)
	}

	rel, _ := filepath.Rel(brainDir, dest)
	return filepath.ToSlash(rel), nil
}

// excerpt returns the first n runes of s, trimmed, single-spaced.
func excerpt(s string, n int) string {
	s = strings.TrimSpace(s)
	r := []rune(s)
	if len(r) > n {
		r = r[:n]
	}
	return strings.TrimSpace(string(r))
}

// readRawSource reads a pending note for promote/reject through the raw root,
// refusing symlinks and a symlinked raw/ (hyperguild#153).
func readRawSource(brainDir, base, queue string) ([]byte, string, error) {
	if queue != "" && queue != QueueInbox && queue != QueueNeedsReview {
		return nil, "", fmt.Errorf("unknown queue %q (want %q or %q)", queue, QueueInbox, QueueNeedsReview)
	}
	root, _, err := OpenRawRoot(brainDir)
	if err != nil {
		if errors.Is(err, os.ErrNotExist) {
			return nil, "", fmt.Errorf("pending note %q does not exist in raw/", base)
		}
		return nil, "", fmt.Errorf("read source: %w", err)
	}
	defer func() { _ = root.Close() }()

	var topRaw, parkedRaw []byte
	var topErr, parkedErr error
	if queue != QueueNeedsReview {
		topRaw, _, topErr = readRawNote(root, base)
	}
	if queue != QueueInbox {
		// The sub-root refuses a symlinked directory like OpenRawRoot does.
		if sub, serr := root.OpenRoot(parkedDir); serr != nil {
			parkedErr = serr
		} else {
			parkedRaw, _, parkedErr = readRawNote(sub, base)
			_ = sub.Close()
		}
	}
	haveTop := queue != QueueNeedsReview && topErr == nil
	haveParked := queue != QueueInbox && parkedErr == nil
	switch {
	case haveTop && haveParked:
		return nil, "", fmt.Errorf("%q is in both raw/ and raw/%s/: name the queue (%q or %q)", base, parkedDir, QueueInbox, QueueNeedsReview)
	case haveTop:
		return topRaw, "raw/" + base, nil
	case haveParked:
		return parkedRaw, "raw/" + parkedDir + "/" + base, nil
	}
	for _, e := range []error{topErr, parkedErr} {
		if e != nil && !errors.Is(e, os.ErrNotExist) {
			return nil, "", fmt.Errorf("read source: %w", e)
		}
	}
	return nil, "", fmt.Errorf("pending note %q does not exist in raw/", base)
}

// parkedDir is where the watcher parks notes it could not place.
const parkedDir = "needs-review"

// The two review queues brain_pending lists. promote and reject take one of
// them (or neither, when the name is unambiguous).
const (
	QueueInbox       = "inbox"        // raw/<name>
	QueueNeedsReview = "needs-review" // raw/needs-review/<name>
)

// unquoteScalar undoes the Go-style double quoting the watcher writes around a
// free-text value (strconv.Quote keeps it on one line); a bare value is
// returned as is.
func unquoteScalar(v string) string {
	if u, err := strconv.Unquote(v); err == nil {
		return u
	}
	return v
}