package api import ( "errors" "fmt" "io/fs" "os" "path/filepath" "regexp" "sort" "strconv" "strings" "time" "github.com/mathiasbq/hyperguild/ingestion/internal/brain" ) // PendingNote describes a raw/ note awaiting human promotion to the wiki. type PendingNote struct { Filename string `json:"filename"` CreatedAt string `json:"created_at"` SizeBytes int64 `json:"size_bytes"` Excerpt string `json:"excerpt"` // Set only for a note parked in raw/needs-review/ by the wing resolver // (hyperguild#205): it was not placed automatically, and the proposal says // what the resolver would have done and why it did not. Approve by moving // the note into raw// and the watcher ingests it there. Queue string `json:"queue"` ProposedWing string `json:"proposed_wing,omitempty"` ProposalReason string `json:"proposal_reason,omitempty"` ProposalConfidence float64 `json:"proposal_confidence,omitempty"` } // datePrefix matches a leading YYYY-MM-DD- on a raw filename, stripped when // deriving the default promoted slug. var datePrefix = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}-`) // ListPending returns the notes in brain/raw/ awaiting review, oldest-first // (natural review order). An absent raw/ dir yields an empty slice, not an // error. Only .md files are listed; tunnel-candidate files are skipped. func ListPending(brainDir string) ([]PendingNote, error) { root, _, err := OpenRawRoot(brainDir) if err != nil { if errors.Is(err, os.ErrNotExist) { return []PendingNote{}, nil } return nil, fmt.Errorf("read raw dir: %w", err) } defer func() { _ = root.Close() }() out, err := listNotes(root, QueueInbox) if err != nil { return nil, err } // raw/needs-review/ is read through its own root, so a symlinked directory // or note there is refused exactly like at the top level (#153). if sub, serr := root.OpenRoot("needs-review"); serr == nil { defer func() { _ = sub.Close() }() parked, perr := listNotes(sub, QueueNeedsReview) if perr != nil { return nil, perr } out = append(out, parked...) } sort.SliceStable(out, func(i, j int) bool { return out[i].CreatedAt < out[j].CreatedAt }) return out, nil } // listNotes lists the regular .md notes directly under root. queue labels the // notes ("" for the top of raw/) and enables reading a wing proposal. func listNotes(root *os.Root, queue string) ([]PendingNote, error) { entries, err := fs.ReadDir(root.FS(), ".") if err != nil { return nil, fmt.Errorf("read raw dir: %w", err) } out := make([]PendingNote, 0, len(entries)) for _, e := range entries { isMD := strings.HasSuffix(e.Name(), ".md") // A parked pdf or txt is listed too (the watcher parks any supported // file); at the top level only markdown ever was. parkedOther := queue == QueueNeedsReview && (strings.HasSuffix(e.Name(), ".txt") || strings.HasSuffix(e.Name(), ".pdf")) if !e.Type().IsRegular() || (!isMD && !parkedOther) || strings.HasPrefix(e.Name(), "tunnel-candidates-") { continue // directories, and symlinks whose target may be anywhere (#153) } if !isMD { info, ierr := root.Lstat(e.Name()) if ierr != nil { continue } out = append(out, PendingNote{ Filename: e.Name(), CreatedAt: info.ModTime().UTC().Format(time.RFC3339), SizeBytes: info.Size(), Excerpt: "(not markdown; the wing proposal is in log.md. Move the file into raw// to ingest it.)", Queue: queue, }) continue } raw, info, readErr := readRawNote(root, e.Name()) if readErr != nil { // Not skipped: a note that cannot be read (over the cap, say) must // not silently vanish from the queue it is waiting in. fi, ierr := root.Lstat(e.Name()) if ierr != nil { continue } out = append(out, PendingNote{ Filename: e.Name(), CreatedAt: fi.ModTime().UTC().Format(time.RFC3339), SizeBytes: fi.Size(), Excerpt: "(cannot be read: " + readErr.Error() + ")", Queue: queue, }) continue } fm, body := parseFrontmatter(string(raw)) created := fm.get("created_at") if created == "" { created = info.ModTime().UTC().Format(time.RFC3339) } n := PendingNote{ Filename: e.Name(), CreatedAt: created, SizeBytes: info.Size(), Excerpt: excerpt(body, 200), Queue: queue, } if queue == QueueNeedsReview { n.ProposedWing = fm.get("proposed_wing") n.ProposalReason = unquoteScalar(fm.get("proposal_reason")) n.ProposalConfidence, _ = strconv.ParseFloat(fm.get("proposal_confidence"), 64) } out = append(out, n) } return out, nil } // PromoteOptions identifies a raw note to promote and its wiki destination. type PromoteOptions struct { Filename string // basename in brain/raw/ Wing string Hall string Slug string // optional; defaults to Filename minus date prefix + .md // Queue names which listing the note is in: QueueInbox or QueueNeedsReview. // Empty is accepted only when the name exists in exactly one of them. Queue string // Origin/WrittenBy re-attribute the promoted note to the promoter: the raw // note's own frontmatter claims are not provenance (hyperguild#145). Origin string WrittenBy string } // PromoteNote moves a note from brain/raw/ into the structured wiki: it // rewrites frontmatter (sets wing/hall/promoted_at, preserves created_at and // any custom fields), writes to brain/wiki///.md, deletes // the source, then rebuilds the wing index and runs auto-tunnel detection. // // It is atomic from the caller's view: validation (hall, wing, slug, // collision) happens before any filesystem change, and the source is deleted // only after the destination write succeeds (write-then-delete, never move). // Returns the promoted note's path relative to brainDir. func PromoteNote(brainDir string, opts PromoteOptions) (string, error) { // Validate filename (basename only — no traversal) before touching fs. base := filepath.Base(opts.Filename) if base != opts.Filename || base == "." || base == ".." || strings.ContainsAny(opts.Filename, `/\`) { return "", fmt.Errorf("invalid filename %q", opts.Filename) } if !strings.HasSuffix(base, ".md") { return "", fmt.Errorf("only a markdown note can be promoted; move %q into raw// to ingest it", base) } slug := opts.Slug if slug == "" { slug = datePrefix.ReplaceAllString(strings.TrimSuffix(base, ".md"), "") } // The retired layout's directory names are not wings: a note promoted into // "sources" would land in wiki/sources//, inside the retired dir. switch brain.Sanitise(opts.Wing) { case "sources", "concepts", "entities", "wiki": return "", fmt.Errorf("%q is not a wing", opts.Wing) } // NotePath validates hall + wing + slug; do this before reading anything. dest, err := brain.NotePath(brainDir, opts.Wing, opts.Hall, slug) if err != nil { return "", err } raw, srcRel, err := readRawSource(brainDir, base, opts.Queue) if err != nil { return "", err } src := filepath.Join(brainDir, filepath.FromSlash(srcRel)) // Collision: never silently overwrite an existing promoted note. if _, statErr := os.Stat(dest); statErr == nil { rel, _ := filepath.Rel(brainDir, dest) return "", fmt.Errorf("target %s already exists; choose a different slug", filepath.ToSlash(rel)) } fm, body := parseFrontmatter(string(raw)) // The resolver's proposal described where the note was going to be placed; // once a human has placed it, it is noise (and `wing` is set below). for _, k := range []string{"proposed_wing", "proposal_confidence", "proposal_reason"} { fm.drop(k) } now := time.Now().UTC().Format(time.RFC3339) fm.set("wing", brain.Sanitise(opts.Wing)) fm.set("hall", opts.Hall) if fm.get("created_at") == "" { fm.set("created_at", now) } fm.set("promoted_at", now) // Write-then-delete: the source survives any write failure. Both go // through the brain root (hyperguild#188). promoted, err := StampProvenance(fm.render()+body, opts.Origin, opts.WrittenBy) if err != nil { return "", err } // Created with O_EXCL: a note that appeared after the collision check // above is never overwritten. if err := createBrainFile(brainDir, dest, []byte(promoted)); err != nil { return "", fmt.Errorf("write promoted note: %w", err) } if err := removeBrainFile(brainDir, src); err != nil { return "", fmt.Errorf("promoted note written but source removal failed: %w", err) } rel, _ := filepath.Rel(brainDir, dest) relSlash := filepath.ToSlash(rel) // Best-effort wiki upkeep — the note is already promoted. _ = brain.BuildWingIndex(brainDir, opts.Wing) _ = brain.AutoTunnel(brainDir, relSlash, body) return relSlash, nil } // RejectOptions identifies a raw note to reject and why. type RejectOptions struct { Filename string // basename in brain/raw/ Reason string // optional Queue string // as PromoteOptions.Queue } // RejectNote records a human decision not to promote a brain/raw/ note, // without silently losing it: it stamps rejected_at (+ reject_reason when // given) into frontmatter and moves the file to brain/raw/rejected/, using // the same write-then-delete safety PromoteNote uses — the source survives // any write failure — and the same basename-only filename validation. // // A rejected note never entered the wiki, so no index rebuild or // auto-tunnel run: unlike PromoteNote, this is the end of the note's // journey, not the start of a new one. It stays out of ListPending's view // (raw/rejected/ is a subdirectory, and ListPending's os.ReadDir on raw/ // does not recurse), so a rejected note is never re-surfaced as pending — // the human decision, once made, stays made, the same principle behind // this estate's "issue status is surfaced, not auto-closed, but once a // human decides, the decision is recorded" rule applied one level earlier. func RejectNote(brainDir string, opts RejectOptions) (string, error) { base := filepath.Base(opts.Filename) if base != opts.Filename || base == "." || base == ".." || strings.ContainsAny(opts.Filename, `/\`) { return "", fmt.Errorf("invalid filename %q", opts.Filename) } raw, srcRel, err := readRawSource(brainDir, base, opts.Queue) if err != nil { return "", err } src := filepath.Join(brainDir, filepath.FromSlash(srcRel)) dest := filepath.Join(brainDir, "raw", "rejected", base) if _, statErr := os.Stat(dest); statErr == nil { rel, _ := filepath.Rel(brainDir, dest) return "", fmt.Errorf("a rejected note named %s already exists", filepath.ToSlash(rel)) } content := string(raw) if strings.HasSuffix(base, ".md") { fm, body := parseFrontmatter(string(raw)) fm.set("rejected_at", time.Now().UTC().Format(time.RFC3339)) if opts.Reason != "" { fm.set("reject_reason", opts.Reason) } content = fm.render() + body } // a pdf or txt is moved byte for byte: prepending frontmatter would alter it // Write-then-delete: the source survives any write failure. Both go // through the brain root (hyperguild#188). if err := createBrainFile(brainDir, dest, []byte(content)); err != nil { return "", fmt.Errorf("write rejected note: %w", err) } if err := removeBrainFile(brainDir, src); err != nil { return "", fmt.Errorf("rejected note written but source removal failed: %w", err) } rel, _ := filepath.Rel(brainDir, dest) return filepath.ToSlash(rel), nil } // excerpt returns the first n runes of s, trimmed, single-spaced. func excerpt(s string, n int) string { s = strings.TrimSpace(s) r := []rune(s) if len(r) > n { r = r[:n] } return strings.TrimSpace(string(r)) } // readRawSource reads a pending note for promote/reject through the raw root, // refusing symlinks and a symlinked raw/ (hyperguild#153). func readRawSource(brainDir, base, queue string) ([]byte, string, error) { if queue != "" && queue != QueueInbox && queue != QueueNeedsReview { return nil, "", fmt.Errorf("unknown queue %q (want %q or %q)", queue, QueueInbox, QueueNeedsReview) } root, _, err := OpenRawRoot(brainDir) if err != nil { if errors.Is(err, os.ErrNotExist) { return nil, "", fmt.Errorf("pending note %q does not exist in raw/", base) } return nil, "", fmt.Errorf("read source: %w", err) } defer func() { _ = root.Close() }() var topRaw, parkedRaw []byte var topErr, parkedErr error if queue != QueueNeedsReview { topRaw, _, topErr = readRawNote(root, base) } if queue != QueueInbox { // The sub-root refuses a symlinked directory like OpenRawRoot does. if sub, serr := root.OpenRoot(parkedDir); serr != nil { parkedErr = serr } else { parkedRaw, _, parkedErr = readRawNote(sub, base) _ = sub.Close() } } haveTop := queue != QueueNeedsReview && topErr == nil haveParked := queue != QueueInbox && parkedErr == nil switch { case haveTop && haveParked: return nil, "", fmt.Errorf("%q is in both raw/ and raw/%s/: name the queue (%q or %q)", base, parkedDir, QueueInbox, QueueNeedsReview) case haveTop: return topRaw, "raw/" + base, nil case haveParked: return parkedRaw, "raw/" + parkedDir + "/" + base, nil } for _, e := range []error{topErr, parkedErr} { if e != nil && !errors.Is(e, os.ErrNotExist) { return nil, "", fmt.Errorf("read source: %w", e) } } return nil, "", fmt.Errorf("pending note %q does not exist in raw/", base) } // parkedDir is where the watcher parks notes it could not place. const parkedDir = "needs-review" // The two review queues brain_pending lists. promote and reject take one of // them (or neither, when the name is unambiguous). const ( QueueInbox = "inbox" // raw/ QueueNeedsReview = "needs-review" // raw/needs-review/ ) // unquoteScalar undoes the Go-style double quoting the watcher writes around a // free-text value (strconv.Quote keeps it on one line); a bare value is // returned as is. func unquoteScalar(v string) string { if u, err := strconv.Unquote(v); err == nil { return u } return v }