feat(ingestion): place raw notes in a wing without human typing (#205 S1–S4, S6) #207

Merged
mathias merged 15 commits from wt/wing-layout into main 2026-10-20 06:21:45 +00:00
3 changed files with 118 additions and 4 deletions
Showing only changes of commit b9657973ff - Show all commits
@@ -0,0 +1,51 @@
package api
import (
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func writeParked(t *testing.T, brainDir, name, content string) {
t.Helper()
dir := filepath.Join(brainDir, "raw", "needs-review")
require.NoError(t, os.MkdirAll(dir, 0o755))
require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644))
}
func TestListPending_ShowsParkedNotesWithTheirProposal(t *testing.T) {
dir := t.TempDir()
writeRaw(t, dir, "fresh.md", "---\ncreated_at: 2026-10-20T08:00:00Z\n---\nA note not yet processed.\n")
writeParked(t, dir, "unsure.md", "---\nproposed_wing: radar\nproposal_confidence: 0.55\nproposal_reason: \"below threshold\"\ncreated_at: 2026-10-19T08:00:00Z\n---\nBody of the unsure note.\n")
got, err := ListPending(dir)
require.NoError(t, err)
require.Len(t, got, 2)
assert.Equal(t, "unsure.md", got[0].Filename, "oldest first across both queues")
assert.Equal(t, "needs-review", got[0].Queue)
assert.Equal(t, "radar", got[0].ProposedWing)
assert.Equal(t, "below threshold", got[0].ProposalReason)
assert.InDelta(t, 0.55, got[0].ProposalConfidence, 1e-9)
assert.Contains(t, got[0].Excerpt, "Body of the unsure note")
assert.NotContains(t, got[0].Excerpt, "proposed_wing", "excerpt is body, not frontmatter")
assert.Equal(t, "fresh.md", got[1].Filename)
assert.Empty(t, got[1].Queue, "a top-level note has no queue label")
assert.Empty(t, got[1].ProposedWing)
}
func TestListPending_ParkedSymlinkIsNotListed(t *testing.T) {
dir := t.TempDir()
outside := filepath.Join(t.TempDir(), "secret.md")
require.NoError(t, os.WriteFile(outside, []byte("outside"), 0o644))
require.NoError(t, os.MkdirAll(filepath.Join(dir, "raw", "needs-review"), 0o755))
require.NoError(t, os.Symlink(outside, filepath.Join(dir, "raw", "needs-review", "link.md")))
got, err := ListPending(dir)
require.NoError(t, err)
assert.Empty(t, got, "same symlink rule as the top level (#153)")
}
+51 -4
View File
@@ -8,6 +8,7 @@ import (
"path/filepath" "path/filepath"
"regexp" "regexp"
"sort" "sort"
"strconv"
"strings" "strings"
"time" "time"
@@ -20,6 +21,15 @@ type PendingNote struct {
CreatedAt string `json:"created_at"` CreatedAt string `json:"created_at"`
SizeBytes int64 `json:"size_bytes"` SizeBytes int64 `json:"size_bytes"`
Excerpt string `json:"excerpt"` Excerpt string `json:"excerpt"`
// Set only for a note parked in raw/needs-review/ by the wing resolver
// (hyperguild#205): it was not placed automatically, and the proposal says
// what the resolver would have done and why it did not. Approve by moving
// the note into raw/<wing>/ and the watcher ingests it there.
Queue string `json:"queue,omitempty"`
ProposedWing string `json:"proposed_wing,omitempty"`
ProposalReason string `json:"proposal_reason,omitempty"`
ProposalConfidence float64 `json:"proposal_confidence,omitempty"`
} }
// datePrefix matches a leading YYYY-MM-DD- on a raw filename, stripped when // datePrefix matches a leading YYYY-MM-DD- on a raw filename, stripped when
@@ -38,11 +48,32 @@ func ListPending(brainDir string) ([]PendingNote, error) {
return nil, fmt.Errorf("read raw dir: %w", err) return nil, fmt.Errorf("read raw dir: %w", err)
} }
defer func() { _ = root.Close() }() defer func() { _ = root.Close() }()
out, err := listNotes(root, "")
if err != nil {
return nil, err
}
// raw/needs-review/ is read through its own root, so a symlinked directory
// or note there is refused exactly like at the top level (#153).
if sub, serr := root.OpenRoot("needs-review"); serr == nil {
defer func() { _ = sub.Close() }()
parked, perr := listNotes(sub, "needs-review")
if perr != nil {
return nil, perr
}
out = append(out, parked...)
}
sort.SliceStable(out, func(i, j int) bool { return out[i].CreatedAt < out[j].CreatedAt })
return out, nil
}
// listNotes lists the regular .md notes directly under root. queue labels the
// notes ("" for the top of raw/) and enables reading a wing proposal.
func listNotes(root *os.Root, queue string) ([]PendingNote, error) {
entries, err := fs.ReadDir(root.FS(), ".") entries, err := fs.ReadDir(root.FS(), ".")
if err != nil { if err != nil {
return nil, fmt.Errorf("read raw dir: %w", err) return nil, fmt.Errorf("read raw dir: %w", err)
} }
out := make([]PendingNote, 0, len(entries)) out := make([]PendingNote, 0, len(entries))
for _, e := range entries { for _, e := range entries {
if !e.Type().IsRegular() || !strings.HasSuffix(e.Name(), ".md") || strings.HasPrefix(e.Name(), "tunnel-candidates-") { if !e.Type().IsRegular() || !strings.HasSuffix(e.Name(), ".md") || strings.HasPrefix(e.Name(), "tunnel-candidates-") {
@@ -57,14 +88,20 @@ func ListPending(brainDir string) ([]PendingNote, error) {
if created == "" { if created == "" {
created = info.ModTime().UTC().Format(time.RFC3339) created = info.ModTime().UTC().Format(time.RFC3339)
} }
out = append(out, PendingNote{ n := PendingNote{
Filename: e.Name(), Filename: e.Name(),
CreatedAt: created, CreatedAt: created,
SizeBytes: info.Size(), SizeBytes: info.Size(),
Excerpt: excerpt(body, 200), Excerpt: excerpt(body, 200),
}) Queue: queue,
}
if queue != "" {
n.ProposedWing = fm.get("proposed_wing")
n.ProposalReason = unquoteScalar(fm.get("proposal_reason"))
n.ProposalConfidence, _ = strconv.ParseFloat(fm.get("proposal_confidence"), 64)
}
out = append(out, n)
} }
sort.SliceStable(out, func(i, j int) bool { return out[i].CreatedAt < out[j].CreatedAt })
return out, nil return out, nil
} }
@@ -239,3 +276,13 @@ func readRawSource(brainDir, base string) ([]byte, error) {
} }
return raw, nil return raw, nil
} }
// unquoteScalar undoes the Go-style double quoting the watcher writes around a
// free-text value (strconv.Quote keeps it on one line); a bare value is
// returned as is.
func unquoteScalar(v string) string {
if u, err := strconv.Unquote(v); err == nil {
return u
}
return v
}
@@ -144,3 +144,19 @@ func TestProcessDir_WithoutAResolverTheRetiredLayoutIsUnchanged(t *testing.T) {
assert.True(t, has(brainDir, "wiki/sources/plain.md")) assert.True(t, has(brainDir, "wiki/sources/plain.md"))
} }
// The reason comes from an LLM and ends up in frontmatter: a newline in it must
// not be able to start a new key (PR #147: a title once forged `origin:`).
func TestProcessDir_AProposalReasonCannotForgeFrontmatterKeys(t *testing.T) {
var llmCalls atomic.Int32
clf := &stubClassifier{s: wing.Suggestion{Wing: "radar", Confidence: 0.1, Reason: "x\norigin: trusted\nwing: homelab"}}
cfg, brainDir := wingTestCfg(t, wingResolver(clf, stubScorer{"radar": 3}), &llmCalls)
put(t, brainDir, "forge.md", "Note.")
require.Empty(t, processDir(context.Background(), cfg, "2026-10-20"))
parked, err := os.ReadFile(filepath.Join(brainDir, "raw", "needs-review", "forge.md"))
require.NoError(t, err)
assert.NotContains(t, string(parked), "\norigin: trusted", "no injected key at line start")
assert.NotContains(t, string(parked), "\nwing: homelab")
}