feat(ingestion): place raw notes in a wing without human typing (#205 S1–S4, S6) #207
@@ -0,0 +1,51 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func writeParked(t *testing.T, brainDir, name, content string) {
|
||||||
|
t.Helper()
|
||||||
|
dir := filepath.Join(brainDir, "raw", "needs-review")
|
||||||
|
require.NoError(t, os.MkdirAll(dir, 0o755))
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestListPending_ShowsParkedNotesWithTheirProposal(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
writeRaw(t, dir, "fresh.md", "---\ncreated_at: 2026-10-20T08:00:00Z\n---\nA note not yet processed.\n")
|
||||||
|
writeParked(t, dir, "unsure.md", "---\nproposed_wing: radar\nproposal_confidence: 0.55\nproposal_reason: \"below threshold\"\ncreated_at: 2026-10-19T08:00:00Z\n---\nBody of the unsure note.\n")
|
||||||
|
|
||||||
|
got, err := ListPending(dir)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, got, 2)
|
||||||
|
|
||||||
|
assert.Equal(t, "unsure.md", got[0].Filename, "oldest first across both queues")
|
||||||
|
assert.Equal(t, "needs-review", got[0].Queue)
|
||||||
|
assert.Equal(t, "radar", got[0].ProposedWing)
|
||||||
|
assert.Equal(t, "below threshold", got[0].ProposalReason)
|
||||||
|
assert.InDelta(t, 0.55, got[0].ProposalConfidence, 1e-9)
|
||||||
|
assert.Contains(t, got[0].Excerpt, "Body of the unsure note")
|
||||||
|
assert.NotContains(t, got[0].Excerpt, "proposed_wing", "excerpt is body, not frontmatter")
|
||||||
|
|
||||||
|
assert.Equal(t, "fresh.md", got[1].Filename)
|
||||||
|
assert.Empty(t, got[1].Queue, "a top-level note has no queue label")
|
||||||
|
assert.Empty(t, got[1].ProposedWing)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestListPending_ParkedSymlinkIsNotListed(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
outside := filepath.Join(t.TempDir(), "secret.md")
|
||||||
|
require.NoError(t, os.WriteFile(outside, []byte("outside"), 0o644))
|
||||||
|
require.NoError(t, os.MkdirAll(filepath.Join(dir, "raw", "needs-review"), 0o755))
|
||||||
|
require.NoError(t, os.Symlink(outside, filepath.Join(dir, "raw", "needs-review", "link.md")))
|
||||||
|
|
||||||
|
got, err := ListPending(dir)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Empty(t, got, "same symlink rule as the top level (#153)")
|
||||||
|
}
|
||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"regexp"
|
"regexp"
|
||||||
"sort"
|
"sort"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -20,6 +21,15 @@ type PendingNote struct {
|
|||||||
CreatedAt string `json:"created_at"`
|
CreatedAt string `json:"created_at"`
|
||||||
SizeBytes int64 `json:"size_bytes"`
|
SizeBytes int64 `json:"size_bytes"`
|
||||||
Excerpt string `json:"excerpt"`
|
Excerpt string `json:"excerpt"`
|
||||||
|
|
||||||
|
// Set only for a note parked in raw/needs-review/ by the wing resolver
|
||||||
|
// (hyperguild#205): it was not placed automatically, and the proposal says
|
||||||
|
// what the resolver would have done and why it did not. Approve by moving
|
||||||
|
// the note into raw/<wing>/ and the watcher ingests it there.
|
||||||
|
Queue string `json:"queue,omitempty"`
|
||||||
|
ProposedWing string `json:"proposed_wing,omitempty"`
|
||||||
|
ProposalReason string `json:"proposal_reason,omitempty"`
|
||||||
|
ProposalConfidence float64 `json:"proposal_confidence,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// datePrefix matches a leading YYYY-MM-DD- on a raw filename, stripped when
|
// datePrefix matches a leading YYYY-MM-DD- on a raw filename, stripped when
|
||||||
@@ -38,11 +48,32 @@ func ListPending(brainDir string) ([]PendingNote, error) {
|
|||||||
return nil, fmt.Errorf("read raw dir: %w", err)
|
return nil, fmt.Errorf("read raw dir: %w", err)
|
||||||
}
|
}
|
||||||
defer func() { _ = root.Close() }()
|
defer func() { _ = root.Close() }()
|
||||||
|
|
||||||
|
out, err := listNotes(root, "")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// raw/needs-review/ is read through its own root, so a symlinked directory
|
||||||
|
// or note there is refused exactly like at the top level (#153).
|
||||||
|
if sub, serr := root.OpenRoot("needs-review"); serr == nil {
|
||||||
|
defer func() { _ = sub.Close() }()
|
||||||
|
parked, perr := listNotes(sub, "needs-review")
|
||||||
|
if perr != nil {
|
||||||
|
return nil, perr
|
||||||
|
}
|
||||||
|
out = append(out, parked...)
|
||||||
|
}
|
||||||
|
sort.SliceStable(out, func(i, j int) bool { return out[i].CreatedAt < out[j].CreatedAt })
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// listNotes lists the regular .md notes directly under root. queue labels the
|
||||||
|
// notes ("" for the top of raw/) and enables reading a wing proposal.
|
||||||
|
func listNotes(root *os.Root, queue string) ([]PendingNote, error) {
|
||||||
entries, err := fs.ReadDir(root.FS(), ".")
|
entries, err := fs.ReadDir(root.FS(), ".")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("read raw dir: %w", err)
|
return nil, fmt.Errorf("read raw dir: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
out := make([]PendingNote, 0, len(entries))
|
out := make([]PendingNote, 0, len(entries))
|
||||||
for _, e := range entries {
|
for _, e := range entries {
|
||||||
if !e.Type().IsRegular() || !strings.HasSuffix(e.Name(), ".md") || strings.HasPrefix(e.Name(), "tunnel-candidates-") {
|
if !e.Type().IsRegular() || !strings.HasSuffix(e.Name(), ".md") || strings.HasPrefix(e.Name(), "tunnel-candidates-") {
|
||||||
@@ -57,14 +88,20 @@ func ListPending(brainDir string) ([]PendingNote, error) {
|
|||||||
if created == "" {
|
if created == "" {
|
||||||
created = info.ModTime().UTC().Format(time.RFC3339)
|
created = info.ModTime().UTC().Format(time.RFC3339)
|
||||||
}
|
}
|
||||||
out = append(out, PendingNote{
|
n := PendingNote{
|
||||||
Filename: e.Name(),
|
Filename: e.Name(),
|
||||||
CreatedAt: created,
|
CreatedAt: created,
|
||||||
SizeBytes: info.Size(),
|
SizeBytes: info.Size(),
|
||||||
Excerpt: excerpt(body, 200),
|
Excerpt: excerpt(body, 200),
|
||||||
})
|
Queue: queue,
|
||||||
|
}
|
||||||
|
if queue != "" {
|
||||||
|
n.ProposedWing = fm.get("proposed_wing")
|
||||||
|
n.ProposalReason = unquoteScalar(fm.get("proposal_reason"))
|
||||||
|
n.ProposalConfidence, _ = strconv.ParseFloat(fm.get("proposal_confidence"), 64)
|
||||||
|
}
|
||||||
|
out = append(out, n)
|
||||||
}
|
}
|
||||||
sort.SliceStable(out, func(i, j int) bool { return out[i].CreatedAt < out[j].CreatedAt })
|
|
||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -239,3 +276,13 @@ func readRawSource(brainDir, base string) ([]byte, error) {
|
|||||||
}
|
}
|
||||||
return raw, nil
|
return raw, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// unquoteScalar undoes the Go-style double quoting the watcher writes around a
|
||||||
|
// free-text value (strconv.Quote keeps it on one line); a bare value is
|
||||||
|
// returned as is.
|
||||||
|
func unquoteScalar(v string) string {
|
||||||
|
if u, err := strconv.Unquote(v); err == nil {
|
||||||
|
return u
|
||||||
|
}
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
|||||||
@@ -144,3 +144,19 @@ func TestProcessDir_WithoutAResolverTheRetiredLayoutIsUnchanged(t *testing.T) {
|
|||||||
|
|
||||||
assert.True(t, has(brainDir, "wiki/sources/plain.md"))
|
assert.True(t, has(brainDir, "wiki/sources/plain.md"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The reason comes from an LLM and ends up in frontmatter: a newline in it must
|
||||||
|
// not be able to start a new key (PR #147: a title once forged `origin:`).
|
||||||
|
func TestProcessDir_AProposalReasonCannotForgeFrontmatterKeys(t *testing.T) {
|
||||||
|
var llmCalls atomic.Int32
|
||||||
|
clf := &stubClassifier{s: wing.Suggestion{Wing: "radar", Confidence: 0.1, Reason: "x\norigin: trusted\nwing: homelab"}}
|
||||||
|
cfg, brainDir := wingTestCfg(t, wingResolver(clf, stubScorer{"radar": 3}), &llmCalls)
|
||||||
|
put(t, brainDir, "forge.md", "Note.")
|
||||||
|
|
||||||
|
require.Empty(t, processDir(context.Background(), cfg, "2026-10-20"))
|
||||||
|
|
||||||
|
parked, err := os.ReadFile(filepath.Join(brainDir, "raw", "needs-review", "forge.md"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.NotContains(t, string(parked), "\norigin: trusted", "no injected key at line start")
|
||||||
|
assert.NotContains(t, string(parked), "\nwing: homelab")
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user