feat(watcher): resolve each note's wing; park the unplaceable in raw/needs-review/ (#205 S3)
With Config.Resolver set, an explicit subfolder/frontmatter wing or an agreeing classifier+scorer routes the note into that wing; anything else is parked with its proposal in frontmatter and never extracted. needs-review/ is skipped by the walk like rejected/. Nil Resolver keeps the retired layout. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -15,6 +15,7 @@ import (
|
||||
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/extract"
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/pipeline"
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/wing"
|
||||
)
|
||||
|
||||
// Config holds watcher configuration.
|
||||
@@ -27,6 +28,11 @@ type Config struct {
|
||||
// from raw/ nor marked (e.g. a read-only raw/), so later polls skip them
|
||||
// instead of ingesting them again. In memory only: a restart retries once.
|
||||
stuck *sync.Map
|
||||
|
||||
// Resolver, when set, decides each note's wing (hyperguild#205): explicit
|
||||
// subfolder/frontmatter, else classifier + scorer. A note it cannot place
|
||||
// is parked in raw/needs-review/ for a human. Nil keeps the retired layout.
|
||||
Resolver *wing.Resolver
|
||||
}
|
||||
|
||||
// Start launches the watcher in a background goroutine.
|
||||
@@ -94,7 +100,7 @@ func processDir(ctx context.Context, cfg Config, date string) []error {
|
||||
// path that treats raw/ as "awaiting processing" — this is that path.
|
||||
if d.IsDir() {
|
||||
name := d.Name()
|
||||
if name == "processed" || name == "failed" || name == "rejected" {
|
||||
if name == "processed" || name == "failed" || name == "rejected" || name == "needs-review" {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
@@ -165,7 +171,16 @@ func processFile(ctx context.Context, cfg Config, root *os.Root, rawDir, path, d
|
||||
return fmt.Errorf("extract text: %w", err)
|
||||
}
|
||||
|
||||
_, runErr := pipeline.Run(ctx, cfg.Pipeline, cfg.BrainDir, content, source, false)
|
||||
pcfg := cfg.Pipeline
|
||||
if cfg.Resolver != nil {
|
||||
d := cfg.Resolver.Resolve(ctx, wing.Note{Dir: topDir(rel), FrontmatterWing: frontmatterWing(content), Text: content})
|
||||
if d.Source == wing.Exception {
|
||||
return parkForReview(cfg, root, rawDir, rel, path, filename, content, d, date)
|
||||
}
|
||||
pcfg.Wing = d.Wing
|
||||
slog.Info("watcher: wing resolved", "file", filename, "wing", d.Wing, "how", d.Source, "reason", d.Reason)
|
||||
}
|
||||
_, runErr := pipeline.Run(ctx, pcfg, cfg.BrainDir, content, source, false)
|
||||
if runErr != nil {
|
||||
// Copy to failed/ and leave a .failed marker so we don't retry.
|
||||
failedDir := filepath.Join(cfg.BrainDir, "raw", "failed")
|
||||
@@ -308,3 +323,65 @@ func copyFromRoot(root *os.Root, rel, dst string) error {
|
||||
}
|
||||
return out.Close()
|
||||
}
|
||||
|
||||
// topDir is the first directory of a raw/-relative path, "" for a top-level file.
|
||||
func topDir(rel string) string {
|
||||
if i := strings.IndexByte(filepath.ToSlash(rel), '/'); i > 0 {
|
||||
return filepath.ToSlash(rel)[:i]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// frontmatterWing reads `wing:` from a note's leading frontmatter, "" if none.
|
||||
func frontmatterWing(content string) string {
|
||||
if !strings.HasPrefix(content, "---\n") {
|
||||
return ""
|
||||
}
|
||||
block, _, _ := strings.Cut(content[4:], "\n---")
|
||||
for _, line := range strings.Split(block, "\n") {
|
||||
if key, val, ok := strings.Cut(line, ":"); ok && strings.TrimSpace(key) == "wing" {
|
||||
return strings.Trim(strings.TrimSpace(val), `"'`)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// parkForReview moves a note the resolver could not place into
|
||||
// raw/needs-review/, recording the proposal in its frontmatter so a human can
|
||||
// approve it (brain_promote). Nothing is extracted or written to the wiki. The
|
||||
// original is removed only after the parked copy is written.
|
||||
func parkForReview(cfg Config, root *os.Root, rawDir, rel, path, filename, content string, d wing.Decision, date string) error {
|
||||
dir := filepath.Join(rawDir, "needs-review")
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return fmt.Errorf("mkdir needs-review: %w", err)
|
||||
}
|
||||
dest := filepath.Join(dir, filename)
|
||||
if _, err := os.Stat(dest); err == nil {
|
||||
ext := filepath.Ext(filename)
|
||||
dest = filepath.Join(dir, strings.TrimSuffix(filename, ext)+"-"+time.Now().UTC().Format("150405")+ext)
|
||||
}
|
||||
if strings.ToLower(filepath.Ext(filename)) == ".pdf" {
|
||||
// Binary: park the bytes untouched; the proposal is in log.md only.
|
||||
if err := copyFromRoot(root, rel, dest); err != nil {
|
||||
return fmt.Errorf("park pdf: %w", err)
|
||||
}
|
||||
} else if err := os.WriteFile(dest, []byte(withProposal(content, d)), 0o644); err != nil {
|
||||
return fmt.Errorf("park note: %w", err)
|
||||
}
|
||||
if err := appendWatcherLog(cfg.BrainDir, filename, fmt.Errorf("needs review (proposed wing %q): %s", d.Wing, d.Reason), date); err != nil {
|
||||
slog.Error("watcher: failed to write log entry", "error", err)
|
||||
}
|
||||
slog.Info("watcher: parked for review", "file", filename, "proposed_wing", d.Wing, "reason", d.Reason)
|
||||
retire(cfg, root, rel, path, ".processed", date)
|
||||
return nil
|
||||
}
|
||||
|
||||
// withProposal adds the resolver's proposal to a note's frontmatter, creating
|
||||
// the block when the note has none.
|
||||
func withProposal(content string, d wing.Decision) string {
|
||||
kv := fmt.Sprintf("proposed_wing: %s\nproposal_confidence: %.2f\nproposal_reason: %q\n", d.Wing, d.Confidence, d.Reason)
|
||||
if strings.HasPrefix(content, "---\n") {
|
||||
return "---\n" + kv + content[4:]
|
||||
}
|
||||
return "---\n" + kv + "---\n" + content
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user