fix(watcher): refusals end in a terminal state; pin every converted writer (#158)
Review round 1 on PR #195: - A refused marker or archive write left no marker, so the note was reprocessed every interval (archive copies piling up; an LLM call per round on the failure path). The marker check now uses Lstat (any entry at the marker path counts, so a dangling marker link skips the note instead of looping), and a note whose archive copy or marker write is refused is remembered (path, size, mtime) and not retried until it changes. - cleanRel no longer refuses a backslash: it is a legal filename byte on Linux, not a traversal, and a raw note named with one was never archived or marked. - Tests now reach every converted writer: the log.md append (real raw/failed so the log is reached, symlinked log.md), a dangling marker link, the refused-retry loop with a positive control, a backslash name, a buildAndWrite swap race, and the claude-session archive with in-brain and outside symlinked archive dirs. The stale lint allowlist entry for appendWatcherLog is removed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -3,12 +3,14 @@ package watcher
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
@@ -110,14 +112,28 @@ func processDir(ctx context.Context, cfg Config, date string) []error {
|
||||
}
|
||||
|
||||
// Skip files that have already been processed or permanently failed.
|
||||
if _, err := os.Stat(path + ".processed"); err == nil {
|
||||
// Any entry at the marker path counts (Lstat, not Stat): a dangling
|
||||
// marker symlink made os.Stat report nothing and the note was
|
||||
// reprocessed every interval (hyperguild#158).
|
||||
if _, err := os.Lstat(path + ".processed"); err == nil {
|
||||
return nil
|
||||
}
|
||||
if _, err := os.Stat(path + ".failed"); err == nil {
|
||||
if _, err := os.Lstat(path + ".failed"); err == nil {
|
||||
return nil
|
||||
}
|
||||
// A note whose archive copy or marker was refused is not retried
|
||||
// until it changes: a refusal has no marker to stop the next round,
|
||||
// and each round can cost an LLM call (hyperguild#158).
|
||||
info, ierr := d.Info()
|
||||
if ierr == nil && refusedUnchanged(path, info) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := processFile(ctx, cfg, root, rawDir, path, date); err != nil {
|
||||
if ierr == nil && errors.Is(err, brainfs.ErrNotRegular) {
|
||||
rememberRefused(path, info)
|
||||
slog.Error("watcher: note refused; not retried until it changes", "file", filepath.Base(path), "error", err)
|
||||
}
|
||||
errs = append(errs, fmt.Errorf("process %s: %w", filepath.Base(path), err))
|
||||
}
|
||||
return nil
|
||||
@@ -152,6 +168,9 @@ func processFile(ctx context.Context, cfg Config, root *os.Root, rawDir, path, d
|
||||
return fmt.Errorf("copy to failed: %w", cpErr)
|
||||
}
|
||||
if mkErr := putBrainFile(cfg.BrainDir, "raw/"+filepath.ToSlash(rel)+".failed", []byte(runErr.Error())); mkErr != nil {
|
||||
if errors.Is(mkErr, brainfs.ErrNotRegular) {
|
||||
return fmt.Errorf("write .failed marker: %w", mkErr)
|
||||
}
|
||||
slog.Error("watcher: failed to write .failed marker", "error", mkErr)
|
||||
}
|
||||
|
||||
@@ -177,6 +196,9 @@ func processFile(ctx context.Context, cfg Config, root *os.Root, rawDir, path, d
|
||||
return fmt.Errorf("copy to processed: %w", err)
|
||||
}
|
||||
if err := putBrainFile(cfg.BrainDir, "raw/"+filepath.ToSlash(rel)+".processed", []byte(date)); err != nil {
|
||||
if errors.Is(err, brainfs.ErrNotRegular) {
|
||||
return fmt.Errorf("write .processed marker: %w", err)
|
||||
}
|
||||
slog.Error("watcher: failed to write .processed marker", "error", err)
|
||||
}
|
||||
|
||||
@@ -184,6 +206,32 @@ func processFile(ctx context.Context, cfg Config, root *os.Root, rawDir, path, d
|
||||
return nil
|
||||
}
|
||||
|
||||
// refused remembers notes whose archive copy or marker write was refused,
|
||||
// by path, with the size and mtime they had, for the life of the process.
|
||||
var refused sync.Map
|
||||
|
||||
type refusedStamp struct {
|
||||
size int64
|
||||
mtime time.Time
|
||||
}
|
||||
|
||||
func rememberRefused(path string, info os.FileInfo) {
|
||||
refused.Store(path, refusedStamp{size: info.Size(), mtime: info.ModTime()})
|
||||
}
|
||||
|
||||
func refusedUnchanged(path string, info os.FileInfo) bool {
|
||||
v, ok := refused.Load(path)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
st := v.(refusedStamp)
|
||||
if st.size == info.Size() && st.mtime.Equal(info.ModTime()) {
|
||||
return true
|
||||
}
|
||||
refused.Delete(path)
|
||||
return false
|
||||
}
|
||||
|
||||
// deriveSource turns a filename into a human-readable source name.
|
||||
// "shape-up-book.md" → "Shape Up Book"
|
||||
func deriveSource(filename string) string {
|
||||
|
||||
Reference in New Issue
Block a user