MaxNoteBytes (default 64 MiB) is checked on the Lstat size before extraction; an oversize note stays in raw/, is reported once and not retried until it changes. The #199 case (marker name over NAME_MAX) is covered by the remember-on-any-failure retire path and now has a test that fails without it. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -27,12 +27,24 @@ type Config struct {
|
||||
Interval time.Duration
|
||||
Pipeline pipeline.Config
|
||||
|
||||
// MaxNoteBytes refuses a note larger than this before anything is read into
|
||||
// memory (hyperguild#198). 0 means defaultMaxNoteBytes.
|
||||
MaxNoteBytes int64
|
||||
|
||||
// Resolver, when set, decides each note's wing (hyperguild#205): explicit
|
||||
// subfolder/frontmatter, else classifier + scorer. A note it cannot place
|
||||
// is parked in raw/needs-review/ for a human. Nil keeps the retired layout.
|
||||
Resolver *wing.Resolver
|
||||
}
|
||||
|
||||
// defaultMaxNoteBytes bounds a single note: extraction and the archive copy both
|
||||
// hold it in memory.
|
||||
const defaultMaxNoteBytes = 64 << 20
|
||||
|
||||
// errTooLarge marks a note refused for its size; like a refused write it is
|
||||
// reported once and not retried until the note changes.
|
||||
var errTooLarge = errors.New("note too large")
|
||||
|
||||
// Start launches the watcher in a background goroutine.
|
||||
// It returns immediately. The watcher stops when ctx is cancelled.
|
||||
func Start(ctx context.Context, cfg Config) {
|
||||
@@ -137,7 +149,7 @@ func processDir(ctx context.Context, cfg Config, date string) []error {
|
||||
}
|
||||
|
||||
if err := processFile(ctx, cfg, root, rawDir, path, date); err != nil {
|
||||
if ierr == nil && errors.Is(err, brainfs.ErrNotRegular) {
|
||||
if ierr == nil && (errors.Is(err, brainfs.ErrNotRegular) || errors.Is(err, errTooLarge)) {
|
||||
rememberRefused(path, info)
|
||||
slog.Error("watcher: note refused; not retried until it changes", "file", filepath.Base(path), "error", err)
|
||||
}
|
||||
@@ -169,6 +181,13 @@ func processFile(ctx context.Context, cfg Config, root *os.Root, rawDir, path, d
|
||||
if err != nil {
|
||||
return fmt.Errorf("stat before ingest: %w", err)
|
||||
}
|
||||
limit := cfg.MaxNoteBytes
|
||||
if limit <= 0 {
|
||||
limit = defaultMaxNoteBytes
|
||||
}
|
||||
if before.Size() > limit {
|
||||
return fmt.Errorf("%s is %d bytes, over the %d byte cap: %w", filename, before.Size(), limit, errTooLarge)
|
||||
}
|
||||
content, err := extract.TextIn(root, rel)
|
||||
if err != nil {
|
||||
return fmt.Errorf("extract text: %w", err)
|
||||
|
||||
Reference in New Issue
Block a user