fix(watcher): unique archive copies, read caps at the read, stale markers (review of #210)
From the independent review of #210: - F1: processed/ and failed/ copies are created O_EXCL with -2,-3 suffixes via createUnique; same-named notes from different folders no longer replace one another's only surviving copy - F2: extract.TextInMax / readAllMax cap the bytes actually read (a note can grow after the Lstat); extract.TextIn gets a 64 MiB default cap - F3: a regular marker older than its note describes an earlier note of the same name; it no longer hides the new one (warn + remove); non-regular markers still count as done (no writes through links) - F4: the planted-marker-link test is renamed to what it observes Each has a test that fails without it; the read-cap test uses an endless reader that errors at 8 MiB so a regression fails instead of exhausting memory. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -2,6 +2,7 @@
|
||||
package extract
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
@@ -50,6 +51,18 @@ func fileExt(path string) string {
|
||||
// external tool (pdf, docmark) are copied to a temp file from the handle
|
||||
// opened through the root, then extracted from the copy.
|
||||
func TextIn(r *os.Root, rel string) (string, error) {
|
||||
return TextInMax(r, rel, DefaultMaxBytes)
|
||||
}
|
||||
|
||||
// DefaultMaxBytes bounds what TextIn reads or copies from one file.
|
||||
const DefaultMaxBytes = 64 << 20
|
||||
|
||||
// ErrTooLarge reports a file that exceeded the read cap while being read: the
|
||||
// size seen at an earlier Lstat is not a bound, the file may have grown.
|
||||
var ErrTooLarge = errors.New("file exceeds the read cap")
|
||||
|
||||
// TextInMax is TextIn with an explicit cap on the bytes read (hyperguild#198).
|
||||
func TextInMax(r *os.Root, rel string, max int64) (string, error) {
|
||||
fi, err := r.Lstat(rel)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("stat %s: %w", rel, err)
|
||||
@@ -65,10 +78,13 @@ func TextIn(r *os.Root, rel string) (string, error) {
|
||||
ext := strings.ToLower(fileExt(rel))
|
||||
switch ext {
|
||||
case ".md", ".txt":
|
||||
b, err := io.ReadAll(f)
|
||||
b, err := io.ReadAll(io.LimitReader(f, max+1))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read %s: %w", rel, err)
|
||||
}
|
||||
if int64(len(b)) > max {
|
||||
return "", fmt.Errorf("%s: %w (%d bytes)", rel, ErrTooLarge, max)
|
||||
}
|
||||
return string(b), nil
|
||||
case ".pdf", ".docx", ".xlsx", ".pptx", ".png", ".jpg", ".jpeg":
|
||||
tmp, err := os.CreateTemp("", "ingest-*"+ext)
|
||||
@@ -76,10 +92,15 @@ func TextIn(r *os.Root, rel string) (string, error) {
|
||||
return "", fmt.Errorf("temp copy: %w", err)
|
||||
}
|
||||
defer func() { _ = os.Remove(tmp.Name()) }()
|
||||
if _, err := io.Copy(tmp, f); err != nil {
|
||||
n, err := io.Copy(tmp, io.LimitReader(f, max+1))
|
||||
if err != nil {
|
||||
_ = tmp.Close()
|
||||
return "", fmt.Errorf("copy %s: %w", rel, err)
|
||||
}
|
||||
if n > max {
|
||||
_ = tmp.Close()
|
||||
return "", fmt.Errorf("%s: %w (%d bytes)", rel, ErrTooLarge, max)
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return "", fmt.Errorf("copy %s: %w", rel, err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user