mathiasandClaude Opus 5.5
87c80e4e82
- Reverting the ingest call sites to extract.Text(path) stayed green; a test now fails if anything outside internal/extract calls extract.Text( (only TextIn, through an os.Root, may be used). - The watcher symlink test passed for a watcher that ingested nothing, or stopped at the first symlink (SkipAll: one synced link halting all ingestion). It now plants a real note after the symlink and asserts exactly one ingest, and the symlinked-raw refusal has its own test. - A FIFO swapped in between Lstat and Open hung open(2) forever. extract.OpenRegular opens non-blocking and checks the type on the handle; readRawNote, TextIn and the watcher's archive copy use it. - The symlinked-root test covers pending, promote, reject, resolve and ReadIngestFile, and is named for exactly those. - The hard-link residual is documented where it is accepted. Out of scope, filed: #157 (symlinks elsewhere in the Syncthing-shared brain dir reach search and other readers), #158 (writes under raw/ follow symlinks). Refs #153 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
113 lines
3.2 KiB
Go
113 lines
3.2 KiB
Go
// ingestion/internal/extract/extract.go
|
|
package extract
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"strings"
|
|
"syscall"
|
|
)
|
|
|
|
// Text reads the file at path and returns its plain-text content.
|
|
// Supported extensions: .md, .txt (passthrough), .pdf (via pdftotext),
|
|
// .docx/.xlsx/.pptx/.png/.jpg/.jpeg (via docmark, ADR-0013 -- requires
|
|
// DOCMARK_URL to be set; see docmark.go).
|
|
func Text(path string) (string, error) {
|
|
ext := strings.ToLower(fileExt(path))
|
|
switch ext {
|
|
case ".md", ".txt":
|
|
b, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return "", fmt.Errorf("read %s: %w", path, err)
|
|
}
|
|
return string(b), nil
|
|
case ".pdf":
|
|
return extractPDF(path)
|
|
case ".docx", ".xlsx", ".pptx", ".png", ".jpg", ".jpeg":
|
|
return extractViaDocmark(path)
|
|
default:
|
|
return "", fmt.Errorf("unsupported file extension: %s", ext)
|
|
}
|
|
}
|
|
|
|
// fileExt returns the file extension including the dot, lowercased.
|
|
func fileExt(path string) string {
|
|
for i := len(path) - 1; i >= 0; i-- {
|
|
if path[i] == '.' {
|
|
return path[i:]
|
|
}
|
|
if path[i] == '/' || path[i] == '\\' {
|
|
break
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// TextIn is Text for a file read through an os.Root (hyperguild#153): the
|
|
// read cannot leave the root, so a path checked earlier cannot be swapped
|
|
// for a symlink pointing elsewhere before it is read. Formats that need an
|
|
// external tool (pdf, docmark) are copied to a temp file from the handle
|
|
// opened through the root, then extracted from the copy.
|
|
func TextIn(r *os.Root, rel string) (string, error) {
|
|
fi, err := r.Lstat(rel)
|
|
if err != nil {
|
|
return "", fmt.Errorf("stat %s: %w", rel, err)
|
|
}
|
|
if !fi.Mode().IsRegular() {
|
|
return "", fmt.Errorf("%s: not a regular file", rel)
|
|
}
|
|
f, _, err := OpenRegular(r, rel)
|
|
if err != nil {
|
|
return "", fmt.Errorf("open %s: %w", rel, err)
|
|
}
|
|
defer func() { _ = f.Close() }()
|
|
ext := strings.ToLower(fileExt(rel))
|
|
switch ext {
|
|
case ".md", ".txt":
|
|
b, err := io.ReadAll(f)
|
|
if err != nil {
|
|
return "", fmt.Errorf("read %s: %w", rel, err)
|
|
}
|
|
return string(b), nil
|
|
case ".pdf", ".docx", ".xlsx", ".pptx", ".png", ".jpg", ".jpeg":
|
|
tmp, err := os.CreateTemp("", "ingest-*"+ext)
|
|
if err != nil {
|
|
return "", fmt.Errorf("temp copy: %w", err)
|
|
}
|
|
defer func() { _ = os.Remove(tmp.Name()) }()
|
|
if _, err := io.Copy(tmp, f); err != nil {
|
|
_ = tmp.Close()
|
|
return "", fmt.Errorf("copy %s: %w", rel, err)
|
|
}
|
|
if err := tmp.Close(); err != nil {
|
|
return "", fmt.Errorf("copy %s: %w", rel, err)
|
|
}
|
|
return Text(tmp.Name())
|
|
default:
|
|
return "", fmt.Errorf("unsupported file extension: %s", ext)
|
|
}
|
|
}
|
|
|
|
// OpenRegular opens rel inside r and returns it only if the OPENED handle is
|
|
// a regular file. The open is non-blocking, so a FIFO swapped in after an
|
|
// Lstat (PR #156 review: it hung open(2) forever) is refused promptly rather
|
|
// than stalling the reader. Callers keep their Lstat check, which refuses
|
|
// symlinks outright; this closes the race between that check and the open.
|
|
func OpenRegular(r *os.Root, rel string) (*os.File, os.FileInfo, error) {
|
|
f, err := r.OpenFile(rel, os.O_RDONLY|syscall.O_NONBLOCK, 0)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
fi, err := f.Stat()
|
|
if err != nil {
|
|
_ = f.Close()
|
|
return nil, nil, err
|
|
}
|
|
if !fi.Mode().IsRegular() {
|
|
_ = f.Close()
|
|
return nil, nil, fmt.Errorf("%s: not a regular file", rel)
|
|
}
|
|
return f, fi, nil
|
|
}
|