mathiasandClaude Opus 5.5
8a91feb02b
raw/ is writable via Syncthing (a sendreceive folder shared with flamingo, symlinks synced) and by the brain CronJobs, and several readers followed symlinks out of it: - /pending and brain_pending showed a symlink target's contents; brain_promote would copy it into wiki/ (then brain-sync to Gitea); brain_reject would copy it into raw/rejected/. - The watcher, which needs no API call at all, ingested symlinked files (WalkDir yields them; extract.Text followed them). Not in the issue; found while fixing it. - Ingest checked a path then reopened it by name (TOCTOU). OpenRawRoot opens brain/raw as an os.Root and refuses a raw/ that is itself a symlink. ListPending, PromoteNote and RejectNote read through it and refuse anything not a regular file. extract.TextIn reads through a root (temp copy for pdf/docmark); /ingest-path, brain_ingest and the watcher use it, and the watcher archives via the root too, so a file swapped for a symlink after a check still cannot lead out. Refs #153 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
90 lines
2.5 KiB
Go
90 lines
2.5 KiB
Go
// ingestion/internal/extract/extract.go
|
|
package extract
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"strings"
|
|
)
|
|
|
|
// Text reads the file at path and returns its plain-text content.
|
|
// Supported extensions: .md, .txt (passthrough), .pdf (via pdftotext),
|
|
// .docx/.xlsx/.pptx/.png/.jpg/.jpeg (via docmark, ADR-0013 -- requires
|
|
// DOCMARK_URL to be set; see docmark.go).
|
|
func Text(path string) (string, error) {
|
|
ext := strings.ToLower(fileExt(path))
|
|
switch ext {
|
|
case ".md", ".txt":
|
|
b, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return "", fmt.Errorf("read %s: %w", path, err)
|
|
}
|
|
return string(b), nil
|
|
case ".pdf":
|
|
return extractPDF(path)
|
|
case ".docx", ".xlsx", ".pptx", ".png", ".jpg", ".jpeg":
|
|
return extractViaDocmark(path)
|
|
default:
|
|
return "", fmt.Errorf("unsupported file extension: %s", ext)
|
|
}
|
|
}
|
|
|
|
// fileExt returns the file extension including the dot, lowercased.
|
|
func fileExt(path string) string {
|
|
for i := len(path) - 1; i >= 0; i-- {
|
|
if path[i] == '.' {
|
|
return path[i:]
|
|
}
|
|
if path[i] == '/' || path[i] == '\\' {
|
|
break
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// TextIn is Text for a file read through an os.Root (hyperguild#153): the
|
|
// read cannot leave the root, so a path checked earlier cannot be swapped
|
|
// for a symlink pointing elsewhere before it is read. Formats that need an
|
|
// external tool (pdf, docmark) are copied to a temp file from the handle
|
|
// opened through the root, then extracted from the copy.
|
|
func TextIn(r *os.Root, rel string) (string, error) {
|
|
fi, err := r.Lstat(rel)
|
|
if err != nil {
|
|
return "", fmt.Errorf("stat %s: %w", rel, err)
|
|
}
|
|
if !fi.Mode().IsRegular() {
|
|
return "", fmt.Errorf("%s: not a regular file", rel)
|
|
}
|
|
f, err := r.Open(rel)
|
|
if err != nil {
|
|
return "", fmt.Errorf("open %s: %w", rel, err)
|
|
}
|
|
defer func() { _ = f.Close() }()
|
|
ext := strings.ToLower(fileExt(rel))
|
|
switch ext {
|
|
case ".md", ".txt":
|
|
b, err := io.ReadAll(f)
|
|
if err != nil {
|
|
return "", fmt.Errorf("read %s: %w", rel, err)
|
|
}
|
|
return string(b), nil
|
|
case ".pdf", ".docx", ".xlsx", ".pptx", ".png", ".jpg", ".jpeg":
|
|
tmp, err := os.CreateTemp("", "ingest-*"+ext)
|
|
if err != nil {
|
|
return "", fmt.Errorf("temp copy: %w", err)
|
|
}
|
|
defer func() { _ = os.Remove(tmp.Name()) }()
|
|
if _, err := io.Copy(tmp, f); err != nil {
|
|
_ = tmp.Close()
|
|
return "", fmt.Errorf("copy %s: %w", rel, err)
|
|
}
|
|
if err := tmp.Close(); err != nil {
|
|
return "", fmt.Errorf("copy %s: %w", rel, err)
|
|
}
|
|
return Text(tmp.Name())
|
|
default:
|
|
return "", fmt.Errorf("unsupported file extension: %s", ext)
|
|
}
|
|
}
|