diff --git a/Taskfile.yml b/Taskfile.yml index 62af499..4fd6585 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -138,6 +138,11 @@ tasks: - golangci-lint run ./... - cd ingestion && golangci-lint run ./... + lint:frontmatter: + desc: Report brain notes whose YAML frontmatter does not parse (BRAIN=path, FIX=1 repairs in place) + cmds: + - cd ingestion && go run ./cmd/frontmatterlint {{if .FIX}}-fix{{end}} {{.BRAIN | default "../../brain"}} + test: cmds: - go test -race -count=1 ./... diff --git a/ingestion/cmd/frontmatterlint/main.go b/ingestion/cmd/frontmatterlint/main.go new file mode 100644 index 0000000..a0c5788 --- /dev/null +++ b/ingestion/cmd/frontmatterlint/main.go @@ -0,0 +1,53 @@ +// Command frontmatterlint reports markdown notes whose YAML frontmatter does +// not parse, and optionally repairs the ones that only need quoting. +// +// go run ./cmd/frontmatterlint ../../brain # report +// go run ./cmd/frontmatterlint -fix ../../brain # repair in place +// +// Exit codes: 0 clean (or everything repaired), 1 findings remain, 2 the scan +// itself failed. +package main + +import ( + "flag" + "fmt" + "os" + + "github.com/mathiasbq/hyperguild/ingestion/internal/frontmatter" +) + +func main() { + fix := flag.Bool("fix", false, "repair files that quoting can rescue, in place") + flag.Parse() + + root := "." + if flag.NArg() > 0 { + root = flag.Arg(0) + } + + findings, err := frontmatter.Scan(root, *fix) + if err != nil { + fmt.Fprintln(os.Stderr, "frontmatterlint:", err) + os.Exit(2) + } + + var unresolved int + for _, f := range findings { + if f.Repaired { + fmt.Printf("repaired %s\n", f.Path) + continue + } + unresolved++ + fmt.Printf("BROKEN %s: %v\n", f.Path, f.Err) + } + + if unresolved > 0 { + fmt.Fprintf(os.Stderr, "\n%d file(s) with unparseable frontmatter", unresolved) + if !*fix { + fmt.Fprint(os.Stderr, " — re-run with -fix") + } + fmt.Fprintln(os.Stderr) + os.Exit(1) + } + fmt.Printf("✓ frontmatter: %d file(s) repaired, none broken\n", len(findings)) +} diff --git a/ingestion/internal/frontmatter/frontmatter.go b/ingestion/internal/frontmatter/frontmatter.go new file mode 100644 index 0000000..0b6f959 --- /dev/null +++ b/ingestion/internal/frontmatter/frontmatter.go @@ -0,0 +1,141 @@ +// Package frontmatter lints and repairs the YAML frontmatter block of a +// markdown note. +// +// Notes written directly — hand-authored, or promoted from brain/raw/ — do not +// pass through pipeline.BuildPages, so nothing quotes their scalars. A title +// containing an unescaped ": " then produces a block that yaml.Unmarshal +// rejects with "mapping values are not allowed here", and every consumer that +// reads frontmatter silently skips the note. +// +// Fix is deliberately conservative: it only adds quoting, never reorders or +// rewrites, and it refuses to return a document that still does not parse. +package frontmatter + +import ( + "bytes" + "fmt" + "regexp" + "strings" + + "gopkg.in/yaml.v3" +) + +const fence = "---" + +// Lint reports whether doc's frontmatter block parses as YAML. +// +// A document with no frontmatter block, or with an unterminated one, is not an +// error: there is nothing claiming to be YAML. +func Lint(doc []byte) error { + b, ok := block(doc) + if !ok { + return nil + } + var v any + if err := yaml.Unmarshal(b, &v); err != nil { + return fmt.Errorf("frontmatter: %w", err) + } + return nil +} + +// Fix quotes unquoted scalars whose value breaks YAML parsing and returns the +// repaired document. +// +// It returns (doc, false, nil) when the document already parses, and +// (doc, false, err) when quoting cannot rescue it — an unfixable document is +// always returned byte-identical, never half-repaired. +func Fix(doc []byte) ([]byte, bool, error) { + if err := Lint(doc); err == nil { + return doc, false, nil + } + + b, ok := block(doc) + if !ok { + // Lint only fails when there is a block, so this is unreachable in + // practice; return the original rather than guess. + return doc, false, nil + } + + fixed := quoteScalars(b) + if bytes.Equal(fixed, b) { + return doc, false, fmt.Errorf("frontmatter: not repairable by quoting: %w", Lint(doc)) + } + + var v any + if err := yaml.Unmarshal(fixed, &v); err != nil { + return doc, false, fmt.Errorf("frontmatter: quoting did not repair the block: %w", err) + } + + _, rest, _ := split(doc) + out := []byte(fence + "\n") + out = append(out, fixed...) + out = append(out, []byte(fence)...) + out = append(out, rest...) + return out, true, nil +} + +// mappingLine matches "key: value" at any indentation, capturing the prefix so +// it can be re-emitted untouched. +// +// Sequence items ("- value") are deliberately not handled: "- foo: bar" is +// valid YAML, so a document containing one does not reach Fix in the first +// place. See the Lint table for why that gap is left open. +var mappingLine = regexp.MustCompile(`^(\s*[A-Za-z_][\w.-]*:[ \t]+)(\S.*)$`) + +// quoteScalars single-quotes every unquoted scalar value in the block that +// contains a colon. Values that are already quoted, or that carry YAML syntax +// this package has no business rewriting, are left alone. +func quoteScalars(block []byte) []byte { + lines := strings.Split(string(block), "\n") + for i, line := range lines { + m := mappingLine.FindStringSubmatch(line) + if m == nil { + continue + } + prefix, value := m[1], m[2] + if !needsQuoting(value) { + continue + } + lines[i] = prefix + "'" + strings.ReplaceAll(value, "'", "''") + "'" + } + return []byte(strings.Join(lines, "\n")) +} + +// needsQuoting reports whether a scalar value must be quoted to parse as the +// string it plainly is. Only the colon case is handled: it is the one this +// package exists for, and every other special case risks rewriting a value +// that was deliberately structured (a nested mapping, an anchor, a number). +func needsQuoting(value string) bool { + if strings.HasPrefix(value, "'") || strings.HasPrefix(value, `"`) { + return false + } + if strings.HasPrefix(value, "|") || strings.HasPrefix(value, ">") { + return false // block scalar; its content is not parsed as YAML + } + if strings.HasPrefix(value, "{") || strings.HasPrefix(value, "[") { + return false // flow collection; quoting would silently make it a string + } + return strings.Contains(value, ": ") || strings.HasSuffix(value, ":") +} + +// block returns the raw YAML between the opening and closing fences. +func block(doc []byte) ([]byte, bool) { + b, _, ok := split(doc) + return b, ok +} + +// split returns the frontmatter block and everything from the closing fence +// onward (the closing fence itself excluded, its trailing newline included). +func split(doc []byte) (block, rest []byte, ok bool) { + s := string(doc) + if !strings.HasPrefix(s, fence+"\n") { + return nil, nil, false + } + body := s[len(fence)+1:] + + idx := strings.Index(body, "\n"+fence) + if idx < 0 { + return nil, nil, false + } + return []byte(body[:idx+1]), []byte(body[idx+1+len(fence):]), true +} diff --git a/ingestion/internal/frontmatter/frontmatter_test.go b/ingestion/internal/frontmatter/frontmatter_test.go new file mode 100644 index 0000000..7edebbf --- /dev/null +++ b/ingestion/internal/frontmatter/frontmatter_test.go @@ -0,0 +1,158 @@ +// ingestion/internal/frontmatter/frontmatter_test.go +package frontmatter + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +func TestLint(t *testing.T) { + tests := []struct { + name string + doc string + wantErr bool + }{ + { + name: "no frontmatter block is not an error", + doc: "# Just a heading\n\nSome body text.\n", + }, + { + name: "quoted title parses", + doc: "---\ntitle: 'Headless dispatch runs: Bash is denied'\n---\n\nbody\n", + }, + { + name: "plain title parses", + doc: "---\ntitle: A simple title\nwing: homelab\n---\n\nbody\n", + }, + { + name: "unquoted colon in title does not parse", + doc: "---\ntitle: Headless dispatch runs: Bash is denied — use gitea MCP\nwing: homelab\n---\n\nbody\n", + wantErr: true, + }, + { + // Known gap, deliberately not fixed here: this parses, but the + // item becomes map[string]any{"Headless dispatch": "denied"} + // rather than a string. It is a silent type corruption, not a + // parse failure, so repairing it would rewrite documents that + // currently load — out of scope for a lint about broken YAML. + name: "unquoted colon in a list item parses as a mapping, not an error", + doc: "---\ntitle: Fine\naliases:\n - Headless dispatch: denied\n---\n\nbody\n", + }, + { + name: "unterminated frontmatter is not treated as frontmatter", + doc: "---\ntitle: no closing fence\n\nbody\n", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := Lint([]byte(tt.doc)) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + }) + } +} + +func TestFix(t *testing.T) { + tests := []struct { + name string + doc string + wantChanged bool + wantKey string + wantValue string + }{ + { + name: "quotes an unquoted title containing a colon", + doc: "---\ntitle: Headless dispatch runs: Bash is denied — use gitea MCP\nwing: homelab\n---\n\nbody\n", + wantChanged: true, + wantKey: "title", + wantValue: "Headless dispatch runs: Bash is denied — use gitea MCP", + }, + { + // Parses already (see the Lint table), so Fix must leave it alone + // rather than rewrite a document that loads. + name: "leaves a colon-bearing list item alone because it already parses", + doc: "---\ntitle: Fine\naliases:\n - Headless dispatch: denied\n---\n\nbody\n", + wantChanged: false, + wantKey: "title", + wantValue: "Fine", + }, + { + name: "escapes an embedded single quote when quoting", + doc: "---\ntitle: Don't do this: it breaks\n---\n\nbody\n", + wantChanged: true, + wantKey: "title", + wantValue: "Don't do this: it breaks", + }, + { + name: "leaves an already-valid document untouched", + doc: "---\ntitle: 'Already quoted: fine'\nwing: homelab\n---\n\nbody\n", + wantChanged: false, + wantKey: "title", + wantValue: "Already quoted: fine", + }, + { + name: "leaves a document with no frontmatter untouched", + doc: "# Heading\n\nbody: with a colon\n", + wantChanged: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, changed, err := Fix([]byte(tt.doc)) + require.NoError(t, err) + assert.Equal(t, tt.wantChanged, changed) + + if !tt.wantChanged { + assert.Equal(t, tt.doc, string(got), "unchanged docs must be returned byte-identical") + } + + // Whatever came back must parse — that is the whole point. + require.NoError(t, Lint(got), "Fix must not return a document that still fails to parse") + + if tt.wantKey == "" { + return + } + block, ok := block(got) + require.True(t, ok) + var m map[string]any + require.NoError(t, yaml.Unmarshal(block, &m)) + assert.Equal(t, tt.wantValue, m[tt.wantKey], "the value must survive quoting unchanged") + }) + } +} + +func TestFix_PreservesBodyExactly(t *testing.T) { + body := "\nThe body has: colons, 'quotes', and\n---\na horizontal rule.\n" + doc := "---\ntitle: Broken: title\n---" + body + + got, changed, err := Fix([]byte(doc)) + require.NoError(t, err) + require.True(t, changed) + assert.True(t, len(got) > 0) + + _, rest, ok := split([]byte(doc)) + require.True(t, ok) + _, gotRest, ok := split(got) + require.True(t, ok) + assert.Equal(t, string(rest), string(gotRest), "the body must not be touched") +} + +func TestFix_RefusesToReturnAnUnparseableResult(t *testing.T) { + // A frontmatter block that quoting cannot rescue: the structure itself is + // broken, not a scalar. Fix must return the original and an error rather + // than a half-repaired document. + doc := "---\ntitle: fine\n\tbad: tab-indented mapping\n - stray list item\n---\n\nbody\n" + + got, changed, err := Fix([]byte(doc)) + require.Error(t, err) + assert.False(t, changed) + assert.Equal(t, doc, string(got), "an unfixable document must come back byte-identical") +} diff --git a/ingestion/internal/frontmatter/scan.go b/ingestion/internal/frontmatter/scan.go new file mode 100644 index 0000000..06192a4 --- /dev/null +++ b/ingestion/internal/frontmatter/scan.go @@ -0,0 +1,84 @@ +// ingestion/internal/frontmatter/scan.go +package frontmatter + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + "unicode/utf8" +) + +// Finding is one markdown file whose frontmatter does not parse. +type Finding struct { + Path string // relative to the scanned root + Err error // why it does not parse, or why the repair was refused + Repaired bool // true only when fix was requested and the file was rewritten +} + +// Scan walks root for *.md files whose frontmatter fails to parse. +// +// With fix set, each finding that quoting can rescue is rewritten in place; +// anything quoting cannot rescue is reported and left byte-identical. A file +// that cannot be read or decoded is reported as a finding rather than aborting +// the walk — one poisoned note must not hide the other findings. +func Scan(root string, fix bool) ([]Finding, error) { + var found []Finding + + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.EqualFold(filepath.Ext(path), ".md") { + return nil + } + + rel, relErr := filepath.Rel(root, path) + if relErr != nil { + rel = path + } + + doc, readErr := os.ReadFile(path) + if readErr != nil { + found = append(found, Finding{Path: rel, Err: readErr}) + return nil + } + if !utf8.Valid(doc) { + found = append(found, Finding{Path: rel, Err: fmt.Errorf("not valid UTF-8")}) + return nil + } + if lintErr := Lint(doc); lintErr == nil { + return nil + } + + if !fix { + found = append(found, Finding{Path: rel, Err: Lint(doc)}) + return nil + } + + repaired, changed, fixErr := Fix(doc) + if fixErr != nil || !changed { + found = append(found, Finding{Path: rel, Err: fixErr}) + return nil + } + if writeErr := writeFilePreservingMode(path, repaired); writeErr != nil { + found = append(found, Finding{Path: rel, Err: writeErr}) + return nil + } + found = append(found, Finding{Path: rel, Repaired: true}) + return nil + }) + if err != nil { + return nil, fmt.Errorf("scan %s: %w", root, err) + } + return found, nil +} + +func writeFilePreservingMode(path string, content []byte) error { + mode := fs.FileMode(0o644) + if info, err := os.Stat(path); err == nil { + mode = info.Mode().Perm() + } + return os.WriteFile(path, content, mode) +} diff --git a/ingestion/internal/frontmatter/scan_test.go b/ingestion/internal/frontmatter/scan_test.go new file mode 100644 index 0000000..92e0849 --- /dev/null +++ b/ingestion/internal/frontmatter/scan_test.go @@ -0,0 +1,103 @@ +// ingestion/internal/frontmatter/scan_test.go +package frontmatter + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func writeTree(t *testing.T, files map[string]string) string { + t.Helper() + root := t.TempDir() + for name, content := range files { + path := filepath.Join(root, name) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) + } + return root +} + +const ( + brokenDoc = "---\ntitle: Broken: title here\nwing: homelab\n---\n\nbody\n" + validDoc = "---\ntitle: 'Fine: quoted'\n---\n\nbody\n" +) + +func TestScan_ReportsOnlyBrokenFiles(t *testing.T) { + root := writeTree(t, map[string]string{ + "wiki/a/broken.md": brokenDoc, + "wiki/a/valid.md": validDoc, + "wiki/b/plain.md": "# no frontmatter\n\nbody: with colon\n", + "wiki/a/notes.txt": brokenDoc, // not markdown — must be ignored + }) + + found, err := Scan(root, false) + require.NoError(t, err) + + require.Len(t, found, 1) + assert.Equal(t, "wiki/a/broken.md", found[0].Path) + assert.False(t, found[0].Repaired) + assert.Error(t, found[0].Err) + + // Dry run must not touch the file. + got, err := os.ReadFile(filepath.Join(root, "wiki/a/broken.md")) + require.NoError(t, err) + assert.Equal(t, brokenDoc, string(got)) +} + +func TestScan_FixWritesRepairedFilesAndLeavesTheRestAlone(t *testing.T) { + root := writeTree(t, map[string]string{ + "wiki/a/broken.md": brokenDoc, + "wiki/a/valid.md": validDoc, + }) + + found, err := Scan(root, true) + require.NoError(t, err) + require.Len(t, found, 1) + assert.True(t, found[0].Repaired) + assert.NoError(t, found[0].Err) + + repaired, err := os.ReadFile(filepath.Join(root, "wiki/a/broken.md")) + require.NoError(t, err) + require.NoError(t, Lint(repaired), "the repaired file must parse") + assert.Contains(t, string(repaired), "title: 'Broken: title here'") + assert.Contains(t, string(repaired), "wing: homelab", "untouched keys must survive") + + untouched, err := os.ReadFile(filepath.Join(root, "wiki/a/valid.md")) + require.NoError(t, err) + assert.Equal(t, validDoc, string(untouched)) + + // Second pass finds nothing — the repair is idempotent and complete. + again, err := Scan(root, true) + require.NoError(t, err) + assert.Empty(t, again) +} + +func TestScan_ReportsUnfixableWithoutWriting(t *testing.T) { + unfixable := "---\ntitle: fine\n\tbad: tab-indented\n - stray\n---\n\nbody\n" + root := writeTree(t, map[string]string{"wiki/x.md": unfixable}) + + found, err := Scan(root, true) + require.NoError(t, err) + require.Len(t, found, 1) + assert.False(t, found[0].Repaired) + assert.Error(t, found[0].Err) + + got, err := os.ReadFile(filepath.Join(root, "wiki/x.md")) + require.NoError(t, err) + assert.Equal(t, unfixable, string(got), "an unfixable file must be left exactly as it was") +} + +func TestScan_ReportsUndecodableFilesInsteadOfFailingTheRun(t *testing.T) { + root := writeTree(t, map[string]string{"wiki/ok.md": validDoc}) + require.NoError(t, os.WriteFile(filepath.Join(root, "wiki/bad-bytes.md"), + []byte{'-', '-', '-', '\n', 0xff, 0xfe, '\n', '-', '-', '-', '\n'}, 0o644)) + + found, err := Scan(root, false) + require.NoError(t, err, "one unreadable file must not abort the whole scan") + require.Len(t, found, 1) + assert.Equal(t, "wiki/bad-bytes.md", found[0].Path) +}