feat(ingestion): lint and repair unparseable note frontmatter (#90)

Hand-authored and promoted notes skip pipeline.BuildPages, so nothing
quotes their scalars. A title carrying an unescaped ": " yields a block
yaml.Unmarshal rejects, and every consumer that reads frontmatter drops
the note silently — 18 files repo-wide today, up from the 8 in #90.

frontmatter.Lint reports it; frontmatter.Fix quotes only what quoting can
rescue and refuses to return a document that still fails to parse, so an
unfixable file comes back byte-identical rather than half-repaired. Scan
walks a tree and reports an unreadable file as a finding instead of
aborting, so one poisoned note cannot hide the rest.

Sequence items are deliberately left alone: "- foo: bar" is valid YAML
that silently becomes a map, which is a type corruption rather than a
parse failure, and repairing it would rewrite notes that currently load.

    task lint:frontmatter          # report
    task lint:frontmatter FIX=1    # repair in place

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BeAp5LTscnz2W7Ubt4eJ6m

Taskfile.yml

@@ -138,6 +138,11 @@ tasks:
       - golangci-lint run ./...
       - cd ingestion && golangci-lint run ./...
 
+  lint:frontmatter:
+    desc: Report brain notes whose YAML frontmatter does not parse (BRAIN=path, FIX=1 repairs in place)
+    cmds:
+      - cd ingestion && go run ./cmd/frontmatterlint {{if .FIX}}-fix{{end}} {{.BRAIN | default "../../brain"}}
+
   test:
     cmds:
       - go test -race -count=1 ./...

ingestion/cmd/frontmatterlint/main.go

@@ -0,0 +1,53 @@
+// Command frontmatterlint reports markdown notes whose YAML frontmatter does
+// not parse, and optionally repairs the ones that only need quoting.
+//
+//	go run ./cmd/frontmatterlint ../../brain          # report
+//	go run ./cmd/frontmatterlint -fix ../../brain     # repair in place
+//
+// Exit codes: 0 clean (or everything repaired), 1 findings remain, 2 the scan
+// itself failed.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/mathiasbq/hyperguild/ingestion/internal/frontmatter"
+)
+
+func main() {
+	fix := flag.Bool("fix", false, "repair files that quoting can rescue, in place")
+	flag.Parse()
+
+	root := "."
+	if flag.NArg() > 0 {
+		root = flag.Arg(0)
+	}
+
+	findings, err := frontmatter.Scan(root, *fix)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "frontmatterlint:", err)
+		os.Exit(2)
+	}
+
+	var unresolved int
+	for _, f := range findings {
+		if f.Repaired {
+			fmt.Printf("repaired  %s\n", f.Path)
+			continue
+		}
+		unresolved++
+		fmt.Printf("BROKEN    %s: %v\n", f.Path, f.Err)
+	}
+
+	if unresolved > 0 {
+		fmt.Fprintf(os.Stderr, "\n%d file(s) with unparseable frontmatter", unresolved)
+		if !*fix {
+			fmt.Fprint(os.Stderr, " — re-run with -fix")
+		}
+		fmt.Fprintln(os.Stderr)
+		os.Exit(1)
+	}
+	fmt.Printf("✓ frontmatter: %d file(s) repaired, none broken\n", len(findings))
+}

ingestion/internal/frontmatter/frontmatter.go

@@ -0,0 +1,141 @@
+// Package frontmatter lints and repairs the YAML frontmatter block of a
+// markdown note.
+//
+// Notes written directly — hand-authored, or promoted from brain/raw/ — do not
+// pass through pipeline.BuildPages, so nothing quotes their scalars. A title
+// containing an unescaped ": " then produces a block that yaml.Unmarshal
+// rejects with "mapping values are not allowed here", and every consumer that
+// reads frontmatter silently skips the note.
+//
+// Fix is deliberately conservative: it only adds quoting, never reorders or
+// rewrites, and it refuses to return a document that still does not parse.
+package frontmatter
+
+import (
+	"bytes"
+	"fmt"
+	"regexp"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+)
+
+const fence = "---"
+
+// Lint reports whether doc's frontmatter block parses as YAML.
+//
+// A document with no frontmatter block, or with an unterminated one, is not an
+// error: there is nothing claiming to be YAML.
+func Lint(doc []byte) error {
+	b, ok := block(doc)
+	if !ok {
+		return nil
+	}
+	var v any
+	if err := yaml.Unmarshal(b, &v); err != nil {
+		return fmt.Errorf("frontmatter: %w", err)
+	}
+	return nil
+}
+
+// Fix quotes unquoted scalars whose value breaks YAML parsing and returns the
+// repaired document.
+//
+// It returns (doc, false, nil) when the document already parses, and
+// (doc, false, err) when quoting cannot rescue it — an unfixable document is
+// always returned byte-identical, never half-repaired.
+func Fix(doc []byte) ([]byte, bool, error) {
+	if err := Lint(doc); err == nil {
+		return doc, false, nil
+	}
+
+	b, ok := block(doc)
+	if !ok {
+		// Lint only fails when there is a block, so this is unreachable in
+		// practice; return the original rather than guess.
+		return doc, false, nil
+	}
+
+	fixed := quoteScalars(b)
+	if bytes.Equal(fixed, b) {
+		return doc, false, fmt.Errorf("frontmatter: not repairable by quoting: %w", Lint(doc))
+	}
+
+	var v any
+	if err := yaml.Unmarshal(fixed, &v); err != nil {
+		return doc, false, fmt.Errorf("frontmatter: quoting did not repair the block: %w", err)
+	}
+
+	_, rest, _ := split(doc)
+	out := []byte(fence + "\n")
+	out = append(out, fixed...)
+	out = append(out, []byte(fence)...)
+	out = append(out, rest...)
+	return out, true, nil
+}
+
+// mappingLine matches "key: value" at any indentation, capturing the prefix so
+// it can be re-emitted untouched.
+//
+// Sequence items ("- value") are deliberately not handled: "- foo: bar" is
+// valid YAML, so a document containing one does not reach Fix in the first
+// place. See the Lint table for why that gap is left open.
+var mappingLine = regexp.MustCompile(`^(\s*[A-Za-z_][\w.-]*:[ \t]+)(\S.*)$`)
+
+// quoteScalars single-quotes every unquoted scalar value in the block that
+// contains a colon. Values that are already quoted, or that carry YAML syntax
+// this package has no business rewriting, are left alone.
+func quoteScalars(block []byte) []byte {
+	lines := strings.Split(string(block), "\n")
+	for i, line := range lines {
+		m := mappingLine.FindStringSubmatch(line)
+		if m == nil {
+			continue
+		}
+		prefix, value := m[1], m[2]
+		if !needsQuoting(value) {
+			continue
+		}
+		lines[i] = prefix + "'" + strings.ReplaceAll(value, "'", "''") + "'"
+	}
+	return []byte(strings.Join(lines, "\n"))
+}
+
+// needsQuoting reports whether a scalar value must be quoted to parse as the
+// string it plainly is. Only the colon case is handled: it is the one this
+// package exists for, and every other special case risks rewriting a value
+// that was deliberately structured (a nested mapping, an anchor, a number).
+func needsQuoting(value string) bool {
+	if strings.HasPrefix(value, "'") || strings.HasPrefix(value, `"`) {
+		return false
+	}
+	if strings.HasPrefix(value, "|") || strings.HasPrefix(value, ">") {
+		return false // block scalar; its content is not parsed as YAML
+	}
+	if strings.HasPrefix(value, "{") || strings.HasPrefix(value, "[") {
+		return false // flow collection; quoting would silently make it a string
+	}
+	return strings.Contains(value, ": ") || strings.HasSuffix(value, ":")
+}
+
+// block returns the raw YAML between the opening and closing fences.
+func block(doc []byte) ([]byte, bool) {
+	b, _, ok := split(doc)
+	return b, ok
+}
+
+// split returns the frontmatter block and everything from the closing fence
+// onward (the closing fence itself excluded, its trailing newline included).
+func split(doc []byte) (block, rest []byte, ok bool) {
+	s := string(doc)
+	if !strings.HasPrefix(s, fence+"\n") {
+		return nil, nil, false
+	}
+	body := s[len(fence)+1:]
+
+	idx := strings.Index(body, "\n"+fence)
+	if idx < 0 {
+		return nil, nil, false
+	}
+	return []byte(body[:idx+1]), []byte(body[idx+1+len(fence):]), true
+}

ingestion/internal/frontmatter/frontmatter_test.go

@@ -0,0 +1,158 @@
+// ingestion/internal/frontmatter/frontmatter_test.go
+package frontmatter
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v3"
+)
+
+func TestLint(t *testing.T) {
+	tests := []struct {
+		name    string
+		doc     string
+		wantErr bool
+	}{
+		{
+			name: "no frontmatter block is not an error",
+			doc:  "# Just a heading\n\nSome body text.\n",
+		},
+		{
+			name: "quoted title parses",
+			doc:  "---\ntitle: 'Headless dispatch runs: Bash is denied'\n---\n\nbody\n",
+		},
+		{
+			name: "plain title parses",
+			doc:  "---\ntitle: A simple title\nwing: homelab\n---\n\nbody\n",
+		},
+		{
+			name:    "unquoted colon in title does not parse",
+			doc:     "---\ntitle: Headless dispatch runs: Bash is denied — use gitea MCP\nwing: homelab\n---\n\nbody\n",
+			wantErr: true,
+		},
+		{
+			// Known gap, deliberately not fixed here: this parses, but the
+			// item becomes map[string]any{"Headless dispatch": "denied"}
+			// rather than a string. It is a silent type corruption, not a
+			// parse failure, so repairing it would rewrite documents that
+			// currently load — out of scope for a lint about broken YAML.
+			name: "unquoted colon in a list item parses as a mapping, not an error",
+			doc:  "---\ntitle: Fine\naliases:\n  - Headless dispatch: denied\n---\n\nbody\n",
+		},
+		{
+			name: "unterminated frontmatter is not treated as frontmatter",
+			doc:  "---\ntitle: no closing fence\n\nbody\n",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := Lint([]byte(tt.doc))
+			if tt.wantErr {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+		})
+	}
+}
+
+func TestFix(t *testing.T) {
+	tests := []struct {
+		name        string
+		doc         string
+		wantChanged bool
+		wantKey     string
+		wantValue   string
+	}{
+		{
+			name:        "quotes an unquoted title containing a colon",
+			doc:         "---\ntitle: Headless dispatch runs: Bash is denied — use gitea MCP\nwing: homelab\n---\n\nbody\n",
+			wantChanged: true,
+			wantKey:     "title",
+			wantValue:   "Headless dispatch runs: Bash is denied — use gitea MCP",
+		},
+		{
+			// Parses already (see the Lint table), so Fix must leave it alone
+			// rather than rewrite a document that loads.
+			name:        "leaves a colon-bearing list item alone because it already parses",
+			doc:         "---\ntitle: Fine\naliases:\n  - Headless dispatch: denied\n---\n\nbody\n",
+			wantChanged: false,
+			wantKey:     "title",
+			wantValue:   "Fine",
+		},
+		{
+			name:        "escapes an embedded single quote when quoting",
+			doc:         "---\ntitle: Don't do this: it breaks\n---\n\nbody\n",
+			wantChanged: true,
+			wantKey:     "title",
+			wantValue:   "Don't do this: it breaks",
+		},
+		{
+			name:        "leaves an already-valid document untouched",
+			doc:         "---\ntitle: 'Already quoted: fine'\nwing: homelab\n---\n\nbody\n",
+			wantChanged: false,
+			wantKey:     "title",
+			wantValue:   "Already quoted: fine",
+		},
+		{
+			name:        "leaves a document with no frontmatter untouched",
+			doc:         "# Heading\n\nbody: with a colon\n",
+			wantChanged: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, changed, err := Fix([]byte(tt.doc))
+			require.NoError(t, err)
+			assert.Equal(t, tt.wantChanged, changed)
+
+			if !tt.wantChanged {
+				assert.Equal(t, tt.doc, string(got), "unchanged docs must be returned byte-identical")
+			}
+
+			// Whatever came back must parse — that is the whole point.
+			require.NoError(t, Lint(got), "Fix must not return a document that still fails to parse")
+
+			if tt.wantKey == "" {
+				return
+			}
+			block, ok := block(got)
+			require.True(t, ok)
+			var m map[string]any
+			require.NoError(t, yaml.Unmarshal(block, &m))
+			assert.Equal(t, tt.wantValue, m[tt.wantKey], "the value must survive quoting unchanged")
+		})
+	}
+}
+
+func TestFix_PreservesBodyExactly(t *testing.T) {
+	body := "\nThe body has: colons, 'quotes', and\n---\na horizontal rule.\n"
+	doc := "---\ntitle: Broken: title\n---" + body
+
+	got, changed, err := Fix([]byte(doc))
+	require.NoError(t, err)
+	require.True(t, changed)
+	assert.True(t, len(got) > 0)
+
+	_, rest, ok := split([]byte(doc))
+	require.True(t, ok)
+	_, gotRest, ok := split(got)
+	require.True(t, ok)
+	assert.Equal(t, string(rest), string(gotRest), "the body must not be touched")
+}
+
+func TestFix_RefusesToReturnAnUnparseableResult(t *testing.T) {
+	// A frontmatter block that quoting cannot rescue: the structure itself is
+	// broken, not a scalar. Fix must return the original and an error rather
+	// than a half-repaired document.
+	doc := "---\ntitle: fine\n\tbad: tab-indented mapping\n  - stray list item\n---\n\nbody\n"
+
+	got, changed, err := Fix([]byte(doc))
+	require.Error(t, err)
+	assert.False(t, changed)
+	assert.Equal(t, doc, string(got), "an unfixable document must come back byte-identical")
+}

ingestion/internal/frontmatter/scan.go

@@ -0,0 +1,84 @@
+// ingestion/internal/frontmatter/scan.go
+package frontmatter
+
+import (
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+	"unicode/utf8"
+)
+
+// Finding is one markdown file whose frontmatter does not parse.
+type Finding struct {
+	Path     string // relative to the scanned root
+	Err      error  // why it does not parse, or why the repair was refused
+	Repaired bool   // true only when fix was requested and the file was rewritten
+}
+
+// Scan walks root for *.md files whose frontmatter fails to parse.
+//
+// With fix set, each finding that quoting can rescue is rewritten in place;
+// anything quoting cannot rescue is reported and left byte-identical. A file
+// that cannot be read or decoded is reported as a finding rather than aborting
+// the walk — one poisoned note must not hide the other findings.
+func Scan(root string, fix bool) ([]Finding, error) {
+	var found []Finding
+
+	err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() || !strings.EqualFold(filepath.Ext(path), ".md") {
+			return nil
+		}
+
+		rel, relErr := filepath.Rel(root, path)
+		if relErr != nil {
+			rel = path
+		}
+
+		doc, readErr := os.ReadFile(path)
+		if readErr != nil {
+			found = append(found, Finding{Path: rel, Err: readErr})
+			return nil
+		}
+		if !utf8.Valid(doc) {
+			found = append(found, Finding{Path: rel, Err: fmt.Errorf("not valid UTF-8")})
+			return nil
+		}
+		if lintErr := Lint(doc); lintErr == nil {
+			return nil
+		}
+
+		if !fix {
+			found = append(found, Finding{Path: rel, Err: Lint(doc)})
+			return nil
+		}
+
+		repaired, changed, fixErr := Fix(doc)
+		if fixErr != nil || !changed {
+			found = append(found, Finding{Path: rel, Err: fixErr})
+			return nil
+		}
+		if writeErr := writeFilePreservingMode(path, repaired); writeErr != nil {
+			found = append(found, Finding{Path: rel, Err: writeErr})
+			return nil
+		}
+		found = append(found, Finding{Path: rel, Repaired: true})
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("scan %s: %w", root, err)
+	}
+	return found, nil
+}
+
+func writeFilePreservingMode(path string, content []byte) error {
+	mode := fs.FileMode(0o644)
+	if info, err := os.Stat(path); err == nil {
+		mode = info.Mode().Perm()
+	}
+	return os.WriteFile(path, content, mode)
+}

ingestion/internal/frontmatter/scan_test.go

@@ -0,0 +1,103 @@
+// ingestion/internal/frontmatter/scan_test.go
+package frontmatter
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func writeTree(t *testing.T, files map[string]string) string {
+	t.Helper()
+	root := t.TempDir()
+	for name, content := range files {
+		path := filepath.Join(root, name)
+		require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755))
+		require.NoError(t, os.WriteFile(path, []byte(content), 0o644))
+	}
+	return root
+}
+
+const (
+	brokenDoc = "---\ntitle: Broken: title here\nwing: homelab\n---\n\nbody\n"
+	validDoc  = "---\ntitle: 'Fine: quoted'\n---\n\nbody\n"
+)
+
+func TestScan_ReportsOnlyBrokenFiles(t *testing.T) {
+	root := writeTree(t, map[string]string{
+		"wiki/a/broken.md": brokenDoc,
+		"wiki/a/valid.md":  validDoc,
+		"wiki/b/plain.md":  "# no frontmatter\n\nbody: with colon\n",
+		"wiki/a/notes.txt": brokenDoc, // not markdown — must be ignored
+	})
+
+	found, err := Scan(root, false)
+	require.NoError(t, err)
+
+	require.Len(t, found, 1)
+	assert.Equal(t, "wiki/a/broken.md", found[0].Path)
+	assert.False(t, found[0].Repaired)
+	assert.Error(t, found[0].Err)
+
+	// Dry run must not touch the file.
+	got, err := os.ReadFile(filepath.Join(root, "wiki/a/broken.md"))
+	require.NoError(t, err)
+	assert.Equal(t, brokenDoc, string(got))
+}
+
+func TestScan_FixWritesRepairedFilesAndLeavesTheRestAlone(t *testing.T) {
+	root := writeTree(t, map[string]string{
+		"wiki/a/broken.md": brokenDoc,
+		"wiki/a/valid.md":  validDoc,
+	})
+
+	found, err := Scan(root, true)
+	require.NoError(t, err)
+	require.Len(t, found, 1)
+	assert.True(t, found[0].Repaired)
+	assert.NoError(t, found[0].Err)
+
+	repaired, err := os.ReadFile(filepath.Join(root, "wiki/a/broken.md"))
+	require.NoError(t, err)
+	require.NoError(t, Lint(repaired), "the repaired file must parse")
+	assert.Contains(t, string(repaired), "title: 'Broken: title here'")
+	assert.Contains(t, string(repaired), "wing: homelab", "untouched keys must survive")
+
+	untouched, err := os.ReadFile(filepath.Join(root, "wiki/a/valid.md"))
+	require.NoError(t, err)
+	assert.Equal(t, validDoc, string(untouched))
+
+	// Second pass finds nothing — the repair is idempotent and complete.
+	again, err := Scan(root, true)
+	require.NoError(t, err)
+	assert.Empty(t, again)
+}
+
+func TestScan_ReportsUnfixableWithoutWriting(t *testing.T) {
+	unfixable := "---\ntitle: fine\n\tbad: tab-indented\n  - stray\n---\n\nbody\n"
+	root := writeTree(t, map[string]string{"wiki/x.md": unfixable})
+
+	found, err := Scan(root, true)
+	require.NoError(t, err)
+	require.Len(t, found, 1)
+	assert.False(t, found[0].Repaired)
+	assert.Error(t, found[0].Err)
+
+	got, err := os.ReadFile(filepath.Join(root, "wiki/x.md"))
+	require.NoError(t, err)
+	assert.Equal(t, unfixable, string(got), "an unfixable file must be left exactly as it was")
+}
+
+func TestScan_ReportsUndecodableFilesInsteadOfFailingTheRun(t *testing.T) {
+	root := writeTree(t, map[string]string{"wiki/ok.md": validDoc})
+	require.NoError(t, os.WriteFile(filepath.Join(root, "wiki/bad-bytes.md"),
+		[]byte{'-', '-', '-', '\n', 0xff, 0xfe, '\n', '-', '-', '-', '\n'}, 0o644))
+
+	found, err := Scan(root, false)
+	require.NoError(t, err, "one unreadable file must not abort the whole scan")
+	require.Len(t, found, 1)
+	assert.Equal(t, "wiki/bad-bytes.md", found[0].Path)
+}