feat(brain): audit-log every tools/call; server-owned write provenance (#147)

Closes #144 (audit line per tools/call) and #145 (origin/written_by stamped on every model-reachable write, surfaced in brain_query/brain_get).

Passed pr-gatekeeper after four rounds of adversarial review, verified by execution: #147

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Mathias <mthbqv@gmail.com>

ingestion/internal/provenance/provenance.go

@@ -0,0 +1,226 @@
+// Package provenance stamps server-owned write provenance (hyperguild#145)
+// into a brain note's frontmatter: origin (static | jwt | unknown) and
+// written_by (the authenticated principal), so a reader can discount what a
+// connector session wrote after reading an untrusted page.
+//
+// The control is a post-condition applied by every write path a model can
+// reach. Two review rounds on PR #147 showed that deciding line by line which
+// lines are provenance loses to YAML's context-dependent features — an alias
+// key resolves only against an anchor elsewhere, and a `...` line ends the
+// document before the stamp. So Apply does not try to recognise provenance.
+// It keeps only frontmatter it can prove is plain:
+//
+//   - a top-level line `key: value` whose key is a bare identifier,
+//   - together with that key's continuation lines (indented, or `- ` items
+//     after an empty value),
+//   - where the group decodes ON ITS OWN, with yaml.v3, to a mapping with
+//     exactly that one key,
+//   - and the key is not origin/written_by and has not been seen before.
+//
+// Everything else — aliases, anchors used as keys, document markers, complex
+// keys, merge keys, flow maps, tags, quoted keys, duplicates — is dropped.
+// Then the stamp is appended and the whole result is decoded by yaml.v3 as a
+// backstop: if it does not decode to exactly the stamped values, Apply fails
+// and the caller refuses the write.
+package provenance
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"regexp"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+
+	sharedfm "github.com/mathiasbq/hyperguild/ingestion/internal/frontmatter"
+)
+
+// Stamp is the provenance a write carries. An empty Origin means "unknown".
+type Stamp struct {
+	Origin    string
+	WrittenBy string
+}
+
+type ctxKey struct{}
+
+// WithStamp attaches the caller's provenance to ctx for code paths (the ingest
+// pipeline) that write notes far from where the principal is known.
+func WithStamp(ctx context.Context, s Stamp) context.Context {
+	return context.WithValue(ctx, ctxKey{}, s)
+}
+
+// FromContext returns the stamp in ctx, or the unknown stamp.
+func FromContext(ctx context.Context) Stamp {
+	if s, ok := ctx.Value(ctxKey{}).(Stamp); ok {
+		return s
+	}
+	return Stamp{}
+}
+
+// ErrUnverifiable means the stamped note would not read back as stamped.
+var ErrUnverifiable = errors.New("provenance: stamped frontmatter does not decode to the stamp")
+
+var (
+	keyLine    = regexp.MustCompile(`^([A-Za-z_][A-Za-z0-9_.-]*)[ ]*:(?:[ ]|$)(.*)$`)
+	bareScalar = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._@-]*$`)
+	reserved   = map[string]bool{"origin": true, "written_by": true}
+)
+
+func (s Stamp) origin() string {
+	if s.Origin == "" {
+		return "unknown"
+	}
+	return s.Origin
+}
+
+// Apply returns content with its frontmatter reduced to plain groups and the
+// stamp appended. Content without frontmatter gains a block.
+func Apply(content string, s Stamp) (string, error) {
+	stamp := "origin: " + scalar(s.origin()) + "\n"
+	if s.WrittenBy != "" {
+		stamp += "written_by: " + scalar(s.WrittenBy) + "\n"
+	}
+	content = foldFrontmatterCR(content)
+	block, rest, ok := sharedfm.Split([]byte(content))
+	var out string
+	switch {
+	case !ok && strings.HasPrefix(strings.ReplaceAll(content, "\r\n", "\n"), "---\n"):
+		// A fence was opened but cannot be closed (e.g. a "---x" line).
+		// Prepending a second block would demote the original frontmatter
+		// into the body, so refuse instead (PR #147 review round 3).
+		return "", fmt.Errorf("%w: unclosed frontmatter fence", ErrUnverifiable)
+	case !ok:
+		out = "---\n" + stamp + "---\n" + content
+	default:
+		out = "---\n" + plainGroups(string(block)) + stamp + "---\n" + strings.TrimPrefix(string(rest), "\n")
+	}
+	if err := verify(out, s); err != nil {
+		return "", err
+	}
+	return out, nil
+}
+
+// ApplyFile applies the stamp to the note at absPath in place.
+func ApplyFile(absPath string, s Stamp) error {
+	b, err := os.ReadFile(absPath)
+	if err != nil {
+		return fmt.Errorf("read for provenance: %w", err)
+	}
+	out, err := Apply(string(b), s)
+	if err != nil {
+		return err
+	}
+	if out == string(b) {
+		return nil
+	}
+	if err := os.WriteFile(absPath, []byte(out), 0o644); err != nil {
+		return fmt.Errorf("write provenance: %w", err)
+	}
+	return nil
+}
+
+// plainGroups keeps the provably plain key groups of a frontmatter block.
+func plainGroups(block string) string {
+	lines := strings.Split(strings.TrimSuffix(block, "\n"), "\n")
+	seen := map[string]bool{}
+	var b strings.Builder
+	for i := 0; i < len(lines); {
+		line := lines[i]
+		m := keyLine.FindStringSubmatch(line)
+		if m == nil {
+			i++ // not a plain key line: dropped, as is anything it would govern
+			for i < len(lines) && isContinuation(lines[i], "") {
+				i++
+			}
+			continue
+		}
+		key, value := m[1], strings.TrimSpace(m[2])
+		j := i + 1
+		for j < len(lines) && isContinuation(lines[j], value) {
+			j++
+		}
+		group := strings.Join(lines[i:j], "\n") + "\n"
+		lk := strings.ToLower(key)
+		if !reserved[lk] && !seen[lk] && decodesToOnly(group, key) {
+			seen[lk] = true
+			b.WriteString(group)
+		}
+		i = j
+	}
+	return b.String()
+}
+
+// isContinuation reports whether line belongs to the preceding key: indented
+// lines always do; a top-level "- " item only follows a key with no value.
+func isContinuation(line, value string) bool {
+	if line == "" {
+		return false
+	}
+	if line[0] == ' ' {
+		return true
+	}
+	return value == "" && strings.HasPrefix(line, "- ")
+}
+
+func decodesToOnly(group, key string) bool {
+	var m map[string]any
+	if err := yaml.Unmarshal([]byte(group), &m); err != nil {
+		return false
+	}
+	_, ok := m[key]
+	return ok && len(m) == 1
+}
+
+func verify(doc string, s Stamp) error {
+	block, _, ok := sharedfm.Split([]byte(doc))
+	if !ok {
+		return ErrUnverifiable
+	}
+	var m map[string]any
+	if err := yaml.Unmarshal(block, &m); err != nil {
+		return fmt.Errorf("%w: %v", ErrUnverifiable, err)
+	}
+	if m["origin"] != s.origin() {
+		return ErrUnverifiable
+	}
+	if s.WrittenBy == "" {
+		if _, has := m["written_by"]; has {
+			return ErrUnverifiable
+		}
+	} else if m["written_by"] != s.WrittenBy {
+		return ErrUnverifiable
+	}
+	return nil
+}
+
+// foldFrontmatterCR turns CRLF and lone CR inside a leading frontmatter block
+// into LF, so every reader agrees on where lines end.
+func foldFrontmatterCR(content string) string {
+	s := strings.ReplaceAll(content, "\r\n", "\n")
+	if !strings.HasPrefix(s, "---\n") {
+		return content
+	}
+	end := strings.Index(s[4:], "\n---")
+	if end < 0 {
+		return content
+	}
+	return strings.ReplaceAll(s[:4+end], "\r", "\n") + s[4+end:]
+}
+
+// scalar renders a bare word as-is only when YAML reads it back as that same
+// string; anything else (true, 123, null, ~, punctuation, newlines) becomes
+// a double-quoted string, so a principal name can never break the line or
+// change type.
+func scalar(v string) string {
+	if bareScalar.MatchString(v) {
+		var back any
+		if yaml.Unmarshal([]byte(v), &back) == nil {
+			if s, ok := back.(string); ok && s == v {
+				return v
+			}
+		}
+	}
+	return fmt.Sprintf("%q", v)
+}

ingestion/internal/provenance/provenance_test.go

@@ -0,0 +1,86 @@
+package provenance
+
+import (
+	"errors"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v3"
+)
+
+func fmOf(t *testing.T, doc string) map[string]any {
+	t.Helper()
+	parts := strings.SplitN(doc, "---\n", 3)
+	require.Len(t, parts, 3, doc)
+	var m map[string]any
+	require.NoError(t, yaml.Unmarshal([]byte(parts[1]), &m), parts[1])
+	return m
+}
+
+// The whole-document backstop: a stamp that would not read back as written
+// is refused, never written. A principal with an invalid UTF-8 byte quotes
+// to "\xff", which YAML decodes as U+00FF, so it cannot round-trip.
+func TestApplyRefusesAStampThatDoesNotReadBack(t *testing.T) {
+	_, err := Apply("---\ntitle: t\n---\nbody\n", Stamp{Origin: "jwt", WrittenBy: "bad\xff"})
+	assert.True(t, errors.Is(err, ErrUnverifiable), "got %v", err)
+}
+
+// A principal that YAML would decode as a non-string (true, 123, null) is
+// quoted, so the stamp reads back as the same string.
+func TestApplyQuotesPrincipalsYAMLWouldNotReadAsStrings(t *testing.T) {
+	for _, p := range []string{"true", "123", "null", "1e3", "0x1F", "yes", "~"} {
+		out, err := Apply("---\ntitle: t\n---\nbody\n", Stamp{Origin: "jwt", WrittenBy: p})
+		require.NoError(t, err, p)
+		assert.Equal(t, p, fmOf(t, out)["written_by"], p)
+	}
+}
+
+// A group that does not decode on its own is dropped; its neighbours stay.
+func TestApplyDropsGroupsThatDoNotDecodeAlone(t *testing.T) {
+	out, err := Apply("---\nkeep: 1\ntitle: a: b\nalso: 2\n---\nbody\n", Stamp{Origin: "static", WrittenBy: "local-cli"})
+	require.NoError(t, err)
+	m := fmOf(t, out)
+	assert.Equal(t, 1, m["keep"])
+	assert.Equal(t, 2, m["also"])
+	assert.NotContains(t, m, "title")
+}
+
+func TestApplyKeepsFirstOfDuplicateKeys(t *testing.T) {
+	out, err := Apply("---\ntitle: first\ntitle: second\nTitle: third\n---\nbody\n", Stamp{})
+	require.NoError(t, err)
+	m := fmOf(t, out)
+	assert.Equal(t, "first", m["title"])
+	assert.NotContains(t, m, "Title")
+}
+
+func TestApplyFoldsLoneCRSoItCannotHideALine(t *testing.T) {
+	out, err := Apply("---\ntitle: x\rorigin: static\n---\nbody\n", Stamp{Origin: "jwt", WrittenBy: "a"})
+	require.NoError(t, err)
+	assert.NotContains(t, out, "\r")
+	assert.Equal(t, "jwt", fmOf(t, out)["origin"])
+	assert.Equal(t, "x", fmOf(t, out)["title"])
+}
+
+func TestApplyDropsTabIndentedContinuations(t *testing.T) {
+	out, err := Apply("---\ntitle: x\n\torigin: static\nk: v\n---\nbody\n", Stamp{Origin: "jwt", WrittenBy: "a"})
+	require.NoError(t, err)
+	m := fmOf(t, out)
+	assert.Equal(t, "jwt", m["origin"])
+	assert.Equal(t, "v", m["k"])
+}
+
+// Content that opens a fence the shared splitter cannot close (e.g. a
+// "---x" line) is refused rather than given a second, prepended block that
+// demotes the original frontmatter into the body.
+func TestApplyRefusesAnUnclosableFrontmatterFence(t *testing.T) {
+	_, err := Apply("---\ntitle: t\n---x\norigin: static\n---\nbody\n", Stamp{Origin: "jwt", WrittenBy: "a"})
+	assert.True(t, errors.Is(err, ErrUnverifiable), "got %v", err)
+}
+
+func TestApplyAddsFrontmatterToContentWithoutAny(t *testing.T) {
+	out, err := Apply("# plain\n\norigin: static is prose\n", Stamp{Origin: "static", WrittenBy: "local-cli"})
+	require.NoError(t, err)
+	assert.True(t, strings.HasPrefix(out, "---\norigin: static\nwritten_by: local-cli\n---\n# plain"))
+}