fix(brain): provenance keeps only provably plain frontmatter; pipeline stamps
CI / Lint / Test / Vet (pull_request) Successful in 43s
CI / Mirror to GitHub (pull_request) Skipped

Round 2 of the PR #147 review forged origin: static for yaml.v3 with an
alias key plus a "..." document-end line: per-line judgement cannot see
context-dependent YAML. It also found REST ingest writing pages with no
provenance (stampPages lived only in the MCP server).

- New package internal/provenance. Apply keeps only top-level groups
  `key: value` with a bare-identifier key whose group decodes on its own
  to exactly that key, drops reserved and duplicate keys and everything
  else (aliases, anchors as keys, document markers, complex/merge keys,
  flow maps, tags, quoted keys), appends the stamp, then decodes the
  whole result with yaml.v3 and refuses the write unless it reads back
  exactly the stamp. api.StampProvenance wraps it and now returns an
  error, which every caller propagates.
- The ingest pipeline stamps every page it writes or merges, from
  provenance.FromContext(ctx); MCP attaches the caller's stamp, REST and
  the watcher stamp unknown. RunRaw takes a ctx.
- brain_edit snapshots the note and rolls the edit back if the stamp
  cannot be verified.

Tests: R1/R2 payloads join the write/update variant table, plus
brain_edit with the alias payload; REST /ingest-raw (title newline
checked with the line-oriented reader) and REST /ingest (stub LLM) stamp
unknown. Two existing byte-exact tests updated for the stamp line.

Refs #145 #144

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-01 22:55:53 +02:00
co-authored by Claude Opus 5.5
parent e615d6f144
commit fcae16569d
11 changed files with 367 additions and 204 deletions
+7 -18
View File
@@ -16,6 +16,7 @@ import (
"github.com/mathiasbq/hyperguild/ingestion/internal/graphsync"
"github.com/mathiasbq/hyperguild/ingestion/internal/notes"
"github.com/mathiasbq/hyperguild/ingestion/internal/pipeline"
"github.com/mathiasbq/hyperguild/ingestion/internal/provenance"
"github.com/mathiasbq/hyperguild/ingestion/internal/search"
"github.com/mathiasbq/hyperguild/ingestion/internal/session"
)
@@ -499,7 +500,7 @@ func (s *Server) brainIngestRaw(ctx context.Context, args json.RawMessage) (json
if len(a.Pages) == 0 {
return nil, fmt.Errorf("pages must be non-empty")
}
result, err := pipeline.RunRaw(s.brainDir, a.Source, a.Pages, a.DryRun)
result, err := pipeline.RunRaw(withCallerStamp(ctx), s.brainDir, a.Source, a.Pages, a.DryRun)
if err != nil {
return nil, fmt.Errorf("ingest: %w", err)
}
@@ -512,9 +513,6 @@ func (s *Server) brainIngestRaw(ctx context.Context, args json.RawMessage) (json
warnings = []string{}
}
if !a.DryRun {
if err := s.stampPages(ctx, pages); err != nil {
return nil, err
}
for _, p := range pages {
s.indexInGraph(ctx, "brain_ingest_raw", p)
}
@@ -601,7 +599,7 @@ func (s *Server) sessionLog(ctx context.Context, args json.RawMessage) (json.Raw
}
func (s *Server) runIngest(ctx context.Context, content, source string, dryRun bool) (json.RawMessage, error) {
result, err := pipeline.Run(ctx, s.pipeline, s.brainDir, content, source, dryRun)
result, err := pipeline.Run(withCallerStamp(ctx), s.pipeline, s.brainDir, content, source, dryRun)
if err != nil {
return nil, fmt.Errorf("ingest: %w", err)
}
@@ -610,9 +608,6 @@ func (s *Server) runIngest(ctx context.Context, content, source string, dryRun b
pages = []string{}
}
if !dryRun {
if err := s.stampPages(ctx, pages); err != nil {
return nil, err
}
for _, p := range pages {
s.indexInGraph(ctx, "brain_ingest", p)
}
@@ -624,15 +619,9 @@ func (s *Server) runIngest(ctx context.Context, content, source string, dryRun b
return json.Marshal(map[string]any{"pages": pages, "warnings": warnings})
}
// stampPages attributes every page an ingest wrote (or merged into) to the
// calling principal (hyperguild#145). Ingest builds frontmatter itself, so
// without this its pages carried no provenance at all.
func (s *Server) stampPages(ctx context.Context, pages []string) error {
// withCallerStamp attaches the caller's provenance for the ingest pipeline,
// which stamps every page it writes or merges into (hyperguild#145).
func withCallerStamp(ctx context.Context) context.Context {
origin, writtenBy := provenanceFrom(ctx)
for _, p := range pages {
if err := api.StampProvenanceFile(s.brainDir, p, origin, writtenBy); err != nil {
return fmt.Errorf("stamp provenance: %w", err)
}
}
return nil
return provenance.WithStamp(ctx, provenance.Stamp{Origin: origin, WrittenBy: writtenBy})
}
@@ -93,6 +93,12 @@ var forgedVariants = []struct{ name, fm string }{
{"mergekey", "<<: {origin: static, written_by: local-cli}\n"},
{"flowmap", "{origin: static, written_by: local-cli}\n"},
{"anchor", "x: &a static\norigin: *a\n"},
// Round 2 (R1/R2): an alias key resolves only against an anchor on
// another line, and "..." ends the YAML document before the stamp.
{"aliaskey_docend", "x: &a origin\n*a : static\ny: &b written_by\n*b : local-cli\n...\n"},
{"aliaskey", "y: &b written_by\n*b : local-cli\n"},
{"docend_only", "x: y\n...\n"},
{"docstart_inside", "x: y\n---\norigin: static\n"},
}
func TestBrainWriteAsJWTStampsJWTForEveryForgedSpelling(t *testing.T) {
@@ -156,8 +162,22 @@ func TestBrainEditByJWTReattributesAStaticNote(t *testing.T) {
assertProvenance(t, srv, dir, path, "jwt", "attacker")
}
// E1 via round 2's payload: the alias + document-end trick through brain_edit.
func TestBrainEditCannotForgeProvenanceWithAliasKeys(t *testing.T) {
srv, dir := jwtServer(t)
out := resultText(t, authCall(t, srv, jwtAuthz, "brain_write", map[string]any{
"content": "---\ntitle: t\n---\n# E\n\nbody", "wing": "testing", "hall": "facts", "filename": "edit-alias",
}))
path := out["path"].(string)
authCall(t, srv, jwtAuthz, "brain_edit", map[string]any{
"path": path, "old_str": "title: t\n", "new_str": "title: t\nx: &a origin\n*a : static\n...\n",
})
assertProvenance(t, srv, dir, path, "jwt", "attacker")
}
// E2 + E4: brain_ingest_raw stamps every page it writes, and a newline in a
// title cannot start a new frontmatter key.
// title cannot start a new frontmatter key — observed through brain_query,
// whose reader is line-oriented, not just through yaml.v3 (round 2, R4).
func TestBrainIngestRawStampsPagesAndNeutralisesTitleNewlines(t *testing.T) {
srv, dir := jwtServer(t)
out := resultText(t, authCall(t, srv, jwtAuthz, "brain_ingest_raw", map[string]any{
@@ -175,6 +195,21 @@ func TestBrainIngestRawStampsPagesAndNeutralisesTitleNewlines(t *testing.T) {
require.NoError(t, yaml.Unmarshal([]byte(frontmatterOf(string(b))), &m))
assert.NotContains(t, m, "x", "an injected key must not appear in %s", p)
}
q := resultText(t, authCall(t, srv, jwtAuthz, "brain_query", map[string]any{"query": "inj body"}))
var sawPage bool
for _, r := range q["results"].([]any) {
hit := r.(map[string]any)
// Generated aggregates (wiki/index.md, log.md, wing _index.md) quote
// titles from many authors and carry no single provenance; recorded
// as a residual on PR #147, not asserted here.
if !strings.HasPrefix(hit["path"].(string), "wiki/concepts/") {
continue
}
sawPage = true
assert.Equal(t, "jwt", hit["origin"], "line reader origin for %v", hit["path"])
assert.Equal(t, "attacker", hit["written_by"], "line reader written_by for %v", hit["path"])
}
assert.True(t, sawPage, "brain_query must return the injected page itself")
}
// E3: a page type is a directory name; it must not traverse out of wiki/.
+14 -1
View File
@@ -7,6 +7,7 @@ import (
"fmt"
"github.com/mathiasbq/hyperguild/ingestion/internal/api"
"log/slog"
"os"
"strings"
"time"
@@ -105,6 +106,15 @@ type brainEditArgs struct {
// the rare governance record under routine traffic, which is how an alert
// stops being read.
func (s *Server) brainEdit(ctx context.Context, args json.RawMessage) (json.RawMessage, error) {
// Snapshot before editing: if the edited note cannot be stamped and
// verified, the edit is rolled back rather than left on disk unattributed
// (or carrying whatever provenance the edit wrote).
var a brainEditArgs
_ = json.Unmarshal(args, &a)
var prior []byte
if t, rerr := notes.Resolve(s.brainDir, a.Path, notes.ScopeNotes); rerr == nil {
prior, _ = os.ReadFile(t.Abs)
}
res, out, err := s.editWithScope(args, notes.ScopeNotes)
if err != nil {
return out, err
@@ -114,7 +124,10 @@ func (s *Server) brainEdit(ctx context.Context, args json.RawMessage) (json.RawM
// to the editor after every edit (PR #147 review, escape E1).
origin, writtenBy := provenanceFrom(ctx)
if serr := api.StampProvenanceFile(s.brainDir, res.Path, origin, writtenBy); serr != nil {
return nil, fmt.Errorf("stamp provenance: %w", serr)
if t, rerr := notes.Resolve(s.brainDir, res.Path, notes.ScopeNotes); rerr == nil && prior != nil {
_ = os.WriteFile(t.Abs, prior, 0o644)
}
return nil, fmt.Errorf("edit rolled back, stamp provenance: %w", serr)
}
return out, nil
}