fix(brain): provenance keeps only provably plain frontmatter; pipeline stamps
Round 2 of the PR #147 review forged origin: static for yaml.v3 with an alias key plus a "..." document-end line: per-line judgement cannot see context-dependent YAML. It also found REST ingest writing pages with no provenance (stampPages lived only in the MCP server). - New package internal/provenance. Apply keeps only top-level groups `key: value` with a bare-identifier key whose group decodes on its own to exactly that key, drops reserved and duplicate keys and everything else (aliases, anchors as keys, document markers, complex/merge keys, flow maps, tags, quoted keys), appends the stamp, then decodes the whole result with yaml.v3 and refuses the write unless it reads back exactly the stamp. api.StampProvenance wraps it and now returns an error, which every caller propagates. - The ingest pipeline stamps every page it writes or merges, from provenance.FromContext(ctx); MCP attaches the caller's stamp, REST and the watcher stamp unknown. RunRaw takes a ctx. - brain_edit snapshots the note and rolls the edit back if the stamp cannot be verified. Tests: R1/R2 payloads join the write/update variant table, plus brain_edit with the alias payload; REST /ingest-raw (title newline checked with the line-oriented reader) and REST /ingest (stub LLM) stamp unknown. Two existing byte-exact tests updated for the stamp line. Refs #145 #144 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -16,6 +16,7 @@ import (
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/graphsync"
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/notes"
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/pipeline"
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/provenance"
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/search"
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/session"
|
||||
)
|
||||
@@ -499,7 +500,7 @@ func (s *Server) brainIngestRaw(ctx context.Context, args json.RawMessage) (json
|
||||
if len(a.Pages) == 0 {
|
||||
return nil, fmt.Errorf("pages must be non-empty")
|
||||
}
|
||||
result, err := pipeline.RunRaw(s.brainDir, a.Source, a.Pages, a.DryRun)
|
||||
result, err := pipeline.RunRaw(withCallerStamp(ctx), s.brainDir, a.Source, a.Pages, a.DryRun)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ingest: %w", err)
|
||||
}
|
||||
@@ -512,9 +513,6 @@ func (s *Server) brainIngestRaw(ctx context.Context, args json.RawMessage) (json
|
||||
warnings = []string{}
|
||||
}
|
||||
if !a.DryRun {
|
||||
if err := s.stampPages(ctx, pages); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, p := range pages {
|
||||
s.indexInGraph(ctx, "brain_ingest_raw", p)
|
||||
}
|
||||
@@ -601,7 +599,7 @@ func (s *Server) sessionLog(ctx context.Context, args json.RawMessage) (json.Raw
|
||||
}
|
||||
|
||||
func (s *Server) runIngest(ctx context.Context, content, source string, dryRun bool) (json.RawMessage, error) {
|
||||
result, err := pipeline.Run(ctx, s.pipeline, s.brainDir, content, source, dryRun)
|
||||
result, err := pipeline.Run(withCallerStamp(ctx), s.pipeline, s.brainDir, content, source, dryRun)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ingest: %w", err)
|
||||
}
|
||||
@@ -610,9 +608,6 @@ func (s *Server) runIngest(ctx context.Context, content, source string, dryRun b
|
||||
pages = []string{}
|
||||
}
|
||||
if !dryRun {
|
||||
if err := s.stampPages(ctx, pages); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, p := range pages {
|
||||
s.indexInGraph(ctx, "brain_ingest", p)
|
||||
}
|
||||
@@ -624,15 +619,9 @@ func (s *Server) runIngest(ctx context.Context, content, source string, dryRun b
|
||||
return json.Marshal(map[string]any{"pages": pages, "warnings": warnings})
|
||||
}
|
||||
|
||||
// stampPages attributes every page an ingest wrote (or merged into) to the
|
||||
// calling principal (hyperguild#145). Ingest builds frontmatter itself, so
|
||||
// without this its pages carried no provenance at all.
|
||||
func (s *Server) stampPages(ctx context.Context, pages []string) error {
|
||||
// withCallerStamp attaches the caller's provenance for the ingest pipeline,
|
||||
// which stamps every page it writes or merges into (hyperguild#145).
|
||||
func withCallerStamp(ctx context.Context) context.Context {
|
||||
origin, writtenBy := provenanceFrom(ctx)
|
||||
for _, p := range pages {
|
||||
if err := api.StampProvenanceFile(s.brainDir, p, origin, writtenBy); err != nil {
|
||||
return fmt.Errorf("stamp provenance: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
return provenance.WithStamp(ctx, provenance.Stamp{Origin: origin, WrittenBy: writtenBy})
|
||||
}
|
||||
|
||||
@@ -93,6 +93,12 @@ var forgedVariants = []struct{ name, fm string }{
|
||||
{"mergekey", "<<: {origin: static, written_by: local-cli}\n"},
|
||||
{"flowmap", "{origin: static, written_by: local-cli}\n"},
|
||||
{"anchor", "x: &a static\norigin: *a\n"},
|
||||
// Round 2 (R1/R2): an alias key resolves only against an anchor on
|
||||
// another line, and "..." ends the YAML document before the stamp.
|
||||
{"aliaskey_docend", "x: &a origin\n*a : static\ny: &b written_by\n*b : local-cli\n...\n"},
|
||||
{"aliaskey", "y: &b written_by\n*b : local-cli\n"},
|
||||
{"docend_only", "x: y\n...\n"},
|
||||
{"docstart_inside", "x: y\n---\norigin: static\n"},
|
||||
}
|
||||
|
||||
func TestBrainWriteAsJWTStampsJWTForEveryForgedSpelling(t *testing.T) {
|
||||
@@ -156,8 +162,22 @@ func TestBrainEditByJWTReattributesAStaticNote(t *testing.T) {
|
||||
assertProvenance(t, srv, dir, path, "jwt", "attacker")
|
||||
}
|
||||
|
||||
// E1 via round 2's payload: the alias + document-end trick through brain_edit.
|
||||
func TestBrainEditCannotForgeProvenanceWithAliasKeys(t *testing.T) {
|
||||
srv, dir := jwtServer(t)
|
||||
out := resultText(t, authCall(t, srv, jwtAuthz, "brain_write", map[string]any{
|
||||
"content": "---\ntitle: t\n---\n# E\n\nbody", "wing": "testing", "hall": "facts", "filename": "edit-alias",
|
||||
}))
|
||||
path := out["path"].(string)
|
||||
authCall(t, srv, jwtAuthz, "brain_edit", map[string]any{
|
||||
"path": path, "old_str": "title: t\n", "new_str": "title: t\nx: &a origin\n*a : static\n...\n",
|
||||
})
|
||||
assertProvenance(t, srv, dir, path, "jwt", "attacker")
|
||||
}
|
||||
|
||||
// E2 + E4: brain_ingest_raw stamps every page it writes, and a newline in a
|
||||
// title cannot start a new frontmatter key.
|
||||
// title cannot start a new frontmatter key — observed through brain_query,
|
||||
// whose reader is line-oriented, not just through yaml.v3 (round 2, R4).
|
||||
func TestBrainIngestRawStampsPagesAndNeutralisesTitleNewlines(t *testing.T) {
|
||||
srv, dir := jwtServer(t)
|
||||
out := resultText(t, authCall(t, srv, jwtAuthz, "brain_ingest_raw", map[string]any{
|
||||
@@ -175,6 +195,21 @@ func TestBrainIngestRawStampsPagesAndNeutralisesTitleNewlines(t *testing.T) {
|
||||
require.NoError(t, yaml.Unmarshal([]byte(frontmatterOf(string(b))), &m))
|
||||
assert.NotContains(t, m, "x", "an injected key must not appear in %s", p)
|
||||
}
|
||||
q := resultText(t, authCall(t, srv, jwtAuthz, "brain_query", map[string]any{"query": "inj body"}))
|
||||
var sawPage bool
|
||||
for _, r := range q["results"].([]any) {
|
||||
hit := r.(map[string]any)
|
||||
// Generated aggregates (wiki/index.md, log.md, wing _index.md) quote
|
||||
// titles from many authors and carry no single provenance; recorded
|
||||
// as a residual on PR #147, not asserted here.
|
||||
if !strings.HasPrefix(hit["path"].(string), "wiki/concepts/") {
|
||||
continue
|
||||
}
|
||||
sawPage = true
|
||||
assert.Equal(t, "jwt", hit["origin"], "line reader origin for %v", hit["path"])
|
||||
assert.Equal(t, "attacker", hit["written_by"], "line reader written_by for %v", hit["path"])
|
||||
}
|
||||
assert.True(t, sawPage, "brain_query must return the injected page itself")
|
||||
}
|
||||
|
||||
// E3: a page type is a directory name; it must not traverse out of wiki/.
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"github.com/mathiasbq/hyperguild/ingestion/internal/api"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -105,6 +106,15 @@ type brainEditArgs struct {
|
||||
// the rare governance record under routine traffic, which is how an alert
|
||||
// stops being read.
|
||||
func (s *Server) brainEdit(ctx context.Context, args json.RawMessage) (json.RawMessage, error) {
|
||||
// Snapshot before editing: if the edited note cannot be stamped and
|
||||
// verified, the edit is rolled back rather than left on disk unattributed
|
||||
// (or carrying whatever provenance the edit wrote).
|
||||
var a brainEditArgs
|
||||
_ = json.Unmarshal(args, &a)
|
||||
var prior []byte
|
||||
if t, rerr := notes.Resolve(s.brainDir, a.Path, notes.ScopeNotes); rerr == nil {
|
||||
prior, _ = os.ReadFile(t.Abs)
|
||||
}
|
||||
res, out, err := s.editWithScope(args, notes.ScopeNotes)
|
||||
if err != nil {
|
||||
return out, err
|
||||
@@ -114,7 +124,10 @@ func (s *Server) brainEdit(ctx context.Context, args json.RawMessage) (json.RawM
|
||||
// to the editor after every edit (PR #147 review, escape E1).
|
||||
origin, writtenBy := provenanceFrom(ctx)
|
||||
if serr := api.StampProvenanceFile(s.brainDir, res.Path, origin, writtenBy); serr != nil {
|
||||
return nil, fmt.Errorf("stamp provenance: %w", serr)
|
||||
if t, rerr := notes.Resolve(s.brainDir, res.Path, notes.ScopeNotes); rerr == nil && prior != nil {
|
||||
_ = os.WriteFile(t.Abs, prior, 0o644)
|
||||
}
|
||||
return nil, fmt.Errorf("edit rolled back, stamp provenance: %w", serr)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user