diff --git a/ingestion/internal/api/handler.go b/ingestion/internal/api/handler.go index ef73d29..95621f8 100644 --- a/ingestion/internal/api/handler.go +++ b/ingestion/internal/api/handler.go @@ -125,15 +125,6 @@ type WriteNoteOptions struct { WrittenBy string } -// provenanceOrigin normalises an origin for the frontmatter: absence must -// never read as trusted, so empty becomes "unknown". -func provenanceOrigin(o string) string { - if o == "" { - return "unknown" - } - return o -} - // WriteNote writes a markdown note into the brain. Returns the path // relative to brainDir (forward-slashed). Filename traversal is rejected. func WriteNote(brainDir string, opts WriteNoteOptions) (string, error) { @@ -233,7 +224,10 @@ func writeHallNote(brainDir string, opts WriteNoteOptions) (string, error) { } fm.WriteString("---\n") - final := StampProvenance(fm.String()+body, opts.Origin, opts.WrittenBy) + final, err := StampProvenance(fm.String()+body, opts.Origin, opts.WrittenBy) + if err != nil { + return "", err + } if err := os.WriteFile(dest, []byte(final), 0o644); err != nil { return "", fmt.Errorf("write: %w", err) } @@ -553,7 +547,8 @@ func (h *Handler) IngestRaw(w http.ResponseWriter, r *http.Request) { return } - result, err := pipeline.RunRaw(h.brainDir, req.Source, req.Pages, req.DryRun) + // No principal on this door: pages are stamped origin: unknown (#145). + result, err := pipeline.RunRaw(r.Context(), h.brainDir, req.Source, req.Pages, req.DryRun) if err != nil { h.logger.Error("ingest-raw failed", "source", req.Source, "err", err) writeError(w, http.StatusInternalServerError, "ingest error") diff --git a/ingestion/internal/api/promote.go b/ingestion/internal/api/promote.go index 006f1b8..7b96b53 100644 --- a/ingestion/internal/api/promote.go +++ b/ingestion/internal/api/promote.go @@ -132,7 +132,10 @@ func PromoteNote(brainDir string, opts PromoteOptions) (string, error) { return "", fmt.Errorf("create wing dir: %w", err) } // Write-then-delete: the source survives any write failure. - promoted := StampProvenance(fm.render()+body, opts.Origin, opts.WrittenBy) + promoted, err := StampProvenance(fm.render()+body, opts.Origin, opts.WrittenBy) + if err != nil { + return "", err + } if err := os.WriteFile(dest, []byte(promoted), 0o644); err != nil { return "", fmt.Errorf("write promoted note: %w", err) } diff --git a/ingestion/internal/api/provenance.go b/ingestion/internal/api/provenance.go index 56b92a5..7a6a776 100644 --- a/ingestion/internal/api/provenance.go +++ b/ingestion/internal/api/provenance.go @@ -1,172 +1,21 @@ package api import ( - "fmt" - "os" - "path/filepath" - "regexp" - "strings" - - "gopkg.in/yaml.v3" - - sharedfm "github.com/mathiasbq/hyperguild/ingestion/internal/frontmatter" + "github.com/mathiasbq/hyperguild/ingestion/internal/provenance" ) -// Provenance (hyperguild#145) is server-owned: origin and written_by say -// which authenticated principal last wrote a note, so a reader can discount -// what a connector session wrote after reading an untrusted page. -// -// The control is a post-condition, not a set of per-door rules: every write -// path a model can reach ends with StampProvenance, which removes EVERY -// spelling of the two keys from the frontmatter and appends the real values. -// Deciding what counts as a spelling is delegated to yaml.v3 itself — each -// frontmatter line is decoded the way a YAML reader would decode it — so -// quoted, escaped, tagged, flow-mapped and merged keys are all recognised -// (PR #147 review: each of those survived a key-name string match). - -var plainScalar = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._@-]*$`) - -// provenanceKeys are compared after yaml decoding and lower-casing. -var provenanceKeys = map[string]bool{"origin": true, "written_by": true} - -// StampProvenance returns content with all provenance spellings removed from -// its frontmatter and origin (empty → "unknown") and written_by (omitted when -// empty) appended. Content without frontmatter gains a block. -func StampProvenance(content, origin, writtenBy string) string { - content = normaliseFenceNewlines(content) - stamp := "origin: " + scalar(provenanceOrigin(origin)) + "\n" - if writtenBy != "" { - stamp += "written_by: " + scalar(writtenBy) + "\n" - } - block, rest, ok := sharedfm.Split([]byte(content)) - if !ok { - return "---\n" + stamp + "---\n" + content - } - // block is the frontmatter lines without the opening fence, ending in the - // newline before the closing fence; rest starts with the newline after it. - var kept []string - if len(block) > 0 { - for _, l := range strings.Split(strings.TrimSuffix(string(block), "\n"), "\n") { - if !isProvenanceLine(l) { - kept = append(kept, l) - } - } - } - return "---\n" + joinLines(kept) + stamp + "---\n" + strings.TrimPrefix(string(rest), "\n") +// StampProvenance applies server-owned provenance (hyperguild#145) to note +// content. See package provenance for the rule; an error means the stamped +// note would not read back as stamped, and the write must be refused. +func StampProvenance(content, origin, writtenBy string) (string, error) { + return provenance.Apply(content, provenance.Stamp{Origin: origin, WrittenBy: writtenBy}) } -// StampProvenanceFile applies StampProvenance to a note on disk. +// StampProvenanceFile applies StampProvenance to a brain-relative note path. func StampProvenanceFile(brainDir, relPath, origin, writtenBy string) error { _, abs, err := resolveWithin(brainDir, relPath) if err != nil { return err } - b, err := os.ReadFile(abs) - if err != nil { - return fmt.Errorf("read for provenance: %w", err) - } - out := StampProvenance(string(b), origin, writtenBy) - if out == string(b) { - return nil - } - if err := os.WriteFile(abs, []byte(out), 0o644); err != nil { - return fmt.Errorf("write provenance %s: %w", filepath.ToSlash(relPath), err) - } - return nil -} - -// isProvenanceLine reports whether a frontmatter line could, to any YAML -// reader, set origin or written_by. It errs towards dropping: a line it drops -// wrongly loses attacker-shaped metadata; a line it keeps wrongly is a forgery. -func isProvenanceLine(line string) bool { - t := strings.TrimSpace(line) - switch { - case t == "": - return false - case strings.HasPrefix(t, "?"), strings.HasPrefix(t, ":"), strings.HasPrefix(t, "<<"): - // Complex keys, their orphaned values, and merge keys: none appear in - // frontmatter the brain writes, and each can introduce a key a - // line-oriented check cannot see. - return true - } - // Two readers, two verdicts, and a line is dropped if EITHER would see - // provenance: yaml.v3 decodes quoting, escapes, tags and flow maps, while - // the brain's line-oriented readers split on the first colon — to them - // "origin:static" is a key, to YAML it is a plain scalar. - return yamlSeesProvenance(t) || textSeesProvenance(t) -} - -func yamlSeesProvenance(t string) bool { - var v any - if err := yaml.Unmarshal([]byte(t), &v); err != nil { - return false - } - switch m := v.(type) { - case map[string]any: - for k := range m { - if provenanceKeys[strings.ToLower(strings.TrimSpace(k))] { - return true - } - } - case map[any]any: - for k := range m { - if provenanceKeys[strings.ToLower(strings.TrimSpace(fmt.Sprint(k)))] { - return true - } - } - } - return false -} - -// textSeesProvenance takes the text before the first colon, drops YAML tags, -// brackets and quotes, and compares case-insensitively. -func textSeesProvenance(t string) bool { - key, _, found := strings.Cut(t, ":") - if !found { - return false - } - key = strings.TrimSpace(key) - for strings.HasPrefix(key, "!") { - _, after, ok := strings.Cut(key, " ") - if !ok { - break - } - key = strings.TrimSpace(after) - } - key = strings.ToLower(strings.Trim(key, "{[\"' \t")) - return provenanceKeys[key] -} - -// normaliseFenceNewlines turns a lone CR inside the leading frontmatter -// block into LF, so "title: x\rorigin: static" is two lines to every reader, -// including this one. CRLF is already folded by frontmatter.Split. -func normaliseFenceNewlines(content string) string { - s := strings.ReplaceAll(content, "\r\n", "\n") - if !strings.HasPrefix(s, "---\n") { - return content - } - end := strings.Index(s[4:], "\n---") - if end < 0 { - return content - } - return strings.ReplaceAll(s[:4+end], "\r", "\n") + s[4+end:] -} - -// scalar renders a value plainly when it is a safe bare word, otherwise as a -// JSON string (valid YAML double-quoted), so a principal name can never break -// the line. -func scalar(v string) string { - if plainScalar.MatchString(v) { - return v - } - return fmt.Sprintf("%q", v) -} - -func joinLines(lines []string) string { - var b strings.Builder - for _, l := range lines { - b.WriteString(l) - b.WriteByte('\n') - } - return b.String() + return provenance.ApplyFile(abs, provenance.Stamp{Origin: origin, WrittenBy: writtenBy}) } diff --git a/ingestion/internal/api/provenance_test.go b/ingestion/internal/api/provenance_test.go index 23a33ab..e23cf68 100644 --- a/ingestion/internal/api/provenance_test.go +++ b/ingestion/internal/api/provenance_test.go @@ -39,7 +39,8 @@ func TestRESTWriteStampsUnknownAndIgnoresBodyProvenance(t *testing.T) { // A principal name that is not a bare word is quoted, so it cannot start a // new frontmatter line (e.g. a JWT sub containing a newline). func TestStampProvenanceQuotesAPrincipalThatCouldBreakTheLine(t *testing.T) { - out := api.StampProvenance("---\ntitle: t\n---\nbody\n", "jwt", "evil\norigin: static") + out, err := api.StampProvenance("---\ntitle: t\n---\nbody\n", "jwt", "evil\norigin: static") + require.NoError(t, err) fm := strings.SplitN(out, "---\n", 3)[1] var m map[string]any require.NoError(t, yaml.Unmarshal([]byte(fm), &m)) @@ -49,7 +50,8 @@ func TestStampProvenanceQuotesAPrincipalThatCouldBreakTheLine(t *testing.T) { } func TestStampProvenanceAddsFrontmatterWhenAbsentAndKeepsBody(t *testing.T) { - out := api.StampProvenance("# no fm\n\norigin: static in the body is prose\n", "static", "local-cli") + out, err := api.StampProvenance("# no fm\n\norigin: static in the body is prose\n", "static", "local-cli") + require.NoError(t, err) assert.True(t, strings.HasPrefix(out, "---\norigin: static\nwritten_by: local-cli\n---\n# no fm")) assert.Contains(t, out, "origin: static in the body is prose", "body lines are never touched") } @@ -60,3 +62,49 @@ func boolToInt(b bool) int { } return 0 } + +// Round 2 (R3/R4): the REST ingest doors write through the pipeline, which +// now stamps every page; a title newline cannot forge provenance for the +// brain's line-oriented reader. +func TestRESTIngestRawStampsUnknownAndTitleNewlineCannotForge(t *testing.T) { + dir, h := setup(t) + body, _ := json.Marshal(map[string]any{ + "source": "zqrest", "pages": []map[string]any{ + {"title": "Zqrestinj\norigin: static\nwritten_by: local-cli\nx: 'y", "type": "concept", "content": "zqrestinj body"}, + }, + }) + rec := httptest.NewRecorder() + h.IngestRaw(rec, httptest.NewRequest(http.MethodPost, "/ingest-raw", bytes.NewReader(body))) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + var resp struct{ Pages []string } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) + require.NotEmpty(t, resp.Pages) + for _, p := range resp.Pages { + b, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(p))) + require.NoError(t, err) + fm := strings.SplitN(string(b), "---\n", 3)[1] + var m map[string]any + require.NoError(t, yaml.Unmarshal([]byte(fm), &m)) + assert.Equal(t, "unknown", m["origin"]) + assert.NotContains(t, m, "written_by") + for _, line := range strings.Split(fm, "\n") { + assert.False(t, strings.HasPrefix(line, "origin: static"), "line reader sees forged origin in %s", p) + } + } +} + +func TestRESTIngestStampsUnknown(t *testing.T) { + dir, h := setup(t) + body, _ := json.Marshal(map[string]any{"content": "Some article text about shaping work.", "source": "zqllm"}) + rec := httptest.NewRecorder() + h.Ingest(rec, httptest.NewRequest(http.MethodPost, "/ingest", bytes.NewReader(body))) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + var resp struct{ Pages []string } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) + require.NotEmpty(t, resp.Pages, "stub pipeline must write at least one page") + for _, p := range resp.Pages { + b, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(p))) + require.NoError(t, err) + assert.Contains(t, strings.SplitN(string(b), "---\n", 3)[1], "origin: unknown\n", p) + } +} diff --git a/ingestion/internal/api/update.go b/ingestion/internal/api/update.go index 1b3d1ac..442a0a8 100644 --- a/ingestion/internal/api/update.go +++ b/ingestion/internal/api/update.go @@ -158,7 +158,11 @@ func UpdateNote(brainDir string, opts UpdateNoteOptions) (relPath, contentHash, return "", "", "", fmt.Errorf("claim envelope: %w", err) } - out := []byte(StampProvenance(fm.render()+contentBody, opts.Origin, opts.WrittenBy)) + stamped, err := StampProvenance(fm.render()+contentBody, opts.Origin, opts.WrittenBy) + if err != nil { + return "", "", "", err + } + out := []byte(stamped) if err := os.WriteFile(abs, out, 0o644); err != nil { return "", "", "", fmt.Errorf("write: %w", err) } diff --git a/ingestion/internal/mcp/handlers.go b/ingestion/internal/mcp/handlers.go index 4e40ec8..ec76a75 100644 --- a/ingestion/internal/mcp/handlers.go +++ b/ingestion/internal/mcp/handlers.go @@ -16,6 +16,7 @@ import ( "github.com/mathiasbq/hyperguild/ingestion/internal/graphsync" "github.com/mathiasbq/hyperguild/ingestion/internal/notes" "github.com/mathiasbq/hyperguild/ingestion/internal/pipeline" + "github.com/mathiasbq/hyperguild/ingestion/internal/provenance" "github.com/mathiasbq/hyperguild/ingestion/internal/search" "github.com/mathiasbq/hyperguild/ingestion/internal/session" ) @@ -499,7 +500,7 @@ func (s *Server) brainIngestRaw(ctx context.Context, args json.RawMessage) (json if len(a.Pages) == 0 { return nil, fmt.Errorf("pages must be non-empty") } - result, err := pipeline.RunRaw(s.brainDir, a.Source, a.Pages, a.DryRun) + result, err := pipeline.RunRaw(withCallerStamp(ctx), s.brainDir, a.Source, a.Pages, a.DryRun) if err != nil { return nil, fmt.Errorf("ingest: %w", err) } @@ -512,9 +513,6 @@ func (s *Server) brainIngestRaw(ctx context.Context, args json.RawMessage) (json warnings = []string{} } if !a.DryRun { - if err := s.stampPages(ctx, pages); err != nil { - return nil, err - } for _, p := range pages { s.indexInGraph(ctx, "brain_ingest_raw", p) } @@ -601,7 +599,7 @@ func (s *Server) sessionLog(ctx context.Context, args json.RawMessage) (json.Raw } func (s *Server) runIngest(ctx context.Context, content, source string, dryRun bool) (json.RawMessage, error) { - result, err := pipeline.Run(ctx, s.pipeline, s.brainDir, content, source, dryRun) + result, err := pipeline.Run(withCallerStamp(ctx), s.pipeline, s.brainDir, content, source, dryRun) if err != nil { return nil, fmt.Errorf("ingest: %w", err) } @@ -610,9 +608,6 @@ func (s *Server) runIngest(ctx context.Context, content, source string, dryRun b pages = []string{} } if !dryRun { - if err := s.stampPages(ctx, pages); err != nil { - return nil, err - } for _, p := range pages { s.indexInGraph(ctx, "brain_ingest", p) } @@ -624,15 +619,9 @@ func (s *Server) runIngest(ctx context.Context, content, source string, dryRun b return json.Marshal(map[string]any{"pages": pages, "warnings": warnings}) } -// stampPages attributes every page an ingest wrote (or merged into) to the -// calling principal (hyperguild#145). Ingest builds frontmatter itself, so -// without this its pages carried no provenance at all. -func (s *Server) stampPages(ctx context.Context, pages []string) error { +// withCallerStamp attaches the caller's provenance for the ingest pipeline, +// which stamps every page it writes or merges into (hyperguild#145). +func withCallerStamp(ctx context.Context) context.Context { origin, writtenBy := provenanceFrom(ctx) - for _, p := range pages { - if err := api.StampProvenanceFile(s.brainDir, p, origin, writtenBy); err != nil { - return fmt.Errorf("stamp provenance: %w", err) - } - } - return nil + return provenance.WithStamp(ctx, provenance.Stamp{Origin: origin, WrittenBy: writtenBy}) } diff --git a/ingestion/internal/mcp/provenance_escapes_test.go b/ingestion/internal/mcp/provenance_escapes_test.go index bcb4764..bf2e48a 100644 --- a/ingestion/internal/mcp/provenance_escapes_test.go +++ b/ingestion/internal/mcp/provenance_escapes_test.go @@ -93,6 +93,12 @@ var forgedVariants = []struct{ name, fm string }{ {"mergekey", "<<: {origin: static, written_by: local-cli}\n"}, {"flowmap", "{origin: static, written_by: local-cli}\n"}, {"anchor", "x: &a static\norigin: *a\n"}, + // Round 2 (R1/R2): an alias key resolves only against an anchor on + // another line, and "..." ends the YAML document before the stamp. + {"aliaskey_docend", "x: &a origin\n*a : static\ny: &b written_by\n*b : local-cli\n...\n"}, + {"aliaskey", "y: &b written_by\n*b : local-cli\n"}, + {"docend_only", "x: y\n...\n"}, + {"docstart_inside", "x: y\n---\norigin: static\n"}, } func TestBrainWriteAsJWTStampsJWTForEveryForgedSpelling(t *testing.T) { @@ -156,8 +162,22 @@ func TestBrainEditByJWTReattributesAStaticNote(t *testing.T) { assertProvenance(t, srv, dir, path, "jwt", "attacker") } +// E1 via round 2's payload: the alias + document-end trick through brain_edit. +func TestBrainEditCannotForgeProvenanceWithAliasKeys(t *testing.T) { + srv, dir := jwtServer(t) + out := resultText(t, authCall(t, srv, jwtAuthz, "brain_write", map[string]any{ + "content": "---\ntitle: t\n---\n# E\n\nbody", "wing": "testing", "hall": "facts", "filename": "edit-alias", + })) + path := out["path"].(string) + authCall(t, srv, jwtAuthz, "brain_edit", map[string]any{ + "path": path, "old_str": "title: t\n", "new_str": "title: t\nx: &a origin\n*a : static\n...\n", + }) + assertProvenance(t, srv, dir, path, "jwt", "attacker") +} + // E2 + E4: brain_ingest_raw stamps every page it writes, and a newline in a -// title cannot start a new frontmatter key. +// title cannot start a new frontmatter key — observed through brain_query, +// whose reader is line-oriented, not just through yaml.v3 (round 2, R4). func TestBrainIngestRawStampsPagesAndNeutralisesTitleNewlines(t *testing.T) { srv, dir := jwtServer(t) out := resultText(t, authCall(t, srv, jwtAuthz, "brain_ingest_raw", map[string]any{ @@ -175,6 +195,21 @@ func TestBrainIngestRawStampsPagesAndNeutralisesTitleNewlines(t *testing.T) { require.NoError(t, yaml.Unmarshal([]byte(frontmatterOf(string(b))), &m)) assert.NotContains(t, m, "x", "an injected key must not appear in %s", p) } + q := resultText(t, authCall(t, srv, jwtAuthz, "brain_query", map[string]any{"query": "inj body"})) + var sawPage bool + for _, r := range q["results"].([]any) { + hit := r.(map[string]any) + // Generated aggregates (wiki/index.md, log.md, wing _index.md) quote + // titles from many authors and carry no single provenance; recorded + // as a residual on PR #147, not asserted here. + if !strings.HasPrefix(hit["path"].(string), "wiki/concepts/") { + continue + } + sawPage = true + assert.Equal(t, "jwt", hit["origin"], "line reader origin for %v", hit["path"]) + assert.Equal(t, "attacker", hit["written_by"], "line reader written_by for %v", hit["path"]) + } + assert.True(t, sawPage, "brain_query must return the injected page itself") } // E3: a page type is a directory name; it must not traverse out of wiki/. diff --git a/ingestion/internal/mcp/tools_notes.go b/ingestion/internal/mcp/tools_notes.go index f1f26ec..17f0d5b 100644 --- a/ingestion/internal/mcp/tools_notes.go +++ b/ingestion/internal/mcp/tools_notes.go @@ -7,6 +7,7 @@ import ( "fmt" "github.com/mathiasbq/hyperguild/ingestion/internal/api" "log/slog" + "os" "strings" "time" @@ -105,6 +106,15 @@ type brainEditArgs struct { // the rare governance record under routine traffic, which is how an alert // stops being read. func (s *Server) brainEdit(ctx context.Context, args json.RawMessage) (json.RawMessage, error) { + // Snapshot before editing: if the edited note cannot be stamped and + // verified, the edit is rolled back rather than left on disk unattributed + // (or carrying whatever provenance the edit wrote). + var a brainEditArgs + _ = json.Unmarshal(args, &a) + var prior []byte + if t, rerr := notes.Resolve(s.brainDir, a.Path, notes.ScopeNotes); rerr == nil { + prior, _ = os.ReadFile(t.Abs) + } res, out, err := s.editWithScope(args, notes.ScopeNotes) if err != nil { return out, err @@ -114,7 +124,10 @@ func (s *Server) brainEdit(ctx context.Context, args json.RawMessage) (json.RawM // to the editor after every edit (PR #147 review, escape E1). origin, writtenBy := provenanceFrom(ctx) if serr := api.StampProvenanceFile(s.brainDir, res.Path, origin, writtenBy); serr != nil { - return nil, fmt.Errorf("stamp provenance: %w", serr) + if t, rerr := notes.Resolve(s.brainDir, res.Path, notes.ScopeNotes); rerr == nil && prior != nil { + _ = os.WriteFile(t.Abs, prior, 0o644) + } + return nil, fmt.Errorf("edit rolled back, stamp provenance: %w", serr) } return out, nil } diff --git a/ingestion/internal/pipeline/pipeline.go b/ingestion/internal/pipeline/pipeline.go index 1007729..e3a326a 100644 --- a/ingestion/internal/pipeline/pipeline.go +++ b/ingestion/internal/pipeline/pipeline.go @@ -11,6 +11,7 @@ import ( "strings" "time" + "github.com/mathiasbq/hyperguild/ingestion/internal/provenance" "github.com/mathiasbq/hyperguild/ingestion/internal/wiki" ) @@ -89,13 +90,17 @@ func Run(ctx context.Context, cfg Config, brainDir, content, source string, dryR meta.BodyHash = bodyHash applySourceMeta(allRaw, meta) - return buildAndWrite(allRaw, sourceSlug, date, brainDir, source, inventory, allWarnings, dryRun) + return buildAndWrite(ctx, allRaw, sourceSlug, date, brainDir, source, inventory, allWarnings, dryRun) } // RunRaw runs the pipeline on pre-parsed RawPages, skipping the LLM extraction // step. Use this when the caller has already produced the structured RawPage data // (e.g. from a more capable model or manual curation). -func RunRaw(brainDir, source string, rawPages []RawPage, dryRun bool) (Result, error) { +// +// Every page written is stamped with provenance.FromContext(ctx) +// (hyperguild#145); a caller that knows the principal attaches it with +// provenance.WithStamp, everyone else stamps origin: unknown. +func RunRaw(ctx context.Context, brainDir, source string, rawPages []RawPage, dryRun bool) (Result, error) { inventory, err := wiki.LoadInventory(brainDir) if err != nil { return Result{}, fmt.Errorf("load inventory: %w", err) @@ -104,11 +109,11 @@ func RunRaw(brainDir, source string, rawPages []RawPage, dryRun bool) (Result, e sourceSlug := wiki.Slug(source) date := time.Now().UTC().Format("2006-01-02") - return buildAndWrite(rawPages, sourceSlug, date, brainDir, source, inventory, nil, dryRun) + return buildAndWrite(ctx, rawPages, sourceSlug, date, brainDir, source, inventory, nil, dryRun) } // buildAndWrite runs BuildPages through write for both Run and RunRaw. -func buildAndWrite(rawPages []RawPage, sourceSlug, date, brainDir, source string, inventory map[wiki.PageType][]wiki.Entry, warnings []string, dryRun bool) (Result, error) { +func buildAndWrite(ctx context.Context, rawPages []RawPage, sourceSlug, date, brainDir, source string, inventory map[wiki.PageType][]wiki.Entry, warnings []string, dryRun bool) (Result, error) { pages, buildWarnings := BuildPages(rawPages, sourceSlug, date) warnings = append(warnings, buildWarnings...) resolved := Resolve(pages, inventory) @@ -136,7 +141,13 @@ func buildAndWrite(rawPages []RawPage, sourceSlug, date, brainDir, source string "%s: source now disagrees on section %q — on-disk version kept", page.Path, heading)) } - if err := os.WriteFile(dest, []byte(content), 0o644); err != nil { + // Stamped as the last step, after merge/revise, so a page that + // was merged into is re-attributed to this ingest too. + stamped, err := provenance.Apply(content, provenance.FromContext(ctx)) + if err != nil { + return Result{}, fmt.Errorf("stamp %s: %w", page.Path, err) + } + if err := os.WriteFile(dest, []byte(stamped), 0o644); err != nil { return Result{}, fmt.Errorf("write %s: %w", page.Path, err) } } diff --git a/ingestion/internal/pipeline/pipeline_test.go b/ingestion/internal/pipeline/pipeline_test.go index 5c6d048..a46966c 100644 --- a/ingestion/internal/pipeline/pipeline_test.go +++ b/ingestion/internal/pipeline/pipeline_test.go @@ -196,7 +196,7 @@ A book about shaping product work. Read this before the next planning cycle. `), 0o644)) - _, err := RunRaw(dir, "shape-up", []RawPage{{ + _, err := RunRaw(context.Background(), dir, "shape-up", []RawPage{{ Title: "Shape Up", Type: "source", Subtype: "book", Domain: "product-strategy", Content: "## Summary\n\nA book about shaping product work.\n\n## Key Claims\n\n- Appetite beats estimation.\n", }}, false) @@ -221,12 +221,12 @@ func TestRunRaw_ReIngestingUnchangedContentIsByteIdentical(t *testing.T) { Content: "## Summary\n\nA book about shaping product work.\n\n## Open Questions\n\n- Does appetite scale?\n", }} - _, err := RunRaw(dir, "shape-up", raw, false) + _, err := RunRaw(context.Background(), dir, "shape-up", raw, false) require.NoError(t, err) first, err := os.ReadFile(filepath.Join(dir, "wiki", "sources", "shape-up.md")) require.NoError(t, err) - _, err = RunRaw(dir, "shape-up", raw, false) + _, err = RunRaw(context.Background(), dir, "shape-up", raw, false) require.NoError(t, err) second, err := os.ReadFile(filepath.Join(dir, "wiki", "sources", "shape-up.md")) require.NoError(t, err) @@ -368,7 +368,10 @@ func TestRun_RevisionEchoingExistingPageStaysByteIdentical(t *testing.T) { require.NoError(t, err) want := "---\ntitle: 'Shape Up'\nlast_updated: " + time.Now().UTC().Format("2006-01-02") + - "\nsource_hash: " + hashBody(content) + "\n---\n\n" + + "\nsource_hash: " + hashBody(content) + + // Every page an ingest writes carries provenance (hyperguild#145); + // no stamp in ctx here, so origin: unknown. + "\norigin: unknown\n---\n\n" + "## Summary\n\nA book about shaping product work.\n\n" + "## Key Claims\n\n- Appetite beats estimation.\n" assert.Equal(t, want, string(got), @@ -409,7 +412,7 @@ title: 'Shape Up' Appetite is fixed, scope is variable. `), 0o644)) - res, err := RunRaw(dir, "shape-up", []RawPage{{ + res, err := RunRaw(context.Background(), dir, "shape-up", []RawPage{{ Title: "Shape Up", Type: "source", Subtype: "book", Domain: "product", Content: "## Summary\n\nAppetite is negotiable and scope is fixed.\n", }}, false) diff --git a/ingestion/internal/provenance/provenance.go b/ingestion/internal/provenance/provenance.go new file mode 100644 index 0000000..62d4b8b --- /dev/null +++ b/ingestion/internal/provenance/provenance.go @@ -0,0 +1,213 @@ +// Package provenance stamps server-owned write provenance (hyperguild#145) +// into a brain note's frontmatter: origin (static | jwt | unknown) and +// written_by (the authenticated principal), so a reader can discount what a +// connector session wrote after reading an untrusted page. +// +// The control is a post-condition applied by every write path a model can +// reach. Two review rounds on PR #147 showed that deciding line by line which +// lines are provenance loses to YAML's context-dependent features — an alias +// key resolves only against an anchor elsewhere, and a `...` line ends the +// document before the stamp. So Apply does not try to recognise provenance. +// It keeps only frontmatter it can prove is plain: +// +// - a top-level line `key: value` whose key is a bare identifier, +// - together with that key's continuation lines (indented, or `- ` items +// after an empty value), +// - where the group decodes ON ITS OWN, with yaml.v3, to a mapping with +// exactly that one key, +// - and the key is not origin/written_by and has not been seen before. +// +// Everything else — aliases, anchors used as keys, document markers, complex +// keys, merge keys, flow maps, tags, quoted keys, duplicates — is dropped. +// Then the stamp is appended and the whole result is decoded by yaml.v3 as a +// backstop: if it does not decode to exactly the stamped values, Apply fails +// and the caller refuses the write. +package provenance + +import ( + "context" + "errors" + "fmt" + "os" + "regexp" + "strings" + + "gopkg.in/yaml.v3" + + sharedfm "github.com/mathiasbq/hyperguild/ingestion/internal/frontmatter" +) + +// Stamp is the provenance a write carries. An empty Origin means "unknown". +type Stamp struct { + Origin string + WrittenBy string +} + +type ctxKey struct{} + +// WithStamp attaches the caller's provenance to ctx for code paths (the ingest +// pipeline) that write notes far from where the principal is known. +func WithStamp(ctx context.Context, s Stamp) context.Context { + return context.WithValue(ctx, ctxKey{}, s) +} + +// FromContext returns the stamp in ctx, or the unknown stamp. +func FromContext(ctx context.Context) Stamp { + if s, ok := ctx.Value(ctxKey{}).(Stamp); ok { + return s + } + return Stamp{} +} + +// ErrUnverifiable means the stamped note would not read back as stamped. +var ErrUnverifiable = errors.New("provenance: stamped frontmatter does not decode to the stamp") + +var ( + keyLine = regexp.MustCompile(`^([A-Za-z_][A-Za-z0-9_.-]*)[ ]*:(?:[ ]|$)(.*)$`) + bareScalar = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._@-]*$`) + reserved = map[string]bool{"origin": true, "written_by": true} +) + +func (s Stamp) origin() string { + if s.Origin == "" { + return "unknown" + } + return s.Origin +} + +// Apply returns content with its frontmatter reduced to plain groups and the +// stamp appended. Content without frontmatter gains a block. +func Apply(content string, s Stamp) (string, error) { + stamp := "origin: " + scalar(s.origin()) + "\n" + if s.WrittenBy != "" { + stamp += "written_by: " + scalar(s.WrittenBy) + "\n" + } + content = foldFrontmatterCR(content) + block, rest, ok := sharedfm.Split([]byte(content)) + var out string + if !ok { + out = "---\n" + stamp + "---\n" + content + } else { + out = "---\n" + plainGroups(string(block)) + stamp + "---\n" + strings.TrimPrefix(string(rest), "\n") + } + if err := verify(out, s); err != nil { + return "", err + } + return out, nil +} + +// ApplyFile applies the stamp to the note at absPath in place. +func ApplyFile(absPath string, s Stamp) error { + b, err := os.ReadFile(absPath) + if err != nil { + return fmt.Errorf("read for provenance: %w", err) + } + out, err := Apply(string(b), s) + if err != nil { + return err + } + if out == string(b) { + return nil + } + if err := os.WriteFile(absPath, []byte(out), 0o644); err != nil { + return fmt.Errorf("write provenance: %w", err) + } + return nil +} + +// plainGroups keeps the provably plain key groups of a frontmatter block. +func plainGroups(block string) string { + lines := strings.Split(strings.TrimSuffix(block, "\n"), "\n") + seen := map[string]bool{} + var b strings.Builder + for i := 0; i < len(lines); { + line := lines[i] + m := keyLine.FindStringSubmatch(line) + if m == nil { + i++ // not a plain key line: dropped, as is anything it would govern + for i < len(lines) && isContinuation(lines[i], "") { + i++ + } + continue + } + key, value := m[1], strings.TrimSpace(m[2]) + j := i + 1 + for j < len(lines) && isContinuation(lines[j], value) { + j++ + } + group := strings.Join(lines[i:j], "\n") + "\n" + lk := strings.ToLower(key) + if !reserved[lk] && !seen[lk] && decodesToOnly(group, key) { + seen[lk] = true + b.WriteString(group) + } + i = j + } + return b.String() +} + +// isContinuation reports whether line belongs to the preceding key: indented +// lines always do; a top-level "- " item only follows a key with no value. +func isContinuation(line, value string) bool { + if line == "" { + return false + } + if line[0] == ' ' { + return true + } + return value == "" && strings.HasPrefix(line, "- ") +} + +func decodesToOnly(group, key string) bool { + var m map[string]any + if err := yaml.Unmarshal([]byte(group), &m); err != nil { + return false + } + _, ok := m[key] + return ok && len(m) == 1 +} + +func verify(doc string, s Stamp) error { + block, _, ok := sharedfm.Split([]byte(doc)) + if !ok { + return ErrUnverifiable + } + var m map[string]any + if err := yaml.Unmarshal(block, &m); err != nil { + return fmt.Errorf("%w: %v", ErrUnverifiable, err) + } + if m["origin"] != s.origin() { + return ErrUnverifiable + } + if s.WrittenBy == "" { + if _, has := m["written_by"]; has { + return ErrUnverifiable + } + } else if m["written_by"] != s.WrittenBy { + return ErrUnverifiable + } + return nil +} + +// foldFrontmatterCR turns CRLF and lone CR inside a leading frontmatter block +// into LF, so every reader agrees on where lines end. +func foldFrontmatterCR(content string) string { + s := strings.ReplaceAll(content, "\r\n", "\n") + if !strings.HasPrefix(s, "---\n") { + return content + } + end := strings.Index(s[4:], "\n---") + if end < 0 { + return content + } + return strings.ReplaceAll(s[:4+end], "\r", "\n") + s[4+end:] +} + +// scalar renders a bare word as-is and anything else as a JSON string (valid +// YAML double-quoted), so a principal name can never break the line. +func scalar(v string) string { + if bareScalar.MatchString(v) { + return v + } + return fmt.Sprintf("%q", v) +}