diff --git a/ingestion/internal/api/handler.go b/ingestion/internal/api/handler.go index bdff89c..6359f74 100644 --- a/ingestion/internal/api/handler.go +++ b/ingestion/internal/api/handler.go @@ -118,6 +118,11 @@ type WriteNoteOptions struct { Wing string Hall string SourceType string // "internal" marks a first-party observation (e.g. claudewatcher) that needs no external citation + // Origin/WrittenBy are server-owned provenance (hyperguild#145): any + // origin/written_by in Content's own frontmatter is discarded, so a + // caller cannot claim a more trusted origin than it authenticated as. + Origin string + WrittenBy string } // WriteNote writes a markdown note into the brain. Returns the path @@ -169,9 +174,6 @@ func writeHallNote(brainDir string, opts WriteNoteOptions) (string, error) { return "", fmt.Errorf("claim envelope: %w", err) } - if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { - return "", fmt.Errorf("create hall dir: %w", err) - } emitted := make(map[string]bool, 6) var fm strings.Builder @@ -219,7 +221,17 @@ func writeHallNote(brainDir string, opts WriteNoteOptions) (string, error) { } fm.WriteString("---\n") - if err := os.WriteFile(dest, []byte(fm.String()+body), 0o644); err != nil { + final, err := StampProvenance(fm.String()+body, opts.Origin, opts.WrittenBy) + if err != nil { + return "", err + } + // The directory is created only once the note is known to be writable: + // a refused write leaves nothing behind, not even an empty hall dir + // (brain#21; provenance refusal joins the envelope refusal here). + if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { + return "", fmt.Errorf("create hall dir: %w", err) + } + if err := os.WriteFile(dest, []byte(final), 0o644); err != nil { return "", fmt.Errorf("write: %w", err) } rel, _ := filepath.Rel(brainDir, dest) @@ -295,7 +307,16 @@ func (h *Handler) Write(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "invalid JSON") return } - relPath, err := WriteNote(h.brainDir, WriteNoteOptions(req)) + // Field by field, not a struct conversion: provenance is server-owned and + // this door carries no principal, so it stamps origin: unknown + // (hyperguild#145). A JSON body must never be able to set it. + relPath, err := WriteNote(h.brainDir, WriteNoteOptions{ + Content: req.Content, + Filename: req.Filename, + Wing: req.Wing, + Hall: req.Hall, + SourceType: req.SourceType, + }) if err != nil { h.logger.Error("write failed", "err", err) writeError(w, http.StatusBadRequest, err.Error()) @@ -529,7 +550,8 @@ func (h *Handler) IngestRaw(w http.ResponseWriter, r *http.Request) { return } - result, err := pipeline.RunRaw(h.brainDir, req.Source, req.Pages, req.DryRun) + // No principal on this door: pages are stamped origin: unknown (#145). + result, err := pipeline.RunRaw(r.Context(), h.brainDir, req.Source, req.Pages, req.DryRun) if err != nil { h.logger.Error("ingest-raw failed", "source", req.Source, "err", err) writeError(w, http.StatusInternalServerError, "ingest error") @@ -585,7 +607,11 @@ func (h *Handler) Promote(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "invalid JSON") return } - rel, err := PromoteNote(h.brainDir, PromoteOptions(req)) + // Field by field: this door carries no principal, so the promoted note is + // stamped origin: unknown and the body cannot set provenance (#145). + rel, err := PromoteNote(h.brainDir, PromoteOptions{ + Filename: req.Filename, Wing: req.Wing, Hall: req.Hall, Slug: req.Slug, + }) if err != nil { writeError(w, http.StatusBadRequest, err.Error()) return diff --git a/ingestion/internal/api/promote.go b/ingestion/internal/api/promote.go index 8def9e0..7b96b53 100644 --- a/ingestion/internal/api/promote.go +++ b/ingestion/internal/api/promote.go @@ -72,6 +72,10 @@ type PromoteOptions struct { Wing string Hall string Slug string // optional; defaults to Filename minus date prefix + .md + // Origin/WrittenBy re-attribute the promoted note to the promoter: the raw + // note's own frontmatter claims are not provenance (hyperguild#145). + Origin string + WrittenBy string } // PromoteNote moves a note from brain/raw/ into the structured wiki: it @@ -128,7 +132,11 @@ func PromoteNote(brainDir string, opts PromoteOptions) (string, error) { return "", fmt.Errorf("create wing dir: %w", err) } // Write-then-delete: the source survives any write failure. - if err := os.WriteFile(dest, []byte(fm.render()+body), 0o644); err != nil { + promoted, err := StampProvenance(fm.render()+body, opts.Origin, opts.WrittenBy) + if err != nil { + return "", err + } + if err := os.WriteFile(dest, []byte(promoted), 0o644); err != nil { return "", fmt.Errorf("write promoted note: %w", err) } if err := os.Remove(src); err != nil { diff --git a/ingestion/internal/api/provenance.go b/ingestion/internal/api/provenance.go new file mode 100644 index 0000000..7a6a776 --- /dev/null +++ b/ingestion/internal/api/provenance.go @@ -0,0 +1,21 @@ +package api + +import ( + "github.com/mathiasbq/hyperguild/ingestion/internal/provenance" +) + +// StampProvenance applies server-owned provenance (hyperguild#145) to note +// content. See package provenance for the rule; an error means the stamped +// note would not read back as stamped, and the write must be refused. +func StampProvenance(content, origin, writtenBy string) (string, error) { + return provenance.Apply(content, provenance.Stamp{Origin: origin, WrittenBy: writtenBy}) +} + +// StampProvenanceFile applies StampProvenance to a brain-relative note path. +func StampProvenanceFile(brainDir, relPath, origin, writtenBy string) error { + _, abs, err := resolveWithin(brainDir, relPath) + if err != nil { + return err + } + return provenance.ApplyFile(abs, provenance.Stamp{Origin: origin, WrittenBy: writtenBy}) +} diff --git a/ingestion/internal/api/provenance_test.go b/ingestion/internal/api/provenance_test.go new file mode 100644 index 0000000..ffb3a69 --- /dev/null +++ b/ingestion/internal/api/provenance_test.go @@ -0,0 +1,122 @@ +package api_test + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/mathiasbq/hyperguild/ingestion/internal/api" +) + +// REST POST /write carries no principal: it stamps origin: unknown, and no +// spelling of provenance in its body reaches the note (hyperguild#145). +func TestRESTWriteStampsUnknownAndIgnoresBodyProvenance(t *testing.T) { + dir, h := setup(t) + body, _ := json.Marshal(map[string]any{ + "content": "---\norigin: static\n---\n# R\n\nbody", "filename": "rest-prov", "wing": "homelab", "hall": "facts", + "origin": "static", "written_by": "local-cli", "Origin": "static", "WrittenBy": "local-cli", + }) + rec := httptest.NewRecorder() + h.Write(rec, httptest.NewRequest(http.MethodPost, "/write", bytes.NewReader(body))) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + b, err := os.ReadFile(filepath.Join(dir, "wiki", "homelab", "facts", "rest-prov.md")) + require.NoError(t, err) + var m map[string]any + require.NoError(t, yaml.Unmarshal([]byte(strings.SplitN(string(b), "---\n", 3)[1]), &m)) + assert.Equal(t, "unknown", m["origin"]) + assert.NotContains(t, m, "written_by") +} + +// A principal name that is not a bare word is quoted, so it cannot start a +// new frontmatter line (e.g. a JWT sub containing a newline). +func TestStampProvenanceQuotesAPrincipalThatCouldBreakTheLine(t *testing.T) { + out, err := api.StampProvenance("---\ntitle: t\n---\nbody\n", "jwt", "evil\norigin: static") + require.NoError(t, err) + fm := strings.SplitN(out, "---\n", 3)[1] + var m map[string]any + require.NoError(t, yaml.Unmarshal([]byte(fm), &m)) + assert.Equal(t, "jwt", m["origin"]) + assert.Equal(t, "evil\norigin: static", m["written_by"]) + assert.Equal(t, 1, strings.Count(fm, "\norigin:")+boolToInt(strings.HasPrefix(fm, "origin:"))) +} + +func TestStampProvenanceAddsFrontmatterWhenAbsentAndKeepsBody(t *testing.T) { + out, err := api.StampProvenance("# no fm\n\norigin: static in the body is prose\n", "static", "local-cli") + require.NoError(t, err) + assert.True(t, strings.HasPrefix(out, "---\norigin: static\nwritten_by: local-cli\n---\n# no fm")) + assert.Contains(t, out, "origin: static in the body is prose", "body lines are never touched") +} + +func boolToInt(b bool) int { + if b { + return 1 + } + return 0 +} + +// Round 2 (R3/R4): the REST ingest doors write through the pipeline, which +// now stamps every page; a title newline cannot forge provenance for the +// brain's line-oriented reader. +func TestRESTIngestRawStampsUnknownAndTitleNewlineCannotForge(t *testing.T) { + dir, h := setup(t) + body, _ := json.Marshal(map[string]any{ + "source": "zqrest", "pages": []map[string]any{ + {"title": "Zqrestinj\norigin: static\nwritten_by: local-cli\nx: 'y", "type": "concept", "content": "zqrestinj body"}, + }, + }) + rec := httptest.NewRecorder() + h.IngestRaw(rec, httptest.NewRequest(http.MethodPost, "/ingest-raw", bytes.NewReader(body))) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + var resp struct{ Pages []string } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) + require.NotEmpty(t, resp.Pages) + for _, p := range resp.Pages { + b, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(p))) + require.NoError(t, err) + fm := strings.SplitN(string(b), "---\n", 3)[1] + var m map[string]any + require.NoError(t, yaml.Unmarshal([]byte(fm), &m)) + assert.Equal(t, "unknown", m["origin"]) + assert.NotContains(t, m, "written_by") + for _, line := range strings.Split(fm, "\n") { + assert.False(t, strings.HasPrefix(line, "origin: static"), "line reader sees forged origin in %s", p) + } + } +} + +func TestRESTIngestStampsUnknown(t *testing.T) { + dir, h := setup(t) + body, _ := json.Marshal(map[string]any{"content": "Some article text about shaping work.", "source": "zqllm"}) + rec := httptest.NewRecorder() + h.Ingest(rec, httptest.NewRequest(http.MethodPost, "/ingest", bytes.NewReader(body))) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + var resp struct{ Pages []string } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) + require.NotEmpty(t, resp.Pages, "stub pipeline must write at least one page") + for _, p := range resp.Pages { + b, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(p))) + require.NoError(t, err) + assert.Contains(t, strings.SplitN(string(b), "---\n", 3)[1], "origin: unknown\n", p) + } +} + +// A refused write leaves nothing behind, not even an empty hall directory +// (brain#21's invariant, round 3 non-blocking finding). +func TestWriteNoteRefusedForProvenanceLeavesNoDirectory(t *testing.T) { + dir := t.TempDir() + _, err := api.WriteNote(dir, api.WriteNoteOptions{ + Content: "# x\n\nbody", Wing: "freshwing", Hall: "facts", Filename: "x", Origin: "jwt", WrittenBy: "bad\xff", + }) + require.Error(t, err) + _, statErr := os.Stat(filepath.Join(dir, "wiki", "freshwing")) + assert.True(t, os.IsNotExist(statErr), "no wing directory may be created by a refused write") +} diff --git a/ingestion/internal/api/update.go b/ingestion/internal/api/update.go index 085796d..442a0a8 100644 --- a/ingestion/internal/api/update.go +++ b/ingestion/internal/api/update.go @@ -46,6 +46,11 @@ type UpdateNoteOptions struct { Slug string Content string // new full body (whole-note replace) Reason string // optional; stamped as supersede_reason + // Origin/WrittenBy re-stamp provenance to the updating principal: the + // body is replaced wholesale, so the last writer is the author of what + // a reader sees (hyperguild#145). Never taken from Content's fragment. + Origin string + WrittenBy string } // UpdateNote supersedes an existing note in place. It replaces the body @@ -122,7 +127,8 @@ func UpdateNote(brainDir string, opts UpdateNoteOptions) (relPath, contentHash, // overwrites below to save the other two keeps the invariant // explicit at the one place it can be silently broken by a future // reordering, not implicit in call order. - if l.key == "" || l.key == "updated_at" || l.key == "supersedes" || l.key == "supersede_reason" { + if l.key == "" || l.key == "updated_at" || l.key == "supersedes" || l.key == "supersede_reason" || + l.key == "origin" || l.key == "written_by" { continue } fm.set(l.key, l.value) @@ -152,7 +158,11 @@ func UpdateNote(brainDir string, opts UpdateNoteOptions) (relPath, contentHash, return "", "", "", fmt.Errorf("claim envelope: %w", err) } - out := []byte(fm.render() + contentBody) + stamped, err := StampProvenance(fm.render()+contentBody, opts.Origin, opts.WrittenBy) + if err != nil { + return "", "", "", err + } + out := []byte(stamped) if err := os.WriteFile(abs, out, 0o644); err != nil { return "", "", "", fmt.Errorf("write: %w", err) } diff --git a/ingestion/internal/brainstore/store.go b/ingestion/internal/brainstore/store.go index 8f30ef7..c7633d8 100644 --- a/ingestion/internal/brainstore/store.go +++ b/ingestion/internal/brainstore/store.go @@ -49,10 +49,12 @@ func (s *Store) Write(ctx context.Context, n capture.Note) (capture.Ref, error) n.Content = content relPath, err := api.WriteNote(s.brainDir, api.WriteNoteOptions{ - Content: n.Content, - Filename: n.Filename, - Wing: n.Wing, - Hall: n.Hall, + Content: n.Content, + Filename: n.Filename, + Wing: n.Wing, + Hall: n.Hall, + Origin: n.Origin, + WrittenBy: n.WrittenBy, }) if err != nil { return capture.Ref{}, err @@ -74,7 +76,7 @@ func (s *Store) Update(ctx context.Context, slug string, n capture.Note) (captur } n.Content = content - opts := api.UpdateNoteOptions{Content: n.Content, Reason: n.Reason} + opts := api.UpdateNoteOptions{Content: n.Content, Reason: n.Reason, Origin: n.Origin, WrittenBy: n.WrittenBy} if strings.Contains(slug, "/") { opts.Path = slug } else { diff --git a/ingestion/internal/capture/entities.go b/ingestion/internal/capture/entities.go index 0b159f8..7541a03 100644 --- a/ingestion/internal/capture/entities.go +++ b/ingestion/internal/capture/entities.go @@ -60,6 +60,7 @@ type CaptureContext struct { Classification string // caller-declared level token ("" = unspecified) Principal string // server-derived (auth); audit identity Origin Zone // server-derived trust zone; the I1 gate input + AuthPath string // server-derived: "static" or "jwt"; stamped as note provenance (#145) } // Insight is one piece of session knowledge bound for the brain. A diff --git a/ingestion/internal/capture/ports.go b/ingestion/internal/capture/ports.go index 7952e3d..d65c51e 100644 --- a/ingestion/internal/capture/ports.go +++ b/ingestion/internal/capture/ports.go @@ -40,6 +40,11 @@ type Note struct { Wing string Hall string Reason string + // Provenance (hyperguild#145) is stamped by the server from the + // authenticated principal, never taken from Content's frontmatter. + // Origin is "static", "jwt" or "unknown"; empty means "unknown". + Origin string + WrittenBy string } // BrainStore is the brain persistence port — the shared implementation of diff --git a/ingestion/internal/capture/service.go b/ingestion/internal/capture/service.go index 59af865..00b0bc8 100644 --- a/ingestion/internal/capture/service.go +++ b/ingestion/internal/capture/service.go @@ -142,7 +142,7 @@ func (s *Service) Capture(ctx context.Context, in CaptureInput) (CaptureReceipt, var landed []string for i, ins := range in.Insights { - res, item, err := s.persistInsight(ctx, ins) + res, item, err := s.persistInsight(ctx, ins, in.Context) receipt.Insights = append(receipt.Insights, res) if err != nil { receipt.Errors = append(receipt.Errors, ItemError{Item: fmt.Sprintf("insight[%d]", i), Error: err.Error()}) @@ -278,8 +278,11 @@ func escalationReason(escalators []string) string { return "required by: " + strings.Join(escalators, ", ") } -func (s *Service) persistInsight(ctx context.Context, ins Insight) (InsightResult, string, error) { - note := Note{Content: ins.Text, Wing: ins.Wing, Hall: ins.Hall, Filename: brain.Sanitise(firstLine(ins.Text))} +func (s *Service) persistInsight(ctx context.Context, ins Insight, cc CaptureContext) (InsightResult, string, error) { + note := Note{ + Content: ins.Text, Wing: ins.Wing, Hall: ins.Hall, Filename: brain.Sanitise(firstLine(ins.Text)), + Origin: cc.AuthPath, WrittenBy: cc.Principal, + } var ref Ref var err error if ins.SupersedeSlug != "" { diff --git a/ingestion/internal/capturehttp/handler.go b/ingestion/internal/capturehttp/handler.go index 15e928d..204bb66 100644 --- a/ingestion/internal/capturehttp/handler.go +++ b/ingestion/internal/capturehttp/handler.go @@ -104,6 +104,10 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // caller may have tried to put in the body. in.Context.Principal = principal in.Context.Origin = h.resolver.Resolve(principal, viaStatic) + in.Context.AuthPath = "jwt" + if viaStatic { + in.Context.AuthPath = "static" + } rec, err := h.svc.Capture(r.Context(), in) switch { diff --git a/ingestion/internal/capturehttp/provenance_test.go b/ingestion/internal/capturehttp/provenance_test.go new file mode 100644 index 0000000..b03eb53 --- /dev/null +++ b/ingestion/internal/capturehttp/provenance_test.go @@ -0,0 +1,51 @@ +package capturehttp_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/mathiasbq/hyperguild/ingestion/internal/audit" + "github.com/mathiasbq/hyperguild/ingestion/internal/brainstore" + "github.com/mathiasbq/hyperguild/ingestion/internal/capture" + "github.com/mathiasbq/hyperguild/ingestion/internal/capturehttp" + "github.com/mathiasbq/hyperguild/ingestion/internal/classification" +) + +// The REST capture door derives AuthPath itself (hyperguild#145); a body +// field cannot set it, and static vs JWT are told apart. +func TestRESTCaptureStampsAuthPathFromAuthentication(t *testing.T) { + for _, tc := range []struct { + name, authz, wantOrigin, wantBy string + v capturehttp.Validator + sovereign []string + }{ + {"static", "Bearer " + staticTok, "origin: static\n", "written_by: local-cli\n", nil, nil}, + {"jwt", "Bearer jwt", "origin: jwt\n", "written_by: koala-cli\n", fakeValidator{subject: "koala-cli"}, []string{"koala-cli"}}, + } { + t.Run(tc.name, func(t *testing.T) { + brainDir := t.TempDir() + cfg, err := classification.Load(t.TempDir()) + require.NoError(t, err) + svc := capture.NewService(brainstore.New(brainDir), fakeTracker{}, cfg, audit.NewSlogSink(nil)) + h := capturehttp.New(svc, tc.v, staticTok, "local-cli", capturehttp.NewOriginResolver(tc.sovereign)) + rr := do(t, h, tc.authz, map[string]any{ + "context": map[string]any{ + "harness": "claude-code", "actor": "mathias", "classification": "internal", + "AuthPath": "static", "auth_path": "static", "principal": "local-cli", + }, + "insights": []map[string]any{{"text": "rest provenance probe", "wing": "hyperguild", "hall": "facts"}}, + }) + require.Equal(t, 200, rr.Code, rr.Body.String()) + matches, _ := filepath.Glob(filepath.Join(brainDir, "wiki/hyperguild/facts/*.md")) + require.Len(t, matches, 1) + b, err := os.ReadFile(matches[0]) + require.NoError(t, err) + assert.Contains(t, string(b), tc.wantOrigin) + assert.Contains(t, string(b), tc.wantBy) + }) + } +} diff --git a/ingestion/internal/mcp/audit.go b/ingestion/internal/mcp/audit.go new file mode 100644 index 0000000..ea95680 --- /dev/null +++ b/ingestion/internal/mcp/audit.go @@ -0,0 +1,48 @@ +package mcp + +import ( + "context" + "encoding/json" + "log/slog" + "time" +) + +// logToolCall emits one structured record per tools/call (hyperguild#144, +// OWASP MCP08): who called which tool, against which wing/hall, and how it +// ended. Brain content is never copied into the record — only its size — so +// the audit trail does not become a second, unscrubbed copy of the brain. +func logToolCall(ctx context.Context, name string, args json.RawMessage, start time.Time, err error) { + origin, principal := provenanceFrom(ctx) + attrs := []any{ + "tool", name, + "principal", principal, + "origin", origin, + "duration_ms", time.Since(start).Milliseconds(), + } + var a struct { + Wing string `json:"wing"` + Hall string `json:"hall"` + Content string `json:"content"` + } + if json.Unmarshal(args, &a) == nil { + if a.Wing != "" { + attrs = append(attrs, "wing", a.Wing) + } + if a.Hall != "" { + attrs = append(attrs, "hall", a.Hall) + } + if a.Content != "" { + attrs = append(attrs, "content_bytes", len(a.Content)) + } + } + // The outcome is a class, never the error text: handler errors echo + // argument values (a rejected half_life, a requested path), which would + // make the audit line a copy of what the caller sent (PR #147 review E6). + // The caller already received the full error in the tool response. + outcome := "ok" + if err != nil { + outcome = "error" + } + attrs = append(attrs, "outcome", outcome) + slog.InfoContext(ctx, "mcp tool call", attrs...) +} diff --git a/ingestion/internal/mcp/audit_test.go b/ingestion/internal/mcp/audit_test.go new file mode 100644 index 0000000..d374853 --- /dev/null +++ b/ingestion/internal/mcp/audit_test.go @@ -0,0 +1,83 @@ +package mcp_test + +import ( + "bytes" + "encoding/json" + "log/slog" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// captureToolCallLog routes slog to a buffer for fn and returns the decoded +// "mcp tool call" records. +func captureToolCallLog(t *testing.T, fn func()) []map[string]any { + t.Helper() + var buf bytes.Buffer + prev := slog.Default() + slog.SetDefault(slog.New(slog.NewJSONHandler(&buf, nil))) + t.Cleanup(func() { slog.SetDefault(prev) }) + fn() + var out []map[string]any + for _, l := range strings.Split(strings.TrimSpace(buf.String()), "\n") { + var rec map[string]any + if json.Unmarshal([]byte(l), &rec) == nil && rec["msg"] == "mcp tool call" { + out = append(out, rec) + } + } + return out +} + +// hyperguild#144: brain_query/brain_read left no record of who read what. +func TestToolsCallLogsOneRecordWithToolPrincipalOriginAndScope(t *testing.T) { + srv, _, _, _ := govServer(t) + recs := captureToolCallLog(t, func() { + authCall(t, srv, "Bearer "+capStaticTok, "brain_query", + map[string]any{"query": "anything", "wing": "homelab", "hall": "facts"}) + }) + require.Len(t, recs, 1) + r := recs[0] + assert.Equal(t, "brain_query", r["tool"]) + assert.Equal(t, "local-cli", r["principal"]) + assert.Equal(t, "static", r["origin"]) + assert.Equal(t, "homelab", r["wing"]) + assert.Equal(t, "facts", r["hall"]) + assert.Equal(t, "ok", r["outcome"]) + assert.Contains(t, r, "duration_ms") +} + +func TestToolsCallWithoutDerivablePrincipalLogsOriginUnknown(t *testing.T) { + srv, _, _, _ := govServer(t) + recs := captureToolCallLog(t, func() { + authCall(t, srv, "", "brain_query", map[string]any{"query": "x"}) + }) + require.Len(t, recs, 1) + assert.Equal(t, "unknown", recs[0]["origin"]) + assert.Equal(t, "", recs[0]["principal"]) +} + +func TestToolsCallLogsErrorOutcome(t *testing.T) { + srv, _, _, _ := govServer(t) + recs := captureToolCallLog(t, func() { + authCall(t, srv, "Bearer "+capStaticTok, "brain_query", map[string]any{}) + }) + require.Len(t, recs, 1) + assert.Equal(t, "error", recs[0]["outcome"]) +} + +// The audit line must not become a second copy of brain content. +func TestToolsCallLogOmitsContentButRecordsItsSize(t *testing.T) { + const sentinel = "SENTINEL-BRAIN-CONTENT-91c2" + srv, _, _, _ := govServer(t) + recs := captureToolCallLog(t, func() { + authCall(t, srv, "Bearer "+capStaticTok, "brain_write", map[string]any{ + "content": "# t\n\n" + sentinel, "wing": "testing", "hall": "facts", "filename": "audit-sentinel", + }) + }) + require.Len(t, recs, 1) + raw, _ := json.Marshal(recs[0]) + assert.NotContains(t, string(raw), sentinel) + assert.EqualValues(t, len("# t\n\n"+sentinel), recs[0]["content_bytes"]) +} diff --git a/ingestion/internal/mcp/handlers.go b/ingestion/internal/mcp/handlers.go index 2163c94..ec76a75 100644 --- a/ingestion/internal/mcp/handlers.go +++ b/ingestion/internal/mcp/handlers.go @@ -16,6 +16,7 @@ import ( "github.com/mathiasbq/hyperguild/ingestion/internal/graphsync" "github.com/mathiasbq/hyperguild/ingestion/internal/notes" "github.com/mathiasbq/hyperguild/ingestion/internal/pipeline" + "github.com/mathiasbq/hyperguild/ingestion/internal/provenance" "github.com/mathiasbq/hyperguild/ingestion/internal/search" "github.com/mathiasbq/hyperguild/ingestion/internal/session" ) @@ -255,11 +256,14 @@ func (s *Server) brainWrite(ctx context.Context, args json.RawMessage) (json.Raw // one implementation (capture uses the same store). The read-after-write // handle {id, path, content_hash} comes back from the store; path is kept // for backward compatibility. + origin, writtenBy := provenanceFrom(ctx) ref, err := s.store.Write(ctx, capture.Note{ - Content: a.Content, - Filename: a.Filename, - Wing: a.Wing, - Hall: a.Hall, + Content: a.Content, + Filename: a.Filename, + Wing: a.Wing, + Hall: a.Hall, + Origin: origin, + WrittenBy: writtenBy, }) if err != nil { return nil, err @@ -304,11 +308,14 @@ func (s *Server) brainUpdate(ctx context.Context, args json.RawMessage) (json.Ra if a.Path != "" { slug = a.Path } + origin, writtenBy := provenanceFrom(ctx) ref, err := s.store.Update(ctx, slug, capture.Note{ - Content: a.Content, - Wing: a.Wing, - Hall: a.Hall, - Reason: a.Reason, + Content: a.Content, + Wing: a.Wing, + Hall: a.Hall, + Reason: a.Reason, + Origin: origin, + WrittenBy: writtenBy, }) if err != nil { return nil, err @@ -387,8 +394,10 @@ func (s *Server) brainPromote(ctx context.Context, args json.RawMessage) (json.R if err := json.Unmarshal(args, &a); err != nil { return nil, fmt.Errorf("parse args: %w", err) } + origin, writtenBy := provenanceFrom(ctx) relPath, err := api.PromoteNote(s.brainDir, api.PromoteOptions{ Filename: a.Filename, Wing: a.Wing, Hall: a.Hall, Slug: a.Slug, + Origin: origin, WrittenBy: writtenBy, }) if err != nil { return nil, err @@ -491,7 +500,7 @@ func (s *Server) brainIngestRaw(ctx context.Context, args json.RawMessage) (json if len(a.Pages) == 0 { return nil, fmt.Errorf("pages must be non-empty") } - result, err := pipeline.RunRaw(s.brainDir, a.Source, a.Pages, a.DryRun) + result, err := pipeline.RunRaw(withCallerStamp(ctx), s.brainDir, a.Source, a.Pages, a.DryRun) if err != nil { return nil, fmt.Errorf("ingest: %w", err) } @@ -590,7 +599,7 @@ func (s *Server) sessionLog(ctx context.Context, args json.RawMessage) (json.Raw } func (s *Server) runIngest(ctx context.Context, content, source string, dryRun bool) (json.RawMessage, error) { - result, err := pipeline.Run(ctx, s.pipeline, s.brainDir, content, source, dryRun) + result, err := pipeline.Run(withCallerStamp(ctx), s.pipeline, s.brainDir, content, source, dryRun) if err != nil { return nil, fmt.Errorf("ingest: %w", err) } @@ -609,3 +618,10 @@ func (s *Server) runIngest(ctx context.Context, content, source string, dryRun b } return json.Marshal(map[string]any{"pages": pages, "warnings": warnings}) } + +// withCallerStamp attaches the caller's provenance for the ingest pipeline, +// which stamps every page it writes or merges into (hyperguild#145). +func withCallerStamp(ctx context.Context) context.Context { + origin, writtenBy := provenanceFrom(ctx) + return provenance.WithStamp(ctx, provenance.Stamp{Origin: origin, WrittenBy: writtenBy}) +} diff --git a/ingestion/internal/mcp/provenance_escapes_test.go b/ingestion/internal/mcp/provenance_escapes_test.go new file mode 100644 index 0000000..0783f1c --- /dev/null +++ b/ingestion/internal/mcp/provenance_escapes_test.go @@ -0,0 +1,325 @@ +package mcp_test + +import ( + "bytes" + "context" + "encoding/json" + "log/slog" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/mathiasbq/hyperguild/ingestion/internal/audit" + "github.com/mathiasbq/hyperguild/ingestion/internal/brainstore" + "github.com/mathiasbq/hyperguild/ingestion/internal/capture" + "github.com/mathiasbq/hyperguild/ingestion/internal/capturehttp" + "github.com/mathiasbq/hyperguild/ingestion/internal/classification" + "github.com/mathiasbq/hyperguild/ingestion/internal/mcp" + "github.com/mathiasbq/hyperguild/ingestion/internal/pipeline" +) + +// Escapes found by the PR #147 gatekeeper review. Each test drives the real +// mcp.Server as a JWT principal "attacker" and asserts the provenance +// invariant on the file every reader sees. + +const jwtAuthz = "Bearer jwt-token" + +func jwtServer(t *testing.T) (*mcp.Server, string) { + t.Helper() + dir := newBrainRepo(t) + cfg, err := classification.Load(dir) + require.NoError(t, err) + svc := capture.NewService(brainstore.New(dir), capFakeTracker{}, cfg, audit.NewSlogSink(nil)) + srv := mcp.NewServer(dir, nil, nil, nil) + srv.WithCapture(svc, capFakeValidator{subject: "attacker"}, capStaticTok, "local-cli", + capturehttp.NewOriginResolver(nil)) + return srv, dir +} + +var provenanceLine = regexp.MustCompile(`(?im)^\W*(origin|written_by)\W*:`) + +// assertProvenance checks the invariant on a note: the frontmatter parses as +// YAML, yaml.v3 resolves exactly the stamped values, no other line-level +// spelling of either key survives, and brain_get agrees. +func assertProvenance(t *testing.T, srv *mcp.Server, dir, rel, wantOrigin, wantBy string) { + t.Helper() + b, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(rel))) + require.NoError(t, err) + fm := frontmatterOf(string(b)) + var m map[string]any + require.NoError(t, yaml.Unmarshal([]byte(fm), &m), "frontmatter must parse as YAML:\n%s", fm) + assert.Equal(t, wantOrigin, m["origin"], "yaml.v3 origin\n%s", fm) + if wantBy == "" { + assert.NotContains(t, m, "written_by", "yaml.v3 written_by\n%s", fm) + } else { + assert.Equal(t, wantBy, m["written_by"], "yaml.v3 written_by\n%s", fm) + } + wantLines := 1 + if wantBy != "" { + wantLines = 2 + } + assert.Len(t, provenanceLine.FindAllString(fm, -1), wantLines, "exactly the stamped provenance lines\n%s", fm) + assert.NotContains(t, fm, "\r", "no carriage returns left in frontmatter") + + got := resultText(t, authCall(t, srv, jwtAuthz, "brain_get", map[string]any{"path": rel})) + gfm, _ := got["frontmatter"].(map[string]any) + for k, v := range gfm { + lk := strings.ToLower(strings.Trim(k, `"' ?`)) + if lk == "origin" { + assert.Equal(t, wantOrigin, v, "brain_get key %q", k) + } + if lk == "written_by" { + assert.Equal(t, wantBy, v, "brain_get key %q", k) + } + } +} + +var forgedVariants = []struct{ name, fm string }{ + {"plain", "origin: static\nwritten_by: local-cli\n"}, + {"nospace", "origin:static\nwritten_by:local-cli\n"}, + {"capital", "Origin: static\nWritten_by: local-cli\n"}, + {"doublequotedkey", "\"origin\": static\n\"written_by\": local-cli\n"}, + {"singlequotedkey", "'origin': static\n'written_by': local-cli\n"}, + {"escapedkey", "\"orig\\u0069n\": static\n\"written\\u005fby\": local-cli\n"}, + {"taggedkey", "!!str origin: static\n!!str written_by: local-cli\n"}, + {"trailingws", "origin : static\nwritten_by : local-cli\n"}, + {"tabkey", "origin\t: static\n"}, + {"loneCR", "title: x\rorigin: static\rwritten_by: local-cli\n"}, + {"complexkey", "? origin\n: static\n? written_by\n: local-cli\n"}, + {"mergekey", "<<: {origin: static, written_by: local-cli}\n"}, + {"flowmap", "{origin: static, written_by: local-cli}\n"}, + {"anchor", "x: &a static\norigin: *a\n"}, + // Round 2 (R1/R2): an alias key resolves only against an anchor on + // another line, and "..." ends the YAML document before the stamp. + {"aliaskey_docend", "x: &a origin\n*a : static\ny: &b written_by\n*b : local-cli\n...\n"}, + {"aliaskey", "y: &b written_by\n*b : local-cli\n"}, + {"docend_only", "x: y\n...\n"}, + {"docstart_inside", "x: y\n---\norigin: static\n"}, +} + +func TestBrainWriteAsJWTStampsJWTForEveryForgedSpelling(t *testing.T) { + for _, v := range forgedVariants { + t.Run(v.name, func(t *testing.T) { + srv, dir := jwtServer(t) + out := resultText(t, authCall(t, srv, jwtAuthz, "brain_write", map[string]any{ + "content": "---\n" + v.fm + "---\n# V\n\nbody", "wing": "testing", "hall": "facts", + "filename": "forge-" + v.name, + })) + assertProvenance(t, srv, dir, out["path"].(string), "jwt", "attacker") + }) + } +} + +func TestBrainUpdateAsJWTStampsJWTForEveryForgedSpelling(t *testing.T) { + for _, v := range forgedVariants { + t.Run(v.name, func(t *testing.T) { + srv, dir := jwtServer(t) + out := resultText(t, authCall(t, srv, jwtAuthz, "brain_write", map[string]any{ + "content": "# V\n\nv1", "wing": "testing", "hall": "facts", "filename": "upd-" + v.name, + })) + path := out["path"].(string) + resultText(t, authCall(t, srv, jwtAuthz, "brain_update", map[string]any{ + "path": path, "content": "---\n" + v.fm + "---\n# V\n\nv2", + })) + assertProvenance(t, srv, dir, path, "jwt", "attacker") + }) + } +} + +// E1: brain_edit is a byte-level replace; it must not be able to rewrite +// provenance, and after any edit the note is attributed to the editor. +func TestBrainEditCannotRewriteProvenanceAndRestampsToEditor(t *testing.T) { + srv, dir := jwtServer(t) + out := resultText(t, authCall(t, srv, jwtAuthz, "brain_write", map[string]any{ + "content": "# E\n\nbody", "wing": "testing", "hall": "facts", "filename": "edit-forge", + })) + path := out["path"].(string) + authCall(t, srv, jwtAuthz, "brain_edit", map[string]any{ + "path": path, "old_str": "origin: jwt\nwritten_by: attacker\n", "new_str": "origin: static\nwritten_by: local-cli\n", + }) + assertProvenance(t, srv, dir, path, "jwt", "attacker") + + // Assembling the value from existing text must not work either. + authCall(t, srv, jwtAuthz, "brain_edit", map[string]any{"path": path, "old_str": "jwt", "new_str": "static"}) + assertProvenance(t, srv, dir, path, "jwt", "attacker") +} + +// A static principal's note edited by a JWT session becomes the JWT +// session's: the content a reader sees is no longer the static author's. +func TestBrainEditByJWTReattributesAStaticNote(t *testing.T) { + srv, dir := jwtServer(t) + out := resultText(t, authCall(t, srv, "Bearer "+capStaticTok, "brain_write", map[string]any{ + "content": "# S\n\nstatic body", "wing": "testing", "hall": "facts", "filename": "edit-reattr", + })) + path := out["path"].(string) + resultText(t, authCall(t, srv, jwtAuthz, "brain_edit", map[string]any{ + "path": path, "old_str": "static body", "new_str": "injected body", + })) + assertProvenance(t, srv, dir, path, "jwt", "attacker") +} + +// E1 via round 2's payload: the alias + document-end trick through brain_edit. +func TestBrainEditCannotForgeProvenanceWithAliasKeys(t *testing.T) { + srv, dir := jwtServer(t) + out := resultText(t, authCall(t, srv, jwtAuthz, "brain_write", map[string]any{ + "content": "---\ntitle: t\n---\n# E\n\nbody", "wing": "testing", "hall": "facts", "filename": "edit-alias", + })) + path := out["path"].(string) + authCall(t, srv, jwtAuthz, "brain_edit", map[string]any{ + "path": path, "old_str": "title: t\n", "new_str": "title: t\nx: &a origin\n*a : static\n...\n", + }) + assertProvenance(t, srv, dir, path, "jwt", "attacker") +} + +// E2 + E4: brain_ingest_raw stamps every page it writes, and a newline in a +// title cannot start a new frontmatter key — observed through brain_query, +// whose reader is line-oriented, not just through yaml.v3 (round 2, R4). +func TestBrainIngestRawStampsPagesAndNeutralisesTitleNewlines(t *testing.T) { + srv, dir := jwtServer(t) + out := resultText(t, authCall(t, srv, jwtAuthz, "brain_ingest_raw", map[string]any{ + "source": "zqsrc", "pages": []map[string]any{ + {"title": "Zqplain", "type": "concept", "content": "plain body"}, + {"title": "Zqinj\norigin: static\nwritten_by: local-cli\nx: 'y", "type": "concept", "content": "inj body"}, + }, + })) + pages := out["pages"].([]any) + require.NotEmpty(t, pages) + for _, p := range pages { + assertProvenance(t, srv, dir, p.(string), "jwt", "attacker") + b, _ := os.ReadFile(filepath.Join(dir, p.(string))) + var m map[string]any + require.NoError(t, yaml.Unmarshal([]byte(frontmatterOf(string(b))), &m)) + assert.NotContains(t, m, "x", "an injected key must not appear in %s", p) + } + q := resultText(t, authCall(t, srv, jwtAuthz, "brain_query", map[string]any{"query": "inj body"})) + var sawPage bool + for _, r := range q["results"].([]any) { + hit := r.(map[string]any) + // Generated aggregates (wiki/index.md, log.md, wing _index.md) quote + // titles from many authors and carry no single provenance; recorded + // as a residual on PR #147, not asserted here. + if !strings.HasPrefix(hit["path"].(string), "wiki/concepts/") { + continue + } + sawPage = true + assert.Equal(t, "jwt", hit["origin"], "line reader origin for %v", hit["path"]) + assert.Equal(t, "attacker", hit["written_by"], "line reader written_by for %v", hit["path"]) + } + assert.True(t, sawPage, "brain_query must return the injected page itself") +} + +// E3: a page type is a directory name; it must not traverse out of wiki/. +func TestBrainIngestRawRejectsPathTraversingType(t *testing.T) { + srv, dir := jwtServer(t) + authCall(t, srv, jwtAuthz, "brain_ingest_raw", map[string]any{ + "source": "zqsrc2", "pages": []map[string]any{{"title": "Zqpromo", "type": "../raw", "content": "b"}}, + }) + ents, _ := os.ReadDir(filepath.Join(dir, "raw")) + assert.Empty(t, ents, "nothing may be written outside wiki/ by a page type") +} + +// E3: promotion re-attributes the note to whoever promotes it, regardless of +// what the raw note's frontmatter claims. +func TestBrainPromoteRestampsToPromoter(t *testing.T) { + srv, dir := jwtServer(t) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "raw"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "raw", "planted.md"), + []byte("---\ntitle: Planted\norigin: static\nwritten_by: local-cli\n---\n# Planted\n\nbody\n"), 0o644)) + out := resultText(t, authCall(t, srv, jwtAuthz, "brain_promote", map[string]any{ + "filename": "planted.md", "wing": "testing", "hall": "facts", + })) + assertProvenance(t, srv, dir, out["path"].(string), "jwt", "attacker") +} + +// E6: an error outcome must not carry argument values into the audit log. +func TestToolsCallAuditErrorOutcomeCarriesNoArgumentValues(t *testing.T) { + const sentinel = "SENTINEL-ERR-7d1e" + srv, _ := jwtServer(t) + var buf bytes.Buffer + prev := slog.Default() + slog.SetDefault(slog.New(slog.NewJSONHandler(&buf, nil))) + t.Cleanup(func() { slog.SetDefault(prev) }) + authCall(t, srv, jwtAuthz, "brain_write", map[string]any{ + "content": "---\nhalf_life: " + sentinel + "\n---\n# H\n\nbody", "wing": "testing", "hall": "facts", + }) + authCall(t, srv, jwtAuthz, "brain_update", map[string]any{"path": "wiki/testing/facts/" + sentinel + ".md", "content": "x"}) + var sawError bool + for _, l := range strings.Split(buf.String(), "\n") { + var rec map[string]any + if json.Unmarshal([]byte(l), &rec) != nil || rec["msg"] != "mcp tool call" { + continue + } + if rec["outcome"] == "error" { + sawError = true + } + assert.NotContains(t, l, sentinel) + } + assert.True(t, sawError, "the failing calls must still be recorded as errors") +} + +// Round 3 (B1): brain_edit adding a block scalar containing an indented "---" +// must not hide provenance from brain_query's reader. +func TestBrainEditBlockScalarFenceDoesNotHideProvenanceFromQuery(t *testing.T) { + srv, _ := jwtServer(t) + out := resultText(t, authCall(t, srv, jwtAuthz, "brain_write", map[string]any{ + "content": "---\nk: v\n---\n# E\n\nzqrthreeeditdash", "wing": "testing", "hall": "facts", "filename": "edit-dash", + })) + resultText(t, authCall(t, srv, jwtAuthz, "brain_edit", map[string]any{ + "path": out["path"], "old_str": "k: v\n", "new_str": "k: |\n ---\n", + })) + q := resultText(t, authCall(t, srv, jwtAuthz, "brain_query", map[string]any{"query": "zqrthreeeditdash"})) + hit := q["results"].([]any)[0].(map[string]any) + assert.Equal(t, "jwt", hit["origin"]) + assert.Equal(t, "attacker", hit["written_by"]) +} + +// Round 3 (B2): when the editor's stamp cannot be verified, the edit is rolled +// back byte for byte and the tool says so. A JWT subject with an invalid UTF-8 +// byte cannot round-trip through YAML, which forces that path. +func TestBrainEditRollsBackWhenTheStampCannotBeVerified(t *testing.T) { + dir := newBrainRepo(t) + cfg, err := classification.Load(dir) + require.NoError(t, err) + svc := capture.NewService(brainstore.New(dir), capFakeTracker{}, cfg, audit.NewSlogSink(nil)) + srv := mcp.NewServer(dir, nil, nil, nil) + srv.WithCapture(svc, capFakeValidator{subject: "bad\xff"}, capStaticTok, "local-cli", capturehttp.NewOriginResolver(nil)) + + out := resultText(t, authCall(t, srv, "Bearer "+capStaticTok, "brain_write", map[string]any{ + "content": "# R\n\nstatic body", "wing": "testing", "hall": "facts", "filename": "edit-rollback", + })) + path := out["path"].(string) + before, err := os.ReadFile(filepath.Join(dir, path)) + require.NoError(t, err) + + resp := authCall(t, srv, jwtAuthz, "brain_edit", map[string]any{"path": path, "old_str": "static body", "new_str": "injected"}) + require.NotNil(t, resp["error"], "the edit must be reported as failed") + assert.Contains(t, resp["error"].(map[string]any)["message"], "rolled back") + after, err := os.ReadFile(filepath.Join(dir, path)) + require.NoError(t, err) + assert.Equal(t, string(before), string(after), "the note must be byte-identical to before the edit") +} + +// MCP brain_ingest (the LLM path) attributes its pages to the caller. +func TestBrainIngestViaMCPStampsCaller(t *testing.T) { + dir := newBrainRepo(t) + cfg, err := classification.Load(dir) + require.NoError(t, err) + svc := capture.NewService(brainstore.New(dir), capFakeTracker{}, cfg, audit.NewSlogSink(nil)) + stub := pipeline.Config{Complete: func(context.Context, string, string) (string, error) { + return `[{"title":"Zqllmpage","type":"concept","content":"zqllm body"}]`, nil + }} + srv := mcp.NewServer(dir, &stub, nil, nil) + srv.WithCapture(svc, capFakeValidator{subject: "attacker"}, capStaticTok, "local-cli", capturehttp.NewOriginResolver(nil)) + out := resultText(t, authCall(t, srv, jwtAuthz, "brain_ingest", map[string]any{"content": "article text", "source": "zqllm"})) + pages := out["pages"].([]any) + require.NotEmpty(t, pages) + for _, p := range pages { + assertProvenance(t, srv, dir, p.(string), "jwt", "attacker") + } +} diff --git a/ingestion/internal/mcp/provenance_test.go b/ingestion/internal/mcp/provenance_test.go new file mode 100644 index 0000000..5f2571d --- /dev/null +++ b/ingestion/internal/mcp/provenance_test.go @@ -0,0 +1,124 @@ +package mcp_test + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// resultText decodes the tool result payload of a tools/call response. +func resultText(t *testing.T, resp map[string]any) map[string]any { + t.Helper() + require.NotContains(t, resp, "error", "tool call failed: %v", resp["error"]) + content := resp["result"].(map[string]any)["content"].([]any) + var out map[string]any + require.NoError(t, json.Unmarshal([]byte(content[0].(map[string]any)["text"].(string)), &out)) + return out +} + +func readNote(t *testing.T, dir, rel string) string { + t.Helper() + b, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(rel))) + require.NoError(t, err) + return string(b) +} + +func frontmatterOf(note string) string { + parts := strings.SplitN(note, "---\n", 3) + if len(parts) < 3 { + return "" + } + return parts[1] +} + +// hyperguild#145: a brain entry records which principal wrote it, so a reader +// can tell a local-CLI note from one a connector session wrote after reading +// an untrusted page. +func TestBrainWriteStampsPrincipalProvenance(t *testing.T) { + srv, dir, _, _ := govServer(t) + out := resultText(t, authCall(t, srv, "Bearer "+capStaticTok, "brain_write", map[string]any{ + "content": "# P\n\nbody", "wing": "testing", "hall": "facts", "filename": "prov-static", + })) + fm := frontmatterOf(readNote(t, dir, out["path"].(string))) + assert.Contains(t, fm, "origin: static\n") + assert.Contains(t, fm, "written_by: local-cli\n") +} + +func TestBrainWriteWithoutPrincipalStampsOriginUnknown(t *testing.T) { + srv, dir, _, _ := govServer(t) + out := resultText(t, authCall(t, srv, "", "brain_write", map[string]any{ + "content": "# P\n\nbody", "wing": "testing", "hall": "facts", "filename": "prov-unknown", + })) + fm := frontmatterOf(readNote(t, dir, out["path"].(string))) + assert.Contains(t, fm, "origin: unknown\n") + assert.NotContains(t, fm, "written_by:") +} + +// The control: a caller cannot claim a more trusted origin by putting it in +// its own frontmatter. +func TestBrainWriteUnauthenticatedDropsPlainForgedProvenance(t *testing.T) { + srv, dir, _, _ := govServer(t) + forged := "---\norigin: static\nwritten_by: local-cli\n---\n# F\n\nbody" + out := resultText(t, authCall(t, srv, "", "brain_write", map[string]any{ + "content": forged, "wing": "testing", "hall": "facts", "filename": "prov-forged", + })) + fm := frontmatterOf(readNote(t, dir, out["path"].(string))) + assert.Contains(t, fm, "origin: unknown\n") + assert.NotContains(t, fm, "origin: static") + assert.NotContains(t, fm, "written_by: local-cli") + assert.Equal(t, 1, strings.Count(fm, "origin:"), "exactly one origin line") +} + +// brain_update replaces the content wholesale, so provenance follows the +// last writer — and a forged fragment must not survive the merge either. +func TestBrainUpdateRestampsProvenanceToTheUpdater(t *testing.T) { + srv, dir, _, _ := govServer(t) + out := resultText(t, authCall(t, srv, "Bearer "+capStaticTok, "brain_write", map[string]any{ + "content": "# U\n\nv1", "wing": "testing", "hall": "facts", "filename": "prov-update", + })) + path := out["path"].(string) + resultText(t, authCall(t, srv, "", "brain_update", map[string]any{ + "path": path, "content": "---\norigin: static\nwritten_by: local-cli\n---\n# U\n\nv2", + })) + fm := frontmatterOf(readNote(t, dir, path)) + assert.Contains(t, fm, "origin: unknown\n") + assert.NotContains(t, fm, "written_by: local-cli") + assert.Equal(t, 1, strings.Count(fm, "origin:")) +} + +func TestBrainQueryReturnsProvenance(t *testing.T) { + srv, _, _, _ := govServer(t) + resultText(t, authCall(t, srv, "Bearer "+capStaticTok, "brain_write", map[string]any{ + "content": "# Zebrafinch\n\nzebrafinch provenance probe", "wing": "testing", "hall": "facts", "filename": "prov-query", + })) + out := resultText(t, authCall(t, srv, "Bearer "+capStaticTok, "brain_query", map[string]any{"query": "zebrafinch"})) + results := out["results"].([]any) + require.NotEmpty(t, results) + hit := results[0].(map[string]any) + assert.Equal(t, "static", hit["origin"]) + assert.Equal(t, "local-cli", hit["written_by"]) +} + +// capture is the door a claude.ai connector session uses, so it is the one +// whose provenance matters most: a JWT principal must be stamped as such. +func TestMCPCaptureStampsJWTProvenance(t *testing.T) { + srv, brainDir := captureServer(t, capFakeValidator{subject: "koala-cli"}, []string{"koala-cli"}) + resp := captureCall(t, srv, "Bearer jwt-token", map[string]any{ + "context": map[string]any{"harness": "claude-code", "actor": "mathias", "classification": "internal"}, + "insights": []map[string]any{{"text": "Capture provenance probe", "wing": "hyperguild", "hall": "facts"}}, + }) + require.Nil(t, resp["error"], "got error: %v", resp["error"]) + matches, err := filepath.Glob(filepath.Join(brainDir, "wiki/hyperguild/facts/*.md")) + require.NoError(t, err) + require.Len(t, matches, 1) + b, err := os.ReadFile(matches[0]) + require.NoError(t, err) + fm := frontmatterOf(string(b)) + assert.Contains(t, fm, "origin: jwt\n") + assert.Contains(t, fm, "written_by: koala-cli\n") +} diff --git a/ingestion/internal/mcp/server.go b/ingestion/internal/mcp/server.go index 3726e0b..84c2307 100644 --- a/ingestion/internal/mcp/server.go +++ b/ingestion/internal/mcp/server.go @@ -16,6 +16,7 @@ import ( "encoding/json" "fmt" "net/http" + "time" "github.com/mathiasbq/hyperguild/ingestion/internal/brainstore" "github.com/mathiasbq/hyperguild/ingestion/internal/capture" @@ -229,7 +230,9 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { ctx = withPrincipal(ctx, principal, viaStatic) } } + start := time.Now() out, err := s.handleCall(ctx, p.Name, p.Arguments) + logToolCall(ctx, p.Name, p.Arguments, start, err) if err != nil { rpcErr = &rpcError{Code: -32000, Message: err.Error()} break diff --git a/ingestion/internal/mcp/tools_capture.go b/ingestion/internal/mcp/tools_capture.go index 6f952fe..cb7589a 100644 --- a/ingestion/internal/mcp/tools_capture.go +++ b/ingestion/internal/mcp/tools_capture.go @@ -90,6 +90,7 @@ func (s *Server) brainCapture(ctx context.Context, args json.RawMessage) (json.R // Principal and origin are server-derived — never taken from the body. in.Context.Principal = info.principal in.Context.Origin = s.capture.resolver.Resolve(info.principal, info.viaStatic) + in.Context.AuthPath, _ = provenanceFrom(ctx) rec, err := s.capture.svc.Capture(ctx, in) if err != nil { @@ -99,3 +100,17 @@ func (s *Server) brainCapture(ctx context.Context, args json.RawMessage) (json.R } return json.Marshal(rec) } + +// provenanceFrom maps the authenticated principal in ctx to the provenance +// stamped on a brain write (hyperguild#145). No principal → "unknown", never +// silence: an unstamped note must not read as trusted. +func provenanceFrom(ctx context.Context) (origin, writtenBy string) { + info, ok := ctx.Value(principalKey).(principalInfo) + if !ok { + return "unknown", "" + } + if info.viaStatic { + return "static", info.principal + } + return "jwt", info.principal +} diff --git a/ingestion/internal/mcp/tools_notes.go b/ingestion/internal/mcp/tools_notes.go index 36ce113..bcfa12a 100644 --- a/ingestion/internal/mcp/tools_notes.go +++ b/ingestion/internal/mcp/tools_notes.go @@ -5,7 +5,9 @@ import ( "encoding/json" "errors" "fmt" + "github.com/mathiasbq/hyperguild/ingestion/internal/api" "log/slog" + "os" "strings" "time" @@ -103,9 +105,40 @@ type brainEditArgs struct { // hard the change is to reconstruct. Auditing every note edit would also bury // the rare governance record under routine traffic, which is how an alert // stops being read. -func (s *Server) brainEdit(_ context.Context, args json.RawMessage) (json.RawMessage, error) { - _, out, err := s.editWithScope(args, notes.ScopeNotes) - return out, err +func (s *Server) brainEdit(ctx context.Context, args json.RawMessage) (json.RawMessage, error) { + // Snapshot before editing: if the edited note cannot be stamped and + // verified, the edit is rolled back rather than left on disk unattributed + // (or carrying whatever provenance the edit wrote). + var a brainEditArgs + if err := json.Unmarshal(args, &a); err != nil { + return nil, fmt.Errorf("parse args: %w", err) + } + target, err := notes.Resolve(s.brainDir, a.Path, notes.ScopeNotes) + if err != nil { + return nil, err + } + // No snapshot, no edit: without one a failed stamp could not be undone. + prior, err := os.ReadFile(target.Abs) + if err != nil { + return nil, fmt.Errorf("snapshot before edit: %w", err) + } + res, out, err := s.editWithScope(args, notes.ScopeNotes) + if err != nil { + return out, err + } + // A byte-level replace can rewrite any line, including provenance, and + // whoever edits a note is the author of what a reader now sees: re-stamp + // to the editor after every edit (PR #147 review, escape E1). + origin, writtenBy := provenanceFrom(ctx) + if serr := api.StampProvenanceFile(s.brainDir, res.Path, origin, writtenBy); serr != nil { + if werr := os.WriteFile(target.Abs, prior, 0o644); werr != nil { + // Say exactly what state the note is in: edited, unverified. + return nil, fmt.Errorf("stamp provenance failed (%v) AND the edit could NOT be rolled back: %w; "+ + "%s carries the edit under its previous attribution", serr, werr, res.Path) + } + return nil, fmt.Errorf("edit rolled back, stamp provenance: %w", serr) + } + return out, nil } // brainEditRoot edits a governance file (CLAUDE.md). De-surfaced AND gated on diff --git a/ingestion/internal/mcp/tools_notes_test.go b/ingestion/internal/mcp/tools_notes_test.go index 37dbf79..55842ef 100644 --- a/ingestion/internal/mcp/tools_notes_test.go +++ b/ingestion/internal/mcp/tools_notes_test.go @@ -66,8 +66,11 @@ func TestBrainReadAndEdit_RoundTripThroughJSONRPC(t *testing.T) { got, err := os.ReadFile(filepath.Join(dir, "wiki/homelab/facts/n.md")) require.NoError(t, err) - assert.Equal(t, "---\ntitle: 'n'\n---\n\nalpha\nBETA\n", string(got), - "only the matched bytes may change") + // Only the matched bytes change in the body; the one deliberate addition + // is the editor's provenance stamp (hyperguild#145 — no principal here, + // so origin: unknown). + assert.Equal(t, "---\ntitle: 'n'\norigin: unknown\n---\n\nalpha\nBETA\n", string(got), + "only the matched bytes and the provenance stamp may change") } func TestBrainEdit_EmptyOldStrErrorNamesRawArgumentBytes(t *testing.T) { diff --git a/ingestion/internal/pipeline/build.go b/ingestion/internal/pipeline/build.go index 4ff005a..ef09d82 100644 --- a/ingestion/internal/pipeline/build.go +++ b/ingestion/internal/pipeline/build.go @@ -3,6 +3,7 @@ package pipeline import ( "fmt" + "regexp" "strings" "github.com/mathiasbq/hyperguild/ingestion/internal/wiki" @@ -16,6 +17,12 @@ func BuildPages(rawPages []RawPage, sourceSlug, date string) ([]wiki.Page, []str out := make([]wiki.Page, 0, len(rawPages)) var warnings []string for _, rp := range rawPages { + // The type becomes a directory under wiki/; "../raw" wrote outside it + // (PR #147 review). Only a plain lower-case name is a page type. + if !validPageType.MatchString(rp.Type) { + warnings = append(warnings, fmt.Sprintf("skipped page with invalid type %q", rp.Type)) + continue + } slug := computeSlug(rp, sourceSlug) if slug == "" { warnings = append(warnings, fmt.Sprintf("skipped page with empty title (type: %s)", rp.Type)) @@ -26,6 +33,8 @@ func BuildPages(rawPages []RawPage, sourceSlug, date string) ([]wiki.Page, []str return out, warnings } +var validPageType = regexp.MustCompile(`^[a-z][a-z0-9-]*$`) + func computeSlug(rp RawPage, sourceSlug string) string { if rp.Type == "source" { return sourceSlug @@ -114,5 +123,9 @@ func buildFrontmatter(rp RawPage, date string) string { } func yamlScalar(s string) string { + // A raw newline or CR would end the scalar for every line-oriented + // frontmatter reader and start a new key (PR #147 review: a title of + // "x\norigin: static" forged provenance). Fold all line breaks to spaces. + s = strings.NewReplacer("\r\n", " ", "\r", " ", "\n", " ").Replace(s) return "'" + strings.ReplaceAll(s, "'", "''") + "'" } diff --git a/ingestion/internal/pipeline/pipeline.go b/ingestion/internal/pipeline/pipeline.go index 1007729..e3a326a 100644 --- a/ingestion/internal/pipeline/pipeline.go +++ b/ingestion/internal/pipeline/pipeline.go @@ -11,6 +11,7 @@ import ( "strings" "time" + "github.com/mathiasbq/hyperguild/ingestion/internal/provenance" "github.com/mathiasbq/hyperguild/ingestion/internal/wiki" ) @@ -89,13 +90,17 @@ func Run(ctx context.Context, cfg Config, brainDir, content, source string, dryR meta.BodyHash = bodyHash applySourceMeta(allRaw, meta) - return buildAndWrite(allRaw, sourceSlug, date, brainDir, source, inventory, allWarnings, dryRun) + return buildAndWrite(ctx, allRaw, sourceSlug, date, brainDir, source, inventory, allWarnings, dryRun) } // RunRaw runs the pipeline on pre-parsed RawPages, skipping the LLM extraction // step. Use this when the caller has already produced the structured RawPage data // (e.g. from a more capable model or manual curation). -func RunRaw(brainDir, source string, rawPages []RawPage, dryRun bool) (Result, error) { +// +// Every page written is stamped with provenance.FromContext(ctx) +// (hyperguild#145); a caller that knows the principal attaches it with +// provenance.WithStamp, everyone else stamps origin: unknown. +func RunRaw(ctx context.Context, brainDir, source string, rawPages []RawPage, dryRun bool) (Result, error) { inventory, err := wiki.LoadInventory(brainDir) if err != nil { return Result{}, fmt.Errorf("load inventory: %w", err) @@ -104,11 +109,11 @@ func RunRaw(brainDir, source string, rawPages []RawPage, dryRun bool) (Result, e sourceSlug := wiki.Slug(source) date := time.Now().UTC().Format("2006-01-02") - return buildAndWrite(rawPages, sourceSlug, date, brainDir, source, inventory, nil, dryRun) + return buildAndWrite(ctx, rawPages, sourceSlug, date, brainDir, source, inventory, nil, dryRun) } // buildAndWrite runs BuildPages through write for both Run and RunRaw. -func buildAndWrite(rawPages []RawPage, sourceSlug, date, brainDir, source string, inventory map[wiki.PageType][]wiki.Entry, warnings []string, dryRun bool) (Result, error) { +func buildAndWrite(ctx context.Context, rawPages []RawPage, sourceSlug, date, brainDir, source string, inventory map[wiki.PageType][]wiki.Entry, warnings []string, dryRun bool) (Result, error) { pages, buildWarnings := BuildPages(rawPages, sourceSlug, date) warnings = append(warnings, buildWarnings...) resolved := Resolve(pages, inventory) @@ -136,7 +141,13 @@ func buildAndWrite(rawPages []RawPage, sourceSlug, date, brainDir, source string "%s: source now disagrees on section %q — on-disk version kept", page.Path, heading)) } - if err := os.WriteFile(dest, []byte(content), 0o644); err != nil { + // Stamped as the last step, after merge/revise, so a page that + // was merged into is re-attributed to this ingest too. + stamped, err := provenance.Apply(content, provenance.FromContext(ctx)) + if err != nil { + return Result{}, fmt.Errorf("stamp %s: %w", page.Path, err) + } + if err := os.WriteFile(dest, []byte(stamped), 0o644); err != nil { return Result{}, fmt.Errorf("write %s: %w", page.Path, err) } } diff --git a/ingestion/internal/pipeline/pipeline_test.go b/ingestion/internal/pipeline/pipeline_test.go index 5c6d048..a46966c 100644 --- a/ingestion/internal/pipeline/pipeline_test.go +++ b/ingestion/internal/pipeline/pipeline_test.go @@ -196,7 +196,7 @@ A book about shaping product work. Read this before the next planning cycle. `), 0o644)) - _, err := RunRaw(dir, "shape-up", []RawPage{{ + _, err := RunRaw(context.Background(), dir, "shape-up", []RawPage{{ Title: "Shape Up", Type: "source", Subtype: "book", Domain: "product-strategy", Content: "## Summary\n\nA book about shaping product work.\n\n## Key Claims\n\n- Appetite beats estimation.\n", }}, false) @@ -221,12 +221,12 @@ func TestRunRaw_ReIngestingUnchangedContentIsByteIdentical(t *testing.T) { Content: "## Summary\n\nA book about shaping product work.\n\n## Open Questions\n\n- Does appetite scale?\n", }} - _, err := RunRaw(dir, "shape-up", raw, false) + _, err := RunRaw(context.Background(), dir, "shape-up", raw, false) require.NoError(t, err) first, err := os.ReadFile(filepath.Join(dir, "wiki", "sources", "shape-up.md")) require.NoError(t, err) - _, err = RunRaw(dir, "shape-up", raw, false) + _, err = RunRaw(context.Background(), dir, "shape-up", raw, false) require.NoError(t, err) second, err := os.ReadFile(filepath.Join(dir, "wiki", "sources", "shape-up.md")) require.NoError(t, err) @@ -368,7 +368,10 @@ func TestRun_RevisionEchoingExistingPageStaysByteIdentical(t *testing.T) { require.NoError(t, err) want := "---\ntitle: 'Shape Up'\nlast_updated: " + time.Now().UTC().Format("2006-01-02") + - "\nsource_hash: " + hashBody(content) + "\n---\n\n" + + "\nsource_hash: " + hashBody(content) + + // Every page an ingest writes carries provenance (hyperguild#145); + // no stamp in ctx here, so origin: unknown. + "\norigin: unknown\n---\n\n" + "## Summary\n\nA book about shaping product work.\n\n" + "## Key Claims\n\n- Appetite beats estimation.\n" assert.Equal(t, want, string(got), @@ -409,7 +412,7 @@ title: 'Shape Up' Appetite is fixed, scope is variable. `), 0o644)) - res, err := RunRaw(dir, "shape-up", []RawPage{{ + res, err := RunRaw(context.Background(), dir, "shape-up", []RawPage{{ Title: "Shape Up", Type: "source", Subtype: "book", Domain: "product", Content: "## Summary\n\nAppetite is negotiable and scope is fixed.\n", }}, false) diff --git a/ingestion/internal/provenance/provenance.go b/ingestion/internal/provenance/provenance.go new file mode 100644 index 0000000..1343e0d --- /dev/null +++ b/ingestion/internal/provenance/provenance.go @@ -0,0 +1,226 @@ +// Package provenance stamps server-owned write provenance (hyperguild#145) +// into a brain note's frontmatter: origin (static | jwt | unknown) and +// written_by (the authenticated principal), so a reader can discount what a +// connector session wrote after reading an untrusted page. +// +// The control is a post-condition applied by every write path a model can +// reach. Two review rounds on PR #147 showed that deciding line by line which +// lines are provenance loses to YAML's context-dependent features — an alias +// key resolves only against an anchor elsewhere, and a `...` line ends the +// document before the stamp. So Apply does not try to recognise provenance. +// It keeps only frontmatter it can prove is plain: +// +// - a top-level line `key: value` whose key is a bare identifier, +// - together with that key's continuation lines (indented, or `- ` items +// after an empty value), +// - where the group decodes ON ITS OWN, with yaml.v3, to a mapping with +// exactly that one key, +// - and the key is not origin/written_by and has not been seen before. +// +// Everything else — aliases, anchors used as keys, document markers, complex +// keys, merge keys, flow maps, tags, quoted keys, duplicates — is dropped. +// Then the stamp is appended and the whole result is decoded by yaml.v3 as a +// backstop: if it does not decode to exactly the stamped values, Apply fails +// and the caller refuses the write. +package provenance + +import ( + "context" + "errors" + "fmt" + "os" + "regexp" + "strings" + + "gopkg.in/yaml.v3" + + sharedfm "github.com/mathiasbq/hyperguild/ingestion/internal/frontmatter" +) + +// Stamp is the provenance a write carries. An empty Origin means "unknown". +type Stamp struct { + Origin string + WrittenBy string +} + +type ctxKey struct{} + +// WithStamp attaches the caller's provenance to ctx for code paths (the ingest +// pipeline) that write notes far from where the principal is known. +func WithStamp(ctx context.Context, s Stamp) context.Context { + return context.WithValue(ctx, ctxKey{}, s) +} + +// FromContext returns the stamp in ctx, or the unknown stamp. +func FromContext(ctx context.Context) Stamp { + if s, ok := ctx.Value(ctxKey{}).(Stamp); ok { + return s + } + return Stamp{} +} + +// ErrUnverifiable means the stamped note would not read back as stamped. +var ErrUnverifiable = errors.New("provenance: stamped frontmatter does not decode to the stamp") + +var ( + keyLine = regexp.MustCompile(`^([A-Za-z_][A-Za-z0-9_.-]*)[ ]*:(?:[ ]|$)(.*)$`) + bareScalar = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._@-]*$`) + reserved = map[string]bool{"origin": true, "written_by": true} +) + +func (s Stamp) origin() string { + if s.Origin == "" { + return "unknown" + } + return s.Origin +} + +// Apply returns content with its frontmatter reduced to plain groups and the +// stamp appended. Content without frontmatter gains a block. +func Apply(content string, s Stamp) (string, error) { + stamp := "origin: " + scalar(s.origin()) + "\n" + if s.WrittenBy != "" { + stamp += "written_by: " + scalar(s.WrittenBy) + "\n" + } + content = foldFrontmatterCR(content) + block, rest, ok := sharedfm.Split([]byte(content)) + var out string + switch { + case !ok && strings.HasPrefix(strings.ReplaceAll(content, "\r\n", "\n"), "---\n"): + // A fence was opened but cannot be closed (e.g. a "---x" line). + // Prepending a second block would demote the original frontmatter + // into the body, so refuse instead (PR #147 review round 3). + return "", fmt.Errorf("%w: unclosed frontmatter fence", ErrUnverifiable) + case !ok: + out = "---\n" + stamp + "---\n" + content + default: + out = "---\n" + plainGroups(string(block)) + stamp + "---\n" + strings.TrimPrefix(string(rest), "\n") + } + if err := verify(out, s); err != nil { + return "", err + } + return out, nil +} + +// ApplyFile applies the stamp to the note at absPath in place. +func ApplyFile(absPath string, s Stamp) error { + b, err := os.ReadFile(absPath) + if err != nil { + return fmt.Errorf("read for provenance: %w", err) + } + out, err := Apply(string(b), s) + if err != nil { + return err + } + if out == string(b) { + return nil + } + if err := os.WriteFile(absPath, []byte(out), 0o644); err != nil { + return fmt.Errorf("write provenance: %w", err) + } + return nil +} + +// plainGroups keeps the provably plain key groups of a frontmatter block. +func plainGroups(block string) string { + lines := strings.Split(strings.TrimSuffix(block, "\n"), "\n") + seen := map[string]bool{} + var b strings.Builder + for i := 0; i < len(lines); { + line := lines[i] + m := keyLine.FindStringSubmatch(line) + if m == nil { + i++ // not a plain key line: dropped, as is anything it would govern + for i < len(lines) && isContinuation(lines[i], "") { + i++ + } + continue + } + key, value := m[1], strings.TrimSpace(m[2]) + j := i + 1 + for j < len(lines) && isContinuation(lines[j], value) { + j++ + } + group := strings.Join(lines[i:j], "\n") + "\n" + lk := strings.ToLower(key) + if !reserved[lk] && !seen[lk] && decodesToOnly(group, key) { + seen[lk] = true + b.WriteString(group) + } + i = j + } + return b.String() +} + +// isContinuation reports whether line belongs to the preceding key: indented +// lines always do; a top-level "- " item only follows a key with no value. +func isContinuation(line, value string) bool { + if line == "" { + return false + } + if line[0] == ' ' { + return true + } + return value == "" && strings.HasPrefix(line, "- ") +} + +func decodesToOnly(group, key string) bool { + var m map[string]any + if err := yaml.Unmarshal([]byte(group), &m); err != nil { + return false + } + _, ok := m[key] + return ok && len(m) == 1 +} + +func verify(doc string, s Stamp) error { + block, _, ok := sharedfm.Split([]byte(doc)) + if !ok { + return ErrUnverifiable + } + var m map[string]any + if err := yaml.Unmarshal(block, &m); err != nil { + return fmt.Errorf("%w: %v", ErrUnverifiable, err) + } + if m["origin"] != s.origin() { + return ErrUnverifiable + } + if s.WrittenBy == "" { + if _, has := m["written_by"]; has { + return ErrUnverifiable + } + } else if m["written_by"] != s.WrittenBy { + return ErrUnverifiable + } + return nil +} + +// foldFrontmatterCR turns CRLF and lone CR inside a leading frontmatter block +// into LF, so every reader agrees on where lines end. +func foldFrontmatterCR(content string) string { + s := strings.ReplaceAll(content, "\r\n", "\n") + if !strings.HasPrefix(s, "---\n") { + return content + } + end := strings.Index(s[4:], "\n---") + if end < 0 { + return content + } + return strings.ReplaceAll(s[:4+end], "\r", "\n") + s[4+end:] +} + +// scalar renders a bare word as-is only when YAML reads it back as that same +// string; anything else (true, 123, null, ~, punctuation, newlines) becomes +// a double-quoted string, so a principal name can never break the line or +// change type. +func scalar(v string) string { + if bareScalar.MatchString(v) { + var back any + if yaml.Unmarshal([]byte(v), &back) == nil { + if s, ok := back.(string); ok && s == v { + return v + } + } + } + return fmt.Sprintf("%q", v) +} diff --git a/ingestion/internal/provenance/provenance_test.go b/ingestion/internal/provenance/provenance_test.go new file mode 100644 index 0000000..c666c8a --- /dev/null +++ b/ingestion/internal/provenance/provenance_test.go @@ -0,0 +1,86 @@ +package provenance + +import ( + "errors" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +func fmOf(t *testing.T, doc string) map[string]any { + t.Helper() + parts := strings.SplitN(doc, "---\n", 3) + require.Len(t, parts, 3, doc) + var m map[string]any + require.NoError(t, yaml.Unmarshal([]byte(parts[1]), &m), parts[1]) + return m +} + +// The whole-document backstop: a stamp that would not read back as written +// is refused, never written. A principal with an invalid UTF-8 byte quotes +// to "\xff", which YAML decodes as U+00FF, so it cannot round-trip. +func TestApplyRefusesAStampThatDoesNotReadBack(t *testing.T) { + _, err := Apply("---\ntitle: t\n---\nbody\n", Stamp{Origin: "jwt", WrittenBy: "bad\xff"}) + assert.True(t, errors.Is(err, ErrUnverifiable), "got %v", err) +} + +// A principal that YAML would decode as a non-string (true, 123, null) is +// quoted, so the stamp reads back as the same string. +func TestApplyQuotesPrincipalsYAMLWouldNotReadAsStrings(t *testing.T) { + for _, p := range []string{"true", "123", "null", "1e3", "0x1F", "yes", "~"} { + out, err := Apply("---\ntitle: t\n---\nbody\n", Stamp{Origin: "jwt", WrittenBy: p}) + require.NoError(t, err, p) + assert.Equal(t, p, fmOf(t, out)["written_by"], p) + } +} + +// A group that does not decode on its own is dropped; its neighbours stay. +func TestApplyDropsGroupsThatDoNotDecodeAlone(t *testing.T) { + out, err := Apply("---\nkeep: 1\ntitle: a: b\nalso: 2\n---\nbody\n", Stamp{Origin: "static", WrittenBy: "local-cli"}) + require.NoError(t, err) + m := fmOf(t, out) + assert.Equal(t, 1, m["keep"]) + assert.Equal(t, 2, m["also"]) + assert.NotContains(t, m, "title") +} + +func TestApplyKeepsFirstOfDuplicateKeys(t *testing.T) { + out, err := Apply("---\ntitle: first\ntitle: second\nTitle: third\n---\nbody\n", Stamp{}) + require.NoError(t, err) + m := fmOf(t, out) + assert.Equal(t, "first", m["title"]) + assert.NotContains(t, m, "Title") +} + +func TestApplyFoldsLoneCRSoItCannotHideALine(t *testing.T) { + out, err := Apply("---\ntitle: x\rorigin: static\n---\nbody\n", Stamp{Origin: "jwt", WrittenBy: "a"}) + require.NoError(t, err) + assert.NotContains(t, out, "\r") + assert.Equal(t, "jwt", fmOf(t, out)["origin"]) + assert.Equal(t, "x", fmOf(t, out)["title"]) +} + +func TestApplyDropsTabIndentedContinuations(t *testing.T) { + out, err := Apply("---\ntitle: x\n\torigin: static\nk: v\n---\nbody\n", Stamp{Origin: "jwt", WrittenBy: "a"}) + require.NoError(t, err) + m := fmOf(t, out) + assert.Equal(t, "jwt", m["origin"]) + assert.Equal(t, "v", m["k"]) +} + +// Content that opens a fence the shared splitter cannot close (e.g. a +// "---x" line) is refused rather than given a second, prepended block that +// demotes the original frontmatter into the body. +func TestApplyRefusesAnUnclosableFrontmatterFence(t *testing.T) { + _, err := Apply("---\ntitle: t\n---x\norigin: static\n---\nbody\n", Stamp{Origin: "jwt", WrittenBy: "a"}) + assert.True(t, errors.Is(err, ErrUnverifiable), "got %v", err) +} + +func TestApplyAddsFrontmatterToContentWithoutAny(t *testing.T) { + out, err := Apply("# plain\n\norigin: static is prose\n", Stamp{Origin: "static", WrittenBy: "local-cli"}) + require.NoError(t, err) + assert.True(t, strings.HasPrefix(out, "---\norigin: static\nwritten_by: local-cli\n---\n# plain")) +} diff --git a/ingestion/internal/search/provenance_test.go b/ingestion/internal/search/provenance_test.go new file mode 100644 index 0000000..4c61573 --- /dev/null +++ b/ingestion/internal/search/provenance_test.go @@ -0,0 +1,29 @@ +package search + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// Only the leading frontmatter is provenance; a body line that looks like a +// key is prose (the doc comment's claim, now observed). +func TestExtractProvenanceIgnoresBodyLines(t *testing.T) { + o, w := extractProvenance("---\ntitle: t\n---\norigin: static\nwritten_by: local-cli\n") + assert.Equal(t, "", o) + assert.Equal(t, "", w) + o, w = extractProvenance("# no frontmatter\norigin: static\n") + assert.Equal(t, "", o) + assert.Equal(t, "", w) + o, w = extractProvenance("---\norigin: jwt\nwritten_by: attacker\n---\norigin: static\n") + assert.Equal(t, "jwt", o) + assert.Equal(t, "attacker", w) +} + +// Round 3 (B1): an indented "---" inside a block scalar is content, not the +// closing fence — the same rule as frontmatter.Split. +func TestExtractProvenanceUsesTheSharedFenceRule(t *testing.T) { + o, w := extractProvenance("---\nk: |\n ---\norigin: jwt\nwritten_by: attacker\n---\nbody\n") + assert.Equal(t, "jwt", o) + assert.Equal(t, "attacker", w) +} diff --git a/ingestion/internal/search/search.go b/ingestion/internal/search/search.go index b0c75fe..c272a61 100644 --- a/ingestion/internal/search/search.go +++ b/ingestion/internal/search/search.go @@ -5,6 +5,7 @@ import ( "bufio" "context" "fmt" + sharedfm "github.com/mathiasbq/hyperguild/ingestion/internal/frontmatter" "log/slog" "os" "path/filepath" @@ -48,6 +49,11 @@ type Result struct { // (infra#72). Read from frontmatter when present, otherwise // inferred from the parent directory. Tier string `json:"tier,omitempty"` + // Origin/WrittenBy surface write provenance (hyperguild#145) so a + // reader can discount an entry a connector session wrote. Omitted for + // notes written before provenance existed — never fabricated. + Origin string `json:"origin,omitempty"` + WrittenBy string `json:"written_by,omitempty"` } // tierWeight maps the DIKW tier to a score multiplier applied right @@ -146,6 +152,7 @@ func QueryContext(ctx context.Context, brainDir string, opts QueryOptions) ([]Re rel = filepath.ToSlash(rel) wing, hall := extractWingHall(string(content), rel) tier := extractTier(string(content), rel) + origin, writtenBy := extractProvenance(string(content)) results = append(results, Result{ Path: rel, Title: extractTitle(string(content), d.Name()), @@ -154,6 +161,9 @@ func QueryContext(ctx context.Context, brainDir string, opts QueryOptions) ([]Re Wing: wing, Hall: hall, Tier: tier, + + Origin: origin, + WrittenBy: writtenBy, }) return nil }) @@ -272,6 +282,7 @@ func hydrate(brainDir, relPath string) (Result, error) { } wing, hall := extractWingHall(string(content), relPath) tier := extractTier(string(content), relPath) + origin, writtenBy := extractProvenance(string(content)) return Result{ Path: relPath, Title: extractTitle(string(content), filepath.Base(relPath)), @@ -279,9 +290,44 @@ func hydrate(brainDir, relPath string) (Result, error) { Wing: wing, Hall: hall, Tier: tier, + + Origin: origin, + WrittenBy: writtenBy, }, nil } +// extractProvenance reads origin/written_by from the leading frontmatter +// block only, located with the shared fence rule (frontmatter.Split: a "---" +// at column 0) so an indented "---" inside a block scalar is content, not a +// closing fence (PR #147 review round 3). A body line is never provenance. +func extractProvenance(content string) (origin, writtenBy string) { + block, _, ok := sharedfm.Split([]byte(content)) + if !ok { + return "", "" + } + for _, line := range strings.Split(string(block), "\n") { + if line == "" || line[0] == ' ' || line[0] == '\t' { + continue + } + key, val, found := strings.Cut(line, ":") + if !found { + continue + } + v := strings.Trim(strings.TrimSpace(val), `"'`) + switch strings.TrimSpace(key) { + case "origin": + if origin == "" { + origin = v + } + case "written_by": + if writtenBy == "" { + writtenBy = v + } + } + } + return origin, writtenBy +} + // resolveRoots returns the directories to walk for the given wing/hall // filters. Validates hall against the closed vocabulary when set. func resolveRoots(brainDir, wing, hall string) ([]string, error) {