mathias
4d6ebb626f
A traceability probe falsified this gate four hours after I shipped it. The check was `grep -q 'session_id' "$dir/SKILL.md"` — file-scoped. So the probe stripped session_id from all three documented payloads in tdd/SKILL.md, added one prose line saying "the tool has a required session_id field", and the gate returned 0. The exact regression it exists to prevent was back in the file, and the gate said nothing. The duration_ms rule was no better: it matched one spelling, the historical one. Both of these passed — `"duration_ms": <wall-clock>` with a JSON-quoted key, and `duration_ms: elapsed wall-clock ms` with no angle brackets. It was a regression test against the nine strings that existed that morning, described in the commit message as a rule. Now each documented payload is checked on its own line and reported with a line number, and duration_ms is rejected in any spelling unless it carries an actual number — the objection was always to a placeholder no agent can fill, not to the field. Five new cases, each one a probe from that run: prose mentioning the word while the payload omits it, one bad payload among several, both duration_ms spellings, and a real measurement being allowed. Also fixed while writing them: my first regex had `[[:space:]]*` before a negated class that did not exclude whitespace, so the star matched empty and the space itself tripped the check — a real measurement failed. The tests caught it immediately, which is the argument for writing them from the probes rather than from the intent. Refs #36, #39 Bump-Type: patch
101 lines
4.0 KiB
Bash
101 lines
4.0 KiB
Bash
#!/bin/sh
|
|
# Tests for bin/check-session-log.
|
|
#
|
|
# The regression: nine skills documented a session_log payload and not one of
|
|
# them named `session_id` — the tool's ONLY required field. Every documented
|
|
# call was incomplete, so an agent following any of them either failed the call
|
|
# or invented an id, and invented ids do not group. Found by a pressure test
|
|
# against `debug`, then found to be fleet-wide (#36).
|
|
#
|
|
# Also gated: `duration_ms: <wall-clock>`, which an agent cannot measure. A
|
|
# documented field nobody can honestly populate gets fabricated or silently
|
|
# dropped; both are worse than omitting it.
|
|
|
|
set -u
|
|
|
|
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
|
SUT="$ROOT/bin/check-session-log"
|
|
|
|
pass=0
|
|
fail=0
|
|
WORK=${TMPDIR:-/tmp}/check_session_log_test.$$
|
|
|
|
if [ ! -f "$SUT" ]; then
|
|
printf 'FAIL: %s does not exist\n' "$SUT"
|
|
exit 1
|
|
fi
|
|
|
|
cleanup() { rm -rf "$WORK"; }
|
|
trap cleanup EXIT INT TERM
|
|
|
|
# fixture <name> <body> — a checkout with one skill whose SKILL.md is <body>.
|
|
fixture() {
|
|
_f="$WORK/$1"
|
|
mkdir -p "$_f/skills/alpha"
|
|
printf -- '---\nname: alpha\ndescription: a\n---\n\n%s\n' "$2" > "$_f/skills/alpha/SKILL.md"
|
|
printf '%s' "$_f"
|
|
}
|
|
|
|
# expect <rc> <label> <fixture> [needle]
|
|
expect() {
|
|
_want=$1; _label=$2; _fx=$3; _needle=${4:-}
|
|
_out=$(sh "$SUT" "$_fx" 2>&1); _rc=$?
|
|
if [ "$_rc" -ne "$_want" ]; then
|
|
fail=$((fail + 1)); printf ' FAIL %s (want rc=%s got %s)\n%s\n' "$_label" "$_want" "$_rc" "$_out"; return
|
|
fi
|
|
if [ -n "$_needle" ] && ! printf '%s' "$_out" | grep -q "$_needle"; then
|
|
fail=$((fail + 1)); printf ' FAIL %s (rc ok, report never says %s)\n%s\n' "$_label" "$_needle" "$_out"; return
|
|
fi
|
|
pass=$((pass + 1)); printf ' ok %s\n' "$_label"
|
|
}
|
|
|
|
expect 0 'a skill that never mentions session_log is fine' \
|
|
"$(fixture quiet 'Just a skill. No logging here.')"
|
|
|
|
expect 0 'a documented call naming session_id passes' \
|
|
"$(fixture good 'Call `session_log` with `{session_id: "<id>", skill: "alpha", phase: "red", final_status: "pass"}`')"
|
|
|
|
expect 1 'a documented call omitting session_id fails' \
|
|
"$(fixture bad 'Call `session_log` with `{skill: "alpha", phase: "red", final_status: "pass"}`')" \
|
|
'session_id'
|
|
|
|
expect 1 'documenting duration_ms as wall-clock fails' \
|
|
"$(fixture clock 'Call `session_log` with `{session_id: "<id>", skill: "alpha", duration_ms: <wall-clock>}`')" \
|
|
'duration_ms'
|
|
|
|
# --- the gate must read the PAYLOAD, not the file ---------------------------
|
|
# A traceability probe falsified the original: it grepped the whole file for the
|
|
# string "session_id", so stripping the field from every documented payload and
|
|
# leaving the word once in prose passed cleanly. The exact regression the gate
|
|
# exists to prevent was back in the file and the gate said nothing.
|
|
|
|
expect 1 'a payload without session_id fails even when prose mentions the word' \
|
|
"$(fixture prose 'The tool has a required session_id field.
|
|
|
|
Call `session_log` with `{skill: "alpha", phase: "red", final_status: "pass"}`')" \
|
|
'session_id'
|
|
|
|
expect 1 'one bad payload among several fails' \
|
|
"$(fixture mixed 'Call `session_log` with `{session_id: "<id>", skill: "alpha", phase: "red"}`
|
|
|
|
Then `session_log` with `{skill: "alpha", phase: "green", final_status: "pass"}`')" \
|
|
'session_id'
|
|
|
|
# duration_ms was banned by matching exactly one spelling of it — the historical
|
|
# one. Both of these passed the original gate.
|
|
expect 1 'duration_ms with a JSON-quoted key fails' \
|
|
"$(fixture jsonkey 'Call `session_log` with `{session_id: "<id>", "duration_ms": <wall-clock>}`')" \
|
|
'duration_ms'
|
|
|
|
expect 1 'duration_ms described in prose rather than angle brackets fails' \
|
|
"$(fixture prosems 'Call `session_log` with `{session_id: "<id>", duration_ms: elapsed wall-clock ms}`')" \
|
|
'duration_ms'
|
|
|
|
# A real measurement is allowed: the objection is to a placeholder no agent can
|
|
# fill, not to the field.
|
|
expect 0 'a duration_ms carrying a real value is allowed' \
|
|
"$(fixture realms 'Call `session_log` with `{session_id: "<id>", duration_ms: 1430}`')"
|
|
|
|
printf '\ncheck_session_log: %d passed, %d failed\n' "$pass" "$fail"
|
|
[ "$fail" -eq 0 ]
|