mathias
b616c51a2c
Nine skills documented the payload. None named `session_id`, which is the tool's only required field. So every documented call was one the tool rejects — an agent following any of them either failed the call or invented an id, and invented ids do not group, which means the pass-rate the /pass-rate endpoint computes was being assembled from entries that cannot be joined. Found by the #19 pressure test against `debug`, which invented `debug-signed-artifact-missing-2026-08-29` and said so: "my three entries are only groupable by luck of naming". The grep afterwards showed it was fleet-wide, not a `debug` bug. Also removed: `duration_ms: <wall-clock>` in eight of them. An agent does not observe a clock. The same run omitted it "rather than fabricate", which is the good outcome; the bad outcome is the run that fabricates, and a made-up number in a metrics stream is worse than a missing one. It is an optional field and is now simply absent. bin/check-session-log gates both, with tests. This is exactly the case #19 has in mind for a small tool over prose: the rule is a grep, so it should be a gate, not a note asking people to remember. Closes part of #36 Bump-Type: patch
68 lines
2.4 KiB
Bash
68 lines
2.4 KiB
Bash
#!/bin/sh
|
|
# Tests for bin/check-session-log.
|
|
#
|
|
# The regression: nine skills documented a session_log payload and not one of
|
|
# them named `session_id` — the tool's ONLY required field. Every documented
|
|
# call was incomplete, so an agent following any of them either failed the call
|
|
# or invented an id, and invented ids do not group. Found by a pressure test
|
|
# against `debug`, then found to be fleet-wide (#36).
|
|
#
|
|
# Also gated: `duration_ms: <wall-clock>`, which an agent cannot measure. A
|
|
# documented field nobody can honestly populate gets fabricated or silently
|
|
# dropped; both are worse than omitting it.
|
|
|
|
set -u
|
|
|
|
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
|
SUT="$ROOT/bin/check-session-log"
|
|
|
|
pass=0
|
|
fail=0
|
|
WORK=${TMPDIR:-/tmp}/check_session_log_test.$$
|
|
|
|
if [ ! -f "$SUT" ]; then
|
|
printf 'FAIL: %s does not exist\n' "$SUT"
|
|
exit 1
|
|
fi
|
|
|
|
cleanup() { rm -rf "$WORK"; }
|
|
trap cleanup EXIT INT TERM
|
|
|
|
# fixture <name> <body> — a checkout with one skill whose SKILL.md is <body>.
|
|
fixture() {
|
|
_f="$WORK/$1"
|
|
mkdir -p "$_f/skills/alpha"
|
|
printf -- '---\nname: alpha\ndescription: a\n---\n\n%s\n' "$2" > "$_f/skills/alpha/SKILL.md"
|
|
printf '%s' "$_f"
|
|
}
|
|
|
|
# expect <rc> <label> <fixture> [needle]
|
|
expect() {
|
|
_want=$1; _label=$2; _fx=$3; _needle=${4:-}
|
|
_out=$(sh "$SUT" "$_fx" 2>&1); _rc=$?
|
|
if [ "$_rc" -ne "$_want" ]; then
|
|
fail=$((fail + 1)); printf ' FAIL %s (want rc=%s got %s)\n%s\n' "$_label" "$_want" "$_rc" "$_out"; return
|
|
fi
|
|
if [ -n "$_needle" ] && ! printf '%s' "$_out" | grep -q "$_needle"; then
|
|
fail=$((fail + 1)); printf ' FAIL %s (rc ok, report never says %s)\n%s\n' "$_label" "$_needle" "$_out"; return
|
|
fi
|
|
pass=$((pass + 1)); printf ' ok %s\n' "$_label"
|
|
}
|
|
|
|
expect 0 'a skill that never mentions session_log is fine' \
|
|
"$(fixture quiet 'Just a skill. No logging here.')"
|
|
|
|
expect 0 'a documented call naming session_id passes' \
|
|
"$(fixture good 'Call `session_log` with `{session_id: "<id>", skill: "alpha", phase: "red", final_status: "pass"}`')"
|
|
|
|
expect 1 'a documented call omitting session_id fails' \
|
|
"$(fixture bad 'Call `session_log` with `{skill: "alpha", phase: "red", final_status: "pass"}`')" \
|
|
'session_id'
|
|
|
|
expect 1 'documenting duration_ms as wall-clock fails' \
|
|
"$(fixture clock 'Call `session_log` with `{session_id: "<id>", skill: "alpha", duration_ms: <wall-clock>}`')" \
|
|
'duration_ms'
|
|
|
|
printf '\ncheck_session_log: %d passed, %d failed\n' "$pass" "$fail"
|
|
[ "$fail" -eq 0 ]
|