diff --git a/bin/check-session-log b/bin/check-session-log index cb8770a..4a33e58 100755 --- a/bin/check-session-log +++ b/bin/check-session-log @@ -25,15 +25,39 @@ for dir in "$ROOT"/skills/*/; do grep -q 'session_log' "$dir/SKILL.md" || continue - if ! grep -q 'session_id' "$dir/SKILL.md"; then - echo "ERROR: skills/$name documents session_log without session_id — the tool's only required field" >&2 - fail=1 + # Read the PAYLOAD, not the file. The first version of this gate grepped the + # whole SKILL.md for "session_id", so a payload missing the field passed as + # long as the word appeared anywhere — including in prose explaining that the + # field is required. A traceability probe demonstrated it in one sed. + payloads=$(grep -n 'session_log' "$dir/SKILL.md" | grep '{' || true) + + if [ -z "$payloads" ]; then + # Mentions the tool but documents no call. Nothing to check, and nothing to + # get wrong. + continue fi - if grep -q 'duration_ms:[[:space:]]*<' "$dir/SKILL.md"; then - echo "ERROR: skills/$name documents duration_ms as a placeholder an agent cannot measure — omit it or pass a real measurement" >&2 + printf '%s\n' "$payloads" | while IFS= read -r line; do + [ -n "$line" ] || continue + body=$(printf '%s' "$line" | sed 's/^[0-9]*://') + lineno=$(printf '%s' "$line" | cut -d: -f1) + + printf '%s' "$body" | grep -q 'session_id' \ + || echo "ERROR: skills/$name:$lineno documents a session_log payload with no session_id — the tool's only required field" >&2 + + # Any spelling of the key, quoted or not, followed by anything that is not a + # bare number. A placeholder nobody can fill gets fabricated or dropped; a + # real measurement is fine. + if printf '%s' "$body" | grep -Eq '"?duration_ms"?[[:space:]]*:[[:space:]]*[^0-9,}[:space:]]'; then + echo "ERROR: skills/$name:$lineno documents duration_ms as something other than a real measurement — an agent does not observe a clock; omit it" >&2 + fi + done > "$ROOT/.session_log_errors.$$" 2>&1 + + if [ -s "$ROOT/.session_log_errors.$$" ]; then + cat "$ROOT/.session_log_errors.$$" >&2 fail=1 fi + rm -f "$ROOT/.session_log_errors.$$" done exit $fail diff --git a/tests/check_session_log_test.sh b/tests/check_session_log_test.sh index c29a214..5f15878 100644 --- a/tests/check_session_log_test.sh +++ b/tests/check_session_log_test.sh @@ -63,5 +63,38 @@ expect 1 'documenting duration_ms as wall-clock fails' \ "$(fixture clock 'Call `session_log` with `{session_id: "", skill: "alpha", duration_ms: }`')" \ 'duration_ms' +# --- the gate must read the PAYLOAD, not the file --------------------------- +# A traceability probe falsified the original: it grepped the whole file for the +# string "session_id", so stripping the field from every documented payload and +# leaving the word once in prose passed cleanly. The exact regression the gate +# exists to prevent was back in the file and the gate said nothing. + +expect 1 'a payload without session_id fails even when prose mentions the word' \ + "$(fixture prose 'The tool has a required session_id field. + +Call `session_log` with `{skill: "alpha", phase: "red", final_status: "pass"}`')" \ + 'session_id' + +expect 1 'one bad payload among several fails' \ + "$(fixture mixed 'Call `session_log` with `{session_id: "", skill: "alpha", phase: "red"}` + +Then `session_log` with `{skill: "alpha", phase: "green", final_status: "pass"}`')" \ + 'session_id' + +# duration_ms was banned by matching exactly one spelling of it — the historical +# one. Both of these passed the original gate. +expect 1 'duration_ms with a JSON-quoted key fails' \ + "$(fixture jsonkey 'Call `session_log` with `{session_id: "", "duration_ms": }`')" \ + 'duration_ms' + +expect 1 'duration_ms described in prose rather than angle brackets fails' \ + "$(fixture prosems 'Call `session_log` with `{session_id: "", duration_ms: elapsed wall-clock ms}`')" \ + 'duration_ms' + +# A real measurement is allowed: the objection is to a placeholder no agent can +# fill, not to the field. +expect 0 'a duration_ms carrying a real value is allowed' \ + "$(fixture realms 'Call `session_log` with `{session_id: "", duration_ms: 1430}`')" + printf '\ncheck_session_log: %d passed, %d failed\n' "$pass" "$fail" [ "$fail" -eq 0 ]