mathias
40405cd95a
"That list only shrinks" was asserted in the commit message, the script header, the pending file's own header, the test header, and the issue comment. It was enforced nowhere. A traceability probe falsified it in one run: extract the commit to a temp tree, add a skill with no `## Escalation` section, append its name to bin/escalation-pending.txt — gate returns 0. The list could grow silently and forever. "Self-cleaning" was true; "only shrinks" was a social convention wearing a mechanism's clothes, and this repo has spent a day on what those cost. The gate now diffs the pending list against the version committed at HEAD and fails on any added entry, naming it. Where the comparison is impossible — not a git checkout — it says so rather than passing silently, because a ratchet that cannot be verified must not look like one that was. Four new cases: unchanged list, shrinking list, growing list, and the unverifiable case. Refs #37 Bump-Type: patch
128 lines
4.8 KiB
Bash
128 lines
4.8 KiB
Bash
#!/bin/sh
|
|
# Tests for bin/check-escalation.
|
|
#
|
|
# #5 asked whether SKILL.md needs first-class failure/escalation fields and
|
|
# decided: not frontmatter (nothing reads it), a body section (the model reads
|
|
# that). #37 then had to define WHICH skills need one, and every obvious answer
|
|
# was bad — a hardcoded list rots, content sniffing is unfalsifiable, a
|
|
# frontmatter flag is the rejected option through the back door.
|
|
#
|
|
# So there is no acting set. EVERY skill answers the question; a read-only one
|
|
# answers it in a line. The classification problem disappears, and "this skill
|
|
# never hands back" becomes a stated claim rather than an absence.
|
|
#
|
|
# bin/escalation-pending.txt is a ratchet for skills whose real rule has not
|
|
# been written yet. It is self-cleaning: a skill that has a section AND is still
|
|
# listed fails, so the list cannot silently outlive its reason.
|
|
|
|
set -u
|
|
|
|
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
|
SUT="$ROOT/bin/check-escalation"
|
|
|
|
pass=0
|
|
fail=0
|
|
WORK=${TMPDIR:-/tmp}/check_escalation_test.$$
|
|
|
|
if [ ! -f "$SUT" ]; then
|
|
printf 'FAIL: %s does not exist\n' "$SUT"
|
|
exit 1
|
|
fi
|
|
|
|
cleanup() { rm -rf "$WORK"; }
|
|
trap cleanup EXIT INT TERM
|
|
|
|
# fixture <name> — an empty checkout with a bin/ for the pending list.
|
|
fixture() {
|
|
_f="$WORK/$1"
|
|
mkdir -p "$_f/skills" "$_f/bin"
|
|
: > "$_f/bin/escalation-pending.txt"
|
|
printf '%s' "$_f"
|
|
}
|
|
|
|
# skill <root> <name> <has-section>
|
|
skill() {
|
|
mkdir -p "$1/skills/$2"
|
|
{
|
|
printf -- '---\nname: %s\ndescription: d\n---\n\n# %s\n' "$2" "$2"
|
|
[ "$3" = yes ] && printf '\n## Escalation\n\nStop and hand back when X. Hand back by: saying so.\n'
|
|
} > "$1/skills/$2/SKILL.md"
|
|
}
|
|
|
|
# expect <rc> <label> <fixture> [needle]
|
|
expect() {
|
|
_want=$1; _label=$2; _fx=$3; _needle=${4:-}
|
|
_out=$(sh "$SUT" "$_fx" 2>&1); _rc=$?
|
|
if [ "$_rc" -ne "$_want" ]; then
|
|
fail=$((fail + 1)); printf ' FAIL %s (want rc=%s got %s)\n%s\n' "$_label" "$_want" "$_rc" "$_out"; return
|
|
fi
|
|
if [ -n "$_needle" ] && ! printf '%s' "$_out" | grep -q "$_needle"; then
|
|
fail=$((fail + 1)); printf ' FAIL %s (rc ok, report never says %s)\n%s\n' "$_label" "$_needle" "$_out"; return
|
|
fi
|
|
pass=$((pass + 1)); printf ' ok %s\n' "$_label"
|
|
}
|
|
|
|
f=$(fixture ok); skill "$f" alpha yes
|
|
expect 0 'a skill with an Escalation section passes' "$f"
|
|
|
|
f=$(fixture missing); skill "$f" alpha no
|
|
expect 1 'a skill with no section fails, naming it' "$f" 'alpha'
|
|
|
|
f=$(fixture pending); skill "$f" alpha no; printf 'alpha\n' > "$f/bin/escalation-pending.txt"
|
|
expect 0 'a listed skill is allowed through the ratchet' "$f"
|
|
|
|
f=$(fixture done); skill "$f" alpha yes; printf 'alpha\n' > "$f/bin/escalation-pending.txt"
|
|
expect 1 'a listed skill that now HAS a section fails — remove it from the list' "$f" 'escalation-pending'
|
|
|
|
f=$(fixture stale); skill "$f" alpha yes; printf 'ghost\n' > "$f/bin/escalation-pending.txt"
|
|
expect 1 'a listed skill that does not exist fails' "$f" 'ghost'
|
|
|
|
f=$(fixture comments); skill "$f" alpha no
|
|
printf '# a comment, and a blank line follow\n\nalpha\n' > "$f/bin/escalation-pending.txt"
|
|
expect 0 'comments and blank lines in the list are ignored' "$f"
|
|
|
|
# --- the ratchet must actually ratchet ---------------------------------------
|
|
# Five places in this repo claimed the pending list "only shrinks" and nothing
|
|
# enforced it. A traceability probe falsified that in one run: add an unsectioned
|
|
# skill, append it to the list, gate returns 0. A convention stated five times is
|
|
# still a convention.
|
|
#
|
|
# Enforcement compares against the list as committed at HEAD, so the fixtures
|
|
# below need a real git repo.
|
|
|
|
git_fixture() {
|
|
_f="$WORK/$1"
|
|
mkdir -p "$_f/skills" "$_f/bin"
|
|
( cd "$_f" && git init -q . && git config user.email t@example.com && git config user.name test )
|
|
skill "$_f" alpha no
|
|
printf 'alpha\n' > "$_f/bin/escalation-pending.txt"
|
|
( cd "$_f" && git add -A && git commit -qm baseline )
|
|
printf '%s' "$_f"
|
|
}
|
|
|
|
f=$(git_fixture ratchet_same)
|
|
expect 0 'an unchanged list passes' "$f"
|
|
|
|
f=$(git_fixture ratchet_shrink)
|
|
skill "$f" alpha yes
|
|
: > "$f/bin/escalation-pending.txt"
|
|
expect 0 'removing an entry (and writing its section) passes' "$f"
|
|
|
|
f=$(git_fixture ratchet_grow)
|
|
skill "$f" beta no
|
|
printf 'alpha\nbeta\n' > "$f/bin/escalation-pending.txt"
|
|
expect 1 'adding a NEW entry fails — the list only shrinks' "$f" 'beta'
|
|
|
|
# Outside a git repo the comparison is impossible; that must not be silent.
|
|
f=$(fixture ratchet_nogit); skill "$f" alpha no
|
|
printf 'alpha\n' > "$f/bin/escalation-pending.txt"
|
|
_out=$(sh "$SUT" "$f" 2>&1)
|
|
if printf '%s' "$_out" | grep -qi 'cannot verify'; then
|
|
pass=$((pass + 1)); printf ' ok an unverifiable ratchet says so rather than passing silently\n'
|
|
else
|
|
fail=$((fail + 1)); printf ' FAIL an unverifiable ratchet says so rather than passing silently\n%s\n' "$_out"
|
|
fi
|
|
|
|
printf '\ncheck_escalation: %d passed, %d failed\n' "$pass" "$fail"
|
|
[ "$fail" -eq 0 ]
|