mathias
1d140ec525
Implements #5's decision. The hard part was #37's, not #5's: which skills need an escalation rule? A hardcoded list rots, sniffing the prose for "does this write anything" is unfalsifiable, and a frontmatter flag is the option #5 rejected wearing a hat. So there is no acting set. Every skill answers the question, and a read-only one answers it in a line. That turns an absence into a claim someone can disagree with, which is the whole point — "this skill never hands back" is a design decision worth stating, and today it was indistinguishable from nobody having thought about it. Eleven written here, all advisory or read-only, where the honest answer is short and the value is in being specific rather than generic. `cognitive-load` stops when the metric disagrees with the people reading the code, because that is evidence about the metric. `test-design` stops when the tests are fine and the design under test is what makes them awkward. `code-review` stops when the diff is big enough that a review would be a skim, and says so rather than producing confident coverage of a fraction. The remaining thirty are the acting set — writes, dispatches, credentials, releases — and each needs a rule someone thought about rather than a rote paragraph, so they sit in bin/escalation-pending.txt. That list only shrinks, and it is self-cleaning: a listed skill that HAS a section fails the gate, so an exemption cannot outlive its reason. Mutation-verified both ways. Six test cases including the stale-entry case, because an exemption for a skill that no longer exists is how a gate quietly stops covering something. Refs #37, closes the mechanism half Bump-Type: minor
86 lines
3.1 KiB
Bash
86 lines
3.1 KiB
Bash
#!/bin/sh
|
|
# Tests for bin/check-escalation.
|
|
#
|
|
# #5 asked whether SKILL.md needs first-class failure/escalation fields and
|
|
# decided: not frontmatter (nothing reads it), a body section (the model reads
|
|
# that). #37 then had to define WHICH skills need one, and every obvious answer
|
|
# was bad — a hardcoded list rots, content sniffing is unfalsifiable, a
|
|
# frontmatter flag is the rejected option through the back door.
|
|
#
|
|
# So there is no acting set. EVERY skill answers the question; a read-only one
|
|
# answers it in a line. The classification problem disappears, and "this skill
|
|
# never hands back" becomes a stated claim rather than an absence.
|
|
#
|
|
# bin/escalation-pending.txt is a ratchet for skills whose real rule has not
|
|
# been written yet. It is self-cleaning: a skill that has a section AND is still
|
|
# listed fails, so the list cannot silently outlive its reason.
|
|
|
|
set -u
|
|
|
|
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
|
SUT="$ROOT/bin/check-escalation"
|
|
|
|
pass=0
|
|
fail=0
|
|
WORK=${TMPDIR:-/tmp}/check_escalation_test.$$
|
|
|
|
if [ ! -f "$SUT" ]; then
|
|
printf 'FAIL: %s does not exist\n' "$SUT"
|
|
exit 1
|
|
fi
|
|
|
|
cleanup() { rm -rf "$WORK"; }
|
|
trap cleanup EXIT INT TERM
|
|
|
|
# fixture <name> — an empty checkout with a bin/ for the pending list.
|
|
fixture() {
|
|
_f="$WORK/$1"
|
|
mkdir -p "$_f/skills" "$_f/bin"
|
|
: > "$_f/bin/escalation-pending.txt"
|
|
printf '%s' "$_f"
|
|
}
|
|
|
|
# skill <root> <name> <has-section>
|
|
skill() {
|
|
mkdir -p "$1/skills/$2"
|
|
{
|
|
printf -- '---\nname: %s\ndescription: d\n---\n\n# %s\n' "$2" "$2"
|
|
[ "$3" = yes ] && printf '\n## Escalation\n\nStop and hand back when X. Hand back by: saying so.\n'
|
|
} > "$1/skills/$2/SKILL.md"
|
|
}
|
|
|
|
# expect <rc> <label> <fixture> [needle]
|
|
expect() {
|
|
_want=$1; _label=$2; _fx=$3; _needle=${4:-}
|
|
_out=$(sh "$SUT" "$_fx" 2>&1); _rc=$?
|
|
if [ "$_rc" -ne "$_want" ]; then
|
|
fail=$((fail + 1)); printf ' FAIL %s (want rc=%s got %s)\n%s\n' "$_label" "$_want" "$_rc" "$_out"; return
|
|
fi
|
|
if [ -n "$_needle" ] && ! printf '%s' "$_out" | grep -q "$_needle"; then
|
|
fail=$((fail + 1)); printf ' FAIL %s (rc ok, report never says %s)\n%s\n' "$_label" "$_needle" "$_out"; return
|
|
fi
|
|
pass=$((pass + 1)); printf ' ok %s\n' "$_label"
|
|
}
|
|
|
|
f=$(fixture ok); skill "$f" alpha yes
|
|
expect 0 'a skill with an Escalation section passes' "$f"
|
|
|
|
f=$(fixture missing); skill "$f" alpha no
|
|
expect 1 'a skill with no section fails, naming it' "$f" 'alpha'
|
|
|
|
f=$(fixture pending); skill "$f" alpha no; printf 'alpha\n' > "$f/bin/escalation-pending.txt"
|
|
expect 0 'a listed skill is allowed through the ratchet' "$f"
|
|
|
|
f=$(fixture done); skill "$f" alpha yes; printf 'alpha\n' > "$f/bin/escalation-pending.txt"
|
|
expect 1 'a listed skill that now HAS a section fails — remove it from the list' "$f" 'escalation-pending'
|
|
|
|
f=$(fixture stale); skill "$f" alpha yes; printf 'ghost\n' > "$f/bin/escalation-pending.txt"
|
|
expect 1 'a listed skill that does not exist fails' "$f" 'ghost'
|
|
|
|
f=$(fixture comments); skill "$f" alpha no
|
|
printf '# a comment, and a blank line follow\n\nalpha\n' > "$f/bin/escalation-pending.txt"
|
|
expect 0 'comments and blank lines in the list are ignored' "$f"
|
|
|
|
printf '\ncheck_escalation: %d passed, %d failed\n' "$pass" "$fail"
|
|
[ "$fail" -eq 0 ]
|