mathias
b2af4e1129
Everything doctor checks compares copies already on the machine, deliberately — it runs in a SessionStart hook where latency shows, and its header promises no network I/O. That leaves exactly one case invisible, and #30 walked straight into it: a host whose checkout, marketplace clone and installed plugin all agree reads as clean while all three sit a release behind origin. `--online` closes it with one `git ls-remote`, opt-in, for a scheduled per-host job rather than the session hook. Comparison is by order, not equality, so a host ahead of the newest tag — normal between a release commit and its tag — stays quiet. An unreachable remote is a WARN naming itself, not a staleness FAIL. "I could not check" and "you are behind" are different answers, and collapsing them would make a network blip look like a stale library. The remote URL is never printed: a remote rewritten by url.insteadOf expands into a live token, which is a documented way this fleet has leaked one before. Five fixture cases through a $DOCTOR_LATEST_TAG_CMD seam, including one that asserts the DEFAULT run performs no lookup at all — the promise in the header is now a test, not a comment. Verified live on koala: $ bin/doctor --online FAIL behind: this host has 0.12.1; origin has released 0.13.4 $ bin/doctor skills doctor: clean — 41 skills Closes #38 Bump-Type: minor
296 lines
12 KiB
Bash
296 lines
12 KiB
Bash
#!/bin/sh
|
|
# Tests for bin/doctor.
|
|
#
|
|
# Every case here is a real state this fleet was found in on 2026-08-23, and
|
|
# every one of them was SILENT — nothing errored, git status was clean, the
|
|
# harness simply ran with the wrong skills or none at all. The point of doctor
|
|
# is that none of these can be silent again, so each fixture asserts both a
|
|
# non-zero exit AND that the report names the fault.
|
|
#
|
|
# Fixtures drive doctor through $HOME and $DOCTOR_CHECKOUT, so no test touches
|
|
# the real host.
|
|
|
|
set -u
|
|
|
|
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
|
DOCTOR="$ROOT/bin/doctor"
|
|
|
|
pass=0
|
|
fail=0
|
|
WORK=${TMPDIR:-/tmp}/doctor_test.$$
|
|
|
|
if [ ! -f "$DOCTOR" ]; then
|
|
printf 'FAIL: %s does not exist\n' "$DOCTOR"
|
|
exit 1
|
|
fi
|
|
|
|
cleanup() { rm -rf "$WORK"; }
|
|
trap cleanup EXIT INT TERM
|
|
|
|
# new_fixture <name> — builds $WORK/<name>/{ck,home} with a 2-skill checkout
|
|
# and an empty fake HOME. Echoes the fixture root.
|
|
new_fixture() {
|
|
_f="$WORK/$1"
|
|
mkdir -p "$_f/ck/skills/alpha" "$_f/ck/skills/beta" "$_f/home"
|
|
printf -- '---\nname: alpha\ndescription: a\n---\n' > "$_f/ck/skills/alpha/SKILL.md"
|
|
printf -- '---\nname: beta\ndescription: b\n---\n' > "$_f/ck/skills/beta/SKILL.md"
|
|
printf '%s' "$_f"
|
|
}
|
|
|
|
# wire_clean <fixture> [extra-source-dir] — the correct wiring since #27: the
|
|
# canonical location is a REAL directory of per-skill links, fed by one or more
|
|
# sources, with each harness keeping its single directory link to it.
|
|
wire_clean() {
|
|
mkdir -p "$1/home/.agents/skills" "$1/home/.config/crush" "$1/home/.config/opencode"
|
|
for _d in "$1"/ck/skills/*/; do
|
|
ln -sfn "${_d%/}" "$1/home/.agents/skills/$(basename "$_d")"
|
|
done
|
|
printf '%s\n' "$1/ck/skills" > "$1/home/.agents/skills-sources"
|
|
if [ -n "${2:-}" ]; then
|
|
for _d in "$2"/*/; do
|
|
ln -sfn "${_d%/}" "$1/home/.agents/skills/$(basename "$_d")"
|
|
done
|
|
printf '%s\n' "$2" >> "$1/home/.agents/skills-sources"
|
|
fi
|
|
ln -sfn "$1/home/.agents/skills" "$1/home/.config/crush/skills"
|
|
ln -sfn "$1/home/.agents/skills" "$1/home/.config/opencode/skills"
|
|
}
|
|
|
|
# assert_fails <fixture> <substring> <label>
|
|
# Note: doctor must be invoked directly here, not via a helper called in a
|
|
# command substitution — $? from inside $( ) belongs to the subshell and the
|
|
# exit status, which is the whole point of these assertions, would be lost.
|
|
assert_fails() {
|
|
_out=$(HOME="$1/home" DOCTOR_CHECKOUT="$1/ck" sh "$DOCTOR" 2>&1); rc=$?
|
|
if [ "$rc" -eq 0 ]; then
|
|
fail=$((fail + 1))
|
|
printf ' FAIL %s\n doctor exited 0; drift went undetected\n' "$3"
|
|
elif ! printf '%s' "$_out" | grep -qi -- "$2"; then
|
|
fail=$((fail + 1))
|
|
printf ' FAIL %s\n exited %s but report never mentions "%s"\n got: %s\n' "$3" "$rc" "$2" "$_out"
|
|
else
|
|
pass=$((pass + 1))
|
|
printf ' ok %s\n' "$3"
|
|
fi
|
|
}
|
|
|
|
assert_clean() {
|
|
_out=$(HOME="$1/home" DOCTOR_CHECKOUT="$1/ck" sh "$DOCTOR" 2>&1); rc=$?
|
|
if [ "$rc" -eq 0 ]; then
|
|
pass=$((pass + 1)); printf ' ok %s\n' "$2"
|
|
else
|
|
fail=$((fail + 1)); printf ' FAIL %s\n exited %s on a healthy fixture\n got: %s\n' "$2" "$rc" "$_out"
|
|
fi
|
|
}
|
|
|
|
# --- 1. healthy host stays silent -------------------------------------------
|
|
f=$(new_fixture clean); wire_clean "$f"
|
|
assert_clean "$f" 'clean fixture exits 0'
|
|
|
|
# --- 2. dangling link (the 243-link case, gitignored and invisible) ----------
|
|
f=$(new_fixture dangling); wire_clean "$f"
|
|
ln -sfn "$f/ck/skills/vanished" "$f/home/.agents/skills/alpha"
|
|
assert_fails "$f" 'dangling' 'dangling per-skill link is reported'
|
|
|
|
# --- 3. resolves, but into a source nobody configured (silent-stale case) ----
|
|
# Worse than dangling: it works, so nothing complains, while serving content
|
|
# from a checkout that is months behind. Since #27 a link may legitimately
|
|
# point outside the checkout — but only into a source the installer recorded.
|
|
f=$(new_fixture stale)
|
|
mkdir -p "$f/other/alpha"
|
|
printf -- '---\nname: alpha\ndescription: a\n---\n' > "$f/other/alpha/SKILL.md"
|
|
wire_clean "$f"
|
|
ln -sfn "$f/other/alpha" "$f/home/.agents/skills/alpha"
|
|
assert_fails "$f" 'unknown source' 'link into an unconfigured source is reported'
|
|
|
|
# --- 3b. a link into a CONFIGURED extra source is healthy -------------------
|
|
# The whole point of #27: Anthropic's official set alongside this library.
|
|
f=$(new_fixture multisource)
|
|
mkdir -p "$f/other/xlsx"
|
|
printf -- '---\nname: xlsx\ndescription: x\n---\n' > "$f/other/xlsx/SKILL.md"
|
|
wire_clean "$f" "$f/other"
|
|
assert_clean "$f" 'a link into a configured extra source is healthy'
|
|
|
|
# --- 3c. the pre-#27 single-link wiring is named, not mistaken for damage ----
|
|
f=$(new_fixture legacy)
|
|
mkdir -p "$f/home/.agents" "$f/home/.config/crush" "$f/home/.config/opencode"
|
|
ln -sfn "$f/ck/skills" "$f/home/.agents/skills"
|
|
ln -sfn "$f/home/.agents/skills" "$f/home/.config/crush/skills"
|
|
ln -sfn "$f/home/.agents/skills" "$f/home/.config/opencode/skills"
|
|
assert_fails "$f" 'single-link' 'legacy single-link wiring is reported as such'
|
|
|
|
# --- 4. a skill in the checkout with no link anywhere ------------------------
|
|
# The multi-source replacement for the old count check: with per-skill links a
|
|
# skill can be missing individually, which a total count would hide if an extra
|
|
# source happened to supply the same number.
|
|
f=$(new_fixture missing); wire_clean "$f"
|
|
rm -f "$f/home/.agents/skills/beta"
|
|
assert_fails "$f" 'beta' 'a checkout skill with no link is reported'
|
|
|
|
# ...but only for skills that ship to the library. A chat-only skill has no
|
|
# business in a Crush directory, so its absence is correct, not drift (#29).
|
|
f=$(new_fixture chatonly); wire_clean "$f"
|
|
mkdir -p "$f/ck/skills/gamma"
|
|
printf -- '---\nname: gamma\nsurfaces: [chat]\ndescription: g\n---\n' > "$f/ck/skills/gamma/SKILL.md"
|
|
assert_clean "$f" 'a chat-only skill needs no library link'
|
|
|
|
# --- 5. stray entry (the loose secret-hygiene.md that loaded nowhere) --------
|
|
f=$(new_fixture stray); wire_clean "$f"
|
|
printf 'orphan\n' > "$f/ck/skills/loose-note.md"
|
|
assert_fails "$f" 'stray' 'stray non-directory entry is reported'
|
|
|
|
# --- 6. legacy Claude symlinks alongside the plugin (duplicate skills) -------
|
|
f=$(new_fixture dupe); wire_clean "$f"
|
|
mkdir -p "$f/home/.claude/skills"
|
|
ln -sfn "$f/ck/skills/alpha" "$f/home/.claude/skills/alpha"
|
|
assert_fails "$f" 'legacy' 'leftover per-skill Claude symlinks are reported'
|
|
|
|
# --- 7b. harness dir is a REAL directory, not a link into the checkout -------
|
|
# Found on flamingo 2026-08-26: ~/.config/crush/skills held real directories
|
|
# from a pre-plugin install. The old message read "resolves to
|
|
# <path>, outside the active checkout" — naming the directory as its own
|
|
# resolution target, which is true but useless. It must say the dir is not a
|
|
# link at all, because the fix (move it aside, re-run install.sh) is different
|
|
# from the fix for a link into a foreign checkout.
|
|
f=$(new_fixture realdir); wire_clean "$f"
|
|
rm -f "$f/home/.config/crush/skills"
|
|
mkdir -p "$f/home/.config/crush/skills/alpha"
|
|
printf -- '---\nname: alpha\ndescription: a\n---\n' > "$f/home/.config/crush/skills/alpha/SKILL.md"
|
|
assert_fails "$f" 'not a link' 'real directory is reported as unmanaged, not as "outside"'
|
|
|
|
# --- 7. dangling link inside a harness dir (not the canonical link itself) --
|
|
# Exercises the per-entry scan rather than the canonical-link check: this is
|
|
# the shape the 243 dead links actually had.
|
|
f=$(new_fixture ghost); wire_clean "$f"
|
|
mkdir -p "$f/home/.claude/skills"
|
|
ln -sfn "$f/ck/skills/vanished" "$f/home/.claude/skills/vanished"
|
|
assert_fails "$f" 'dangling' 'dangling link inside a harness dir is reported'
|
|
|
|
# --- 8. Claude Code plugin pinned to an older version than is available -----
|
|
# The plugin is the ONLY delivery channel for Claude Code, and Claude Code
|
|
# gates updates on plugin.json's version field. A host that never picked the
|
|
# new one up loads every skill successfully — just the wrong generation. Found
|
|
# live on koala 2026-08-29: installed 0.6.0, marketplace 0.7.0, repo 0.8.5,
|
|
# and doctor said "clean" (#30).
|
|
|
|
# installed_state <fixture> <version> — the Claude Code install record.
|
|
installed_state() {
|
|
mkdir -p "$1/home/.claude/plugins"
|
|
cat > "$1/home/.claude/plugins/installed_plugins.json" <<JSON
|
|
{
|
|
"version": 2,
|
|
"plugins": {
|
|
"skills@mathias": [
|
|
{
|
|
"scope": "user",
|
|
"installPath": "/somewhere/$2",
|
|
"version": "$2"
|
|
}
|
|
]
|
|
}
|
|
}
|
|
JSON
|
|
}
|
|
|
|
# manifest <dir> <version> — a .claude-plugin/plugin.json under <dir>.
|
|
manifest() {
|
|
mkdir -p "$1/.claude-plugin"
|
|
printf '{\n "name": "skills",\n "version": "%s"\n}\n' "$2" > "$1/.claude-plugin/plugin.json"
|
|
}
|
|
|
|
f=$(new_fixture pluginmatch); wire_clean "$f"
|
|
installed_state "$f" 0.8.0
|
|
manifest "$f/ck" 0.8.0
|
|
assert_clean "$f" 'plugin on the current version stays silent'
|
|
|
|
f=$(new_fixture pluginstale); wire_clean "$f"
|
|
installed_state "$f" 0.7.0
|
|
manifest "$f/ck" 0.8.0
|
|
assert_fails "$f" '0.7.0' 'stale plugin version is reported'
|
|
assert_fails "$f" '0.8.0' 'stale plugin report names the available version'
|
|
|
|
# A host that only uses the symlink path has no install record at all. That is
|
|
# a valid configuration and must not fail.
|
|
f=$(new_fixture pluginabsent); wire_clean "$f"
|
|
manifest "$f/ck" 0.8.0
|
|
assert_clean "$f" 'no Claude Code install record is not a fault'
|
|
|
|
# The marketplace clone is what Claude Code compares against, so a newer
|
|
# version there means an update is waiting even when the local checkout agrees
|
|
# with what is installed.
|
|
f=$(new_fixture pluginmarket); wire_clean "$f"
|
|
installed_state "$f" 0.7.0
|
|
manifest "$f/ck" 0.7.0
|
|
manifest "$f/home/.claude/plugins/marketplaces/mathias" 0.9.0
|
|
assert_fails "$f" '0.9.0' 'a newer marketplace clone is reported'
|
|
|
|
# --- 9. --online: the release this host has never fetched ------------------
|
|
# Everything above compares copies already on the host, so a host where the
|
|
# checkout, the marketplace clone and the installed plugin all agree reads as
|
|
# clean while being a release behind origin. Offline by design (#30); this is
|
|
# the opt-in that closes it (#38).
|
|
#
|
|
# The remote lookup is stubbed through $DOCTOR_LATEST_TAG_CMD so no test needs
|
|
# a network or a real remote.
|
|
|
|
online_fixture() {
|
|
_f=$(new_fixture "$1"); wire_clean "$_f"
|
|
installed_state "$_f" "$2"
|
|
manifest "$_f/ck" "$2"
|
|
printf '%s' "$_f"
|
|
}
|
|
|
|
# assert_online <fixture> <stub-tag> <want-rc> <needle> <label>
|
|
assert_online() {
|
|
_out=$(HOME="$1/home" DOCTOR_CHECKOUT="$1/ck" \
|
|
DOCTOR_LATEST_TAG_CMD="printf %s\\n $2" sh "$DOCTOR" --online 2>&1); _rc=$?
|
|
if [ "$3" -eq 0 ] && [ "$_rc" -ne 0 ]; then
|
|
fail=$((fail + 1)); printf ' FAIL %s\n exited %s on a current host\n got: %s\n' "$5" "$_rc" "$_out"; return
|
|
fi
|
|
if [ "$3" -ne 0 ] && [ "$_rc" -eq 0 ]; then
|
|
fail=$((fail + 1)); printf ' FAIL %s\n exited 0; staleness went unreported\n' "$5"; return
|
|
fi
|
|
if [ -n "$4" ] && ! printf '%s' "$_out" | grep -q -- "$4"; then
|
|
fail=$((fail + 1)); printf ' FAIL %s\n report never mentions "%s"\n got: %s\n' "$5" "$4" "$_out"; return
|
|
fi
|
|
pass=$((pass + 1)); printf ' ok %s\n' "$5"
|
|
}
|
|
|
|
f=$(online_fixture behind 0.8.0)
|
|
assert_online "$f" v0.9.0 1 '0.9.0' '--online reports a release the host has not fetched'
|
|
|
|
f=$(online_fixture current 0.9.0)
|
|
assert_online "$f" v0.9.0 0 '' '--online stays quiet when the host is current'
|
|
|
|
# A host AHEAD of the newest tag is the normal state between a release commit
|
|
# and its tag; that is not drift (same reasoning as check's ordering rule).
|
|
f=$(online_fixture ahead 0.9.1)
|
|
assert_online "$f" v0.9.0 0 '' '--online does not complain about being ahead'
|
|
|
|
# Without the flag, no lookup happens at all — the stub would report a newer
|
|
# tag and must be ignored.
|
|
f=$(online_fixture offline 0.8.0)
|
|
_out=$(HOME="$f/home" DOCTOR_CHECKOUT="$f/ck" \
|
|
DOCTOR_LATEST_TAG_CMD="printf v9.9.9\\n" sh "$DOCTOR" 2>&1); _rc=$?
|
|
if [ "$_rc" -eq 0 ]; then
|
|
pass=$((pass + 1)); printf ' ok the default run performs no remote lookup\n'
|
|
else
|
|
fail=$((fail + 1)); printf ' FAIL the default run performs no remote lookup\n got: %s\n' "$_out"
|
|
fi
|
|
|
|
# A remote that cannot be reached is not drift and must not be reported as it.
|
|
# An unreachable remote produces no output at all, so it gets its own run
|
|
# rather than an empty stub argument — "prints nothing" and "prints an empty
|
|
# string" are different things and only the first is the case under test.
|
|
f=$(online_fixture unreachable 0.9.0)
|
|
_out=$(HOME="$f/home" DOCTOR_CHECKOUT="$f/ck" \
|
|
DOCTOR_LATEST_TAG_CMD="true" sh "$DOCTOR" --online 2>&1); _rc=$?
|
|
if [ "$_rc" -ne 0 ] && printf '%s' "$_out" | grep -q 'could not'; then
|
|
pass=$((pass + 1)); printf ' ok --online reports an unreachable remote as its own fault\n'
|
|
else
|
|
fail=$((fail + 1)); printf ' FAIL --online reports an unreachable remote as its own fault\n rc=%s got: %s\n' "$_rc" "$_out"
|
|
fi
|
|
|
|
printf '\ndoctor: %d passed, %d failed\n' "$pass" "$fail"
|
|
[ "$fail" -eq 0 ]
|