mathias/skills
1
0
Fork 0
Code Issues 17 Pull Requests Actions Packages Projects Releases Wiki Activity
Files
025f01aa80cc55f74ec33c3ba0e75718003fe46a
skills/tests/doctor_test.sh
T
mathias 025f01aa80
CI / Check / Test (push) Successful in 0s
Details
CI / Tag release (push) Failing after 1s
Details
feat(install): serve several sources instead of displacing them 
~/.agents/skills was a single link straight at this checkout. That encoded an
assumption nobody had stated — that this checkout is the only source of skills
on the host. On flamingo it was not: Crush's directory held Anthropic's
official set, and the only way to install this library was to move that set
aside. A capability loss dressed up as a tidy-up.

The canonical location becomes a real directory of per-skill symlinks, fed by
the checkout plus anything in SKILLS_EXTRA_SOURCES. Harnesses keep their one
directory link, so the count is N + one per harness rather than N x harnesses,
which is what #23 was actually about.

The three requirements #27 named, each with a test:
- collisions are logged with both paths and the checkout wins, never resolved
  in silence;
- removing a skill from a source prunes its link;
- anything in the managed directory that is not our symlink is reported and
  left in place — the installer does not delete content it did not create.

doctor learns the same model: sources are recorded at ~/.agents/skills-sources,
a link into a configured source is healthy, a link into an unknown one is not,
and the old count check becomes a per-skill check because a per-skill layout
can lose one skill without changing any total. The pre-#27 wiring is named as
legacy rather than reported as damage. count_skills needed find -L; without it
symlinked skill directories count as zero.

Verified live on koala: migrated in place to 42 per-skill links, then a second
source added a 43rd alongside them, then a re-run without that source pruned it
back to 42. Only doctor finding throughout was the genuine stale plugin.

Closes #27
Bump-Type: minor
2026-08-29 10:05:46 +02:00

222 lines
9.1 KiB
Bash
Raw Blame History

#!/bin/sh
# Tests for bin/doctor.
#
# Every case here is a real state this fleet was found in on 2026-08-23, and
# every one of them was SILENT — nothing errored, git status was clean, the
# harness simply ran with the wrong skills or none at all. The point of doctor
# is that none of these can be silent again, so each fixture asserts both a
# non-zero exit AND that the report names the fault.
#
# Fixtures drive doctor through $HOME and $DOCTOR_CHECKOUT, so no test touches
# the real host.
set -u
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
DOCTOR="$ROOT/bin/doctor"
pass=0
fail=0
WORK=${TMPDIR:-/tmp}/doctor_test.$$
if [ ! -f "$DOCTOR" ]; then
printf 'FAIL: %s does not exist\n' "$DOCTOR"
exit 1
fi
cleanup() { rm -rf "$WORK"; }
trap cleanup EXIT INT TERM
# new_fixture <name> — builds $WORK/<name>/{ck,home} with a 2-skill checkout
# and an empty fake HOME. Echoes the fixture root.
new_fixture() {
_f="$WORK/$1"
mkdir -p "$_f/ck/skills/alpha" "$_f/ck/skills/beta" "$_f/home"
printf -- '---\nname: alpha\ndescription: a\n---\n' > "$_f/ck/skills/alpha/SKILL.md"
printf -- '---\nname: beta\ndescription: b\n---\n' > "$_f/ck/skills/beta/SKILL.md"
printf '%s' "$_f"
}
# wire_clean <fixture> [extra-source-dir] — the correct wiring since #27: the
# canonical location is a REAL directory of per-skill links, fed by one or more
# sources, with each harness keeping its single directory link to it.
wire_clean() {
mkdir -p "$1/home/.agents/skills" "$1/home/.config/crush" "$1/home/.config/opencode"
for _d in "$1"/ck/skills/*/; do
ln -sfn "${_d%/}" "$1/home/.agents/skills/$(basename "$_d")"
done
printf '%s\n' "$1/ck/skills" > "$1/home/.agents/skills-sources"
if [ -n "${2:-}" ]; then
for _d in "$2"/*/; do
ln -sfn "${_d%/}" "$1/home/.agents/skills/$(basename "$_d")"
done
printf '%s\n' "$2" >> "$1/home/.agents/skills-sources"
fi
ln -sfn "$1/home/.agents/skills" "$1/home/.config/crush/skills"
ln -sfn "$1/home/.agents/skills" "$1/home/.config/opencode/skills"
}
# assert_fails <fixture> <substring> <label>
# Note: doctor must be invoked directly here, not via a helper called in a
# command substitution — $? from inside $( ) belongs to the subshell and the
# exit status, which is the whole point of these assertions, would be lost.
assert_fails() {
_out=$(HOME="$1/home" DOCTOR_CHECKOUT="$1/ck" sh "$DOCTOR" 2>&1); rc=$?
if [ "$rc" -eq 0 ]; then
fail=$((fail + 1))
printf ' FAIL %s\n doctor exited 0; drift went undetected\n' "$3"
elif ! printf '%s' "$_out" | grep -qi -- "$2"; then
fail=$((fail + 1))
printf ' FAIL %s\n exited %s but report never mentions "%s"\n got: %s\n' "$3" "$rc" "$2" "$_out"
else
pass=$((pass + 1))
printf ' ok %s\n' "$3"
fi
}
assert_clean() {
_out=$(HOME="$1/home" DOCTOR_CHECKOUT="$1/ck" sh "$DOCTOR" 2>&1); rc=$?
if [ "$rc" -eq 0 ]; then
pass=$((pass + 1)); printf ' ok %s\n' "$2"
else
fail=$((fail + 1)); printf ' FAIL %s\n exited %s on a healthy fixture\n got: %s\n' "$2" "$rc" "$_out"
fi
}
# --- 1. healthy host stays silent -------------------------------------------
f=$(new_fixture clean); wire_clean "$f"
assert_clean "$f" 'clean fixture exits 0'
# --- 2. dangling link (the 243-link case, gitignored and invisible) ----------
f=$(new_fixture dangling); wire_clean "$f"
ln -sfn "$f/ck/skills/vanished" "$f/home/.agents/skills/alpha"
assert_fails "$f" 'dangling' 'dangling per-skill link is reported'
# --- 3. resolves, but into a source nobody configured (silent-stale case) ----
# Worse than dangling: it works, so nothing complains, while serving content
# from a checkout that is months behind. Since #27 a link may legitimately
# point outside the checkout — but only into a source the installer recorded.
f=$(new_fixture stale)
mkdir -p "$f/other/alpha"
printf -- '---\nname: alpha\ndescription: a\n---\n' > "$f/other/alpha/SKILL.md"
wire_clean "$f"
ln -sfn "$f/other/alpha" "$f/home/.agents/skills/alpha"
assert_fails "$f" 'unknown source' 'link into an unconfigured source is reported'
# --- 3b. a link into a CONFIGURED extra source is healthy -------------------
# The whole point of #27: Anthropic's official set alongside this library.
f=$(new_fixture multisource)
mkdir -p "$f/other/xlsx"
printf -- '---\nname: xlsx\ndescription: x\n---\n' > "$f/other/xlsx/SKILL.md"
wire_clean "$f" "$f/other"
assert_clean "$f" 'a link into a configured extra source is healthy'
# --- 3c. the pre-#27 single-link wiring is named, not mistaken for damage ----
f=$(new_fixture legacy)
mkdir -p "$f/home/.agents" "$f/home/.config/crush" "$f/home/.config/opencode"
ln -sfn "$f/ck/skills" "$f/home/.agents/skills"
ln -sfn "$f/home/.agents/skills" "$f/home/.config/crush/skills"
ln -sfn "$f/home/.agents/skills" "$f/home/.config/opencode/skills"
assert_fails "$f" 'single-link' 'legacy single-link wiring is reported as such'
# --- 4. a skill in the checkout with no link anywhere ------------------------
# The multi-source replacement for the old count check: with per-skill links a
# skill can be missing individually, which a total count would hide if an extra
# source happened to supply the same number.
f=$(new_fixture missing); wire_clean "$f"
rm -f "$f/home/.agents/skills/beta"
assert_fails "$f" 'beta' 'a checkout skill with no link is reported'
# --- 5. stray entry (the loose secret-hygiene.md that loaded nowhere) --------
f=$(new_fixture stray); wire_clean "$f"
printf 'orphan\n' > "$f/ck/skills/loose-note.md"
assert_fails "$f" 'stray' 'stray non-directory entry is reported'
# --- 6. legacy Claude symlinks alongside the plugin (duplicate skills) -------
f=$(new_fixture dupe); wire_clean "$f"
mkdir -p "$f/home/.claude/skills"
ln -sfn "$f/ck/skills/alpha" "$f/home/.claude/skills/alpha"
assert_fails "$f" 'legacy' 'leftover per-skill Claude symlinks are reported'
# --- 7b. harness dir is a REAL directory, not a link into the checkout -------
# Found on flamingo 2026-08-26: ~/.config/crush/skills held real directories
# from a pre-plugin install. The old message read "resolves to
# <path>, outside the active checkout" — naming the directory as its own
# resolution target, which is true but useless. It must say the dir is not a
# link at all, because the fix (move it aside, re-run install.sh) is different
# from the fix for a link into a foreign checkout.
f=$(new_fixture realdir); wire_clean "$f"
rm -f "$f/home/.config/crush/skills"
mkdir -p "$f/home/.config/crush/skills/alpha"
printf -- '---\nname: alpha\ndescription: a\n---\n' > "$f/home/.config/crush/skills/alpha/SKILL.md"
assert_fails "$f" 'not a link' 'real directory is reported as unmanaged, not as "outside"'
# --- 7. dangling link inside a harness dir (not the canonical link itself) --
# Exercises the per-entry scan rather than the canonical-link check: this is
# the shape the 243 dead links actually had.
f=$(new_fixture ghost); wire_clean "$f"
mkdir -p "$f/home/.claude/skills"
ln -sfn "$f/ck/skills/vanished" "$f/home/.claude/skills/vanished"
assert_fails "$f" 'dangling' 'dangling link inside a harness dir is reported'
# --- 8. Claude Code plugin pinned to an older version than is available -----
# The plugin is the ONLY delivery channel for Claude Code, and Claude Code
# gates updates on plugin.json's version field. A host that never picked the
# new one up loads every skill successfully — just the wrong generation. Found
# live on koala 2026-08-29: installed 0.6.0, marketplace 0.7.0, repo 0.8.5,
# and doctor said "clean" (#30).
# installed_state <fixture> <version> — the Claude Code install record.
installed_state() {
mkdir -p "$1/home/.claude/plugins"
cat > "$1/home/.claude/plugins/installed_plugins.json" <<JSON
{
"version": 2,
"plugins": {
"skills@mathias": [
{
"scope": "user",
"installPath": "/somewhere/$2",
"version": "$2"
}
]
}
}
JSON
}
# manifest <dir> <version> — a .claude-plugin/plugin.json under <dir>.
manifest() {
mkdir -p "$1/.claude-plugin"
printf '{\n "name": "skills",\n "version": "%s"\n}\n' "$2" > "$1/.claude-plugin/plugin.json"
}
f=$(new_fixture pluginmatch); wire_clean "$f"
installed_state "$f" 0.8.0
manifest "$f/ck" 0.8.0
assert_clean "$f" 'plugin on the current version stays silent'
f=$(new_fixture pluginstale); wire_clean "$f"
installed_state "$f" 0.7.0
manifest "$f/ck" 0.8.0
assert_fails "$f" '0.7.0' 'stale plugin version is reported'
assert_fails "$f" '0.8.0' 'stale plugin report names the available version'
# A host that only uses the symlink path has no install record at all. That is
# a valid configuration and must not fail.
f=$(new_fixture pluginabsent); wire_clean "$f"
manifest "$f/ck" 0.8.0
assert_clean "$f" 'no Claude Code install record is not a fault'
# The marketplace clone is what Claude Code compares against, so a newer
# version there means an update is waiting even when the local checkout agrees
# with what is installed.
f=$(new_fixture pluginmarket); wire_clean "$f"
installed_state "$f" 0.7.0
manifest "$f/ck" 0.7.0
manifest "$f/home/.claude/plugins/marketplaces/mathias" 0.9.0
assert_fails "$f" '0.9.0' 'a newer marketplace clone is reported'
printf '\ndoctor: %d passed, %d failed\n' "$pass" "$fail"
[ "$fail" -eq 0 ]
Reference in New Issue View Git Blame Copy Permalink