mathiasandClaude Opus 5
6f1e0be09b
On 2026-08-25 a deploy job ran with an empty IMAGE_TAG, wrote
`image: localhost:5000/gitea-mcp:` into the infra repo and pushed it.
That is invalid YAML, so Flux failed the ENTIRE apps Kustomization --
every app stopped reconciling for about an hour, including three
unrelated deploys queued behind it.
This job's existing check could not catch it. It greps for
`image: <repo>:${IMAGE_TAG}` after the sed, and with IMAGE_TAG empty that
pattern matches the corrupted line itself, so the verification passes
because of the bug rather than despite it.
Asserting the input up front is the fix: an empty tag now fails the job
before anything is written, let alone committed.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011PL5XcG68SvQqz3vGVbWbf
224 lines
9.8 KiB
YAML
224 lines
9.8 KiB
YAML
name: CI
|
||
|
||
"on":
|
||
push:
|
||
branches: [main]
|
||
tags: ["v*"]
|
||
pull_request:
|
||
branches: [main]
|
||
|
||
env:
|
||
IMAGE: tapir
|
||
|
||
jobs:
|
||
# ── 1. Quality gate ─────────────────────────────────────────────────────────
|
||
# NOTE: the acceptance suite is intentionally RED in the scaffold (the engine
|
||
# returns ErrNotImplemented). This job will fail until the first build task
|
||
# implements Engine.ProcessNewVideo. That is the expected TDD starting state.
|
||
check:
|
||
name: Lint / Test / Vet
|
||
runs-on: self-hosted
|
||
steps:
|
||
- uses: actions/checkout@v4
|
||
|
||
- uses: actions/setup-go@v5
|
||
with:
|
||
go-version-file: go.mod
|
||
cache: false
|
||
|
||
- name: Verify toolchain
|
||
run: |
|
||
go version
|
||
task --version
|
||
|
||
- name: Install golangci-lint
|
||
run: |
|
||
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh \
|
||
| sh -s -- -b "$(go env GOPATH)/bin" v2.11.4
|
||
golangci-lint --version
|
||
|
||
- name: Run checks
|
||
run: task check
|
||
|
||
# ── 2. Build image ──────────────────────────────────────────────────────────
|
||
build:
|
||
name: Build & Import
|
||
needs: check
|
||
runs-on: self-hosted
|
||
if: github.event_name != 'pull_request'
|
||
outputs:
|
||
image-tag: ${{ steps.meta.outputs.sha-tag }}
|
||
steps:
|
||
- uses: actions/checkout@v4
|
||
|
||
- name: Derive image tags
|
||
id: meta
|
||
run: |
|
||
SHA=$(git rev-parse --short HEAD)
|
||
echo "sha-tag=${SHA}" >> "$GITHUB_OUTPUT"
|
||
REF="${{ github.ref }}"
|
||
if [[ "$REF" == refs/tags/v* ]]; then
|
||
echo "version-tag=${REF#refs/tags/}" >> "$GITHUB_OUTPUT"
|
||
fi
|
||
|
||
# infra#241: rootless buildah inside the gitea-runner pod can't remap the
|
||
# UID/GID range newuidmap/newgidmap need (the pod drops ALL capabilities
|
||
# and sets allowPrivilegeEscalation:false — both deliberate, it also holds
|
||
# CI secrets), so extracting a base-image layer that chowns outside the
|
||
# single available mapping fails with `ApplyLayer ... requested 0:42 for
|
||
# /etc/gshadow`. Main has been red on exactly this since 2026-08-13.
|
||
# buildctl instead talks to koala's host buildkitd over the socket the
|
||
# runner pod already mounts: the privileged extraction happens inside
|
||
# buildkitd on the host, never in this pod. Same migration agentsquad
|
||
# (infra#233) and segment-embedder (infra#243) already made — zero
|
||
# security tradeoff, unlike loosening the pod's capabilities.
|
||
#
|
||
# buildctl has no registry-push flag of its own: it writes an OCI archive
|
||
# and skopeo pushes that, which is also why every tag below is one copy of
|
||
# a single build rather than a rebuild.
|
||
- name: Build and push to local registry
|
||
# infra#309: implementation lives in .gitea/build-image.sh, a
|
||
# byte-identical copy of mathias/infra's ci-templates/build-image.sh.
|
||
# Do not edit it here -- edit the canonical file, run `task ci:sync`,
|
||
# and infra's drift gate names any repo still carrying an old copy.
|
||
env:
|
||
SHA_TAG: ${{ steps.meta.outputs.sha-tag }}
|
||
VERSION_TAG: ${{ steps.meta.outputs.version-tag }}
|
||
GIT_SHA: ${{ github.sha }}
|
||
IMAGE_SOURCE: ${{ github.repositoryUrl }}
|
||
run: bash .gitea/build-image.sh
|
||
|
||
# The buildah-based smoke test that used to sit here is gone with the
|
||
# buildctl migration above: `buildah from ${REF}` would pull the image
|
||
# back into the pod's buildah store and hit the very uid/gid wall the
|
||
# migration exists to avoid — it would fail the job for a reason that has
|
||
# nothing to do with the image being good. The `Verify rollout` step in
|
||
# the deploy job below is the stronger check anyway: it requires the real
|
||
# pod to reach Ready in-cluster, not just the entrypoint to print its own
|
||
# name for 5s. Tag builds lose their only image-level check (deploy is
|
||
# main-push only) — accepted; a tag is always built from a main commit
|
||
# that already deployed.
|
||
|
||
# ── 3. Deploy via infra repo + Flux ─────────────────────────────────────────
|
||
# Flux native image-automation can't scan localhost:5000 from inside k3s pods
|
||
# (mathias/infra k3s/flux/flux-system/image-automation.yaml) — this job
|
||
# mirrors cobalt-dingo's proven pattern instead: patch the infra repo's
|
||
# manifest directly on every push to main, then annotate Flux for a fast
|
||
# reconcile. Fixes infra#111 (image built+pushed but manifest bump was
|
||
# manual, so merged features silently didn't deploy).
|
||
deploy:
|
||
name: Deploy via GitOps
|
||
needs: build
|
||
runs-on: self-hosted
|
||
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
|
||
environment: staging
|
||
steps:
|
||
- name: Update image tag in infra repo
|
||
env:
|
||
IMAGE_TAG: ${{ needs.build.outputs.image-tag }}
|
||
DEPLOY_KEY: ${{ secrets.INFRA_DEPLOY_KEY }}
|
||
run: |
|
||
set -euo pipefail
|
||
# infra#311: fail on an empty tag BEFORE patching anything. The
|
||
# grep verification further down CANNOT catch it -- with IMAGE_TAG
|
||
# empty its pattern matches the very corruption it exists to reject.
|
||
# The result is `image: <repo>:` in the manifest, which is invalid
|
||
# YAML, so Flux fails the ENTIRE apps Kustomization and every app
|
||
# stops reconciling, not just this one. That happened on 2026-08-25.
|
||
[ -n "${IMAGE_TAG:-}" ] || { echo "IMAGE_TAG is empty - refusing to patch"; exit 1; }
|
||
|
||
# INFRA_DEPLOY_KEY is a Gitea org secret (mathias org), already
|
||
# configured per docs/cd-pipeline.md in the infra repo — same key
|
||
# cobalt-dingo and brain-gardener use, no new secret needed.
|
||
mkdir -p ~/.ssh
|
||
echo "$DEPLOY_KEY" > ~/.ssh/id_infra
|
||
chmod 600 ~/.ssh/id_infra
|
||
ssh-keyscan -p 30022 10.0.1.20 >> ~/.ssh/known_hosts 2>/dev/null
|
||
|
||
export GIT_SSH_COMMAND="ssh -i ~/.ssh/id_infra -o IdentitiesOnly=yes"
|
||
rm -rf /tmp/infra
|
||
git clone -b main ssh://git@10.0.1.20:30022/mathias/infra.git /tmp/infra
|
||
cd /tmp/infra
|
||
|
||
DEPLOYMENT="k3s/apps/tapir/deployment.yaml"
|
||
# In-place update of the image tag. sed (not yq) so we don't
|
||
# depend on additional tooling on the runner — same as cobalt-dingo.
|
||
sed -i "s|image: localhost:5000/tapir:.*|image: localhost:5000/tapir:${IMAGE_TAG}|" "$DEPLOYMENT"
|
||
|
||
# Verify the patch took effect.
|
||
grep -q "localhost:5000/tapir:${IMAGE_TAG}" "$DEPLOYMENT" \
|
||
|| { echo "✗ image tag patch failed"; exit 1; }
|
||
|
||
if git diff --quiet "$DEPLOYMENT"; then
|
||
echo "ℹ image tag unchanged — skipping push"
|
||
else
|
||
git -c user.name="tapir CI" \
|
||
-c user.email="ci@tapir.local" \
|
||
commit -m "chore(deploy): tapir → ${IMAGE_TAG}" "$DEPLOYMENT"
|
||
git push origin main
|
||
echo "✓ pushed to infra repo"
|
||
fi
|
||
|
||
shred -u ~/.ssh/id_infra
|
||
|
||
- name: Trigger Flux reconcile (immediate)
|
||
run: |
|
||
# Without these annotations, Flux would still pick up the change
|
||
# within 30s (the apps Kustomization interval). The annotations
|
||
# cut latency to ~1s.
|
||
kubectl -n flux-system annotate gitrepository flux-system \
|
||
reconcile.fluxcd.io/requestedAt="$(date +%s)" --overwrite
|
||
kubectl -n flux-system annotate kustomization apps \
|
||
reconcile.fluxcd.io/requestedAt="$(date +%s)" --overwrite
|
||
|
||
- name: Wait for Flux to apply new image
|
||
env:
|
||
IMAGE_TAG: ${{ needs.build.outputs.image-tag }}
|
||
run: |
|
||
# Poll the Deployment spec until it reflects the new tag.
|
||
# Bound to 60s so a stuck Flux doesn't hang CI.
|
||
EXPECTED="localhost:5000/tapir:${IMAGE_TAG}"
|
||
for i in $(seq 1 60); do
|
||
CURRENT=$(kubectl get deploy tapir -n tapir \
|
||
-o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null || echo "")
|
||
if [ "$CURRENT" = "$EXPECTED" ]; then
|
||
echo "✓ Flux applied new image after ${i}s"
|
||
break
|
||
fi
|
||
sleep 1
|
||
done
|
||
# Final assertion (in case the loop exited without matching).
|
||
kubectl get deploy tapir -n tapir \
|
||
-o jsonpath='{.spec.template.spec.containers[0].image}' \
|
||
| grep -qx "$EXPECTED" \
|
||
|| { echo "✗ Flux did not apply new image within 60s"; exit 1; }
|
||
|
||
- name: Verify rollout
|
||
run: |
|
||
kubectl rollout status deployment/tapir \
|
||
--namespace tapir \
|
||
--timeout=120s \
|
||
|| {
|
||
echo "── pod status ──"
|
||
kubectl get pods -n tapir -o wide
|
||
echo "── events ──"
|
||
kubectl get events -n tapir --sort-by='.lastTimestamp' | tail -20
|
||
echo "── describe ──"
|
||
kubectl describe pods -n tapir -l app=tapir | tail -40
|
||
exit 1
|
||
}
|
||
|
||
- name: Confirm pod running new image
|
||
env:
|
||
IMAGE_TAG: ${{ needs.build.outputs.image-tag }}
|
||
run: |
|
||
kubectl get pods -n tapir \
|
||
-l app=tapir \
|
||
--field-selector=status.phase=Running \
|
||
-o jsonpath='{.items[*].spec.containers[0].image}' \
|
||
| grep -q "localhost:5000/tapir:${IMAGE_TAG}" \
|
||
&& echo "✓ pod running new image" \
|
||
|| { echo "✗ pod image mismatch"; exit 1; }
|
||
|
||
# ── 4. Mirror to GitHub — skipped for now (SSH key rotation pending) ─
|