Compare commits
235
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f704258fdf | ||
|
|
e3288d0654 | ||
|
|
e0c53f8719 | ||
|
|
0828f0debf | ||
|
|
7282893877 | ||
|
|
cc6fe7083d | ||
|
|
0338b18a17 | ||
|
|
ed9d76b0cf | ||
|
|
0b807c6bcd | ||
|
|
8e4872d2e7 | ||
|
|
4affeb12e8 | ||
|
|
460e07719d | ||
|
|
7b7da650d4 | ||
|
|
85f4cc3236 | ||
|
|
41ab1f5d50 | ||
|
|
67bd0350a3 | ||
|
|
c24573bd9f | ||
|
|
fc1e8e7ba8 | ||
|
|
4a0226d8bd | ||
|
|
2e54d80a78 | ||
|
|
01c1606caa | ||
|
|
9aa01468ce | ||
|
|
25628a9994 | ||
|
|
a6691ac46b | ||
|
|
c909b4e775 | ||
|
|
f309a39027 | ||
|
|
89611164e3 | ||
|
|
93860210ef | ||
|
|
8509004eb7 | ||
|
|
453464a22b | ||
|
|
4ef3e85d16 | ||
|
|
b8af7f32c7 | ||
|
|
813192e5ba | ||
|
|
2ec1557a8c | ||
|
|
68815755e8 | ||
|
|
ba18668796 | ||
|
|
a65f2f2492 | ||
|
|
a43ee19505 | ||
|
|
4301e691f5 | ||
|
|
61d1ff2255 | ||
|
|
9cc954df40 | ||
|
|
e037c6b683 | ||
|
|
5baffd0189 | ||
|
|
228bcc77da | ||
|
|
4ef446548d | ||
|
|
145949875e | ||
|
|
82aeac7c19 | ||
|
|
1b4747ed03 | ||
|
|
9939afb90c | ||
|
|
3e69c00a64 | ||
|
|
253aa567aa | ||
|
|
5644dc3fc8 | ||
|
|
7df1de6628 | ||
|
|
7e920102f6 | ||
|
|
480c7227b5 | ||
|
|
0eb3be9631 | ||
|
|
e8c5ac50bf | ||
|
|
6101dd4e2f | ||
|
|
089444c19f | ||
|
|
d44db88787 | ||
|
|
1f8bd849fe | ||
|
|
7b19801bfa | ||
|
|
1260500d32 | ||
|
|
10b4c43bd2 | ||
|
|
f1f8b6d02e | ||
|
|
0d4c5c9d1d | ||
|
|
24d65b2085 | ||
|
|
df7bd459ed | ||
|
|
4a2067063d | ||
|
|
4bafd5f082 | ||
|
|
e3e78e042b | ||
|
|
6f1e0be09b | ||
|
|
d45348ad2e | ||
|
|
1c2ad2990f | ||
|
|
489b4bb2a1 | ||
|
|
21e7d6c74b | ||
|
|
8814ba6673 | ||
|
|
4b557a4325 | ||
|
|
72cb25111f | ||
|
|
38f222c931 | ||
|
|
e042b6d26d | ||
|
|
c85a32e770 | ||
|
|
b246c0e688 | ||
|
|
c7896cb3cc | ||
|
|
b79fb892c8 | ||
|
|
c315e3e003 | ||
|
|
64e3368f5f | ||
|
|
b01f8d5783 | ||
|
|
8ca767c144 | ||
|
|
f98b640531 | ||
|
|
607a8cbe8d | ||
|
|
54d60e53b9 | ||
|
|
9298e0c686 | ||
|
|
cd461b95f8 | ||
|
|
9c2a04406b | ||
|
|
a5a8cf6f6d | ||
|
|
64d11af9ef | ||
|
|
b590d2708d | ||
|
|
7314895ec4 | ||
|
|
36dd182fb5 | ||
|
|
40808f2d4b | ||
|
|
9232f49555 | ||
|
|
582c1a2065 | ||
|
|
9f0d8cf198 | ||
|
|
d21077303d | ||
|
|
9b2ee2e765 | ||
|
|
b8fbc5a805 | ||
|
|
19ca4282a8 | ||
|
|
71df696448 | ||
|
|
cc3cda4ab8 | ||
|
|
69a49bc603 | ||
|
|
137804b0b1 | ||
|
|
a9be5f285b | ||
|
|
fe56e2fe01 | ||
|
|
beeb5bc31b | ||
|
|
09eb31d1fe | ||
|
|
1665a1e7c4 | ||
|
|
5219561a91 | ||
|
|
9db06d8a63 | ||
|
|
1aa8a97f95 | ||
|
|
cc69a912f4 | ||
|
|
f4a0544903 | ||
|
|
e2a52789b9 | ||
|
|
e696b6405b | ||
|
|
e9b5a3f3e7 | ||
|
|
0ba78e8868 | ||
|
|
821d5f99cd | ||
|
|
5c70408e75 | ||
|
|
cb6917ca59 | ||
|
|
099b2d4c68 | ||
|
|
f66c1bcdcc | ||
|
|
1e65c3b413 | ||
|
|
87c978774f | ||
|
|
70a9f1d4cd | ||
|
|
1d5b2c6365 | ||
|
|
62acfee2ed | ||
|
|
2907801aca | ||
|
|
59050c4db6 | ||
|
|
c320ed88aa | ||
|
|
bddd75d92e | ||
|
|
e4c701c6f1 | ||
|
|
b527db9739 | ||
|
|
c5f556d1d6 | ||
|
|
a884e7e9c5 | ||
|
|
27fd33c99c | ||
|
|
2c96926ff7 | ||
|
|
2cda62b3ad | ||
|
|
f29927f50d | ||
|
|
51aa5d940c | ||
|
|
f775441a62 | ||
|
|
12fb031b6c | ||
|
|
980638d80a | ||
|
|
40b703e02a | ||
|
|
3df0459fed | ||
|
|
2384c47b81 | ||
|
|
4a0a56e152 | ||
|
|
9bf1c31605 | ||
|
|
a1a5217d77 | ||
|
|
9c7e3be984 | ||
|
|
8e45f21d23 | ||
|
|
e7c2e575d3 | ||
|
|
9cd3f7e934 | ||
|
|
c812c71ecc | ||
|
|
084b73907d | ||
|
|
0b04e487ad | ||
|
|
2fedc45443 | ||
|
|
58cd68c1eb | ||
|
|
e472015c76 | ||
|
|
0c0225f9c6 | ||
|
|
8403e8e524 | ||
|
|
57e29ca06c | ||
|
|
24f2a69eaa | ||
|
|
241eebfd9f | ||
|
|
317b0d4834 | ||
|
|
252a4ebd9e | ||
|
|
1ad1966672 | ||
|
|
ccadcecfef | ||
|
|
4c5d3cca81 | ||
|
|
10233ee881 | ||
|
|
f1e9739900 | ||
|
|
940f80899a | ||
|
|
f35c2a85a5 | ||
|
|
5d029a2823 | ||
|
|
f6623afd41 | ||
|
|
149ec2adae | ||
|
|
f5021a8436 | ||
|
|
88294d38bc | ||
|
|
ca9bf2657f | ||
|
|
4706c508e9 | ||
|
|
b4da97e0e2 | ||
|
|
561ba79360 | ||
|
|
2fac735837 | ||
|
|
de54cd33b2 | ||
|
|
b070347597 | ||
|
|
99743af182 | ||
|
|
070491261d | ||
|
|
72bf8a5553 | ||
|
|
dece5dec44 | ||
|
|
893886a60a | ||
|
|
e44485df16 | ||
|
|
8b7ef07ba3 | ||
|
|
1cf58768ed | ||
|
|
f45ba35e25 | ||
|
|
943554a96c | ||
|
|
40a614c8d4 | ||
|
|
ce2fc62ef8 | ||
|
|
0ceacc8230 | ||
|
|
1e81965519 | ||
|
|
f50c072d65 | ||
|
|
e6f508824b | ||
|
|
689500c85e | ||
|
|
4678d473b8 | ||
|
|
c63b2de66d | ||
|
|
27aa319f1d | ||
|
|
61d4d5bc4a | ||
|
|
483730cd03 | ||
|
|
477701fea2 | ||
|
|
17fad140a6 | ||
|
|
eb24a24b9c | ||
|
|
21e6ddd61e | ||
|
|
152aab7a4a | ||
|
|
1018dc0df9 | ||
|
|
74f4fd7f2a | ||
|
|
0cc441d6ce | ||
|
|
8415a97d15 | ||
|
|
fb425cbf9a | ||
|
|
e77edf58ef | ||
|
|
f15f57f9ed | ||
|
|
d208110002 | ||
|
|
3a27bf1126 | ||
|
|
8ca374e657 | ||
|
|
0fdf2f7218 | ||
|
|
d83943c86a | ||
|
|
6b817f11b9 | ||
|
|
672a0c8580 |
@@ -46,3 +46,31 @@ TAPIR_SECRETS_FILE=
|
|||||||
# --- run loop -------------------------------------------------------------
|
# --- run loop -------------------------------------------------------------
|
||||||
# Empty/0 = single pass. Set (e.g. 15m) to poll on that cadence.
|
# Empty/0 = single pass. Set (e.g. 15m) to poll on that cadence.
|
||||||
TAPIR_POLL_INTERVAL=
|
TAPIR_POLL_INTERVAL=
|
||||||
|
# How long to wait before re-fetching a transcript that returned HTTP 429
|
||||||
|
# (rate_limited). Inside the window the video is skipped without hitting the
|
||||||
|
# caption endpoint; after it expires the video is retried. 0 = always retry.
|
||||||
|
# Go duration; default 1h.
|
||||||
|
TAPIR_FETCH_BACKOFF=
|
||||||
|
# Recency bound for AUTO summarization: in automatic mode only videos published
|
||||||
|
# within this window of now are summarized; older ones are discovered + listed
|
||||||
|
# but wait for a manual "Summarize" (so a back-catalogue doesn't self-inflict
|
||||||
|
# 429s). An explicit request bypasses it. Go duration; default 168h (~7d).
|
||||||
|
# 0 = no bound (summarize every unseen video).
|
||||||
|
TAPIR_AUTO_SUMMARIZE_WINDOW=
|
||||||
|
# Minimum interval between outbound caption fetches across the WHOLE process —
|
||||||
|
# the shared per-egress-IP rate gate (ADR-014). Scheduler runners and the web
|
||||||
|
# "Summarize" click-path serialise through it so they cannot collectively trip
|
||||||
|
# 429s. Go duration; default 2s. 0 = unlimited (dev/tests).
|
||||||
|
TAPIR_FETCH_RATE=
|
||||||
|
|
||||||
|
# --- scheduled discovery (tapir serve, ADR-018) ---------------------------
|
||||||
|
# When > 0, `serve` runs in-process discovery for ALL users on this cadence
|
||||||
|
# (e.g. 2h): one runner pass per user per tick, run-once-on-startup then ticked.
|
||||||
|
# Empty/0 = disabled (dev/tests never auto-fetch). SINGLE-REPLICA assumption —
|
||||||
|
# >1 replica double-runs discovery. Go duration.
|
||||||
|
TAPIR_DISCOVERY_INTERVAL=
|
||||||
|
|
||||||
|
# --- invitations (tapir invite) -------------------------------------------
|
||||||
|
# Public base URL used to build the invite link `tapir invite <email>` prints.
|
||||||
|
# Default https://tapir.d-ma.be; no trailing slash needed.
|
||||||
|
TAPIR_PUBLIC_URL=
|
||||||
|
|||||||
Executable
+121
@@ -0,0 +1,121 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Build an OCI image with buildctl and push it to the local registry.
|
||||||
|
#
|
||||||
|
# CANONICAL COPY: mathias/infra ci-templates/build-image.sh (infra#309).
|
||||||
|
# Every repo carries a byte-identical copy at .gitea/build-image.sh, written by
|
||||||
|
# `task ci:sync` and held in place by the drift gate. DO NOT EDIT THE COPIES --
|
||||||
|
# edit here, re-sync, and the gate will tell you which repos still disagree.
|
||||||
|
#
|
||||||
|
# Why this exists: this step was hand-copied into a dozen repos, so migrating it
|
||||||
|
# off rootless buildah (infra#241) meant eleven near-identical commits, eleven
|
||||||
|
# verifications, and three separate rediscoveries of the same fix in the two
|
||||||
|
# repo templates and the gitea-ci skill. The differences between the copies were
|
||||||
|
# incidental -- a label here, a build-arg there -- never intentional. Those are
|
||||||
|
# inputs now, not forks of the implementation.
|
||||||
|
#
|
||||||
|
# Why buildctl and not buildah: the gitea-runner pod runs with
|
||||||
|
# `capabilities: drop [ALL]` and `allowPrivilegeEscalation: false`, which
|
||||||
|
# unconditionally blocks the setuid-root exec that rootless buildah's uid/gid
|
||||||
|
# remapping needs, and the pod has no OCI runtime at all. buildctl is a thin
|
||||||
|
# client -- it hands the build to the host's buildkitd over a socket, so the
|
||||||
|
# privileged layer work happens on the host and this pod's securityContext is
|
||||||
|
# not in the picture. buildah is absent from the runner image as of baked-v14
|
||||||
|
# and its absence is asserted at pod start (infra#308).
|
||||||
|
#
|
||||||
|
# Inputs, all via environment:
|
||||||
|
#
|
||||||
|
# IMAGE (required) image name, e.g. cobalt-dingo
|
||||||
|
# REGISTRY default localhost:5000
|
||||||
|
# SHA_TAG default `git rev-parse --short HEAD`
|
||||||
|
# VERSION_TAG optional extra tag, e.g. v1.2.3 on a tag build
|
||||||
|
# PUSH_LATEST default 1; set 0 to skip the :latest tag
|
||||||
|
# CONTEXT default . -- build context directory
|
||||||
|
# DOCKERFILE_DIR default $CONTEXT -- directory holding the Dockerfile
|
||||||
|
# DOCKERFILE_NAME optional, when the file is not named `Dockerfile`
|
||||||
|
# TARGET optional, multi-stage build target (--opt target=)
|
||||||
|
# BUILD_ARGS optional, space-separated K=V pairs
|
||||||
|
# BUILD_SECRETS optional, space-separated buildkit secret specs, each
|
||||||
|
# `id=NAME,src=PATH`. Pass the SPEC only -- the caller
|
||||||
|
# writes the secret to PATH itself. A secret VALUE must
|
||||||
|
# never be passed here: it would land in this script's
|
||||||
|
# argv and therefore in the CI log.
|
||||||
|
# EXTRA_LABELS optional, space-separated K=V pairs
|
||||||
|
# GIT_SHA optional, stamped as org.opencontainers.image.revision
|
||||||
|
# IMAGE_SOURCE optional, stamped as org.opencontainers.image.source
|
||||||
|
#
|
||||||
|
# Note on shell style below: every conditional is a full `if` block, never
|
||||||
|
# `[ -n "$X" ] && cmd`. Under `set -e` the && form exits the script whenever the
|
||||||
|
# test is false, which is a real bug this estate has already shipped once.
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
if [ -z "${IMAGE:-}" ]; then
|
||||||
|
echo "build-image: IMAGE is required" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
REGISTRY="${REGISTRY:-localhost:5000}"
|
||||||
|
BUILDKIT_HOST="${BUILDKIT_HOST:-unix:///run/buildkit/buildkitd.sock}"
|
||||||
|
CONTEXT="${CONTEXT:-.}"
|
||||||
|
DOCKERFILE_DIR="${DOCKERFILE_DIR:-$CONTEXT}"
|
||||||
|
PUSH_LATEST="${PUSH_LATEST:-1}"
|
||||||
|
VERSION_TAG="${VERSION_TAG:-}"
|
||||||
|
|
||||||
|
if [ -z "${SHA_TAG:-}" ]; then
|
||||||
|
SHA_TAG="$(git rev-parse --short HEAD)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
REF="${REGISTRY}/${IMAGE}:${SHA_TAG}"
|
||||||
|
|
||||||
|
TAR_FILE="$(mktemp)"
|
||||||
|
trap 'rm -f "$TAR_FILE"' EXIT
|
||||||
|
|
||||||
|
args=(
|
||||||
|
--frontend dockerfile.v0
|
||||||
|
--local "context=${CONTEXT}"
|
||||||
|
--local "dockerfile=${DOCKERFILE_DIR}"
|
||||||
|
)
|
||||||
|
|
||||||
|
if [ -n "${DOCKERFILE_NAME:-}" ]; then
|
||||||
|
args+=(--opt "filename=${DOCKERFILE_NAME}")
|
||||||
|
fi
|
||||||
|
if [ -n "${TARGET:-}" ]; then
|
||||||
|
args+=(--opt "target=${TARGET}")
|
||||||
|
fi
|
||||||
|
for spec in ${BUILD_SECRETS:-}; do
|
||||||
|
# Opaque id=NAME,src=PATH -- buildkit reads the file, nothing is echoed.
|
||||||
|
args+=(--secret "${spec}")
|
||||||
|
done
|
||||||
|
if [ -n "${GIT_SHA:-}" ]; then
|
||||||
|
args+=(--opt "label:org.opencontainers.image.revision=${GIT_SHA}")
|
||||||
|
fi
|
||||||
|
if [ -n "${IMAGE_SOURCE:-}" ]; then
|
||||||
|
args+=(--opt "label:org.opencontainers.image.source=${IMAGE_SOURCE}")
|
||||||
|
fi
|
||||||
|
for kv in ${BUILD_ARGS:-}; do
|
||||||
|
args+=(--opt "build-arg:${kv}")
|
||||||
|
done
|
||||||
|
for kv in ${EXTRA_LABELS:-}; do
|
||||||
|
args+=(--opt "label:${kv}")
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "build-image: building ${REF}"
|
||||||
|
buildctl --addr "$BUILDKIT_HOST" build "${args[@]}" --output "type=oci,dest=${TAR_FILE}"
|
||||||
|
|
||||||
|
# buildctl has no registry-push equivalent to `buildah push`, so skopeo copies
|
||||||
|
# the OCI archive up -- once per tag, from the same archive, so every tag points
|
||||||
|
# at bit-identical content.
|
||||||
|
push_tag() {
|
||||||
|
echo "build-image: pushing ${1}"
|
||||||
|
skopeo copy --dest-tls-verify=false "oci-archive:${TAR_FILE}" "docker://${1}"
|
||||||
|
}
|
||||||
|
|
||||||
|
push_tag "$REF"
|
||||||
|
if [ "$PUSH_LATEST" = "1" ]; then
|
||||||
|
push_tag "${REGISTRY}/${IMAGE}:latest"
|
||||||
|
fi
|
||||||
|
if [ -n "$VERSION_TAG" ]; then
|
||||||
|
push_tag "${REGISTRY}/${IMAGE}:${VERSION_TAG}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "✓ build-image: ${REF} pushed"
|
||||||
+151
-41
@@ -61,53 +61,163 @@ jobs:
|
|||||||
echo "version-tag=${REF#refs/tags/}" >> "$GITHUB_OUTPUT"
|
echo "version-tag=${REF#refs/tags/}" >> "$GITHUB_OUTPUT"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# infra#241: rootless buildah inside the gitea-runner pod can't remap the
|
||||||
|
# UID/GID range newuidmap/newgidmap need (the pod drops ALL capabilities
|
||||||
|
# and sets allowPrivilegeEscalation:false — both deliberate, it also holds
|
||||||
|
# CI secrets), so extracting a base-image layer that chowns outside the
|
||||||
|
# single available mapping fails with `ApplyLayer ... requested 0:42 for
|
||||||
|
# /etc/gshadow`. Main has been red on exactly this since 2026-08-13.
|
||||||
|
# buildctl instead talks to koala's host buildkitd over the socket the
|
||||||
|
# runner pod already mounts: the privileged extraction happens inside
|
||||||
|
# buildkitd on the host, never in this pod. Same migration agentsquad
|
||||||
|
# (infra#233) and segment-embedder (infra#243) already made — zero
|
||||||
|
# security tradeoff, unlike loosening the pod's capabilities.
|
||||||
|
#
|
||||||
|
# buildctl has no registry-push flag of its own: it writes an OCI archive
|
||||||
|
# and skopeo pushes that, which is also why every tag below is one copy of
|
||||||
|
# a single build rather than a rebuild.
|
||||||
- name: Build and push to local registry
|
- name: Build and push to local registry
|
||||||
run: |
|
# infra#309: implementation lives in .gitea/build-image.sh, a
|
||||||
REGISTRY="localhost:5000"
|
# byte-identical copy of mathias/infra's ci-templates/build-image.sh.
|
||||||
REF="${REGISTRY}/${{ env.IMAGE }}:${{ steps.meta.outputs.sha-tag }}"
|
# Do not edit it here -- edit the canonical file, run `task ci:sync`,
|
||||||
buildah build \
|
# and infra's drift gate names any repo still carrying an old copy.
|
||||||
--label "org.opencontainers.image.revision=${{ github.sha }}" \
|
env:
|
||||||
--label "org.opencontainers.image.source=${{ github.repositoryUrl }}" \
|
SHA_TAG: ${{ steps.meta.outputs.sha-tag }}
|
||||||
-t ${REF} \
|
VERSION_TAG: ${{ steps.meta.outputs.version-tag }}
|
||||||
-t ${REGISTRY}/${{ env.IMAGE }}:latest \
|
GIT_SHA: ${{ github.sha }}
|
||||||
.
|
IMAGE_SOURCE: ${{ github.repositoryUrl }}
|
||||||
buildah push --tls-verify=false ${REF}
|
run: bash .gitea/build-image.sh
|
||||||
buildah push --tls-verify=false ${REGISTRY}/${{ env.IMAGE }}:latest
|
|
||||||
[[ -n "${{ steps.meta.outputs.version-tag }}" ]] && \
|
|
||||||
buildah push --tls-verify=false \
|
|
||||||
${REGISTRY}/${{ env.IMAGE }}:${{ steps.meta.outputs.version-tag }} || true
|
|
||||||
echo "Image pushed to ${REF}"
|
|
||||||
|
|
||||||
- name: Smoke test
|
# The buildah-based smoke test that used to sit here is gone with the
|
||||||
run: |
|
# buildctl migration above: `buildah from ${REF}` would pull the image
|
||||||
REGISTRY="localhost:5000"
|
# back into the pod's buildah store and hit the very uid/gid wall the
|
||||||
REF="${REGISTRY}/${{ env.IMAGE }}:${{ steps.meta.outputs.sha-tag }}"
|
# migration exists to avoid — it would fail the job for a reason that has
|
||||||
CNAME="smoke-${{ steps.meta.outputs.sha-tag }}"
|
# nothing to do with the image being good. The `Verify rollout` step in
|
||||||
sudo k3s ctr images pull --plain-http ${REF}
|
# the deploy job below is the stronger check anyway: it requires the real
|
||||||
OUTPUT=$(timeout 5 sudo k3s ctr run --rm ${REF} ${CNAME} /tapir 2>&1 || true)
|
# pod to reach Ready in-cluster, not just the entrypoint to print its own
|
||||||
sudo k3s ctr containers delete ${CNAME} 2>/dev/null || true
|
# name for 5s. Tag builds lose their only image-level check (deploy is
|
||||||
echo "$OUTPUT" | grep -q "tapir" \
|
# main-push only) — accepted; a tag is always built from a main commit
|
||||||
&& echo "Smoke test passed" \
|
# that already deployed.
|
||||||
|| echo "Smoke test inconclusive: $OUTPUT"
|
|
||||||
|
|
||||||
# ── 3. Mirror to GitHub (deploy intentionally omitted until manifests exist) ─
|
# ── 3. Deploy via infra repo + Flux ─────────────────────────────────────────
|
||||||
mirror:
|
# Flux native image-automation can't scan localhost:5000 from inside k3s pods
|
||||||
name: Mirror to GitHub
|
# (mathias/infra k3s/flux/flux-system/image-automation.yaml) — this job
|
||||||
|
# mirrors cobalt-dingo's proven pattern instead: patch the infra repo's
|
||||||
|
# manifest directly on every push to main, then annotate Flux for a fast
|
||||||
|
# reconcile. Fixes infra#111 (image built+pushed but manifest bump was
|
||||||
|
# manual, so merged features silently didn't deploy).
|
||||||
|
deploy:
|
||||||
|
name: Deploy via GitOps
|
||||||
needs: build
|
needs: build
|
||||||
runs-on: self-hosted
|
runs-on: self-hosted
|
||||||
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
|
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
|
||||||
|
environment: staging
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- name: Update image tag in infra repo
|
||||||
with:
|
env:
|
||||||
fetch-depth: 0
|
IMAGE_TAG: ${{ needs.build.outputs.image-tag }}
|
||||||
|
DEPLOY_KEY: ${{ secrets.INFRA_DEPLOY_KEY }}
|
||||||
- name: Push to GitHub
|
|
||||||
run: |
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
# infra#311: fail on an empty tag BEFORE patching anything. The
|
||||||
|
# grep verification further down CANNOT catch it -- with IMAGE_TAG
|
||||||
|
# empty its pattern matches the very corruption it exists to reject.
|
||||||
|
# The result is `image: <repo>:` in the manifest, which is invalid
|
||||||
|
# YAML, so Flux fails the ENTIRE apps Kustomization and every app
|
||||||
|
# stops reconciling, not just this one. That happened on 2026-08-25.
|
||||||
|
[ -n "${IMAGE_TAG:-}" ] || { echo "IMAGE_TAG is empty - refusing to patch"; exit 1; }
|
||||||
|
|
||||||
|
# INFRA_DEPLOY_KEY is a Gitea org secret (mathias org), already
|
||||||
|
# configured per docs/cd-pipeline.md in the infra repo — same key
|
||||||
|
# cobalt-dingo and brain-gardener use, no new secret needed.
|
||||||
mkdir -p ~/.ssh
|
mkdir -p ~/.ssh
|
||||||
echo '${{ secrets.GH_DEPLOY_KEY }}' > ~/.ssh/id_rsa_gh_mirror
|
echo "$DEPLOY_KEY" > ~/.ssh/id_infra
|
||||||
chmod 600 ~/.ssh/id_rsa_gh_mirror
|
chmod 600 ~/.ssh/id_infra
|
||||||
ssh-keyscan github.com >> ~/.ssh/known_hosts 2>/dev/null
|
ssh-keyscan -p 30022 10.0.1.20 >> ~/.ssh/known_hosts 2>/dev/null
|
||||||
GIT_SSH_COMMAND="ssh -i ~/.ssh/id_rsa_gh_mirror -o IdentitiesOnly=yes" \
|
|
||||||
git push git@github.com:mathiasb/tapir.git HEAD:main
|
export GIT_SSH_COMMAND="ssh -i ~/.ssh/id_infra -o IdentitiesOnly=yes"
|
||||||
rm ~/.ssh/id_rsa_gh_mirror
|
rm -rf /tmp/infra
|
||||||
echo "Mirrored to GitHub"
|
git clone -b main ssh://git@10.0.1.20:30022/mathias/infra.git /tmp/infra
|
||||||
|
cd /tmp/infra
|
||||||
|
|
||||||
|
DEPLOYMENT="k3s/apps/tapir/deployment.yaml"
|
||||||
|
# In-place update of the image tag. sed (not yq) so we don't
|
||||||
|
# depend on additional tooling on the runner — same as cobalt-dingo.
|
||||||
|
sed -i "s|image: localhost:5000/tapir:.*|image: localhost:5000/tapir:${IMAGE_TAG}|" "$DEPLOYMENT"
|
||||||
|
|
||||||
|
# Verify the patch took effect.
|
||||||
|
grep -q "localhost:5000/tapir:${IMAGE_TAG}" "$DEPLOYMENT" \
|
||||||
|
|| { echo "✗ image tag patch failed"; exit 1; }
|
||||||
|
|
||||||
|
if git diff --quiet "$DEPLOYMENT"; then
|
||||||
|
echo "ℹ image tag unchanged — skipping push"
|
||||||
|
else
|
||||||
|
git -c user.name="tapir CI" \
|
||||||
|
-c user.email="ci@tapir.local" \
|
||||||
|
commit -m "chore(deploy): tapir → ${IMAGE_TAG}" "$DEPLOYMENT"
|
||||||
|
git push origin main
|
||||||
|
echo "✓ pushed to infra repo"
|
||||||
|
fi
|
||||||
|
|
||||||
|
shred -u ~/.ssh/id_infra
|
||||||
|
|
||||||
|
- name: Trigger Flux reconcile (immediate)
|
||||||
|
run: |
|
||||||
|
# Without these annotations, Flux would still pick up the change
|
||||||
|
# within 30s (the apps Kustomization interval). The annotations
|
||||||
|
# cut latency to ~1s.
|
||||||
|
kubectl -n flux-system annotate gitrepository flux-system \
|
||||||
|
reconcile.fluxcd.io/requestedAt="$(date +%s)" --overwrite
|
||||||
|
kubectl -n flux-system annotate kustomization apps \
|
||||||
|
reconcile.fluxcd.io/requestedAt="$(date +%s)" --overwrite
|
||||||
|
|
||||||
|
- name: Wait for Flux to apply new image
|
||||||
|
env:
|
||||||
|
IMAGE_TAG: ${{ needs.build.outputs.image-tag }}
|
||||||
|
run: |
|
||||||
|
# Poll the Deployment spec until it reflects the new tag.
|
||||||
|
# Bound to 60s so a stuck Flux doesn't hang CI.
|
||||||
|
EXPECTED="localhost:5000/tapir:${IMAGE_TAG}"
|
||||||
|
for i in $(seq 1 60); do
|
||||||
|
CURRENT=$(kubectl get deploy tapir -n tapir \
|
||||||
|
-o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null || echo "")
|
||||||
|
if [ "$CURRENT" = "$EXPECTED" ]; then
|
||||||
|
echo "✓ Flux applied new image after ${i}s"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
# Final assertion (in case the loop exited without matching).
|
||||||
|
kubectl get deploy tapir -n tapir \
|
||||||
|
-o jsonpath='{.spec.template.spec.containers[0].image}' \
|
||||||
|
| grep -qx "$EXPECTED" \
|
||||||
|
|| { echo "✗ Flux did not apply new image within 60s"; exit 1; }
|
||||||
|
|
||||||
|
- name: Verify rollout
|
||||||
|
run: |
|
||||||
|
kubectl rollout status deployment/tapir \
|
||||||
|
--namespace tapir \
|
||||||
|
--timeout=120s \
|
||||||
|
|| {
|
||||||
|
echo "── pod status ──"
|
||||||
|
kubectl get pods -n tapir -o wide
|
||||||
|
echo "── events ──"
|
||||||
|
kubectl get events -n tapir --sort-by='.lastTimestamp' | tail -20
|
||||||
|
echo "── describe ──"
|
||||||
|
kubectl describe pods -n tapir -l app=tapir | tail -40
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
- name: Confirm pod running new image
|
||||||
|
env:
|
||||||
|
IMAGE_TAG: ${{ needs.build.outputs.image-tag }}
|
||||||
|
run: |
|
||||||
|
kubectl get pods -n tapir \
|
||||||
|
-l app=tapir \
|
||||||
|
--field-selector=status.phase=Running \
|
||||||
|
-o jsonpath='{.items[*].spec.containers[0].image}' \
|
||||||
|
| grep -q "localhost:5000/tapir:${IMAGE_TAG}" \
|
||||||
|
&& echo "✓ pod running new image" \
|
||||||
|
|| { echo "✗ pod image mismatch"; exit 1; }
|
||||||
|
|
||||||
|
# ── 4. Mirror to GitHub — skipped for now (SSH key rotation pending) ─
|
||||||
|
|||||||
+11
@@ -11,3 +11,14 @@
|
|||||||
.env.*
|
.env.*
|
||||||
!.env.example
|
!.env.example
|
||||||
*.local
|
*.local
|
||||||
|
|
||||||
|
# Spike media (#28): real recordings, their transcripts and derived analyses are
|
||||||
|
# private third-party content and this repo is public. Only synthetic fixtures
|
||||||
|
# are committed — see scripts/spike-media/README.md.
|
||||||
|
/scripts/spike-media/*.mov
|
||||||
|
/scripts/spike-media/*.mp4
|
||||||
|
/scripts/spike-media/*.wav
|
||||||
|
/scripts/spike-media/*.srt
|
||||||
|
/scripts/spike-media/*.json
|
||||||
|
/scripts/spike-media/*.html
|
||||||
|
!/scripts/spike-media/fixtures/
|
||||||
|
|||||||
@@ -0,0 +1,18 @@
|
|||||||
|
{
|
||||||
|
"mcpServers": {
|
||||||
|
"brain": {
|
||||||
|
"type": "http",
|
||||||
|
"url": "https://brain-mcp.d-ma.be/mcp",
|
||||||
|
"headers": {
|
||||||
|
"Authorization": "Bearer ${BRAIN_MCP_TOKEN}"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"gitea": {
|
||||||
|
"type": "http",
|
||||||
|
"url": "https://git-mcp.d-ma.be/mcp",
|
||||||
|
"headers": {
|
||||||
|
"Authorization": "Bearer ${GITEA_MCP_TOKEN}"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -12,6 +12,7 @@ docs it indexes.
|
|||||||
3. `DECISIONS.md` — the ADRs. Decisions are settled here; do not re-litigate without a new ADR.
|
3. `DECISIONS.md` — the ADRs. Decisions are settled here; do not re-litigate without a new ADR.
|
||||||
4. `docs/architecture/architecture.md`, `docs/data-model.md`, `docs/use-cases/*.feature`.
|
4. `docs/architecture/architecture.md`, `docs/data-model.md`, `docs/use-cases/*.feature`.
|
||||||
5. `docs/homelab-integration.md` — the concrete endpoints/conventions you'll need.
|
5. `docs/homelab-integration.md` — the concrete endpoints/conventions you'll need.
|
||||||
|
6. `LANGUAGE.md` — the project vocabulary. Apply the caveman rubric before destructive operations.
|
||||||
|
|
||||||
## How to work in this repo
|
## How to work in this repo
|
||||||
|
|
||||||
@@ -46,10 +47,14 @@ These caused real mistakes that were caught and corrected; the corrections are l
|
|||||||
(See `DECISIONS.md` for full rationale. Listed here so you don't propose them.)
|
(See `DECISIONS.md` for full rationale. Listed here so you don't propose them.)
|
||||||
|
|
||||||
- **No Supabase** — reuse Dex / ESO+1Password / Postgres (ADR-002).
|
- **No Supabase** — reuse Dex / ESO+1Password / Postgres (ADR-002).
|
||||||
- **No global cross-tenant video/transcript table** — per-user isolation (data-model). Dedup
|
- **No global cross-tenant *video* table** — videos stay per-user (data-model). Transcripts ARE
|
||||||
across users is a Future C concern, not a Stage 0/1 default.
|
shared since ADR-021 (public caption content, keyed by `(provider, provider_video_id)`, non-RLS)
|
||||||
- **No audio-download + speech-to-text in the core path** — captions-first (ADR-007). STT is a
|
so re-analysis never re-fetches; the *videos* half of cross-tenant dedup stays a Future C concern.
|
||||||
deferred, bounded optional component.
|
- **No YouTube audio-download + speech-to-text in the core path** — captions-first (ADR-007).
|
||||||
|
That rejection stands. ADR-033 superseded the STT deferral **for user-supplied audio only**:
|
||||||
|
an authenticated user may upload a recording and get a *private* transcript (own RLS table,
|
||||||
|
never the shared `transcripts`). It is off by default (`TAPIR_UPLOADS_ENABLED`) and does not
|
||||||
|
open the door to scraping audio.
|
||||||
- **No public SaaS / sign-up / billing / Google OAuth verification at scale** — Future C,
|
- **No public SaaS / sign-up / billing / Google OAuth verification at scale** — Future C,
|
||||||
deferred behind the Stage 0 gate (ADR-008).
|
deferred behind the Stage 0 gate (ADR-008).
|
||||||
|
|
||||||
@@ -60,8 +65,14 @@ These caused real mistakes that were caught and corrected; the corrections are l
|
|||||||
stores, and sinks are adapters. Adding a video provider or a sink = a new adapter implementing
|
stores, and sinks are adapters. Adding a video provider or a sink = a new adapter implementing
|
||||||
the interface, nothing in the engine changes. This is what keeps "standalone vs homelab" a
|
the interface, nothing in the engine changes. This is what keeps "standalone vs homelab" a
|
||||||
wiring choice (ADR-003).
|
wiring choice (ADR-003).
|
||||||
- **BDD.** The `docs/use-cases/*.feature` files are the behavior spec. New behavior gets a
|
- **BDD.** The `docs/use-cases/*.feature` files are the behavior spec (design records — there is
|
||||||
scenario; the use-case core is tested through fake adapters, not live YouTube/brain.
|
no godog runner). New behavior gets a scenario; the use-case core is tested through fake
|
||||||
|
adapters, not live YouTube/brain. A name-coverage gate (`test/acceptance/scenario_coverage_test.go`,
|
||||||
|
`TestScenarioCoverage`) keeps the two from drifting: every non-`@pending` scenario must be
|
||||||
|
mapped to an existing Go test in `scenarioCoverage`. When you add a scenario, either map it to
|
||||||
|
its covering test or tag it `@pending` in the `.feature` with a one-line reason. It checks the
|
||||||
|
*link*, not that the test exercises the scenario — that's the deliberate trade for not running
|
||||||
|
godog (see issue #5 / the BDD-runner decision).
|
||||||
|
|
||||||
## Skills (engineering discipline)
|
## Skills (engineering discipline)
|
||||||
|
|
||||||
@@ -79,23 +90,61 @@ Skills live in the canonical library `mathias/skills` and are wired into this re
|
|||||||
|
|
||||||
## Current build state (start here for the first task)
|
## Current build state (start here for the first task)
|
||||||
|
|
||||||
The repo is **scaffolded and intentionally RED**:
|
The repo is **green and shipping** — last tag `v0.15.0`. `task check` passes (fmt, vet, lint,
|
||||||
|
`go test -p 1 ./...`). Go is `1.26.1` (see `go.mod`).
|
||||||
|
|
||||||
- Clean Architecture skeleton exists: `internal/domain` (entities), `internal/ports`
|
- Clean Architecture core is implemented: `internal/domain` (entities), `internal/ports`
|
||||||
(interfaces), `internal/usecase` (engine), `cmd/tapir` (entrypoint stub),
|
(interfaces), `internal/usecase.Engine.ProcessNewVideo` (resolve transcript → summarize →
|
||||||
`internal/adapters` (empty — concrete adapters go here).
|
deliver to sinks | skip on no-transcript). The acceptance tests in `test/acceptance/` are
|
||||||
- `usecase.Engine.ProcessNewVideo` returns `ErrNotImplemented`.
|
green against it.
|
||||||
- `test/acceptance/summarize_new_video_test.go` translates the first two Gherkin scenarios and
|
- Adapters present under `internal/adapters/`: `youtube` (captions-first `VideoSource`,
|
||||||
**fails** against the stub. `task check` is therefore red on `test`.
|
timedtext/InnerTube acquisition per ADR-010), `summarizer` + `llm` (the copied AI router,
|
||||||
- **First build task:** implement `ProcessNewVideo` (resolve transcript -> summarize -> deliver to
|
now a resilient endpoint chain — local primary → local fallback → external worst-case,
|
||||||
sinks | skip on no-transcript) to make the acceptance tests green, following the `.feature`
|
parse-failure-aware, ADR-004 + ADR-022), `store` (Postgres, golang-migrate migrations 001–019),
|
||||||
files. Then add the AI-router `Summarizer` (copy `llm` per ADR-004), the YouTube `VideoSource`
|
`secrets` (file-backed `SecretStore`). The brain HTTP sink (ADR-005) is the remaining
|
||||||
adapter (captions-first), and the store + brain sinks.
|
optional sink.
|
||||||
|
- Stage 1 is open (ADR-012): multi-user with **DB-enforced** isolation — Postgres RLS `FORCE`d
|
||||||
|
on all user-owned tables (migration 003), two-user isolation test in
|
||||||
|
`internal/adapters/store/rls_test.go`. Registration gate, per-user YouTube web connect, and
|
||||||
|
account management (disconnect / delete, ADR-013) all shipped.
|
||||||
|
- Transcript persistence (ADR-021, migration 015): transcripts are a **shared, non-RLS** store
|
||||||
|
keyed by `(provider, provider_video_id)` — the single exception to the isolation boundary
|
||||||
|
(`TestTranscriptsTableIsSharedNotRLS`). The engine reads stored transcripts before any caption
|
||||||
|
fetch (`usecase.resolveTranscript`), so re-analysis — re-summarize, paste-a-URL, onboarding
|
||||||
|
burst — never re-touches YouTube. Per-user summaries/videos stay RLS-scoped.
|
||||||
|
- Audio upload (ADR-033, migration 017): `internal/uploads` (validation by **probing**, intake,
|
||||||
|
one-at-a-time in-process worker), `internal/adapters/audio` (ffprobe/ffmpeg), and
|
||||||
|
`internal/adapters/transcribe` (gateway client + iguana→berget chain). Transcripts are private
|
||||||
|
in `uploaded_transcripts`; the audio is never retained; chat reuses `chat.Service`. The
|
||||||
|
isolation test pins the **exact set** of non-RLS tables (`TestRowSecurityOffTablesAreExactlyTheKnownSet`).
|
||||||
|
The runtime image does not yet contain ffmpeg — that is a pending decision recorded in ADR-033.
|
||||||
|
Two-engine reconciliation (ADR-034, migration 019, off unless `TAPIR_RECONCILE_BASE_MODEL`):
|
||||||
|
`internal/reconcile` (word diff, disputes, a local-model adjudicator) and
|
||||||
|
`transcribe.Ensemble`; every model must be a local alias, enforced at start-up.
|
||||||
|
Progress and logging (ADR-035, migration 020): `internal/progress.Tracker` is the ONE writer of a
|
||||||
|
job's phases (saved to the row for the page, logged by the same call); engines and the reconciler
|
||||||
|
report through `progress.From(ctx)`, so `uploads.Transcriber` is unchanged, and a transcriber may
|
||||||
|
implement `uploads.Planner` to declare its own phases. The page tells running / quiet / lost from
|
||||||
|
two timestamps (`alive_at` heartbeat vs `progress_at`) and never shows a percentage that does not
|
||||||
|
come from real counts. **Nothing logged may carry transcript text, words from it, or the spool path.**
|
||||||
|
Errors reach a log, a row or a page as `errkind.Of(err)` (a kind such as `timeout` or `http_502`),
|
||||||
|
never `err.Error()`: error text can hold a provider's response body, the spool path or transcript
|
||||||
|
words (ADR-035 decision 8). Local-only for anything that carries transcript text: `config.IsLocalModel`.
|
||||||
|
`TAPIR_SHOT_DIR=… go test -run TestRenderUploadProgressStatesForReview ./internal/web` renders every
|
||||||
|
state for a screenshot review.
|
||||||
|
**Lint caveat:** `task check` does not fail on lint (#41: the Taskfile `|| echo` swallows
|
||||||
|
findings) — run `golangci-lint run ./...` yourself until that is fixed.
|
||||||
|
- `cmd/tapir` subcommands: `list`, `show`, `auth` (interactive host-side OAuth), `run` (batch
|
||||||
|
watch→summarize), `serve` (the HTMX+Templ web reader/writer under `internal/web`, a new
|
||||||
|
transport over the unchanged engine/ports — ADR-003). `tapir env` prints config.
|
||||||
|
- **Build/run:** `task check` is the gate; `task build` produces the binary. Local dev uses
|
||||||
|
`StubAuth` (allow-all) and a `TAPIR_DB_DSN` Postgres; the deployed service uses Dex OIDC.
|
||||||
|
|
||||||
**Unverified setup items** (see `docs/homelab-integration.md`, marked `confirm`): the Go version
|
**Setup facts** (resolved — see `docs/homelab-integration.md` for the live values): LiteLLM is
|
||||||
in `go.mod` (1.23 — match the koala runner; estate elsewhere uses 1.26.1), the brain-mcp URL, the
|
off-cluster at `koala:30401/v1/` with `LITELLM_MASTER_KEY` from 1Password; the summarization
|
||||||
exact ESO secret-ref naming, and the summarization model alias. Resolve against the live cluster
|
model is config (`TAPIR_SUMMARIZER_MODEL`, default `koala/phi4-mini`), never hardcoded. The
|
||||||
before depending on them, and pin answers back into `docs/homelab-integration.md`.
|
brain-mcp base URL and ESO ref scheme are pinned in that doc; check it before wiring rather than
|
||||||
|
re-deriving.
|
||||||
|
|
||||||
## Provenance (where this design came from)
|
## Provenance (where this design came from)
|
||||||
|
|
||||||
|
|||||||
+1564
-19
File diff suppressed because it is too large
Load Diff
+40
@@ -16,12 +16,52 @@ COPY . .
|
|||||||
# no libc. Trim symbols/DWARF to shrink the layer.
|
# no libc. Trim symbols/DWARF to shrink the layer.
|
||||||
RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags='-s -w' -o /out/tapir ./cmd/tapir
|
RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags='-s -w' -o /out/tapir ./cmd/tapir
|
||||||
|
|
||||||
|
# ── ffmpeg ──────────────────────────────────────────────────────────────────
|
||||||
|
# Audio upload probes files with ffprobe and converts them with ffmpeg (ADR-033,
|
||||||
|
# #34 option A). Static builds, so they run in the distroless image below.
|
||||||
|
#
|
||||||
|
# Pinned by version AND sha256: a changed upstream file fails the build instead of
|
||||||
|
# shipping. The checksum proves these are the bytes that were reviewed (2026-10-02;
|
||||||
|
# the publisher's md5 matched too). It does NOT prove authenticity: the publisher
|
||||||
|
# signs nothing we can check. amd64 only (the cluster is one amd64 node).
|
||||||
|
#
|
||||||
|
# These builds are GPLv3 (--enable-gpl --enable-version3). The service is private and
|
||||||
|
# the image is not distributed; the licence text is copied into the image anyway.
|
||||||
|
#
|
||||||
|
# The release URL moves on when upstream releases a newer version, so the pinned
|
||||||
|
# version is tried in releases/ and then old-releases/. If neither has it the build
|
||||||
|
# fails; vendoring the verified tarball is the durable fix.
|
||||||
|
FROM golang:1.26 AS ffmpeg
|
||||||
|
|
||||||
|
ARG FFMPEG_VERSION=7.0.2
|
||||||
|
ARG FFMPEG_SHA256=abda8d77ce8309141f83ab8edf0596834087c52467f6badf376a6a2a4c87cf67
|
||||||
|
|
||||||
|
# xz is not in the golang image (found by building it: `tar: xz: Cannot exec`).
|
||||||
|
RUN set -eu; \
|
||||||
|
apt-get update; \
|
||||||
|
apt-get install -y --no-install-recommends xz-utils; \
|
||||||
|
rm -rf /var/lib/apt/lists/*; \
|
||||||
|
mkdir -p /out; \
|
||||||
|
for dir in releases old-releases; do \
|
||||||
|
curl -fsSL -o /out/ffmpeg.tar.xz "https://johnvansickle.com/ffmpeg/${dir}/ffmpeg-${FFMPEG_VERSION}-amd64-static.tar.xz" && break; \
|
||||||
|
done; \
|
||||||
|
echo "${FFMPEG_SHA256} /out/ffmpeg.tar.xz" | sha256sum -c -; \
|
||||||
|
tar -xJf /out/ffmpeg.tar.xz -C /out --strip-components=1 --wildcards '*/ffmpeg' '*/ffprobe' '*/GPLv3.txt'; \
|
||||||
|
rm /out/ffmpeg.tar.xz
|
||||||
|
|
||||||
# ── runtime ─────────────────────────────────────────────────────────────────
|
# ── runtime ─────────────────────────────────────────────────────────────────
|
||||||
# distroless static + nonroot: no shell, no package manager, runs as uid 65532.
|
# distroless static + nonroot: no shell, no package manager, runs as uid 65532.
|
||||||
# ca-certificates are bundled, which the OIDC/HTTPS clients need.
|
# ca-certificates are bundled, which the OIDC/HTTPS clients need.
|
||||||
FROM gcr.io/distroless/static-debian12:nonroot
|
FROM gcr.io/distroless/static-debian12:nonroot
|
||||||
|
|
||||||
COPY --from=build /out/tapir /tapir
|
COPY --from=build /out/tapir /tapir
|
||||||
|
COPY --from=ffmpeg /out/ffmpeg /out/ffprobe /usr/local/bin/
|
||||||
|
COPY --from=ffmpeg /out/GPLv3.txt /usr/share/doc/ffmpeg/GPLv3.txt
|
||||||
|
|
||||||
|
# Where the service finds them. Uploads stay OFF until TAPIR_UPLOADS_ENABLED is set
|
||||||
|
# (a deployment decision, not an image one).
|
||||||
|
ENV TAPIR_FFMPEG_BIN=/usr/local/bin/ffmpeg \
|
||||||
|
TAPIR_FFPROBE_BIN=/usr/local/bin/ffprobe
|
||||||
|
|
||||||
EXPOSE 8080
|
EXPOSE 8080
|
||||||
USER nonroot:nonroot
|
USER nonroot:nonroot
|
||||||
|
|||||||
+26
@@ -0,0 +1,26 @@
|
|||||||
|
# Vocabulary (hand-written pilot 2026-06 — will be generated by langgen in Phase 2)
|
||||||
|
|
||||||
|
| Term | Means | Never say |
|
||||||
|
|---|---|---|
|
||||||
|
| transcript | Captions-first text of a video; the source for summarizing. Shared store keyed by `(provider, video_id)`. | "subtitles file", "the audio" |
|
||||||
|
| video | Per-user record of a seen video; the unit of work. Not globally deduped. | "the shared video" |
|
||||||
|
| subscription | A watched channel on a user's connection that Tapir polls for new videos. | "feed", "follow" |
|
||||||
|
| summary | A video's produced output: summary text + highlights + takeaways + AI provenance. | "transcript" |
|
||||||
|
| highlight | A notable point pulled from a video (`Summary.Highlights`). | "takeaway" |
|
||||||
|
| takeaway | An actionable conclusion from a video (`Summary.Takeaways`). | "highlight" |
|
||||||
|
| sink | A delivery destination for a summary (`store`, `brain`). New one = new adapter. | "the database" |
|
||||||
|
| AI router | Local-first chain: local Primary → local fallback → external worst-case. | "the API" |
|
||||||
|
| BYO-AI fallback | User's own external AI key, used only when local fails; opt-in. Without it, content is never sent externally. | "the default AI" |
|
||||||
|
| brain | Persistent homelab knowledge store; in Tapir, one optional HTTP sink (`brain_ingest`), not the filesystem package. | "the database", "the store" |
|
||||||
|
| LiteLLM gateway | Local AI gateway (`koala:30401/v1`); the Primary in the AI router. | "piguard:4000", "koala:4000", "the cloud" |
|
||||||
|
| connection | A connected video account a user authorizes via OAuth; subscriptions hang off it. | "login", "session" |
|
||||||
|
|
||||||
|
## Caveman rubric
|
||||||
|
Before any HIGH/CRITICAL operation, output one line:
|
||||||
|
`caveman: me <verb> <object>, not <excluded thing>`
|
||||||
|
Valid iff: (1) only vocabulary terms + plain verbs, (2) a stranger could identify
|
||||||
|
the exact operation, (3) names one thing explicitly NOT being done.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
- `caveman: me delete user connection, not the shared transcript`
|
||||||
|
- `caveman: me send transcript to BYO-AI fallback, not the LiteLLM gateway`
|
||||||
@@ -37,7 +37,7 @@ model, behavior specs) remain the source of intent.
|
|||||||
|
|
||||||
## Running the Stage-0 demo
|
## Running the Stage-0 demo
|
||||||
|
|
||||||
Tapir runs on **your own** YouTube account: authorize once, then run the
|
Tapir runs on your YouTube account(s): authorize once, then run the
|
||||||
watch→summarize→deliver loop. All configuration is via `TAPIR_*` environment
|
watch→summarize→deliver loop. All configuration is via `TAPIR_*` environment
|
||||||
variables — copy [`.env.example`](.env.example) to `.env` and fill it in (no
|
variables — copy [`.env.example`](.env.example) to `.env` and fill it in (no
|
||||||
secrets are committed; at demo time source them from op, e.g. `op run -- ...`).
|
secrets are committed; at demo time source them from op, e.g. `op run -- ...`).
|
||||||
@@ -56,7 +56,7 @@ go build -o bin/tapir ./cmd/tapir
|
|||||||
./bin/tapir auth
|
./bin/tapir auth
|
||||||
|
|
||||||
# 3. run: detect new videos across your subscriptions, summarize, deliver to the
|
# 3. run: detect new videos across your subscriptions, summarize, deliver to the
|
||||||
# store. Unset TAPIR_POLL_INTERVAL = single pass; set it (e.g. 15m) to loop.
|
# store. Single pass; set TAPIR_DISCOVERY_INTERVAL (e.g. 2h) for the serve loop.
|
||||||
./bin/tapir run
|
./bin/tapir run
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -68,6 +68,18 @@ URI matches `TAPIR_OAUTH_REDIRECT_ADDR`. The summarizer model
|
|||||||
(`TAPIR_SUMMARIZER_MODEL`, default `koala/phi4-mini`) is overridable; pick the
|
(`TAPIR_SUMMARIZER_MODEL`, default `koala/phi4-mini`) is overridable; pick the
|
||||||
final alias when the gateway is reachable (see `docs/homelab-integration.md`).
|
final alias when the gateway is reachable (see `docs/homelab-integration.md`).
|
||||||
|
|
||||||
|
### Web surface (`tapir serve`)
|
||||||
|
|
||||||
|
`tapir serve` starts the HTMX+Templ web UI on `:8080`. Users log in via Dex OIDC (local
|
||||||
|
password or Google); a new Dex subject is routed to `/register` to create a Tapir account.
|
||||||
|
Stage 1 is multi-user: each user connects their own YouTube account from the browser and
|
||||||
|
manages their own summaries under DB-enforced RLS isolation. When `TAPIR_DISCOVERY_INTERVAL`
|
||||||
|
is set (e.g. `2h`), the serve process runs a scheduled discovery pass for every registered
|
||||||
|
user automatically — no CronJob required. In auto mode only videos published within
|
||||||
|
`TAPIR_AUTO_SUMMARIZE_WINDOW` (default ~7d, ADR-020) are summarised automatically; older videos
|
||||||
|
are listed and summarised on demand, so a large back-catalogue doesn't keep re-driving the
|
||||||
|
caption rate gate. See `docs/homelab-integration.md` for the full config reference.
|
||||||
|
|
||||||
### Headless on koala
|
### Headless on koala
|
||||||
|
|
||||||
koala has no browser and no interactive `op` session, so the two interactive
|
koala has no browser and no interactive `op` session, so the two interactive
|
||||||
|
|||||||
+1
-1
@@ -34,7 +34,7 @@ tasks:
|
|||||||
lint:
|
lint:
|
||||||
desc: golangci-lint (installed by CI; skipped locally if absent).
|
desc: golangci-lint (installed by CI; skipped locally if absent).
|
||||||
cmds:
|
cmds:
|
||||||
- 'command -v golangci-lint >/dev/null 2>&1 && golangci-lint run ./... || echo "golangci-lint not installed — skipping (CI installs it)"'
|
- 'if command -v golangci-lint >/dev/null 2>&1; then golangci-lint run ./...; else echo "golangci-lint not installed — skipping (CI installs it)"; fi'
|
||||||
|
|
||||||
test:
|
test:
|
||||||
desc: Run all tests. -p 1 serializes packages so the embedded-postgres tests
|
desc: Run all tests. -p 1 serializes packages so the embedded-postgres tests
|
||||||
|
|||||||
@@ -44,50 +44,74 @@ fallback — their key, their choice.
|
|||||||
|
|
||||||
## Who it is for
|
## Who it is for
|
||||||
|
|
||||||
- **Now (the first customer):** the maintainer — one person, their own subscriptions,
|
- **Now (the first customers):** the maintainer and a small number of known, trusted
|
||||||
summaries delivered to their own store and brain.
|
friends — each with their own account, isolated data, optional BYO-AI. The maintainer is
|
||||||
- **Soon (Future B):** a small number of known, trusted users (friends / beta) — each with
|
the first customer; friendly users provide the earliest real-world signal.
|
||||||
their own account, isolated data, optional BYO-AI.
|
|
||||||
- **Maybe (Future C, explicitly not built yet):** a public multi-tenant service. Deferred
|
- **Maybe (Future C, explicitly not built yet):** a public multi-tenant service. Deferred
|
||||||
until there is evidence of sustained personal use **and** real demand. Building for C
|
until there is evidence of sustained use **and** real demand. Building for C before that
|
||||||
before that evidence is a known anti-goal.
|
evidence is a known anti-goal.
|
||||||
|
|
||||||
## Definition of Success
|
## Definition of Success
|
||||||
|
|
||||||
Success is staged. Each stage has a single, falsifiable headline test. We do not advance
|
Success is staged. Each stage has a single, falsifiable headline test. We do not advance
|
||||||
to the next stage's ambition until the current stage's test passes.
|
to the next stage's ambition until the current stage's test passes.
|
||||||
|
|
||||||
### Stage 0 — Useful to me (the gate)
|
### Stage 0 — Useful to me or a friend (the gate)
|
||||||
|
|
||||||
> **Headline test:** For four consecutive weeks, the maintainer reads Tapir-produced
|
> **Headline test:** Over a 3–4 week window, *either* the maintainer *or* at least one
|
||||||
> summaries for their own subscriptions at least weekly, and at least once acts on a
|
> onboarded friend returns to Tapir and reads/acts on summaries in **≥2 separate weeks**.
|
||||||
> summary (watches / skips / saves a video *because of* the summary).
|
> The test is *return usage* (behavioural), not stated approval. The ideal signal is an
|
||||||
|
> **unprompted** return (organic, not because the maintainer nudged them) — but see the
|
||||||
|
> measurement note below: we currently cannot distinguish prompted from organic returns, so
|
||||||
|
> in practice we count all returns and read the result with that caveat.
|
||||||
|
|
||||||
- Captions-first summarization works end-to-end for the maintainer's real subscriptions.
|
- Captions-first summarization works end-to-end for real subscriptions (the maintainer's
|
||||||
- Summaries land in the maintainer's store and (optionally) brain.
|
and onboarded friends').
|
||||||
|
- Summaries land in each user's own store and (optionally) brain.
|
||||||
- Local-first AI produces summaries of acceptable quality without manual intervention
|
- Local-first AI produces summaries of acceptable quality without manual intervention
|
||||||
most of the time.
|
most of the time.
|
||||||
- **This is the gate.** Multi-user, BYO-AI-for-others, and any SaaS ambition stay deferred
|
- **Why behavioural, not feedback.** Friend *feedback* is gathered and genuinely valuable —
|
||||||
until Stage 0 holds. (Ties to the 2026-07-01 self-use check-in.)
|
but it is **not** the gate. Asked-for feedback from friendly users is the least reliable
|
||||||
|
signal in product development (politeness bias); whether they *come back* is the thing we
|
||||||
|
actually care about. So the gate measures returns, not nice words.
|
||||||
|
- **Measurement note — "unprompted" is an ideal we can't yet measure.** Whether a return was
|
||||||
|
organic or prompted by a nudge is not captured by any data Tapir holds (it's context only
|
||||||
|
the maintainer has). Rather than waive the standard, we name the gap: *unprompted* return
|
||||||
|
is the signal we genuinely want; *returns* (prompted or not) is what the data can show. A
|
||||||
|
return that needed a nudge is a weaker signal than one that didn't, and the result is read
|
||||||
|
with that in mind. If distinguishing them ever matters enough, the maintainer tracks nudges
|
||||||
|
manually or a future build records prompt events — neither is in scope now.
|
||||||
|
- **Why "me OR a friend".** This replaces the original "useful to *me*, specifically" gate
|
||||||
|
(2026-06-03 decision, recorded in DECISIONS.md ADR-016). Getting signal from friendly
|
||||||
|
users is valuable enough to count — but the bar stays behavioural so it can't be cleared
|
||||||
|
by a polite reaction. (Ties to the 2026-07-01 check-in.)
|
||||||
|
- **How it's measured.** Return usage is read from two sources: `summary_actions` (timestamped
|
||||||
|
watch/skip/save per user) answers "acted in ≥2 distinct weeks"; an append-only login-events
|
||||||
|
table (see infra/Tapir build) answers "returned/read in ≥2 distinct weeks" even without an
|
||||||
|
action click — the honest signal for a *reading* product. Login events accrue only from their
|
||||||
|
deploy date onward, so the gate window's data begins then.
|
||||||
|
- **Gate-clock reset (ADR-018).** The 3–4 week window starts when in-process scheduled discovery
|
||||||
|
+ auto-summarize ship — before that, unprompted use was impossible, so the prior window
|
||||||
|
measured nothing (this is starting the clock when the experiment can actually run, not a reset
|
||||||
|
to dodge a failing gate). The 2026-07-01 check-in referenced above moves accordingly to ~3–4
|
||||||
|
weeks after this deploys. See DECISIONS.md ADR-018.
|
||||||
|
- **This is the gate.** Hardening (Stage 1) and any SaaS ambition stay deferred until this
|
||||||
|
behavioural signal exists. Note: multi-user machinery was deliberately built *ahead* of
|
||||||
|
this gate (ADR-012) with isolation enforced — that was an explicit, recorded call, not a
|
||||||
|
sign the gate had passed. The gate is about *evidence of use*, which is still open.
|
||||||
|
|
||||||
### Stage 1 — Useful to a few (Future B)
|
### Stage 1 — Trustworthy at rest (hardening, Future B)
|
||||||
|
|
||||||
> **Headline test:** At least one trusted user other than the maintainer connects their
|
|
||||||
> own account and, within their first month, keeps using it (returns to read summaries in
|
|
||||||
> ≥2 separate weeks) without the maintainer hand-holding each summary.
|
|
||||||
|
|
||||||
- Multiple users, each with isolated accounts, credentials, and summaries.
|
|
||||||
- A new user can self-connect a YouTube/Vimeo account and get summaries with no code change.
|
|
||||||
- Optional BYO-AI works per-user.
|
|
||||||
- No cross-user data leakage — demonstrable, not assumed.
|
|
||||||
|
|
||||||
### Stage 2 — Trustworthy at rest (hardening, still Future B)
|
|
||||||
|
|
||||||
> **Headline test:** Credentials (OAuth tokens, BYO-AI keys) are encrypted at rest via the
|
> **Headline test:** Credentials (OAuth tokens, BYO-AI keys) are encrypted at rest via the
|
||||||
> homelab's existing secrets convention; a documented, rehearsed recovery path exists; and
|
> homelab's existing secrets convention; a documented, rehearsed recovery path exists; and
|
||||||
> a deliberate isolation test (user A cannot read user B's data) passes in CI or a
|
> a deliberate isolation test (user A cannot read user B's data) passes in CI or a
|
||||||
> documented manual drill.
|
> documented manual drill.
|
||||||
|
|
||||||
|
- Per-user data isolation is enforced and tested (delivered early via ADR-012 RLS).
|
||||||
|
- Per-user credentials are encrypted at rest (ADR-015 envelope encryption; build in infra#89).
|
||||||
|
- A new user can self-connect a YouTube/Vimeo account and get summaries with no code change.
|
||||||
|
- Optional BYO-AI works per-user.
|
||||||
|
|
||||||
### Non-goals (current)
|
### Non-goals (current)
|
||||||
|
|
||||||
- Public sign-up / billing / a marketing surface.
|
- Public sign-up / billing / a marketing surface.
|
||||||
@@ -98,7 +122,11 @@ to the next stage's ambition until the current stage's test passes.
|
|||||||
|
|
||||||
## How we will know we are drifting
|
## How we will know we are drifting
|
||||||
|
|
||||||
- We are building Stage 1+ machinery before the Stage 0 gate has passed.
|
- We declare the Stage 0 gate "passed" on the strength of polite feedback rather than
|
||||||
|
behavioural return-usage (the politeness-bias trap the gate is designed to resist).
|
||||||
|
- We build Stage 1 hardening or Future C machinery while the Stage 0 use-evidence is still
|
||||||
|
absent. (Multi-user machinery already shipped ahead of the gate via ADR-012 — a recorded,
|
||||||
|
deliberate exception, not a precedent for more.)
|
||||||
- A user's content reaches a third-party model without that user's explicit, per-user opt-in.
|
- A user's content reaches a third-party model without that user's explicit, per-user opt-in.
|
||||||
- "Brain ingestion" starts dictating the architecture instead of being one sink behind an
|
- "Brain ingestion" starts dictating the architecture instead of being one sink behind an
|
||||||
interface.
|
interface.
|
||||||
|
|||||||
@@ -0,0 +1,64 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestChatModelsReuseTheChainLocalFirst: the switcher offers the ADR-022 chain in
|
||||||
|
// order, deduped — primary, local fallback, cloud.
|
||||||
|
func TestChatModelsReuseTheChainLocalFirst(t *testing.T) {
|
||||||
|
got := chatModels(config.Config{
|
||||||
|
SummarizerModel: "koala/phi4-mini",
|
||||||
|
FallbackModel: "iguana/gemma4-26b",
|
||||||
|
CloudFallbackModel: "berget/mistral-small",
|
||||||
|
})
|
||||||
|
want := []string{"koala/phi4-mini", "iguana/gemma4-26b", "berget/mistral-small"}
|
||||||
|
if strings.Join(got, ",") != strings.Join(want, ",") {
|
||||||
|
t.Fatalf("chatModels = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestChatCloudModelAbsentWhenDisabled: the local-first / NDA lever — with the
|
||||||
|
// cloud fallback empty (TAPIR_CLOUD_FALLBACK_MODEL=""), no external model is
|
||||||
|
// offered in the switcher, so chat content never leaves the local stack (ADR-027,
|
||||||
|
// honouring ADR-022's "content stays local" guarantee).
|
||||||
|
func TestChatCloudModelAbsentWhenDisabled(t *testing.T) {
|
||||||
|
got := chatModels(config.Config{
|
||||||
|
SummarizerModel: "koala/phi4-mini",
|
||||||
|
FallbackModel: "iguana/gemma4-26b",
|
||||||
|
CloudFallbackModel: "",
|
||||||
|
})
|
||||||
|
for _, m := range got {
|
||||||
|
if strings.HasPrefix(m, "berget/") || strings.Contains(m, "mistral") {
|
||||||
|
t.Fatalf("cloud model %q offered though the cloud fallback is disabled", m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
want := []string{"koala/phi4-mini", "iguana/gemma4-26b"}
|
||||||
|
if strings.Join(got, ",") != strings.Join(want, ",") {
|
||||||
|
t.Fatalf("chatModels = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestChatModelsDedup: a config that reuses one alias across slots collapses to a
|
||||||
|
// single switcher entry (no duplicate options).
|
||||||
|
func TestChatModelsDedup(t *testing.T) {
|
||||||
|
got := chatModels(config.Config{
|
||||||
|
SummarizerModel: "koala/phi4-mini",
|
||||||
|
FallbackModel: "koala/phi4-mini",
|
||||||
|
CloudFallbackModel: "",
|
||||||
|
})
|
||||||
|
if len(got) != 1 || got[0] != "koala/phi4-mini" {
|
||||||
|
t.Fatalf("chatModels = %v, want a single deduped entry", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestBuildChatNilWithoutGateway: no gateway → no chat backend (routes unmounted,
|
||||||
|
// read path unaffected).
|
||||||
|
func TestBuildChatNilWithoutGateway(t *testing.T) {
|
||||||
|
if c := buildChat(config.Config{GatewayURL: ""}); c != nil {
|
||||||
|
t.Fatal("buildChat must return nil without a gateway URL")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log/slog"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/runner"
|
||||||
|
)
|
||||||
|
|
||||||
|
// discoveryRunner runs one user's discovery pass.
|
||||||
|
type discoveryRunner func(ctx context.Context, userID string) (runner.Stats, error)
|
||||||
|
|
||||||
|
// serialize wraps run so calls never overlap: every discovery pass — scheduled
|
||||||
|
// or connect-triggered (#6) — acquires the same lock, preserving the
|
||||||
|
// one-fetcher-at-a-time invariant the scheduler relies on (ADR-018, the
|
||||||
|
// single-replica assumption). Locking is per-user, so a connect-triggered pass
|
||||||
|
// interleaves between the scheduler's users instead of waiting for a whole pass.
|
||||||
|
func serialize(mu *sync.Mutex, run discoveryRunner) discoveryRunner {
|
||||||
|
return func(ctx context.Context, userID string) (runner.Stats, error) {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
return run(ctx, userID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// discoveryTrigger fires an out-of-band discovery pass for one user without
|
||||||
|
// blocking the caller (the connect HTTP handler). The pass runs on the server's
|
||||||
|
// long-lived ctx — not the request ctx — so it survives the post-connect
|
||||||
|
// redirect. run is the serialized runner, so a trigger never overlaps the
|
||||||
|
// scheduler. Satisfies web.DiscoveryTrigger.
|
||||||
|
type discoveryTrigger struct {
|
||||||
|
ctx context.Context
|
||||||
|
run discoveryRunner
|
||||||
|
// onboard, when set, runs after the discovery pass to summarize a capped number
|
||||||
|
// of the user's newest videos (Feature 1). Optional.
|
||||||
|
onboard func(ctx context.Context, userID string)
|
||||||
|
log *slog.Logger
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *discoveryTrigger) Enqueue(userID string) {
|
||||||
|
go func() {
|
||||||
|
if _, err := t.run(t.ctx, userID); err != nil {
|
||||||
|
t.log.Warn("discovery: connect-triggered pass had errors", "user", userID, "err", err)
|
||||||
|
}
|
||||||
|
if t.onboard != nil {
|
||||||
|
t.onboard(t.ctx, userID)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/runner"
|
||||||
|
)
|
||||||
|
|
||||||
|
// serialize must guarantee at most one discovery pass runs at a time, so a
|
||||||
|
// connect-triggered pass never fetches concurrently with the scheduler.
|
||||||
|
func TestSerializeRunsOneAtATime(t *testing.T) {
|
||||||
|
var active, maxActive int32
|
||||||
|
run := func(_ context.Context, _ string) (runner.Stats, error) {
|
||||||
|
n := atomic.AddInt32(&active, 1)
|
||||||
|
for { // record the high-water mark of concurrent runs
|
||||||
|
m := atomic.LoadInt32(&maxActive)
|
||||||
|
if n <= m || atomic.CompareAndSwapInt32(&maxActive, m, n) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
time.Sleep(2 * time.Millisecond)
|
||||||
|
atomic.AddInt32(&active, -1)
|
||||||
|
return runner.Stats{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
s := serialize(&sync.Mutex{}, run)
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for i := 0; i < 20; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func(i int) { defer wg.Done(); _, _ = s(context.Background(), fmt.Sprintf("u%d", i)) }(i)
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
require.Equal(t, int32(1), atomic.LoadInt32(&maxActive),
|
||||||
|
"serialize must run at most one pass at a time")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Enqueue runs the user's pass out-of-band (non-blocking) on the trigger's ctx.
|
||||||
|
func TestDiscoveryTriggerEnqueueRunsUser(t *testing.T) {
|
||||||
|
done := make(chan string, 1)
|
||||||
|
run := func(_ context.Context, userID string) (runner.Stats, error) {
|
||||||
|
done <- userID
|
||||||
|
return runner.Stats{}, nil
|
||||||
|
}
|
||||||
|
tr := &discoveryTrigger{ctx: context.Background(), run: run, log: quietLog()}
|
||||||
|
|
||||||
|
tr.Enqueue("u1")
|
||||||
|
|
||||||
|
select {
|
||||||
|
case got := <-done:
|
||||||
|
require.Equal(t, "u1", got)
|
||||||
|
case <-time.After(2 * time.Second):
|
||||||
|
t.Fatal("Enqueue did not run the user's pass")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,209 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"mime/multipart"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
embeddedpostgres "github.com/fergusstrange/embedded-postgres"
|
||||||
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/adapters/store"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/config"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/web"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestE2E_UploadTranscribeAndChat is the end-to-end check for tapir#33: upload a
|
||||||
|
// real audio file, watch it become a transcript through the REAL ffprobe, ffmpeg
|
||||||
|
// and gateway, then ask a question and get a grounded answer from a REAL model.
|
||||||
|
// It is opt-in — it needs real binaries and the live gateway — and skipped unless
|
||||||
|
// TAPIR_E2E_AUDIO is set:
|
||||||
|
//
|
||||||
|
// TAPIR_E2E_AUDIO=/path/meeting.m4a TAPIR_E2E_FFMPEG=/path/ffmpeg \
|
||||||
|
// TAPIR_E2E_FFPROBE=/path/ffprobe TAPIR_E2E_QUESTION='Vem ansvarar för virket?' \
|
||||||
|
// TAPIR_E2E_EXPECT=anders op run --env-file ~/.op-env -- \
|
||||||
|
// go test ./cmd/tapir -run E2E_Upload -v -count=1
|
||||||
|
//
|
||||||
|
// Use SYNTHETIC audio only: this repo is public, and a fallback to berget sends
|
||||||
|
// the audio to a cloud endpoint (#33).
|
||||||
|
func TestE2E_UploadTranscribeAndChat(t *testing.T) {
|
||||||
|
audioPath := os.Getenv("TAPIR_E2E_AUDIO")
|
||||||
|
if audioPath == "" {
|
||||||
|
t.Skip("set TAPIR_E2E_AUDIO (and friends) to run the live end-to-end check")
|
||||||
|
}
|
||||||
|
key := os.Getenv("LITELLM_MASTER_KEY")
|
||||||
|
require.NotEmpty(t, key, "LITELLM_MASTER_KEY must be set (via op run)")
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
t.Cleanup(cancel)
|
||||||
|
|
||||||
|
// A private embedded Postgres: nothing here may touch a real database.
|
||||||
|
port := uint32(55000 + os.Getpid()%1000)
|
||||||
|
rt := filepath.Join(os.TempDir(), fmt.Sprintf("tapir-epg-e2e-%d", os.Getpid()))
|
||||||
|
pg := embeddedpostgres.NewDatabase(embeddedpostgres.DefaultConfig().Port(port).
|
||||||
|
RuntimePath(rt).DataPath(filepath.Join(rt, "data")).BinariesPath(filepath.Join(rt, "bin")).
|
||||||
|
CachePath(filepath.Join(os.TempDir(), "tapir-epg-cache")))
|
||||||
|
require.NoError(t, pg.Start())
|
||||||
|
t.Cleanup(func() { _ = pg.Stop(); _ = os.RemoveAll(rt) })
|
||||||
|
dsn := fmt.Sprintf("postgres://postgres:postgres@localhost:%d/postgres?sslmode=disable", port)
|
||||||
|
|
||||||
|
st, err := store.New(ctx, dsn)
|
||||||
|
require.NoError(t, err)
|
||||||
|
t.Cleanup(st.Close)
|
||||||
|
raw, err := pgxpool.New(ctx, dsn)
|
||||||
|
require.NoError(t, err)
|
||||||
|
t.Cleanup(raw.Close)
|
||||||
|
const uid = "11111111-1111-1111-1111-111111111111"
|
||||||
|
_, err = raw.Exec(ctx, `INSERT INTO users (id) VALUES ($1)`, uid)
|
||||||
|
require.NoError(t, err)
|
||||||
|
_, err = raw.Exec(ctx, `INSERT INTO user_identities (dex_subject, user_id) VALUES ('e2e', $1)`, uid)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
cfg, err := config.Load()
|
||||||
|
require.NoError(t, err)
|
||||||
|
cfg.GatewayKey = key
|
||||||
|
cfg.UploadsEnabled = true
|
||||||
|
cfg.UploadTmpDir = filepath.Join(t.TempDir(), "spool")
|
||||||
|
if os.Getenv("TAPIR_E2E_RECONCILE") == "1" {
|
||||||
|
// Two engines and a local model, as ADR-034 describes.
|
||||||
|
cfg.ReconcileBaseModel = envOrDefault("TAPIR_E2E_BASE_MODEL", "iguana/pianissimo")
|
||||||
|
cfg.ReconcileModel = envOrDefault("TAPIR_E2E_RECONCILE_MODEL", "iguana/gemma4-26b")
|
||||||
|
}
|
||||||
|
cfg.FFmpegBin = envOrDefault("TAPIR_E2E_FFMPEG", "ffmpeg")
|
||||||
|
cfg.FFprobeBin = envOrDefault("TAPIR_E2E_FFPROBE", "ffprobe")
|
||||||
|
|
||||||
|
intake, worker, err := buildUploads(ctx, cfg, st, quietLog())
|
||||||
|
require.NoError(t, err)
|
||||||
|
go worker.Run(ctx)
|
||||||
|
app := &web.App{
|
||||||
|
Store: st, Identity: st, Auth: web.StubAuth{U: web.User{Subject: "e2e"}},
|
||||||
|
Chat: buildChat(cfg),
|
||||||
|
}
|
||||||
|
wireUploads(app, cfg, intake) // the same wiring cmdServe uses
|
||||||
|
srv := httptest.NewServer(app.Router())
|
||||||
|
t.Cleanup(srv.Close)
|
||||||
|
noRedirect := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
|
||||||
|
|
||||||
|
// 1. Upload.
|
||||||
|
f, err := os.Open(audioPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer func() { _ = f.Close() }()
|
||||||
|
pr, pw := io.Pipe()
|
||||||
|
mw := multipart.NewWriter(pw)
|
||||||
|
go func() {
|
||||||
|
_ = mw.WriteField("title", "E2E recording")
|
||||||
|
fw, _ := mw.CreateFormFile("file", filepath.Base(audioPath))
|
||||||
|
_, _ = io.Copy(fw, f)
|
||||||
|
_ = mw.Close()
|
||||||
|
_ = pw.Close()
|
||||||
|
}()
|
||||||
|
req, err := http.NewRequest(http.MethodPost, srv.URL+"/uploads", pr)
|
||||||
|
require.NoError(t, err)
|
||||||
|
req.Header.Set("Content-Type", mw.FormDataContentType())
|
||||||
|
resp, err := noRedirect.Do(req)
|
||||||
|
require.NoError(t, err)
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
require.Equal(t, http.StatusSeeOther, resp.StatusCode, "an accepted upload redirects to its page")
|
||||||
|
loc := resp.Header.Get("Location")
|
||||||
|
require.True(t, strings.HasPrefix(loc, "/uploads/"), loc)
|
||||||
|
t.Logf("accepted -> %s", loc)
|
||||||
|
|
||||||
|
// 2. Wait for the transcript: unfinished states poll, terminal states do not.
|
||||||
|
var page string
|
||||||
|
require.Eventually(t, func() bool {
|
||||||
|
r, err := http.Get(srv.URL + loc + "/status")
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
b, _ := io.ReadAll(r.Body)
|
||||||
|
_ = r.Body.Close()
|
||||||
|
page = string(b)
|
||||||
|
return !strings.Contains(page, "hx-trigger")
|
||||||
|
}, 12*time.Minute, 3*time.Second, "the upload never reached a terminal state")
|
||||||
|
require.Contains(t, page, "Transcribed", "terminal state was not 'done': %.300s", page)
|
||||||
|
|
||||||
|
var status, endpoint, content string
|
||||||
|
var secs float64
|
||||||
|
var hadVideo bool
|
||||||
|
require.NoError(t, raw.QueryRow(ctx,
|
||||||
|
`SELECT status, endpoint, content, audio_seconds, had_video FROM uploaded_transcripts WHERE user_id = $1`, uid).
|
||||||
|
Scan(&status, &endpoint, &content, &secs, &hadVideo))
|
||||||
|
t.Logf("done: endpoint=%s audio_s=%.1f transcript_chars=%d had_video=%v", endpoint, secs, len(content), hadVideo)
|
||||||
|
if os.Getenv("TAPIR_E2E_RECONCILE") == "1" {
|
||||||
|
var recJSON string
|
||||||
|
require.NoError(t, raw.QueryRow(ctx,
|
||||||
|
`SELECT COALESCE(reconciliation::text, 'NULL') FROM uploaded_transcripts WHERE user_id = $1`, uid).Scan(&recJSON))
|
||||||
|
require.NotEqual(t, "NULL", recJSON, "a two-engine run must record its reconciliation")
|
||||||
|
t.Logf("reconciliation: %.600s", recJSON)
|
||||||
|
assert.Contains(t, endpoint, "reconciled by", "every contributor is named in the endpoint")
|
||||||
|
assert.Contains(t, page, "Reconciled from two transcripts")
|
||||||
|
}
|
||||||
|
if os.Getenv("TAPIR_E2E_HAD_VIDEO") == "1" {
|
||||||
|
// A Teams meeting with the camera off: a video file whose audio is what matters.
|
||||||
|
assert.True(t, hadVideo, "the file had a video track and the row must say so")
|
||||||
|
assert.Contains(t, page, "Only its audio was used", "and the user must be told")
|
||||||
|
}
|
||||||
|
assert.Equal(t, "done", status)
|
||||||
|
assert.NotEmpty(t, endpoint)
|
||||||
|
assert.NotEmpty(t, content)
|
||||||
|
|
||||||
|
// ADR-035: a finished job's phases are all closed, and it was reporting while it ran.
|
||||||
|
var progJSON string
|
||||||
|
var progressAt, aliveAt *time.Time
|
||||||
|
require.NoError(t, raw.QueryRow(ctx,
|
||||||
|
`SELECT COALESCE(progress::text, '[]'), progress_at, alive_at FROM uploaded_transcripts WHERE user_id = $1`, uid).
|
||||||
|
Scan(&progJSON, &progressAt, &aliveAt))
|
||||||
|
var steps []domain.ProgressStep
|
||||||
|
require.NoError(t, json.Unmarshal([]byte(progJSON), &steps))
|
||||||
|
require.NotEmpty(t, steps, "the job recorded no phases")
|
||||||
|
for _, st := range steps {
|
||||||
|
assert.Contains(t, []domain.StepState{domain.StepDone, domain.StepSkipped}, st.State,
|
||||||
|
"phase %q was left %q after the job finished", st.Key, st.State)
|
||||||
|
}
|
||||||
|
assert.Equal(t, "save", steps[len(steps)-1].Key)
|
||||||
|
assert.NotNil(t, progressAt, "progress_at must have been stamped")
|
||||||
|
t.Logf("phases: %d, heartbeat stamped: %v", len(steps), aliveAt != nil)
|
||||||
|
entries, err := os.ReadDir(cfg.UploadTmpDir)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Empty(t, entries, "the audio must be deleted once transcribed")
|
||||||
|
|
||||||
|
// 3. Ask a question; the answer must come from the transcript.
|
||||||
|
q := envOrDefault("TAPIR_E2E_QUESTION", "Who is responsible for the timber order?")
|
||||||
|
chatResp, err := http.PostForm(srv.URL+loc+"/chat", url.Values{"question": {q}})
|
||||||
|
require.NoError(t, err)
|
||||||
|
answerPage, _ := io.ReadAll(chatResp.Body)
|
||||||
|
_ = chatResp.Body.Close()
|
||||||
|
require.Equal(t, http.StatusOK, chatResp.StatusCode)
|
||||||
|
t.Logf("question: %s", q)
|
||||||
|
if want := os.Getenv("TAPIR_E2E_EXPECT"); want != "" {
|
||||||
|
assert.Contains(t, strings.ToLower(string(answerPage)), strings.ToLower(want),
|
||||||
|
"the answer should be grounded in the transcript")
|
||||||
|
}
|
||||||
|
t.Logf("answer page excerpt: %.600s", afterMarker(string(answerPage), `class="chat-turn chat-a"`))
|
||||||
|
}
|
||||||
|
|
||||||
|
func envOrDefault(k, d string) string {
|
||||||
|
if v := os.Getenv(k); v != "" {
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
|
||||||
|
func afterMarker(s, m string) string {
|
||||||
|
if i := strings.Index(s, m); i >= 0 {
|
||||||
|
return s[i:]
|
||||||
|
}
|
||||||
|
return "(no answer turn found)"
|
||||||
|
}
|
||||||
+1
-1
@@ -5,7 +5,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
"gitea.d-ma.be/mathias/tapir/internal/adapters/store"
|
"git.d-ma.be/mathias/tapir/internal/adapters/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Env var names for the read-only CLI. DSN and user id are never hardcoded — the
|
// Env var names for the read-only CLI. DSN and user id are never hardcoded — the
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import (
|
|||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
"gitea.d-ma.be/mathias/tapir/internal/adapters/store"
|
"git.d-ma.be/mathias/tapir/internal/adapters/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestFormatListColumnsAndOrdering(t *testing.T) {
|
func TestFormatListColumnsAndOrdering(t *testing.T) {
|
||||||
|
|||||||
+1
-1
@@ -9,7 +9,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"text/tabwriter"
|
"text/tabwriter"
|
||||||
|
|
||||||
"gitea.d-ma.be/mathias/tapir/internal/adapters/store"
|
"git.d-ma.be/mathias/tapir/internal/adapters/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
// runList prints the user's stored summaries as a table, most recent first.
|
// runList prints the user's stored summaries as a table, most recent first.
|
||||||
|
|||||||
+162
-21
@@ -17,18 +17,21 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.d-ma.be/mathias/tapir/internal/adapters/secrets"
|
"git.d-ma.be/mathias/tapir/internal/adapters/secrets"
|
||||||
"gitea.d-ma.be/mathias/tapir/internal/adapters/store"
|
"git.d-ma.be/mathias/tapir/internal/adapters/store"
|
||||||
"gitea.d-ma.be/mathias/tapir/internal/auth"
|
"git.d-ma.be/mathias/tapir/internal/adapters/youtube"
|
||||||
"gitea.d-ma.be/mathias/tapir/internal/config"
|
"git.d-ma.be/mathias/tapir/internal/auth"
|
||||||
"gitea.d-ma.be/mathias/tapir/internal/runner"
|
"git.d-ma.be/mathias/tapir/internal/config"
|
||||||
"gitea.d-ma.be/mathias/tapir/internal/web"
|
"git.d-ma.be/mathias/tapir/internal/metrics"
|
||||||
"gitea.d-ma.be/mathias/tapir/internal/web/oidc"
|
"git.d-ma.be/mathias/tapir/internal/runner"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/web"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/web/oidc"
|
||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
@@ -38,7 +41,7 @@ func main() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
log := slog.New(slog.NewTextHandler(os.Stderr, nil))
|
log := slog.New(slog.NewTextHandler(os.Stderr, nil))
|
||||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
|
ctx, stop := shutdownContext(context.Background())
|
||||||
defer stop()
|
defer stop()
|
||||||
|
|
||||||
var err error
|
var err error
|
||||||
@@ -53,6 +56,8 @@ func main() {
|
|||||||
err = cmdRun(ctx, log)
|
err = cmdRun(ctx, log)
|
||||||
case "serve":
|
case "serve":
|
||||||
err = cmdServe(ctx, log)
|
err = cmdServe(ctx, log)
|
||||||
|
case "report":
|
||||||
|
err = runReport(ctx, os.Args[2:])
|
||||||
default:
|
default:
|
||||||
usage()
|
usage()
|
||||||
os.Exit(2)
|
os.Exit(2)
|
||||||
@@ -73,6 +78,7 @@ usage:
|
|||||||
tapir serve run the web UI (read summaries, record watch/skip/save)
|
tapir serve run the web UI (read summaries, record watch/skip/save)
|
||||||
tapir list [-limit N] list stored summaries, recent first
|
tapir list [-limit N] list stored summaries, recent first
|
||||||
tapir show <video-id> show one summary in full
|
tapir show <video-id> show one summary in full
|
||||||
|
tapir report Stage-0 usage gate: per-user distinct active weeks
|
||||||
|
|
||||||
configuration is via TAPIR_* environment variables (see .env.example).
|
configuration is via TAPIR_* environment variables (see .env.example).
|
||||||
`)
|
`)
|
||||||
@@ -125,10 +131,18 @@ func cmdRun(ctx context.Context, log *slog.Logger) error {
|
|||||||
if engine == nil {
|
if engine == nil {
|
||||||
return fmt.Errorf("run: incomplete summarization config (gateway, youtube credentials, secrets file)")
|
return fmt.Errorf("run: incomplete summarization config (gateway, youtube credentials, secrets file)")
|
||||||
}
|
}
|
||||||
r := runner.New(engine.Source, st, engine, cfg.UserID, log)
|
// Process-wide caption-fetch rate gate (ADR-014 item 2): the batch path shares
|
||||||
|
// the same per-egress-IP limiter as the web click-path.
|
||||||
|
youtube.SetFetchRate(cfg.FetchRate)
|
||||||
|
|
||||||
|
r := runner.New(engine.Source, st, engine, cfg.UserID, log,
|
||||||
|
runner.WithBackoff(cfg.FetchBackoff),
|
||||||
|
runner.WithAutoWindow(cfg.AutoSummarizeWindow),
|
||||||
|
runner.WithCaptionMemory(cfg.ChannelCaptionlessThreshold, cfg.ChannelCaptionlessWindow))
|
||||||
|
|
||||||
log.Info("starting run", "user", cfg.UserID, "model", cfg.SummarizerModel,
|
log.Info("starting run", "user", cfg.UserID, "model", cfg.SummarizerModel,
|
||||||
"gateway", cfg.GatewayURL, "poll_interval", cfg.PollInterval)
|
"gateway", cfg.GatewayURL, "poll_interval", cfg.PollInterval, "fetch_backoff", cfg.FetchBackoff,
|
||||||
|
"fetch_rate", cfg.FetchRate, "auto_window", cfg.AutoSummarizeWindow)
|
||||||
return r.Loop(ctx, cfg.PollInterval)
|
return r.Loop(ctx, cfg.PollInterval)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -152,6 +166,11 @@ func cmdServe(ctx context.Context, log *slog.Logger) error {
|
|||||||
}
|
}
|
||||||
defer st.Close()
|
defer st.Close()
|
||||||
|
|
||||||
|
// Process-wide caption-fetch rate gate (ADR-014 item 2): the web click-path
|
||||||
|
// and the scheduled-discovery runners share one per-egress-IP limiter so they
|
||||||
|
// cannot collectively trip 429s. Must be set before either path fetches.
|
||||||
|
youtube.SetFetchRate(cfg.FetchRate)
|
||||||
|
|
||||||
// Auth seam (handlers depend on web.Auth only). With Dex configured
|
// Auth seam (handlers depend on web.Auth only). With Dex configured
|
||||||
// (TAPIR_OIDC_ISSUER set) serve uses real OIDC login — any Dex subject may
|
// (TAPIR_OIDC_ISSUER set) serve uses real OIDC login — any Dex subject may
|
||||||
// authenticate, then registers a tapir user (ADR-012); otherwise it falls
|
// authenticate, then registers a tapir user (ADR-012); otherwise it falls
|
||||||
@@ -177,7 +196,42 @@ func cmdServe(ctx context.Context, log *slog.Logger) error {
|
|||||||
// The file-backed SecretStore is shared by the connect flow (writes tokens)
|
// The file-backed SecretStore is shared by the connect flow (writes tokens)
|
||||||
// and account management (deletes them on disconnect / delete-account).
|
// and account management (deletes them on disconnect / delete-account).
|
||||||
secretStore := secrets.NewFileStore(cfg.SecretsFile)
|
secretStore := secrets.NewFileStore(cfg.SecretsFile)
|
||||||
app := &web.App{Store: st, Identity: st, Auth: authn, Secrets: secretStore, Log: log}
|
app := &web.App{Store: st, Identity: st, Auth: authn, Secrets: secretStore, Log: log, RecencyWindow: cfg.AutoSummarizeWindow}
|
||||||
|
|
||||||
|
// Per-video deeper-dive chat over the STORED transcript (ADR-027). Enabled
|
||||||
|
// whenever a gateway is configured — it needs no YouTube credentials because it
|
||||||
|
// never fetches. Guarded so a typed-nil never lands in the interface field
|
||||||
|
// (which would mount the routes over a nil backend).
|
||||||
|
if c := buildChat(cfg); c != nil {
|
||||||
|
app.Chat = c
|
||||||
|
log.Info("web chat enabled (stored-transcript only)", "models", chatModels(cfg))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Audio upload -> private transcript (ADR-033). Off unless TAPIR_UPLOADS_ENABLED:
|
||||||
|
// the image must carry ffmpeg first. The worker is IN-PROCESS and one job at a
|
||||||
|
// time, so it inherits the single-replica assumption below (its queue is
|
||||||
|
// process-local); it stops with the same ctx as the server.
|
||||||
|
intake, worker, err := buildUploads(ctx, cfg, st, log)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var workerDone chan struct{} // nil without uploads: nothing to wait for at shutdown
|
||||||
|
wireUploads(app, cfg, intake)
|
||||||
|
if intake != nil {
|
||||||
|
workerDone = make(chan struct{})
|
||||||
|
go func() { worker.Run(ctx); close(workerDone) }()
|
||||||
|
log.Info("audio upload enabled", "model", cfg.TranscribeModel, "fallback", cfg.TranscribeFallbackModel,
|
||||||
|
"max_mb", cfg.UploadMaxBytes>>20, "max_minutes", int(cfg.UploadMaxDuration.Minutes()))
|
||||||
|
log.Warn("audio upload assumes a SINGLE replica: the transcription queue is process-local (ADR-033, ADR-018)")
|
||||||
|
if cfg.TranscribeFallbackModel != "" {
|
||||||
|
log.Warn("audio upload fallback is a CLOUD endpoint: if the primary fails, private audio leaves the homelab",
|
||||||
|
"fallback", cfg.TranscribeFallbackModel)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// User onboarding is handled by the IdP (Authentik invite flow), not Tapir —
|
||||||
|
// the Dex local-password provisioning path was removed (ADR-019). An
|
||||||
|
// authenticated subject with no Tapir user is routed to /register.
|
||||||
|
|
||||||
// Web-initiated YouTube connect (ADR-006). Mounted only when the OAuth client
|
// Web-initiated YouTube connect (ADR-006). Mounted only when the OAuth client
|
||||||
// credentials are present; the refresh token persists through the SecretStore
|
// credentials are present; the refresh token persists through the SecretStore
|
||||||
@@ -189,7 +243,10 @@ func cmdServe(ctx context.Context, log *slog.Logger) error {
|
|||||||
ClientSecret: cfg.YTClientSecret,
|
ClientSecret: cfg.YTClientSecret,
|
||||||
RedirectURL: cfg.YTConnectRedirectURL,
|
RedirectURL: cfg.YTConnectRedirectURL,
|
||||||
}, secretStore, st, log)
|
}, secretStore, st, log)
|
||||||
log.Info("web youtube connect enabled", "redirect", cfg.YTConnectRedirectURL)
|
// Paste-a-URL (Feature 2): same YouTube credentials, per-user adapter built
|
||||||
|
// per request. Mounting the /paste route keys off app.Fetcher being set.
|
||||||
|
app.Fetcher = videoFetcher{cfg: cfg, secrets: secretStore}
|
||||||
|
log.Info("web youtube connect + paste enabled", "redirect", cfg.YTConnectRedirectURL)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Immediate summarization for the web "Summarize" button. When the engine can
|
// Immediate summarization for the web "Summarize" button. When the engine can
|
||||||
@@ -207,23 +264,107 @@ func cmdServe(ctx context.Context, log *slog.Logger) error {
|
|||||||
log.Info("web summarization is queue-only (incomplete engine config)")
|
log.Info("web summarization is queue-only (incomplete engine config)")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// In-process scheduled discovery (ADR-018): when enabled, a background
|
||||||
|
// goroutine runs a discovery pass for ALL users on TAPIR_DISCOVERY_INTERVAL,
|
||||||
|
// reusing the per-user runner.Runner. Cancelled by the same ctx as the server.
|
||||||
|
//
|
||||||
|
// SINGLE-REPLICA ASSUMPTION (load-bearing): this loop lives in the web process.
|
||||||
|
// Running serve at >1 replica would make every replica fetch every user in
|
||||||
|
// parallel — duplicate work and self-inflicted 429s. replicas: 1 is required in
|
||||||
|
// the deployment manifest; scaling up needs a CronJob or leader election first.
|
||||||
|
if cfg.DiscoveryInterval > 0 {
|
||||||
|
log.Info("scheduled discovery enabled", "interval", cfg.DiscoveryInterval, "fetch_rate", cfg.FetchRate)
|
||||||
|
log.Warn("scheduled discovery assumes a SINGLE replica — running serve at >1 replica double-runs discovery (ADR-018)")
|
||||||
|
rawRunUser := func(ctx context.Context, userID string) (runner.Stats, error) {
|
||||||
|
r, err := buildUserRunner(cfg, st, secretStore, userID, log)
|
||||||
|
if err != nil {
|
||||||
|
return runner.Stats{}, err
|
||||||
|
}
|
||||||
|
return r.RunOnce(ctx)
|
||||||
|
}
|
||||||
|
// One lock shared by the scheduler and connect-triggered passes (#6) so
|
||||||
|
// they never fetch concurrently — the single-fetcher invariant (ADR-018).
|
||||||
|
runUser := serialize(&sync.Mutex{}, rawRunUser)
|
||||||
|
// Onboarding burst (Feature 1, refined by ADR-028): after the connect-triggered
|
||||||
|
// discovery pass, summarize up to OnboardSummarizeCount of the user's newest
|
||||||
|
// LIKELY-GOOD unsummarized videos so a fresh account gets a strong first
|
||||||
|
// session. Selection avoids known-junk (Shorts/over-long/livestream VODs via
|
||||||
|
// the persisted duration); the burst leads its chain with the stronger onboard
|
||||||
|
// model. Hard cap; explicit, so it bypasses the recency window — but every
|
||||||
|
// fetch still goes through globalFetchGate. No-op when disabled (count 0) or
|
||||||
|
// queue-only (no processor).
|
||||||
|
//
|
||||||
|
// burstProcessor leads with the stronger model (ADR-028); it collapses onto the
|
||||||
|
// shared Processor when the onboard model is empty/equal-to-primary or the
|
||||||
|
// engine config is incomplete.
|
||||||
|
burstProcessor := app.Processor
|
||||||
|
if burstEngine, berr := buildBurstProcessor(cfg, st); berr != nil {
|
||||||
|
return berr
|
||||||
|
} else if burstEngine != nil {
|
||||||
|
burstProcessor = &engineProcessor{engine: burstEngine, store: st}
|
||||||
|
log.Info("onboarding burst uses a stronger model", "onboard_model", cfg.OnboardSummarizerModel)
|
||||||
|
}
|
||||||
|
onboard := func(ctx context.Context, userID string) {
|
||||||
|
if cfg.OnboardSummarizeCount <= 0 || burstProcessor == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ids, err := st.OnboardBurstVideoIDs(ctx, userID, cfg.OnboardSummarizeCount, cfg.MinVideoSeconds, cfg.OnboardMaxVideoSeconds)
|
||||||
|
if err != nil {
|
||||||
|
log.Warn("onboarding: list burst candidates", "user", userID, "err", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, id := range ids {
|
||||||
|
if err := burstProcessor.ProcessVideo(ctx, userID, id); err != nil {
|
||||||
|
log.Warn("onboarding: summarize", "user", userID, "video", id, "err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(ids) > 0 {
|
||||||
|
log.Info("onboarding burst complete", "user", userID, "summarized", len(ids), "cap", cfg.OnboardSummarizeCount)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if app.Connect != nil {
|
||||||
|
app.Connect.Discovery = &discoveryTrigger{ctx: ctx, run: runUser, onboard: onboard, log: log}
|
||||||
|
log.Info("connect-triggered discovery enabled", "onboard_cap", cfg.OnboardSummarizeCount)
|
||||||
|
}
|
||||||
|
go runScheduler(ctx, cfg.DiscoveryInterval, st, runUser, log)
|
||||||
|
} else {
|
||||||
|
log.Info("scheduled discovery disabled (TAPIR_DISCOVERY_INTERVAL unset or 0)")
|
||||||
|
}
|
||||||
|
|
||||||
srv := &http.Server{
|
srv := &http.Server{
|
||||||
Addr: cfg.HTTPAddr,
|
Addr: cfg.HTTPAddr,
|
||||||
Handler: app.Router(),
|
Handler: metrics.HTTPMiddleware(app.Router()),
|
||||||
ReadHeaderTimeout: 10 * time.Second,
|
ReadHeaderTimeout: 10 * time.Second,
|
||||||
}
|
}
|
||||||
|
|
||||||
// Graceful shutdown on signal: stop accepting, drain in-flight requests.
|
// Prometheus /metrics on a SEPARATE port (ADR-030) — never on the public app
|
||||||
|
// mux, so a scrape is in-cluster only. Empty TAPIR_METRICS_ADDR disables it.
|
||||||
|
var metricsSrv *http.Server
|
||||||
|
if cfg.MetricsAddr != "" {
|
||||||
|
mmux := http.NewServeMux()
|
||||||
|
mmux.Handle("GET /metrics", metrics.Handler())
|
||||||
|
metricsSrv = &http.Server{Addr: cfg.MetricsAddr, Handler: mmux, ReadHeaderTimeout: 10 * time.Second}
|
||||||
go func() {
|
go func() {
|
||||||
<-ctx.Done()
|
log.Info("serving metrics", "addr", cfg.MetricsAddr)
|
||||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
if err := metricsSrv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||||
defer cancel()
|
log.Error("metrics server", "err", err)
|
||||||
_ = srv.Shutdown(shutdownCtx)
|
}
|
||||||
}()
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Graceful shutdown on signal: stop accepting, drain in-flight requests, and let the
|
||||||
|
// worker record the job it was running as interrupted (#50).
|
||||||
|
var extra []*http.Server
|
||||||
|
if metricsSrv != nil {
|
||||||
|
extra = append(extra, metricsSrv)
|
||||||
|
}
|
||||||
|
ln, err := net.Listen("tcp", srv.Addr)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("serve: listen: %w", err)
|
||||||
|
}
|
||||||
log.Info("serving web ui", "addr", cfg.HTTPAddr, "user", cfg.UserID)
|
log.Info("serving web ui", "addr", cfg.HTTPAddr, "user", cfg.UserID)
|
||||||
if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
if err := serveHTTP(ctx, srv, ln, extra, workerDone, shutdownGrace); err != nil {
|
||||||
return fmt.Errorf("serve: %w", err)
|
return err
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
+233
-19
@@ -3,17 +3,40 @@ package main
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
"gitea.d-ma.be/mathias/tapir/internal/adapters/llm"
|
"git.d-ma.be/mathias/tapir/internal/adapters/chat"
|
||||||
"gitea.d-ma.be/mathias/tapir/internal/adapters/secrets"
|
"git.d-ma.be/mathias/tapir/internal/adapters/llm"
|
||||||
"gitea.d-ma.be/mathias/tapir/internal/adapters/store"
|
"git.d-ma.be/mathias/tapir/internal/adapters/secrets"
|
||||||
"gitea.d-ma.be/mathias/tapir/internal/adapters/summarizer"
|
"git.d-ma.be/mathias/tapir/internal/adapters/store"
|
||||||
"gitea.d-ma.be/mathias/tapir/internal/adapters/youtube"
|
"git.d-ma.be/mathias/tapir/internal/adapters/summarizer"
|
||||||
"gitea.d-ma.be/mathias/tapir/internal/config"
|
"git.d-ma.be/mathias/tapir/internal/adapters/youtube"
|
||||||
"gitea.d-ma.be/mathias/tapir/internal/domain"
|
"git.d-ma.be/mathias/tapir/internal/config"
|
||||||
"gitea.d-ma.be/mathias/tapir/internal/usecase"
|
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/metrics"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/ports"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/usecase"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/web"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// videoFetcher adapts the YouTube adapter to web.VideoFetcher for the paste flow
|
||||||
|
// (Feature 2). It builds a per-user adapter bound to that user's token ref and
|
||||||
|
// resolves a single video's metadata via the Data API — ungated; only the later
|
||||||
|
// transcript fetch goes through globalFetchGate.
|
||||||
|
type videoFetcher struct {
|
||||||
|
cfg config.Config
|
||||||
|
secrets ports.SecretStore
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f videoFetcher) FetchVideo(ctx context.Context, userID, videoID string) (domain.Video, error) {
|
||||||
|
a := youtube.New(youtube.Config{
|
||||||
|
ClientID: f.cfg.YTClientID,
|
||||||
|
ClientSecret: f.cfg.YTClientSecret,
|
||||||
|
TokenSecretRef: web.YouTubeTokenRef(userID),
|
||||||
|
}, f.secrets)
|
||||||
|
return a.VideoByID(ctx, userID, videoID)
|
||||||
|
}
|
||||||
|
|
||||||
// buildProcessor wires the summarization engine — YouTube source (captions-first),
|
// buildProcessor wires the summarization engine — YouTube source (captions-first),
|
||||||
// AI-router summarizer, store sink — shared by `tapir run` and the web
|
// AI-router summarizer, store sink — shared by `tapir run` and the web
|
||||||
// "Summarize now" path so the wiring lives in one place. It returns (nil, nil) —
|
// "Summarize now" path so the wiring lives in one place. It returns (nil, nil) —
|
||||||
@@ -22,28 +45,193 @@ import (
|
|||||||
// queue-only fallback: the web UI keeps working (the button just queues) and
|
// queue-only fallback: the web UI keeps working (the button just queues) and
|
||||||
// `tapir run` reports the gap via its own ValidateForRun. Missing engine config
|
// `tapir run` reports the gap via its own ValidateForRun. Missing engine config
|
||||||
// is never an error here.
|
// is never an error here.
|
||||||
|
// buildSummarizer wires the summarization endpoint chain (ADR-022) shared by the
|
||||||
|
// web "Summarize now" path and the scheduler's per-user runners. The chain is:
|
||||||
|
// primary (local, fast) → local fallback → cloud fallback (worst case). Each
|
||||||
|
// endpoint reaches the same LiteLLM gateway with a different model alias — the
|
||||||
|
// gateway fronts both llama-swap and berget — so a fallback is just a different
|
||||||
|
// alias, not a second client config. Empty model entries are skipped, so a
|
||||||
|
// client deployment can set the cloud fallback empty to keep content local.
|
||||||
|
func buildSummarizer(cfg config.Config) *summarizer.Summarizer {
|
||||||
|
mk := summarizerEndpoint(cfg)
|
||||||
|
eps := []summarizer.Endpoint{mk(cfg.SummarizerModel)}
|
||||||
|
if cfg.FallbackModel != "" && cfg.FallbackModel != cfg.SummarizerModel {
|
||||||
|
eps = append(eps, mk(cfg.FallbackModel))
|
||||||
|
}
|
||||||
|
if cfg.CloudFallbackModel != "" && cfg.CloudFallbackModel != cfg.SummarizerModel {
|
||||||
|
eps = append(eps, mk(cfg.CloudFallbackModel))
|
||||||
|
}
|
||||||
|
return summarizer.NewChain(eps, cfg.MaxTranscriptChars)
|
||||||
|
}
|
||||||
|
|
||||||
|
// summarizerEndpoint returns a constructor for a chain endpoint over the one
|
||||||
|
// LiteLLM gateway, varying only the model alias (the gateway fronts both
|
||||||
|
// llama-swap and berget). Shared by the standard and burst chains.
|
||||||
|
func summarizerEndpoint(cfg config.Config) func(model string) summarizer.Endpoint {
|
||||||
|
return func(model string) summarizer.Endpoint {
|
||||||
|
return summarizer.Endpoint{
|
||||||
|
Client: llm.New(cfg.GatewayURL, cfg.GatewayKey, model, cfg.SummarizerTimeout, llm.WithMaxTokens(cfg.SummaryMaxTokens), llm.WithUsageHook(metrics.RecordTokens)),
|
||||||
|
Provider: providerOf(model),
|
||||||
|
Model: model,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// burstChainModels is the ordered, deduped model list for the onboarding burst
|
||||||
|
// (ADR-028): the stronger onboard model leads, then the standard ADR-022 chain
|
||||||
|
// (primary → local fallback → cloud) follows as resilience. Empty entries are
|
||||||
|
// dropped and duplicates collapsed, so the NDA lever (empty cloud fallback) keeps
|
||||||
|
// the burst chain fully local exactly as the standard chain does.
|
||||||
|
func burstChainModels(cfg config.Config) []string {
|
||||||
|
var models []string
|
||||||
|
add := func(m string) {
|
||||||
|
if m == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, e := range models {
|
||||||
|
if e == m {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
models = append(models, m)
|
||||||
|
}
|
||||||
|
add(cfg.OnboardSummarizerModel)
|
||||||
|
add(cfg.SummarizerModel)
|
||||||
|
add(cfg.FallbackModel)
|
||||||
|
add(cfg.CloudFallbackModel)
|
||||||
|
return models
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildBurstSummarizer builds the onboarding-burst summarizer chain (ADR-028):
|
||||||
|
// the onboard model first, then the standard chain as fallback, deduped.
|
||||||
|
func buildBurstSummarizer(cfg config.Config) *summarizer.Summarizer {
|
||||||
|
mk := summarizerEndpoint(cfg)
|
||||||
|
var eps []summarizer.Endpoint
|
||||||
|
for _, m := range burstChainModels(cfg) {
|
||||||
|
eps = append(eps, mk(m))
|
||||||
|
}
|
||||||
|
return summarizer.NewChain(eps, cfg.MaxTranscriptChars)
|
||||||
|
}
|
||||||
|
|
||||||
|
// chatModels is the ordered, local-first set of models offered in the chat
|
||||||
|
// switcher (ADR-027), reusing the ADR-022 chain: primary → local fallback →
|
||||||
|
// cloud. Empty entries are dropped and duplicates collapsed, so a client/NDA
|
||||||
|
// deployment that sets the cloud fallback empty simply has no external model in
|
||||||
|
// the switcher — the same local-first lever the summarizer honours.
|
||||||
|
func chatModels(cfg config.Config) []string {
|
||||||
|
var models []string
|
||||||
|
add := func(m string) {
|
||||||
|
if m == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, e := range models {
|
||||||
|
if e == m {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
models = append(models, m)
|
||||||
|
}
|
||||||
|
add(cfg.SummarizerModel)
|
||||||
|
add(cfg.FallbackModel)
|
||||||
|
add(cfg.CloudFallbackModel)
|
||||||
|
// The model chat over an uploaded recording opens on must be one the shared service can
|
||||||
|
// call: it re-maps a model it does not offer, which silently swapped a configured 31B for
|
||||||
|
// phi4-mini, whose context cannot hold an hour of speech. Local only, and only when
|
||||||
|
// uploads are on, so the video chat is unchanged otherwise (second adversarial review).
|
||||||
|
if cfg.UploadsEnabled && config.IsLocalModel(cfg.UploadChatModel) {
|
||||||
|
add(cfg.UploadChatModel)
|
||||||
|
}
|
||||||
|
return models
|
||||||
|
}
|
||||||
|
|
||||||
|
// uploadChatModels is the chat switcher for a PRIVATE uploaded recording: the same
|
||||||
|
// ordered set as chatModels, local models only. The whole transcript (up to 90,000
|
||||||
|
// characters) is sent to the one chosen, so a cloud model the video chat offers must not
|
||||||
|
// be offered here; with none local, none is offered (ADR-035 review).
|
||||||
|
func uploadChatModels(cfg config.Config) []string {
|
||||||
|
var local []string
|
||||||
|
for _, m := range chatModels(cfg) {
|
||||||
|
if config.IsLocalModel(m) {
|
||||||
|
local = append(local, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return local
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildChat wires the per-video chat service (ADR-027): a Completer factory over
|
||||||
|
// the SAME LiteLLM gateway the summarizer uses (a different alias per model, not a
|
||||||
|
// second client config) and the same transcript-truncation budget. It returns nil
|
||||||
|
// when no gateway is configured — chat is simply not mounted, the read path is
|
||||||
|
// unaffected. It deliberately takes NO YouTube source: chat is stored-only.
|
||||||
|
func buildChat(cfg config.Config) *chat.Service {
|
||||||
|
if cfg.GatewayURL == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
models := chatModels(cfg)
|
||||||
|
if len(models) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
newClient := func(model string) chat.Completer {
|
||||||
|
return llm.New(cfg.GatewayURL, cfg.GatewayKey, model, cfg.SummarizerTimeout, llm.WithMaxTokens(cfg.SummaryMaxTokens), llm.WithUsageHook(metrics.RecordTokens))
|
||||||
|
}
|
||||||
|
return chat.New(newClient, models, cfg.MaxTranscriptChars)
|
||||||
|
}
|
||||||
|
|
||||||
|
// providerOf maps a model alias to the domain AIProvider recorded on summaries.
|
||||||
|
// A "berget/" alias is an external provider; everything else is the local stack.
|
||||||
|
func providerOf(model string) string {
|
||||||
|
if strings.HasPrefix(model, "berget/") {
|
||||||
|
return "berget"
|
||||||
|
}
|
||||||
|
return "local"
|
||||||
|
}
|
||||||
|
|
||||||
func buildProcessor(cfg config.Config, st *store.Store) (*usecase.Engine, error) {
|
func buildProcessor(cfg config.Config, st *store.Store) (*usecase.Engine, error) {
|
||||||
if cfg.GatewayURL == "" || cfg.YTClientID == "" || cfg.YTClientSecret == "" || cfg.SecretsFile == "" {
|
if cfg.GatewayURL == "" || cfg.YTClientID == "" || cfg.YTClientSecret == "" || cfg.SecretsFile == "" {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
secretStore := secrets.NewFileStore(cfg.SecretsFile)
|
src := newYouTubeSource(cfg, secrets.NewFileStore(cfg.SecretsFile))
|
||||||
src := youtube.New(youtube.Config{
|
sum := buildSummarizer(cfg)
|
||||||
|
|
||||||
|
// The store is both the summary sink and the shared transcript cache (ADR-021):
|
||||||
|
// the engine reads stored transcripts before any caption fetch and writes
|
||||||
|
// resolved ones back, so re-analysis never re-touches YouTube.
|
||||||
|
eng := usecase.NewEngine(src, sum, st)
|
||||||
|
eng.Transcripts = st
|
||||||
|
return eng, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// newYouTubeSource builds the captions-first VideoSource shared by the standard
|
||||||
|
// and burst processors — same per-process YouTube credentials and ADR-023 Shorts
|
||||||
|
// filter; only the summarizer chain differs between them.
|
||||||
|
func newYouTubeSource(cfg config.Config, secretStore ports.SecretStore) ports.VideoSource {
|
||||||
|
return youtube.New(youtube.Config{
|
||||||
ClientID: cfg.YTClientID,
|
ClientID: cfg.YTClientID,
|
||||||
ClientSecret: cfg.YTClientSecret,
|
ClientSecret: cfg.YTClientSecret,
|
||||||
TokenSecretRef: cfg.YTTokenRef,
|
TokenSecretRef: cfg.YTTokenRef,
|
||||||
PreferredLanguages: []string{"en"},
|
PreferredLanguages: []string{"en"},
|
||||||
|
MinVideoSeconds: cfg.MinVideoSeconds,
|
||||||
}, secretStore)
|
}, secretStore)
|
||||||
|
}
|
||||||
|
|
||||||
// Local Primary only; no BYO fallback for the demo (fallback nil).
|
// buildBurstProcessor wires a processor whose summarizer leads with the stronger
|
||||||
primary := summarizer.Endpoint{
|
// onboard model (ADR-028), used only by the connect-time burst over the SAME
|
||||||
Client: llm.New(cfg.GatewayURL, cfg.GatewayKey, cfg.SummarizerModel, cfg.SummarizerTimeout),
|
// store / transcript cache / sink — a wiring choice; the engine and ports are
|
||||||
Provider: "local",
|
// unchanged. Returns (nil, nil) — the collapse lever — when the onboard model is
|
||||||
Model: cfg.SummarizerModel,
|
// empty or equal to the primary (the burst then reuses the shared processor), or
|
||||||
|
// when the engine config is incomplete (queue-only, same as buildProcessor).
|
||||||
|
func buildBurstProcessor(cfg config.Config, st *store.Store) (*usecase.Engine, error) {
|
||||||
|
if cfg.OnboardSummarizerModel == "" || cfg.OnboardSummarizerModel == cfg.SummarizerModel {
|
||||||
|
return nil, nil
|
||||||
}
|
}
|
||||||
sum := summarizer.New(primary, nil)
|
if cfg.GatewayURL == "" || cfg.YTClientID == "" || cfg.YTClientSecret == "" || cfg.SecretsFile == "" {
|
||||||
|
return nil, nil
|
||||||
return usecase.NewEngine(src, sum, st), nil
|
}
|
||||||
|
src := newYouTubeSource(cfg, secrets.NewFileStore(cfg.SecretsFile))
|
||||||
|
eng := usecase.NewEngine(src, buildBurstSummarizer(cfg), st)
|
||||||
|
eng.Transcripts = st
|
||||||
|
return eng, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// engineProcessor adapts the engine (which works in terms of a domain.Video) to
|
// engineProcessor adapts the engine (which works in terms of a domain.Video) to
|
||||||
@@ -58,6 +246,11 @@ type engineProcessor struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (p *engineProcessor) ProcessVideo(ctx context.Context, userID, videoID string) error {
|
func (p *engineProcessor) ProcessVideo(ctx context.Context, userID, videoID string) error {
|
||||||
|
// This is the user-initiated (foreground) path — a click on "Summarize",
|
||||||
|
// "Try now", or a pasted URL. Mark the context so the caption gate gives it
|
||||||
|
// priority over the background sweep (ADR-026, Pillar A).
|
||||||
|
ctx = youtube.ForegroundContext(ctx)
|
||||||
|
|
||||||
row, err := p.store.GetVideoRow(ctx, userID, videoID)
|
row, err := p.store.GetVideoRow(ctx, userID, videoID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("load video %q: %w", videoID, err)
|
return fmt.Errorf("load video %q: %w", videoID, err)
|
||||||
@@ -77,7 +270,28 @@ func (p *engineProcessor) ProcessVideo(ctx context.Context, userID, videoID stri
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("process video %q: %w", videoID, err)
|
return fmt.Errorf("process video %q: %w", videoID, err)
|
||||||
}
|
}
|
||||||
if res.Summary != nil {
|
|
||||||
|
// Record the outcome so the status endpoint can show honest state (ADR-025):
|
||||||
|
// a 429'd or caption-less click used to leave transcript_status unset, so the
|
||||||
|
// poll silently reverted to the "Summarize" button. Mirror the runner: stamp
|
||||||
|
// rate_limited / none / fetched. A rate-limited video keeps its requested flag
|
||||||
|
// so the background sweep retries it; none and fetched are terminal here.
|
||||||
|
switch {
|
||||||
|
case res.Skipped && res.TranscriptSource == string(domain.SourceRateLimited):
|
||||||
|
if err := p.store.SetTranscriptStatus(ctx, userID, videoID, "rate_limited"); err != nil {
|
||||||
|
return fmt.Errorf("set rate_limited status %q: %w", videoID, err)
|
||||||
|
}
|
||||||
|
case res.Skipped:
|
||||||
|
if err := p.store.SetTranscriptStatus(ctx, userID, videoID, "none"); err != nil {
|
||||||
|
return fmt.Errorf("set none status %q: %w", videoID, err)
|
||||||
|
}
|
||||||
|
if err := p.store.ClearSummarizeRequested(ctx, userID, videoID); err != nil {
|
||||||
|
return fmt.Errorf("clear summarize flag %q: %w", videoID, err)
|
||||||
|
}
|
||||||
|
case res.Summary != nil:
|
||||||
|
if err := p.store.SetTranscriptStatus(ctx, userID, videoID, "fetched"); err != nil {
|
||||||
|
return fmt.Errorf("set fetched status %q: %w", videoID, err)
|
||||||
|
}
|
||||||
if err := p.store.ClearSummarizeRequested(ctx, userID, videoID); err != nil {
|
if err := p.store.ClearSummarizeRequested(ctx, userID, videoID); err != nil {
|
||||||
return fmt.Errorf("clear summarize flag %q: %w", videoID, err)
|
return fmt.Errorf("clear summarize flag %q: %w", videoID, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ package main
|
|||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.d-ma.be/mathias/tapir/internal/config"
|
"git.d-ma.be/mathias/tapir/internal/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestBuildProcessorNilOnIncompleteConfig asserts the queue-only fallback: when a
|
// TestBuildProcessorNilOnIncompleteConfig asserts the queue-only fallback: when a
|
||||||
@@ -45,3 +45,65 @@ func TestBuildProcessorNilOnIncompleteConfig(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestBurstChainModelsLeadsWithOnboardModel: the onboarding burst chain (ADR-028)
|
||||||
|
// leads with the stronger onboard model, then falls back through the standard
|
||||||
|
// ADR-022 chain (primary -> local fallback -> cloud), deduped.
|
||||||
|
func TestBurstChainModelsLeadsWithOnboardModel(t *testing.T) {
|
||||||
|
got := burstChainModels(config.Config{
|
||||||
|
OnboardSummarizerModel: "iguana/gemma4-26b",
|
||||||
|
SummarizerModel: "koala/phi4-mini",
|
||||||
|
FallbackModel: "iguana/gemma4-26b", // also the onboard model -> dedup
|
||||||
|
CloudFallbackModel: "berget/mistral-small",
|
||||||
|
})
|
||||||
|
want := []string{"iguana/gemma4-26b", "koala/phi4-mini", "berget/mistral-small"}
|
||||||
|
if len(got) != len(want) {
|
||||||
|
t.Fatalf("burstChainModels = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
for i := range want {
|
||||||
|
if got[i] != want[i] {
|
||||||
|
t.Fatalf("burstChainModels = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestBurstChainModelsCloudAbsentWhenDisabled: the NDA lever holds for the burst
|
||||||
|
// too — empty cloud fallback keeps the burst chain fully local.
|
||||||
|
func TestBurstChainModelsCloudAbsentWhenDisabled(t *testing.T) {
|
||||||
|
got := burstChainModels(config.Config{
|
||||||
|
OnboardSummarizerModel: "iguana/gemma4-26b",
|
||||||
|
SummarizerModel: "koala/phi4-mini",
|
||||||
|
CloudFallbackModel: "",
|
||||||
|
})
|
||||||
|
for _, m := range got {
|
||||||
|
if m == "" || m == "berget/mistral-small" {
|
||||||
|
t.Fatalf("cloud model leaked into burst chain: %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestBuildBurstProcessorNilWhenCollapsed: an empty or primary-equal onboard model
|
||||||
|
// collapses the burst onto the shared processor (buildBurstProcessor returns nil).
|
||||||
|
func TestBuildBurstProcessorNilWhenCollapsed(t *testing.T) {
|
||||||
|
base := config.Config{
|
||||||
|
GatewayURL: "http://gw/v1",
|
||||||
|
YTClientID: "id",
|
||||||
|
YTClientSecret: "secret",
|
||||||
|
SecretsFile: "/tmp/secrets.json",
|
||||||
|
SummarizerModel: "koala/phi4-mini",
|
||||||
|
}
|
||||||
|
t.Run("empty onboard model", func(t *testing.T) {
|
||||||
|
base.OnboardSummarizerModel = ""
|
||||||
|
eng, err := buildBurstProcessor(base, nil)
|
||||||
|
if err != nil || eng != nil {
|
||||||
|
t.Fatalf("buildBurstProcessor = (%v, %v), want (nil, nil)", eng, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
t.Run("onboard model equals primary", func(t *testing.T) {
|
||||||
|
base.OnboardSummarizerModel = "koala/phi4-mini"
|
||||||
|
eng, err := buildBurstProcessor(base, nil)
|
||||||
|
if err != nil || eng != nil {
|
||||||
|
t.Fatalf("buildBurstProcessor = (%v, %v), want (nil, nil)", eng, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,97 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"text/tabwriter"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/adapters/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// gateThreshold is the Stage-0 gate (VISION/ADR-016): usage in >= 2 distinct
|
||||||
|
// weeks. The gate passes when any user reaches it.
|
||||||
|
const gateThreshold = 2
|
||||||
|
|
||||||
|
// defaultGateStart is the date Stage-0 return-usage tracking begins: the morning
|
||||||
|
// the pilot was actually unblocked and summaries started flowing (2026-06-11).
|
||||||
|
// Activity before this — testing, the period the pilot was stuck on zero — is
|
||||||
|
// noise and must not count toward the gate. Override with TAPIR_USAGE_GATE_START
|
||||||
|
// (YYYY-MM-DD). The gate measures whether users RETURN once it genuinely works.
|
||||||
|
const defaultGateStart = "2026-06-11"
|
||||||
|
|
||||||
|
// gateStart resolves the baseline date from TAPIR_USAGE_GATE_START or the default,
|
||||||
|
// parsed as a UTC calendar day.
|
||||||
|
func gateStart() (time.Time, error) {
|
||||||
|
v := os.Getenv("TAPIR_USAGE_GATE_START")
|
||||||
|
if v == "" {
|
||||||
|
v = defaultGateStart
|
||||||
|
}
|
||||||
|
t, err := time.Parse("2006-01-02", v)
|
||||||
|
if err != nil {
|
||||||
|
return time.Time{}, fmt.Errorf("TAPIR_USAGE_GATE_START=%q: want YYYY-MM-DD: %w", v, err)
|
||||||
|
}
|
||||||
|
return t, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// runReport prints the Stage-0 usage gate: per-user distinct active weeks (reads
|
||||||
|
// UNION acts) and the pass/fail verdict. Read-only, cross-user — needs only
|
||||||
|
// TAPIR_DB_DSN (not TAPIR_USER_ID; the report enumerates all users itself).
|
||||||
|
func runReport(ctx context.Context, _ []string) error {
|
||||||
|
dsn := os.Getenv(envDSN)
|
||||||
|
if dsn == "" {
|
||||||
|
return fmt.Errorf("%s is required", envDSN)
|
||||||
|
}
|
||||||
|
s, err := store.New(ctx, dsn)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer s.Close()
|
||||||
|
|
||||||
|
since, err := gateStart()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
rows, err := s.ActiveWeeks(ctx, since)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return formatReport(os.Stdout, rows, since)
|
||||||
|
}
|
||||||
|
|
||||||
|
// formatReport renders the per-user week counts and the gate verdict. Pure: no DB,
|
||||||
|
// no env — so the layout and verdict logic are unit-testable without Postgres.
|
||||||
|
func formatReport(w io.Writer, rows []store.UserActiveWeeks, since time.Time) error {
|
||||||
|
if _, err := fmt.Fprintf(w, "Counting usage since %s (Stage-0 gate baseline)\n\n", since.Format("2006-01-02")); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(rows) == 0 {
|
||||||
|
_, err := fmt.Fprintln(w, "no users yet")
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0)
|
||||||
|
_, _ = fmt.Fprintln(tw, "USER\tNAME\tACTIVE_WEEKS\tGATE")
|
||||||
|
passed := false
|
||||||
|
for _, r := range rows {
|
||||||
|
gate := "-"
|
||||||
|
if r.ActiveWeeks >= gateThreshold {
|
||||||
|
gate = "PASS"
|
||||||
|
passed = true
|
||||||
|
}
|
||||||
|
_, _ = fmt.Fprintf(tw, "%s\t%s\t%d\t%s\n", r.UserID, orDash(r.DisplayName), r.ActiveWeeks, gate)
|
||||||
|
}
|
||||||
|
if err := tw.Flush(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
verdict := fmt.Sprintf("\nGate (usage in >= %d distinct weeks): NOT YET MET\n", gateThreshold)
|
||||||
|
if passed {
|
||||||
|
verdict = fmt.Sprintf("\nGate (usage in >= %d distinct weeks): PASSED\n", gateThreshold)
|
||||||
|
}
|
||||||
|
_, err := fmt.Fprint(w, verdict)
|
||||||
|
return err
|
||||||
|
}
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/adapters/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
var testSince = time.Date(2026, 6, 11, 0, 0, 0, 0, time.UTC)
|
||||||
|
|
||||||
|
func TestFormatReportColumnsAndGatePass(t *testing.T) {
|
||||||
|
rows := []store.UserActiveWeeks{
|
||||||
|
{UserID: "user-a", DisplayName: "Ada", ActiveWeeks: 3},
|
||||||
|
{UserID: "user-b", DisplayName: "", ActiveWeeks: 1},
|
||||||
|
}
|
||||||
|
|
||||||
|
var b strings.Builder
|
||||||
|
require.NoError(t, formatReport(&b, rows, testSince))
|
||||||
|
out := b.String()
|
||||||
|
|
||||||
|
require.Contains(t, out, "since 2026-06-11", "report states the gate baseline date")
|
||||||
|
require.Contains(t, out, "USER")
|
||||||
|
require.Contains(t, out, "ACTIVE_WEEKS")
|
||||||
|
require.Contains(t, out, "Ada")
|
||||||
|
require.Contains(t, out, "PASSED", "a user at >= 2 weeks passes the gate")
|
||||||
|
|
||||||
|
// The >=2 user is marked PASS; the 1-week user is not.
|
||||||
|
require.Contains(t, lineContaining(t, out, "user-a"), "PASS")
|
||||||
|
require.NotContains(t, lineContaining(t, out, "user-b"), "PASS")
|
||||||
|
require.Contains(t, lineContaining(t, out, "user-b"), "-", "no display name falls back to dash")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFormatReportGateNotMet(t *testing.T) {
|
||||||
|
rows := []store.UserActiveWeeks{{UserID: "user-a", ActiveWeeks: 1}}
|
||||||
|
|
||||||
|
var b strings.Builder
|
||||||
|
require.NoError(t, formatReport(&b, rows, testSince))
|
||||||
|
require.Contains(t, b.String(), "NOT YET MET", "no user at >= 2 weeks fails the gate")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFormatReportEmpty(t *testing.T) {
|
||||||
|
var b strings.Builder
|
||||||
|
require.NoError(t, formatReport(&b, nil, testSince))
|
||||||
|
require.Contains(t, b.String(), "no users yet")
|
||||||
|
}
|
||||||
@@ -0,0 +1,206 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/adapters/store"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/adapters/youtube"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/config"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/ports"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/runner"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/usecase"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/web"
|
||||||
|
)
|
||||||
|
|
||||||
|
// buildUserRunner constructs a runner.Runner for one user, reusing the same
|
||||||
|
// engine wiring as buildProcessor but bound to that user's own YouTube refresh
|
||||||
|
// token (web.YouTubeTokenRef(userID)) — the Stage-1 per-tenant ref, not the
|
||||||
|
// Stage-0 single ref. It returns an error (not nil) when the global config can't
|
||||||
|
// support live summarization (gateway, YouTube client creds, secrets file), so
|
||||||
|
// the scheduler can skip that user gracefully. A user who simply hasn't connected
|
||||||
|
// YouTube yet builds fine here; their token ref fails to resolve at RunOnce time,
|
||||||
|
// surfacing as a per-user error the scheduler logs and skips.
|
||||||
|
func buildUserRunner(cfg config.Config, st *store.Store, secretStore ports.SecretStore, userID string, log *slog.Logger) (*runner.Runner, error) {
|
||||||
|
if cfg.GatewayURL == "" || cfg.YTClientID == "" || cfg.YTClientSecret == "" || cfg.SecretsFile == "" {
|
||||||
|
return nil, fmt.Errorf("buildUserRunner: incomplete summarization config (gateway, youtube credentials, secrets file)")
|
||||||
|
}
|
||||||
|
|
||||||
|
src := youtube.New(youtube.Config{
|
||||||
|
ClientID: cfg.YTClientID,
|
||||||
|
ClientSecret: cfg.YTClientSecret,
|
||||||
|
TokenSecretRef: web.YouTubeTokenRef(userID),
|
||||||
|
PreferredLanguages: []string{"en"},
|
||||||
|
MinVideoSeconds: cfg.MinVideoSeconds,
|
||||||
|
}, secretStore)
|
||||||
|
|
||||||
|
engine := usecase.NewEngine(src, buildSummarizer(cfg), st)
|
||||||
|
// Share the transcript cache (ADR-021) on the scheduler path too — without
|
||||||
|
// this every scheduled pass re-fetches transcripts it already had, burning the
|
||||||
|
// scarce per-IP caption budget (ADR-014) and starving other users. The web
|
||||||
|
// "Summarize now" path already sets this; the scheduler omitting it was a bug.
|
||||||
|
engine.Transcripts = st
|
||||||
|
|
||||||
|
return runner.New(src, st, engine, userID, log,
|
||||||
|
runner.WithBackoff(cfg.FetchBackoff),
|
||||||
|
runner.WithAutoWindow(cfg.AutoSummarizeWindow),
|
||||||
|
runner.WithCaptionMemory(cfg.ChannelCaptionlessThreshold, cfg.ChannelCaptionlessWindow)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// userLister enumerates every registered user and reports a user's video
|
||||||
|
// connections. *store.Store satisfies it via ListAllUsers + ConnectionsForUser.
|
||||||
|
// A small local interface keeps the scheduler testable with a fake.
|
||||||
|
type userLister interface {
|
||||||
|
ListAllUsers(ctx context.Context) ([]store.UserIdentity, error)
|
||||||
|
ConnectionsForUser(ctx context.Context, userID string) ([]store.Connection, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// runDiscoveryPass runs one discovery pass for every user. runUser performs a
|
||||||
|
// single user's pass (production: build a runner and RunOnce). Per-user failures
|
||||||
|
// — including a buildUserRunner error or a RunOnce error — are logged and skipped
|
||||||
|
// so one bad user, channel, or video never aborts the others (ADR-018 failure
|
||||||
|
// isolation). Returns the stats summed across users.
|
||||||
|
func runDiscoveryPass(
|
||||||
|
ctx context.Context,
|
||||||
|
pass int,
|
||||||
|
lister userLister,
|
||||||
|
runUser func(context.Context, string) (runner.Stats, error),
|
||||||
|
log *slog.Logger,
|
||||||
|
) runner.Stats {
|
||||||
|
users, err := lister.ListAllUsers(ctx)
|
||||||
|
if err != nil {
|
||||||
|
log.Error("scheduler: list users failed", "err", err)
|
||||||
|
return runner.Stats{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Keep only users with a video connection. A pass for a connectionless user
|
||||||
|
// (e.g. a stale Dex-era orphan identity) only tries to resolve a token that
|
||||||
|
// was never minted, logging a spurious "ref not found" every tick. Filtering
|
||||||
|
// here — BEFORE rotation — also keeps fairness honest: rotation is over the
|
||||||
|
// users that actually consume the caption budget, so a dead identity can't eat
|
||||||
|
// a rotation slot and skew the lead share.
|
||||||
|
var connected []store.UserIdentity
|
||||||
|
for _, u := range users {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return runner.Stats{} // shutting down
|
||||||
|
}
|
||||||
|
conns, err := lister.ConnectionsForUser(ctx, u.UserID)
|
||||||
|
if err != nil {
|
||||||
|
log.Warn("scheduler: list connections failed", "user", u.UserID, "err", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if len(conns) == 0 {
|
||||||
|
log.Debug("scheduler: skipping user with no video connections", "user", u.UserID)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
connected = append(connected, u)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rotate who goes first each pass. Caption fetches share one per-egress-IP
|
||||||
|
// rate budget (ADR-014); whoever runs first each pass spends the pre-throttle
|
||||||
|
// window, so a FIXED order permanently starves whoever is last (a new pilot
|
||||||
|
// user got 0 fetches for 12h while the first-listed user got all of them).
|
||||||
|
// Rotation over the connected set gives each real user the lead in turn.
|
||||||
|
connected = rotateUsers(connected, pass)
|
||||||
|
|
||||||
|
log.Info("scheduler: starting discovery pass", "users", len(connected))
|
||||||
|
var total runner.Stats
|
||||||
|
for _, u := range connected {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
break // shutting down: stop enumerating
|
||||||
|
}
|
||||||
|
stats, err := runUser(ctx, u.UserID)
|
||||||
|
total = sumStats(total, stats)
|
||||||
|
if err != nil {
|
||||||
|
log.Warn("scheduler: user discovery pass had errors", "user", u.UserID, "err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
log.Info("scheduler: pass complete",
|
||||||
|
"candidates", total.Candidates, "summarized", total.Summarized,
|
||||||
|
"skipped_seen", total.SkippedSeen, "skipped_no_text", total.SkippedNoText,
|
||||||
|
"skipped_manual", total.SkippedManual, "skipped_too_old", total.SkippedTooOld,
|
||||||
|
"skipped_rate_limited", total.SkippedRateLimited,
|
||||||
|
"skipped_no_caption_channel", total.SkippedNoCaptionChannel,
|
||||||
|
"channel_unavailable", total.ChannelUnavailable, "errors", total.Errors)
|
||||||
|
return total
|
||||||
|
}
|
||||||
|
|
||||||
|
// runScheduler runs a discovery pass on startup, then once every interval until
|
||||||
|
// ctx is cancelled (pod SIGTERM exits the loop cleanly). A non-positive interval
|
||||||
|
// disables scheduling entirely (no startup pass) so dev/tests never auto-fetch.
|
||||||
|
// It reuses the existing runner.Runner via runUser — the only new behaviour over
|
||||||
|
// runner.Loop is iterating all users per tick (ADR-018).
|
||||||
|
func runScheduler(
|
||||||
|
ctx context.Context,
|
||||||
|
interval time.Duration,
|
||||||
|
lister userLister,
|
||||||
|
runUser func(context.Context, string) (runner.Stats, error),
|
||||||
|
log *slog.Logger,
|
||||||
|
) {
|
||||||
|
if interval <= 0 {
|
||||||
|
return // disabled
|
||||||
|
}
|
||||||
|
|
||||||
|
// Derive the rotation offset from wall-clock, NOT an in-memory counter. A
|
||||||
|
// counter reset to 0 on every pod restart always hands the lead to the
|
||||||
|
// first-listed user — so frequent deploys re-starve whoever is last (exactly
|
||||||
|
// what happened to the first pilot user during a deploy-heavy session). A
|
||||||
|
// time-based offset advances with real time and is identical across restarts,
|
||||||
|
// so the lead rotates fairly regardless of how often the pod bounces.
|
||||||
|
runPass := func() {
|
||||||
|
pass := int(time.Now().Unix() / int64(interval/time.Second))
|
||||||
|
runDiscoveryPass(ctx, pass, lister, runUser, log)
|
||||||
|
}
|
||||||
|
|
||||||
|
runPass()
|
||||||
|
|
||||||
|
ticker := time.NewTicker(interval)
|
||||||
|
defer ticker.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
runPass()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// rotateUsers left-rotates users by pass positions so a different user leads each
|
||||||
|
// pass. With n users, user i leads on every pass where pass ≡ i (mod n). A pass
|
||||||
|
// offset that is negative or exceeds n is normalised. Order within the rotation
|
||||||
|
// is otherwise preserved, so the set of users run is unchanged — only who is
|
||||||
|
// first (and thus wins the scarce caption-fetch budget) rotates.
|
||||||
|
func rotateUsers(users []store.UserIdentity, pass int) []store.UserIdentity {
|
||||||
|
n := len(users)
|
||||||
|
if n <= 1 {
|
||||||
|
return users
|
||||||
|
}
|
||||||
|
off := ((pass % n) + n) % n
|
||||||
|
if off == 0 {
|
||||||
|
return users
|
||||||
|
}
|
||||||
|
out := make([]store.UserIdentity, 0, n)
|
||||||
|
out = append(out, users[off:]...)
|
||||||
|
out = append(out, users[:off]...)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// sumStats adds two passes' stats field-wise, so runDiscoveryPass can report a
|
||||||
|
// per-tick aggregate across all users.
|
||||||
|
func sumStats(a, b runner.Stats) runner.Stats {
|
||||||
|
return runner.Stats{
|
||||||
|
Candidates: a.Candidates + b.Candidates,
|
||||||
|
Summarized: a.Summarized + b.Summarized,
|
||||||
|
SkippedSeen: a.SkippedSeen + b.SkippedSeen,
|
||||||
|
SkippedNoText: a.SkippedNoText + b.SkippedNoText,
|
||||||
|
SkippedManual: a.SkippedManual + b.SkippedManual,
|
||||||
|
SkippedTooOld: a.SkippedTooOld + b.SkippedTooOld,
|
||||||
|
SkippedRateLimited: a.SkippedRateLimited + b.SkippedRateLimited,
|
||||||
|
SkippedNoCaptionChannel: a.SkippedNoCaptionChannel + b.SkippedNoCaptionChannel,
|
||||||
|
ChannelUnavailable: a.ChannelUnavailable + b.ChannelUnavailable,
|
||||||
|
Errors: a.Errors + b.Errors,
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,216 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/adapters/store"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/runner"
|
||||||
|
)
|
||||||
|
|
||||||
|
func quietLog() *slog.Logger {
|
||||||
|
return slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||||
|
}
|
||||||
|
|
||||||
|
// fakeLister returns a fixed user set (or an error) for the scheduler under test.
|
||||||
|
type fakeLister struct {
|
||||||
|
users []store.UserIdentity
|
||||||
|
err error
|
||||||
|
noConn map[string]bool // users that have NOT connected a video source
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f fakeLister) ListAllUsers(context.Context) ([]store.UserIdentity, error) {
|
||||||
|
return f.users, f.err
|
||||||
|
}
|
||||||
|
|
||||||
|
// ConnectionsForUser reports a single youtube connection for every user except
|
||||||
|
// those in noConn, which return zero — the connection-less case the scheduler
|
||||||
|
// must skip instead of running (and failing to resolve a token for).
|
||||||
|
func (f fakeLister) ConnectionsForUser(_ context.Context, userID string) ([]store.Connection, error) {
|
||||||
|
if f.noConn[userID] {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return []store.Connection{{Provider: "youtube"}}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// countingRunUser records how many passes each user got, optionally failing for
|
||||||
|
// specific users, under a mutex so it is safe across the scheduler goroutine.
|
||||||
|
type countingRunUser struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
calls map[string]int
|
||||||
|
order []string // userIDs in the order they were run, across all passes
|
||||||
|
failFor map[string]bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func newCountingRunUser(failFor ...string) *countingRunUser {
|
||||||
|
c := &countingRunUser{calls: map[string]int{}, failFor: map[string]bool{}}
|
||||||
|
for _, u := range failFor {
|
||||||
|
c.failFor[u] = true
|
||||||
|
}
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *countingRunUser) run(_ context.Context, userID string) (runner.Stats, error) {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
c.calls[userID]++
|
||||||
|
c.order = append(c.order, userID)
|
||||||
|
if c.failFor[userID] {
|
||||||
|
return runner.Stats{Errors: 1}, errors.New("boom")
|
||||||
|
}
|
||||||
|
return runner.Stats{Summarized: 1}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *countingRunUser) runOrder() []string {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
return append([]string(nil), c.order...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *countingRunUser) count(userID string) int {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
return c.calls[userID]
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *countingRunUser) total() int {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
n := 0
|
||||||
|
for _, v := range c.calls {
|
||||||
|
n += v
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
func usersN(ids ...string) []store.UserIdentity {
|
||||||
|
out := make([]store.UserIdentity, len(ids))
|
||||||
|
for i, id := range ids {
|
||||||
|
out[i] = store.UserIdentity{UserID: id, DexSubject: "dex|" + id}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDiscoveryPassRunsEveryUserOnce(t *testing.T) {
|
||||||
|
lister := fakeLister{users: usersN("a", "b", "c")}
|
||||||
|
rc := newCountingRunUser()
|
||||||
|
|
||||||
|
stats := runDiscoveryPass(context.Background(), 0, lister, rc.run, quietLog())
|
||||||
|
|
||||||
|
require.Equal(t, 1, rc.count("a"))
|
||||||
|
require.Equal(t, 1, rc.count("b"))
|
||||||
|
require.Equal(t, 1, rc.count("c"))
|
||||||
|
require.Equal(t, 3, stats.Summarized, "stats are summed across users")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Caption fetches share one per-IP budget; a fixed user order starves whoever is
|
||||||
|
// last. Each pass must rotate which user leads so the lead slot is shared.
|
||||||
|
func TestDiscoveryPassRotatesLeadUser(t *testing.T) {
|
||||||
|
lister := fakeLister{users: usersN("a", "b", "c")}
|
||||||
|
rc := newCountingRunUser()
|
||||||
|
|
||||||
|
runDiscoveryPass(context.Background(), 0, lister, rc.run, quietLog())
|
||||||
|
runDiscoveryPass(context.Background(), 1, lister, rc.run, quietLog())
|
||||||
|
runDiscoveryPass(context.Background(), 2, lister, rc.run, quietLog())
|
||||||
|
|
||||||
|
require.Equal(t, []string{"a", "b", "c", "b", "c", "a", "c", "a", "b"}, rc.runOrder(),
|
||||||
|
"each pass left-rotates the user order so every user leads in turn")
|
||||||
|
// Fairness: over a full rotation cycle every user ran the same number of times.
|
||||||
|
require.Equal(t, 3, rc.count("a"))
|
||||||
|
require.Equal(t, 3, rc.count("b"))
|
||||||
|
require.Equal(t, 3, rc.count("c"))
|
||||||
|
}
|
||||||
|
|
||||||
|
// A connectionless orphan must not consume a rotation slot: rotation is over the
|
||||||
|
// connected users only, so two real users alternate the lead 50/50 even with a
|
||||||
|
// dead identity listed between them.
|
||||||
|
func TestDiscoveryPassRotationIgnoresConnectionlessUsers(t *testing.T) {
|
||||||
|
lister := fakeLister{users: usersN("a", "orphan", "c"), noConn: map[string]bool{"orphan": true}}
|
||||||
|
rc := newCountingRunUser()
|
||||||
|
|
||||||
|
runDiscoveryPass(context.Background(), 0, lister, rc.run, quietLog())
|
||||||
|
runDiscoveryPass(context.Background(), 1, lister, rc.run, quietLog())
|
||||||
|
|
||||||
|
require.Equal(t, []string{"a", "c", "c", "a"}, rc.runOrder(),
|
||||||
|
"only connected users rotate; the orphan never runs and never holds a slot")
|
||||||
|
require.Equal(t, 0, rc.count("orphan"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDiscoveryPassSkipsUsersWithoutConnections(t *testing.T) {
|
||||||
|
// b never connected a video source (e.g. a stale Dex-era orphan identity).
|
||||||
|
// It must be skipped silently — not run and logged as a token error every pass.
|
||||||
|
lister := fakeLister{users: usersN("a", "b", "c"), noConn: map[string]bool{"b": true}}
|
||||||
|
rc := newCountingRunUser()
|
||||||
|
|
||||||
|
stats := runDiscoveryPass(context.Background(), 0, lister, rc.run, quietLog())
|
||||||
|
|
||||||
|
require.Equal(t, 1, rc.count("a"))
|
||||||
|
require.Equal(t, 0, rc.count("b"), "a user with no connection must be skipped, not run")
|
||||||
|
require.Equal(t, 1, rc.count("c"))
|
||||||
|
require.Equal(t, 2, stats.Summarized, "only connected users contribute")
|
||||||
|
require.Equal(t, 0, stats.Errors, "skipping is silent — no spurious error stat")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDiscoveryPassOneUserFailureDoesNotStopOthers(t *testing.T) {
|
||||||
|
lister := fakeLister{users: usersN("a", "b", "c")}
|
||||||
|
rc := newCountingRunUser("b") // user b's pass errors
|
||||||
|
|
||||||
|
stats := runDiscoveryPass(context.Background(), 0, lister, rc.run, quietLog())
|
||||||
|
|
||||||
|
require.Equal(t, 1, rc.count("a"))
|
||||||
|
require.Equal(t, 1, rc.count("b"))
|
||||||
|
require.Equal(t, 1, rc.count("c"), "a failing user must not abort the rest")
|
||||||
|
require.Equal(t, 2, stats.Summarized) // a + c
|
||||||
|
require.Equal(t, 1, stats.Errors) // b
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDiscoveryPassListerErrorIsContained(t *testing.T) {
|
||||||
|
lister := fakeLister{err: errors.New("db down")}
|
||||||
|
rc := newCountingRunUser()
|
||||||
|
|
||||||
|
stats := runDiscoveryPass(context.Background(), 0, lister, rc.run, quietLog())
|
||||||
|
|
||||||
|
require.Equal(t, 0, rc.total(), "no users enumerated → no passes")
|
||||||
|
require.Equal(t, runner.Stats{}, stats)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSchedulerIntervalZeroDisablesEntirely(t *testing.T) {
|
||||||
|
lister := fakeLister{users: usersN("a", "b")}
|
||||||
|
rc := newCountingRunUser()
|
||||||
|
|
||||||
|
runScheduler(context.Background(), 0, lister, rc.run, quietLog())
|
||||||
|
|
||||||
|
require.Equal(t, 0, rc.total(), "interval 0 must not run even a startup pass")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSchedulerRunsStartupPassThenStopsOnCancel(t *testing.T) {
|
||||||
|
lister := fakeLister{users: usersN("a", "b", "c")}
|
||||||
|
rc := newCountingRunUser()
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
// A long interval so only the startup pass runs before we cancel.
|
||||||
|
runScheduler(ctx, time.Hour, lister, rc.run, quietLog())
|
||||||
|
close(done)
|
||||||
|
}()
|
||||||
|
|
||||||
|
// The startup pass is synchronous at the top of runScheduler; once all three
|
||||||
|
// users have a pass it has completed and the loop is parked on the ticker.
|
||||||
|
require.Eventually(t, func() bool { return rc.total() == 3 }, time.Second, 5*time.Millisecond)
|
||||||
|
|
||||||
|
cancel()
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("scheduler did not exit after ctx cancel")
|
||||||
|
}
|
||||||
|
require.Equal(t, 3, rc.total(), "no extra passes fired between startup and cancel")
|
||||||
|
}
|
||||||
+1
-1
@@ -8,7 +8,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.d-ma.be/mathias/tapir/internal/adapters/store"
|
"git.d-ma.be/mathias/tapir/internal/adapters/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
// runShow prints the full summary for one video: text, highlights, takeaways,
|
// runShow prints the full summary for one video: text, highlights, takeaways,
|
||||||
|
|||||||
@@ -0,0 +1,55 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"os/signal"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// shutdownGrace bounds each wait at shutdown (in-flight requests, then the worker). Two of
|
||||||
|
// them stay inside Kubernetes' default 30 s termination grace.
|
||||||
|
const shutdownGrace = 10 * time.Second
|
||||||
|
|
||||||
|
// shutdownContext is cancelled by SIGINT (a terminal) or SIGTERM (what Kubernetes sends
|
||||||
|
// on every deploy). Handling only SIGINT meant a deploy killed the process before any
|
||||||
|
// shutdown code ran (#50).
|
||||||
|
func shutdownContext(parent context.Context) (context.Context, context.CancelFunc) {
|
||||||
|
return signal.NotifyContext(parent, os.Interrupt, syscall.SIGTERM)
|
||||||
|
}
|
||||||
|
|
||||||
|
// serveHTTP serves srv on ln until it fails or ctx ends, then shuts every server down and
|
||||||
|
// waits for in-flight requests and for workerDone (nil: no worker) to finish, each for at
|
||||||
|
// most grace. It must not return earlier: Serve returns http.ErrServerClosed as soon as
|
||||||
|
// Shutdown STARTS, and returning from main kills the drain and the worker's last write.
|
||||||
|
func serveHTTP(ctx context.Context, srv *http.Server, ln net.Listener, extra []*http.Server, workerDone <-chan struct{}, grace time.Duration) error {
|
||||||
|
errc := make(chan error, 1)
|
||||||
|
go func() { errc <- srv.Serve(ln) }()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case err := <-errc:
|
||||||
|
if !errors.Is(err, http.ErrServerClosed) {
|
||||||
|
return fmt.Errorf("serve: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
case <-ctx.Done():
|
||||||
|
}
|
||||||
|
|
||||||
|
shutdownCtx, cancel := context.WithTimeout(context.Background(), grace)
|
||||||
|
defer cancel()
|
||||||
|
for _, s := range append([]*http.Server{srv}, extra...) {
|
||||||
|
_ = s.Shutdown(shutdownCtx)
|
||||||
|
}
|
||||||
|
if workerDone != nil {
|
||||||
|
select {
|
||||||
|
case <-workerDone:
|
||||||
|
case <-time.After(grace):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,124 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"sync"
|
||||||
|
"syscall"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Kubernetes stops a pod with SIGTERM; only SIGINT was handled, so a deploy killed the
|
||||||
|
// process without running any shutdown code (#50).
|
||||||
|
func TestShutdownContext_IsCancelledBySIGTERMAndSIGINT(t *testing.T) {
|
||||||
|
for _, sig := range []syscall.Signal{syscall.SIGTERM, syscall.SIGINT} {
|
||||||
|
t.Run(sig.String(), func(t *testing.T) {
|
||||||
|
ctx, stop := shutdownContext(context.Background())
|
||||||
|
defer stop()
|
||||||
|
|
||||||
|
if err := syscall.Kill(os.Getpid(), sig); err != nil {
|
||||||
|
t.Fatalf("send %v: %v", sig, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
case <-time.After(2 * time.Second):
|
||||||
|
t.Fatalf("context not cancelled by %v", sig)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// serveHTTP must not return while a request is in flight or the worker is still recording
|
||||||
|
// its job: Serve returns http.ErrServerClosed as soon as Shutdown STARTS, and returning
|
||||||
|
// from it let the process exit before either finished (#50). Each half is tested with the
|
||||||
|
// other already satisfied, so neither can hold the call open on the other's behalf.
|
||||||
|
|
||||||
|
func pendingFor(t *testing.T, returned <-chan error, why string) {
|
||||||
|
t.Helper()
|
||||||
|
select {
|
||||||
|
case err := <-returned:
|
||||||
|
t.Fatalf("serveHTTP returned (%v) %s", err, why)
|
||||||
|
case <-time.After(150 * time.Millisecond):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestServeHTTP_DrainsAnInFlightRequestBeforeReturning(t *testing.T) {
|
||||||
|
release := make(chan struct{})
|
||||||
|
entered := make(chan struct{})
|
||||||
|
srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
close(entered)
|
||||||
|
<-release
|
||||||
|
_, _ = w.Write([]byte("ok"))
|
||||||
|
}))
|
||||||
|
srv.Start()
|
||||||
|
defer srv.Close()
|
||||||
|
var once sync.Once
|
||||||
|
releaseHandler := func() { once.Do(func() { close(release) }) }
|
||||||
|
defer releaseHandler() // runs before srv.Close: a failing test must not hang on the blocked handler
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
workerDone := make(chan struct{})
|
||||||
|
close(workerDone) // the worker is already done: only the request can hold serveHTTP open
|
||||||
|
returned := make(chan error, 1)
|
||||||
|
go func() { returned <- serveHTTP(ctx, srv.Config, srv.Listener, nil, workerDone, 5*time.Second) }()
|
||||||
|
|
||||||
|
go func() { _, _ = http.Get(srv.URL) }()
|
||||||
|
<-entered
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
pendingFor(t, returned, "while a request was still in flight")
|
||||||
|
releaseHandler()
|
||||||
|
select {
|
||||||
|
case err := <-returned:
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("serveHTTP: %v", err)
|
||||||
|
}
|
||||||
|
case <-time.After(2 * time.Second):
|
||||||
|
t.Fatal("serveHTTP did not return once the request finished")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestServeHTTP_WaitsForTheWorkerBeforeReturning(t *testing.T) {
|
||||||
|
srv := httptest.NewUnstartedServer(http.NotFoundHandler())
|
||||||
|
srv.Start()
|
||||||
|
defer srv.Close()
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
workerDone := make(chan struct{})
|
||||||
|
returned := make(chan error, 1)
|
||||||
|
go func() { returned <- serveHTTP(ctx, srv.Config, srv.Listener, nil, workerDone, 5*time.Second) }() // no request in flight
|
||||||
|
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
pendingFor(t, returned, "before the worker finished")
|
||||||
|
close(workerDone)
|
||||||
|
select {
|
||||||
|
case err := <-returned:
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("serveHTTP: %v", err)
|
||||||
|
}
|
||||||
|
case <-time.After(2 * time.Second):
|
||||||
|
t.Fatal("serveHTTP did not return once the worker finished")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestServeHTTP_GivesUpOnAWorkerThatNeverFinishesAfterTheGrace(t *testing.T) {
|
||||||
|
srv := httptest.NewUnstartedServer(http.NotFoundHandler())
|
||||||
|
srv.Start()
|
||||||
|
defer srv.Close()
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
returned := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
returned <- serveHTTP(ctx, srv.Config, srv.Listener, nil, make(chan struct{}), 200*time.Millisecond)
|
||||||
|
}()
|
||||||
|
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-returned:
|
||||||
|
case <-time.After(3 * time.Second):
|
||||||
|
t.Fatal("a stuck worker must not hold the process past the grace period")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,142 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/adapters/audio"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/adapters/llm"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/adapters/transcribe"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/config"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/reconcile"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/web"
|
||||||
|
)
|
||||||
|
|
||||||
|
// uploadsStore is the store surface the upload wiring needs. *store.Store
|
||||||
|
// satisfies it.
|
||||||
|
type uploadsStore interface {
|
||||||
|
uploads.Store
|
||||||
|
uploads.WorkerStore
|
||||||
|
FailInterruptedUploads(ctx context.Context) (int, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildUploads wires audio upload and private transcription (ADR-033). It returns
|
||||||
|
// nils when uploads are disabled — never a typed-nil Intake, which would mount the
|
||||||
|
// /uploads routes over nothing. When enabled it, in order: prepares a private spool
|
||||||
|
// directory, deletes any audio a crashed run left in it, fails the rows such a run
|
||||||
|
// left queued or transcribing (the audio is not retained, so they can never
|
||||||
|
// resume), and only then builds the intake and the worker. The caller runs the
|
||||||
|
// worker. The fallback endpoint is a cloud service; an empty
|
||||||
|
// TAPIR_TRANSCRIBE_FALLBACK_MODEL keeps private audio on the homelab.
|
||||||
|
func buildUploads(ctx context.Context, cfg config.Config, st uploadsStore, log *slog.Logger) (*uploads.Intake, *uploads.Worker, error) {
|
||||||
|
if !cfg.UploadsEnabled {
|
||||||
|
return nil, nil, nil
|
||||||
|
}
|
||||||
|
if cfg.GatewayURL == "" {
|
||||||
|
return nil, nil, fmt.Errorf("uploads: TAPIR_UPLOADS_ENABLED is set but there is no TAPIR_GATEWAY_URL to transcribe through")
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.MkdirAll(cfg.UploadTmpDir, 0o700); err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("uploads: spool dir: %w", err)
|
||||||
|
}
|
||||||
|
removeSpooledAudio(cfg.UploadTmpDir, log)
|
||||||
|
|
||||||
|
n, err := st.FailInterruptedUploads(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("uploads: fail interrupted rows: %w", err)
|
||||||
|
}
|
||||||
|
if n > 0 {
|
||||||
|
log.Warn("uploads: marked interrupted by a restart (audio is not retained; they must be re-uploaded)", "count", n)
|
||||||
|
}
|
||||||
|
|
||||||
|
worker := uploads.NewWorker(st, newConverter(cfg), buildTranscriber(cfg),
|
||||||
|
cfg.UploadTmpDir, cfg.TranscribeTimeout, uploads.WithMaxDuration(cfg.UploadMaxDuration))
|
||||||
|
intake := uploads.NewIntake(st, &audio.FFprobe{Bin: cfg.FFprobeBin}, worker,
|
||||||
|
uploadLimits(cfg), cfg.UploadTmpDir)
|
||||||
|
return intake, worker, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func uploadLimits(cfg config.Config) uploads.Limits {
|
||||||
|
return uploads.Limits{MaxBytes: cfg.UploadMaxBytes, MaxDuration: cfg.UploadMaxDuration}
|
||||||
|
}
|
||||||
|
|
||||||
|
// removeSpooledAudio deletes the spool files this process family creates
|
||||||
|
// (upload-*, wav-*) and nothing else.
|
||||||
|
func removeSpooledAudio(dir string, log *slog.Logger) {
|
||||||
|
for _, pat := range []string{"upload-*", "wav-*"} {
|
||||||
|
matches, _ := filepath.Glob(filepath.Join(dir, pat))
|
||||||
|
for _, m := range matches {
|
||||||
|
if err := os.Remove(m); err != nil {
|
||||||
|
log.Warn("uploads: remove leftover audio", "path", m, "err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// wireUploads connects a built Intake to the web App: the routes, the limits the
|
||||||
|
// form states, and the chat settings for an uploaded recording. A nil intake
|
||||||
|
// leaves the App untouched — assigning a nil *Intake to the Uploader interface
|
||||||
|
// would produce a non-nil interface and mount the routes over nothing.
|
||||||
|
func wireUploads(app *web.App, cfg config.Config, in *uploads.Intake) {
|
||||||
|
if in == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
app.Uploads = in
|
||||||
|
app.UploadLimits = uploadLimits(cfg)
|
||||||
|
app.UploadChatMaxChars = cfg.MaxUploadedTranscriptChars
|
||||||
|
app.UploadChatModel = cfg.UploadChatModel
|
||||||
|
app.UploadChatModels = uploadChatModels(cfg)
|
||||||
|
}
|
||||||
|
|
||||||
|
// newConverter is the ffmpeg adapter capped at the configured maximum duration, so a file
|
||||||
|
// whose header lies about its length cannot become hours of WAV (ADR-035 review).
|
||||||
|
func newConverter(cfg config.Config) *audio.FFmpeg {
|
||||||
|
return &audio.FFmpeg{Bin: cfg.FFmpegBin, MaxSeconds: int(cfg.UploadMaxDuration.Seconds())}
|
||||||
|
}
|
||||||
|
|
||||||
|
// adjudicatorTimeout bounds one adjudication call. A call takes ~2 s on gemma4-26b and ~10 s
|
||||||
|
// on 31b; this is the wait before an unresponsive model is counted as an error.
|
||||||
|
const adjudicatorTimeout = 2 * time.Minute
|
||||||
|
|
||||||
|
// buildTranscriber builds what the worker transcribes with. By default that is the
|
||||||
|
// single-engine chain (the primary, then the optional cloud fallback). When
|
||||||
|
// TAPIR_RECONCILE_BASE_MODEL is set it is the two-engine ensemble (ADR-034): the base
|
||||||
|
// engine and the primary run on the same WAV and a LOCAL model reconciles them. Config
|
||||||
|
// has already refused any non-local model, so nothing here can send private audio or
|
||||||
|
// text to a cloud model for reconciliation; the cloud fallback is unchanged.
|
||||||
|
func buildTranscriber(cfg config.Config) uploads.Transcriber {
|
||||||
|
// No overall client timeout: a 90 minute recording takes minutes. The deadline is
|
||||||
|
// the worker's per-job context.
|
||||||
|
client := &http.Client{}
|
||||||
|
// Each LOCAL engine call gets its own deadline: a wedged engine does not answer, and
|
||||||
|
// without one it used the whole job before the fallback could run.
|
||||||
|
local := func(model string) transcribe.Backend {
|
||||||
|
return transcribe.WithTimeout(transcribe.NewGateway(cfg.GatewayURL, cfg.GatewayKey, model, client), cfg.TranscribeEngineTimeout)
|
||||||
|
}
|
||||||
|
primary := local(cfg.TranscribeModel)
|
||||||
|
var fallback transcribe.Backend
|
||||||
|
if cfg.TranscribeFallbackModel != "" && cfg.TranscribeFallbackModel != cfg.TranscribeModel {
|
||||||
|
fallback = transcribe.NewGateway(cfg.GatewayURL, cfg.GatewayKey, cfg.TranscribeFallbackModel, client)
|
||||||
|
}
|
||||||
|
if cfg.ReconcileBaseModel == "" {
|
||||||
|
return transcribe.NewChain(primary, fallback)
|
||||||
|
}
|
||||||
|
|
||||||
|
base := local(cfg.ReconcileBaseModel)
|
||||||
|
// Temperature 0 and a short reply: the adjudicator's accuracy was measured that way.
|
||||||
|
chat := llm.New(cfg.GatewayURL, cfg.GatewayKey, cfg.ReconcileModel, adjudicatorTimeout,
|
||||||
|
llm.WithTemperature(0), llm.WithMaxTokens(200))
|
||||||
|
var adj reconcile.Adjudicator = reconcile.NewLLMAdjudicator(chat)
|
||||||
|
if cfg.ReconcileCheckOrder {
|
||||||
|
adj = reconcile.NewOrderChecked(adj)
|
||||||
|
}
|
||||||
|
return transcribe.NewEnsemble(base, primary, fallback, adj, reconcile.Options{
|
||||||
|
BaseEngine: cfg.ReconcileBaseModel, AltEngine: cfg.TranscribeModel, Reconciler: cfg.ReconcileModel,
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The chat over an uploaded recording sends the whole private transcript to the model
|
||||||
|
// the user picks, so the switcher for it may offer local models only (ADR-035 review).
|
||||||
|
|
||||||
|
func TestUploadChatModels_NeverOffersACloudModelButKeepsTheLocalOrder(t *testing.T) {
|
||||||
|
var cfg config.Config
|
||||||
|
cfg.SummarizerModel = "koala/phi4-mini"
|
||||||
|
cfg.FallbackModel = "iguana/gemma4-26b"
|
||||||
|
cfg.CloudFallbackModel = "berget/mistral-small"
|
||||||
|
|
||||||
|
got := uploadChatModels(cfg)
|
||||||
|
|
||||||
|
want := []string{"koala/phi4-mini", "iguana/gemma4-26b"}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("uploadChatModels = %v, want %v: the cloud model that the video chat offers must not be offered for a private recording", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUploadChatModels_ACloudPrimaryIsDroppedToo(t *testing.T) {
|
||||||
|
var cfg config.Config
|
||||||
|
cfg.SummarizerModel = "berget/mistral-small"
|
||||||
|
cfg.FallbackModel = "iguana/gemma4-26b"
|
||||||
|
|
||||||
|
got := uploadChatModels(cfg)
|
||||||
|
|
||||||
|
if !reflect.DeepEqual(got, []string{"iguana/gemma4-26b"}) {
|
||||||
|
t.Errorf("uploadChatModels = %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUploadChatModels_NoLocalModelMeansNoneOfferedNotACloudDefault(t *testing.T) {
|
||||||
|
var cfg config.Config
|
||||||
|
cfg.SummarizerModel = "berget/mistral-small"
|
||||||
|
|
||||||
|
if got := uploadChatModels(cfg); len(got) != 0 {
|
||||||
|
t.Errorf("uploadChatModels = %v, want none", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A valid local TAPIR_UPLOAD_CHAT_MODEL that is not in the video chat's chain used to be
|
||||||
|
// accepted by config and then silently ignored: the shared chat service re-maps a model it
|
||||||
|
// does not offer, so the user got koala/phi4-mini, whose 8,192-token context cannot hold an
|
||||||
|
// hour of speech (second adversarial review).
|
||||||
|
|
||||||
|
func TestChatModels_AnUploadChatModelOutsideTheChainIsOfferedWhenUploadsAreOn(t *testing.T) {
|
||||||
|
var cfg config.Config
|
||||||
|
cfg.SummarizerModel = "koala/phi4-mini"
|
||||||
|
cfg.FallbackModel = "iguana/gemma4-26b"
|
||||||
|
cfg.UploadsEnabled = true
|
||||||
|
cfg.UploadChatModel = "iguana/gemma4-31b"
|
||||||
|
|
||||||
|
service := chatModels(cfg)
|
||||||
|
upload := uploadChatModels(cfg)
|
||||||
|
|
||||||
|
if !reflect.DeepEqual(service, []string{"koala/phi4-mini", "iguana/gemma4-26b", "iguana/gemma4-31b"}) {
|
||||||
|
t.Errorf("chatModels = %v: the shared service must be able to call the configured upload model", service)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(upload, service) {
|
||||||
|
t.Errorf("uploadChatModels = %v, want %v", upload, service)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChatModels_NothingIsAddedWhenUploadsAreOffOrTheModelIsNotLocal(t *testing.T) {
|
||||||
|
var cfg config.Config
|
||||||
|
cfg.SummarizerModel = "koala/phi4-mini"
|
||||||
|
cfg.UploadChatModel = "iguana/gemma4-31b"
|
||||||
|
|
||||||
|
if got := chatModels(cfg); !reflect.DeepEqual(got, []string{"koala/phi4-mini"}) {
|
||||||
|
t.Errorf("uploads off: chatModels = %v, want the video chain unchanged", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg.UploadsEnabled = true
|
||||||
|
cfg.UploadChatModel = "berget/mistral-small"
|
||||||
|
if got := chatModels(cfg); !reflect.DeepEqual(got, []string{"koala/phi4-mini"}) {
|
||||||
|
t.Errorf("a cloud upload chat model was added to the service: %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The converter is told the configured maximum, so a file whose header lies about its
|
||||||
|
// length cannot become hours of WAV (adversarial review F2).
|
||||||
|
|
||||||
|
func TestNewConverter_IsCappedAtTheConfiguredMaximumDuration(t *testing.T) {
|
||||||
|
var cfg config.Config
|
||||||
|
cfg.FFmpegBin = "/usr/local/bin/ffmpeg"
|
||||||
|
cfg.UploadMaxDuration = 90 * time.Minute
|
||||||
|
|
||||||
|
c := newConverter(cfg)
|
||||||
|
|
||||||
|
if c.Bin != "/usr/local/bin/ffmpeg" {
|
||||||
|
t.Errorf("Bin = %q", c.Bin)
|
||||||
|
}
|
||||||
|
if c.MaxSeconds != 5400 {
|
||||||
|
t.Errorf("MaxSeconds = %v, want 5400", c.MaxSeconds)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,196 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/adapters/transcribe"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/config"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/web"
|
||||||
|
)
|
||||||
|
|
||||||
|
// fakeUploadsStore satisfies the store surface buildUploads needs, and counts the
|
||||||
|
// start-up sweep.
|
||||||
|
type fakeUploadsStore struct {
|
||||||
|
sweeps int
|
||||||
|
sweepErr error
|
||||||
|
failedKinds []domain.UploadErrorKind
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeUploadsStore) CreateUpload(context.Context, string, string, domain.Language, bool) (string, error) {
|
||||||
|
return "id", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeUploadsStore) FailUpload(_ context.Context, _, _ string, k domain.UploadErrorKind) error {
|
||||||
|
f.failedKinds = append(f.failedKinds, k)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeUploadsStore) StartTranscribing(context.Context, string, string) error { return nil }
|
||||||
|
|
||||||
|
func (f *fakeUploadsStore) CompleteUpload(context.Context, string, string, domain.Transcription) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeUploadsStore) SaveProgress(context.Context, string, string, []domain.ProgressStep, time.Time) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeUploadsStore) Heartbeat(context.Context, string, string, time.Time) error { return nil }
|
||||||
|
|
||||||
|
func (f *fakeUploadsStore) FailInterruptedUploads(context.Context) (int, error) {
|
||||||
|
f.sweeps++
|
||||||
|
return 2, f.sweepErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func uploadCfg(t *testing.T, enabled bool) config.Config {
|
||||||
|
t.Helper()
|
||||||
|
var cfg config.Config
|
||||||
|
cfg.GatewayURL = "http://gateway.invalid/v1"
|
||||||
|
cfg.UploadsEnabled = enabled
|
||||||
|
cfg.UploadMaxBytes = 200 << 20
|
||||||
|
cfg.UploadTmpDir = filepath.Join(t.TempDir(), "spool")
|
||||||
|
cfg.TranscribeModel = "iguana/kb-whisper"
|
||||||
|
cfg.TranscribeFallbackModel = "berget/kb-whisper"
|
||||||
|
return cfg
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildUploads_DisabledMountsNothingAndSweepsNothing(t *testing.T) {
|
||||||
|
st := &fakeUploadsStore{}
|
||||||
|
|
||||||
|
in, w, err := buildUploads(context.Background(), uploadCfg(t, false), st, quietLog())
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Nil(t, in, "a typed-nil Intake must never reach web.App.Uploads: it would mount the routes over nothing")
|
||||||
|
assert.Nil(t, w)
|
||||||
|
assert.Zero(t, st.sweeps)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildUploads_EnabledFailsInterruptedRowsBeforeAnythingElseStarts(t *testing.T) {
|
||||||
|
st := &fakeUploadsStore{}
|
||||||
|
|
||||||
|
in, w, err := buildUploads(context.Background(), uploadCfg(t, true), st, quietLog())
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, in)
|
||||||
|
require.NotNil(t, w)
|
||||||
|
assert.Equal(t, 1, st.sweeps,
|
||||||
|
"audio is not retained, so rows left queued/transcribing by a restart can never resume: fail them at start-up, not never")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildUploads_SweepFailureStopsStartup(t *testing.T) {
|
||||||
|
st := &fakeUploadsStore{sweepErr: errors.New("db down")}
|
||||||
|
|
||||||
|
_, _, err := buildUploads(context.Background(), uploadCfg(t, true), st, quietLog())
|
||||||
|
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Contains(t, err.Error(), "db down")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildUploads_CreatesTheSpoolDirectoryPrivately(t *testing.T) {
|
||||||
|
cfg := uploadCfg(t, true)
|
||||||
|
|
||||||
|
_, _, err := buildUploads(context.Background(), cfg, &fakeUploadsStore{}, quietLog())
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
info, err := os.Stat(cfg.UploadTmpDir)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.True(t, info.IsDir())
|
||||||
|
assert.Equal(t, os.FileMode(0o700), info.Mode().Perm(), "spooled audio is private third-party content")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildUploads_EnabledWithoutAGatewayIsAnError(t *testing.T) {
|
||||||
|
cfg := uploadCfg(t, true)
|
||||||
|
cfg.GatewayURL = ""
|
||||||
|
|
||||||
|
_, _, err := buildUploads(context.Background(), cfg, &fakeUploadsStore{}, quietLog())
|
||||||
|
|
||||||
|
assert.Error(t, err, "uploads on with nowhere to transcribe must fail loudly at start-up, not at the first upload")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildUploads_RemovesAudioLeftBehindByACrash(t *testing.T) {
|
||||||
|
cfg := uploadCfg(t, true)
|
||||||
|
require.NoError(t, os.MkdirAll(cfg.UploadTmpDir, 0o700))
|
||||||
|
leftovers := []string{"upload-123", "wav-456"}
|
||||||
|
for _, n := range leftovers {
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(cfg.UploadTmpDir, n), []byte("audio"), 0o600))
|
||||||
|
}
|
||||||
|
unrelated := filepath.Join(cfg.UploadTmpDir, "someone-elses-file")
|
||||||
|
require.NoError(t, os.WriteFile(unrelated, []byte("keep"), 0o600))
|
||||||
|
|
||||||
|
_, _, err := buildUploads(context.Background(), cfg, &fakeUploadsStore{}, quietLog())
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
for _, n := range leftovers {
|
||||||
|
_, err := os.Stat(filepath.Join(cfg.UploadTmpDir, n))
|
||||||
|
assert.True(t, os.IsNotExist(err), "%s: audio is never retained, so a spooled file from a crashed run must not survive start-up", n)
|
||||||
|
}
|
||||||
|
_, err = os.Stat(unrelated)
|
||||||
|
assert.NoError(t, err, "only our own spool files are removed")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWireUploads_SetsEverythingTheWebAppNeedsForUploadsAndChat(t *testing.T) {
|
||||||
|
cfg := uploadCfg(t, true)
|
||||||
|
cfg.UploadMaxDuration = 90 * 60 * 1e9
|
||||||
|
cfg.MaxUploadedTranscriptChars = 90000
|
||||||
|
cfg.UploadChatModel = "iguana/gemma4-26b"
|
||||||
|
in, _, err := buildUploads(context.Background(), cfg, &fakeUploadsStore{}, quietLog())
|
||||||
|
require.NoError(t, err)
|
||||||
|
app := &web.App{}
|
||||||
|
|
||||||
|
wireUploads(app, cfg, in)
|
||||||
|
|
||||||
|
assert.NotNil(t, app.Uploads)
|
||||||
|
assert.Equal(t, cfg.UploadMaxBytes, app.UploadLimits.MaxBytes, "the form states the limits Intake enforces")
|
||||||
|
assert.Equal(t, cfg.UploadMaxDuration, app.UploadLimits.MaxDuration)
|
||||||
|
assert.Equal(t, 90000, app.UploadChatMaxChars,
|
||||||
|
"without this the chat would use the 18,000-char caption default and cut an hour to its first third")
|
||||||
|
assert.Equal(t, "iguana/gemma4-26b", app.UploadChatModel)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWireUploads_NilIntakeLeavesTheAppUntouched(t *testing.T) {
|
||||||
|
app := &web.App{}
|
||||||
|
|
||||||
|
wireUploads(app, uploadCfg(t, false), nil)
|
||||||
|
|
||||||
|
assert.Nil(t, app.Uploads, "a nil *Intake must not become a non-nil interface that mounts the routes")
|
||||||
|
assert.Zero(t, app.UploadChatMaxChars)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildTranscriber_DefaultIsTheSingleEngineChainAsBefore(t *testing.T) {
|
||||||
|
cfg := uploadCfg(t, true)
|
||||||
|
|
||||||
|
tr := buildTranscriber(cfg)
|
||||||
|
|
||||||
|
assert.IsType(t, &transcribe.Chain{}, tr, "with no base engine configured nothing changes")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildTranscriber_ABaseEngineTurnsOnTheEnsemble(t *testing.T) {
|
||||||
|
cfg := uploadCfg(t, true)
|
||||||
|
cfg.ReconcileBaseModel = "iguana/pianissimo"
|
||||||
|
cfg.ReconcileModel = "iguana/gemma4-26b"
|
||||||
|
|
||||||
|
tr := buildTranscriber(cfg)
|
||||||
|
|
||||||
|
assert.IsType(t, &transcribe.Ensemble{}, tr)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildUploads_WiresTheEnsembleWhenConfigured(t *testing.T) {
|
||||||
|
cfg := uploadCfg(t, true)
|
||||||
|
cfg.ReconcileBaseModel = "iguana/pianissimo"
|
||||||
|
cfg.ReconcileModel = "iguana/gemma4-26b"
|
||||||
|
|
||||||
|
in, w, err := buildUploads(context.Background(), cfg, &fakeUploadsStore{}, quietLog())
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.NotNil(t, in)
|
||||||
|
assert.NotNil(t, w)
|
||||||
|
}
|
||||||
@@ -0,0 +1,78 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The wiring of the per-engine deadline: a gateway whose primary model never answers
|
||||||
|
// must still reach the fallback model inside the job's own deadline (adversarial review
|
||||||
|
// F4). The fake gateway answers per model from the multipart `model` field.
|
||||||
|
|
||||||
|
func TestBuildTranscriber_AWedgedPrimaryReachesTheFallbackBeforeTheJobDeadline(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
_ = r.ParseMultipartForm(1 << 20)
|
||||||
|
if r.FormValue("model") == "iguana/kb-whisper" {
|
||||||
|
<-r.Context().Done() // wedged: never answers
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = w.Write([]byte(`{"text":"hej","duration":1,"segments":[{"start":0,"end":1,"text":"hej"}]}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
cfg := uploadCfg(t, true)
|
||||||
|
cfg.GatewayURL = srv.URL
|
||||||
|
cfg.TranscribeModel = "iguana/kb-whisper"
|
||||||
|
cfg.TranscribeFallbackModel = "berget/kb-whisper"
|
||||||
|
cfg.TranscribeEngineTimeout = 150 * time.Millisecond
|
||||||
|
wav := filepath.Join(t.TempDir(), "a.wav")
|
||||||
|
require.NoError(t, os.WriteFile(wav, []byte("RIFF....WAVEfmt "), 0o600))
|
||||||
|
job, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
got, err := buildTranscriber(cfg).Transcribe(job, wav)
|
||||||
|
|
||||||
|
require.NoError(t, err, "the wedged primary used the whole job and the fallback never ran")
|
||||||
|
assert.Equal(t, "berget/kb-whisper", got.Endpoint)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildTranscriber_AWedgedBaseEngineIsASoftFailureInTheEnsemble(t *testing.T) {
|
||||||
|
// The same deadline must guard the ensemble's BASE engine, not only the chain's primary
|
||||||
|
// (second adversarial review: the wiring was tested for the chain alone).
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
_ = r.ParseMultipartForm(1 << 20)
|
||||||
|
if r.FormValue("model") == "iguana/pianissimo" {
|
||||||
|
<-r.Context().Done()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = w.Write([]byte(`{"text":"hej","duration":1,"segments":[{"start":0,"end":1,"text":"hej"}]}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
cfg := uploadCfg(t, true)
|
||||||
|
cfg.GatewayURL = srv.URL
|
||||||
|
cfg.TranscribeModel = "iguana/kb-whisper"
|
||||||
|
cfg.ReconcileBaseModel = "iguana/pianissimo"
|
||||||
|
cfg.ReconcileModel = "iguana/gemma4-31b"
|
||||||
|
cfg.TranscribeEngineTimeout = 150 * time.Millisecond
|
||||||
|
wav := filepath.Join(t.TempDir(), "a.wav")
|
||||||
|
require.NoError(t, os.WriteFile(wav, []byte("RIFF....WAVEfmt "), 0o600))
|
||||||
|
job, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
got, err := buildTranscriber(cfg).Transcribe(job, wav)
|
||||||
|
|
||||||
|
require.NoError(t, err, "a wedged base engine must not use up the job")
|
||||||
|
assert.Equal(t, "hej", got.Text)
|
||||||
|
require.NotNil(t, got.Reconciliation)
|
||||||
|
assert.Equal(t, "single_engine", string(got.Reconciliation.Status))
|
||||||
|
assert.Contains(t, got.Reconciliation.Reason, "timeout")
|
||||||
|
}
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The worker buildUploads returns enforces the configured maximum on what was really
|
||||||
|
// converted (adversarial review F2). A stub ffmpeg writes a sparse 100-minute WAV however
|
||||||
|
// short the file claimed to be; the gateway is never reached because the job fails first.
|
||||||
|
|
||||||
|
func TestBuildUploads_TheWorkerRefusesAudioLongerThanTheConfiguredMaximum(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
stub := filepath.Join(dir, "ffmpeg")
|
||||||
|
require.NoError(t, os.WriteFile(stub, []byte("#!/bin/sh\nfor a; do last=\"$a\"; done\ntruncate -s 192000044 \"$last\"\n"), 0o700))
|
||||||
|
cfg := uploadCfg(t, true)
|
||||||
|
cfg.FFmpegBin = stub
|
||||||
|
cfg.UploadMaxDuration = 90 * time.Minute
|
||||||
|
cfg.TranscribeTimeout = time.Minute
|
||||||
|
st := &fakeUploadsStore{}
|
||||||
|
_, w, err := buildUploads(context.Background(), cfg, st, quietLog())
|
||||||
|
require.NoError(t, err)
|
||||||
|
src := filepath.Join(cfg.UploadTmpDir, "upload-x")
|
||||||
|
require.NoError(t, os.WriteFile(src, []byte("audio"), 0o600))
|
||||||
|
|
||||||
|
w.Process(context.Background(), uploads.Job{UserID: "u", UploadID: "id", Path: src, Duration: 10 * time.Second})
|
||||||
|
|
||||||
|
assert.Equal(t, []domain.UploadErrorKind{domain.UploadErrTooLong}, st.failedKinds,
|
||||||
|
"a file that claimed 10 s but converted to 100 minutes must fail, not be sent to an engine")
|
||||||
|
}
|
||||||
@@ -45,7 +45,7 @@ adapter behind an interface (Clean Architecture ports & adapters).
|
|||||||
```mermaid
|
```mermaid
|
||||||
graph TB
|
graph TB
|
||||||
subgraph tapir["Tapir (Go)"]
|
subgraph tapir["Tapir (Go)"]
|
||||||
http["HTTP server<br/>OAuth callbacks +<br/>user-facing API"]
|
http["tapir serve<br/>(HTMX+Templ web surface:<br/>read summaries, connect,<br/>account, summarize)"]
|
||||||
watcher["Watcher<br/>detects new videos<br/>(WebSub + poll)"]
|
watcher["Watcher<br/>detects new videos<br/>(WebSub + poll)"]
|
||||||
engine["Summarization engine<br/>(use-case core)"]
|
engine["Summarization engine<br/>(use-case core)"]
|
||||||
resolver["Transcript resolver<br/>(captions-first)"]
|
resolver["Transcript resolver<br/>(captions-first)"]
|
||||||
@@ -95,6 +95,189 @@ two codebases (ADR-003).
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
## Web surface — `tapir serve` (Stage 1, ADR-011 → ADR-012)
|
||||||
|
|
||||||
|
A later transport added over the **unchanged** engine/ports/sinks core (ADR-003): `tapir serve`
|
||||||
|
is an HTMX+Templ reader/writer (`internal/web`) over the existing `store`. It added no business
|
||||||
|
logic to the engine — it reads the store and, for one action, kicks the existing engine. ADR-011
|
||||||
|
shipped it single-user; ADR-012 opened multi-user with DB-enforced (RLS) isolation.
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
graph TB
|
||||||
|
browser["Browser<br/>(Dex-authenticated user)"]
|
||||||
|
subgraph web["internal/web (tapir serve)"]
|
||||||
|
oidc["oidc<br/>Dex OIDC session<br/>(authenticate-only)"]
|
||||||
|
gate["registration gate<br/>new subject -> /register"]
|
||||||
|
pages["summary list + detail<br/>(read) + actions"]
|
||||||
|
connect["/oauth/youtube/callback<br/>per-user token connect"]
|
||||||
|
account["account<br/>(disconnect, delete)"]
|
||||||
|
summarize["Summarize button<br/>-> background goroutine"]
|
||||||
|
end
|
||||||
|
store[("store<br/>(Postgres, RLS per user)")]
|
||||||
|
engine["Summarization engine<br/>(unchanged core)"]
|
||||||
|
secrets["SecretStore<br/>(per-user token refs)"]
|
||||||
|
|
||||||
|
browser --> oidc
|
||||||
|
oidc --> gate
|
||||||
|
gate --> pages
|
||||||
|
pages --> store
|
||||||
|
connect --> secrets
|
||||||
|
connect --> store
|
||||||
|
account --> store
|
||||||
|
account --> secrets
|
||||||
|
summarize -->|background| engine
|
||||||
|
summarize -->|HTMX status poll| store
|
||||||
|
engine --> store
|
||||||
|
```
|
||||||
|
|
||||||
|
- **Dex OIDC session layer** (`internal/web/oidc`) — **authenticate-only** (ADR-012). It proves
|
||||||
|
*who*; authorization/isolation is the DB's job (RLS), not the session's.
|
||||||
|
- **Registration gate** — a Dex subject with no `users` row is routed to `/register`, which
|
||||||
|
creates the `users` row + the `user_identities` mapping (migration 004). Returning subjects
|
||||||
|
pass straight through.
|
||||||
|
- **Web-initiated YouTube connect** — `/oauth/youtube/connect` → `/oauth/youtube/callback`
|
||||||
|
persists a **per-user** refresh-token ref (`youtube/<userID>/refresh_token`) via `SecretStore`
|
||||||
|
and a `video_connections` row (ADR-006, migration 005). Distinct from the CLI `tapir auth`.
|
||||||
|
- **Account management** — `/account` offers disconnect and **delete account**. Delete removes
|
||||||
|
only Tapir-side state (cascade across the user's tables + secret refs); the shared Dex identity
|
||||||
|
is left intact (ADR-013).
|
||||||
|
- **Immediate summarization** — the web "Summarize" button (`POST /v/{id}/summarize`) fires the
|
||||||
|
engine in a **background goroutine** inside `serve`; the page HTMX-polls `/v/{id}/status`,
|
||||||
|
showing a Charmbracelet spinner while in-flight (and an honest "queued/waiting" state under
|
||||||
|
rate-limiting — ADR-014).
|
||||||
|
- **Summarization mode** — `users.auto_summarize` (migration 006). Default is **true** for new
|
||||||
|
users (migration 011, ADR-018); all existing rows were back-filled via migration 012. Auto:
|
||||||
|
new videos **published within the recency window** (`TAPIR_AUTO_SUMMARIZE_WINDOW`, default ~7d,
|
||||||
|
ADR-020) are summarized automatically; older videos are discovered and listed but wait for an
|
||||||
|
explicit "Summarize". Manual: new videos appear unsummarized; the button sets
|
||||||
|
`videos.summarize_requested`, which the next `tapir run` processes and clears. A manual request
|
||||||
|
bypasses the recency bound. Both the click path and the batch `tapir run` drive the same
|
||||||
|
unchanged engine.
|
||||||
|
- **List surface (ADR-020)** — the list reads `ListVideos` ordered summarized-first, then
|
||||||
|
`published_at DESC NULLS LAST`. The web layer collapses the noise so summaries are not buried:
|
||||||
|
un-summarized videos older than the recency window fold into one "Show N older videos"
|
||||||
|
disclosure, and caption-less videos collapse to a single count line. Copy surfaces scarcity
|
||||||
|
honestly (queue counts, gradual-fill note) — it never implies the feed is fuller than it is.
|
||||||
|
|
||||||
|
The engine, ports, and sink adapters are **untouched** by all of the above — the web surface only
|
||||||
|
reads the store and triggers the existing engine. Adding it changed wiring, not the core (ADR-003).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## In-process scheduler (ADR-018)
|
||||||
|
|
||||||
|
`cmdServe` launches a background goroutine when `TAPIR_DISCOVERY_INTERVAL > 0`. On each tick
|
||||||
|
it calls `store.ListAllUsers` (un-RLS'd admin query), builds a per-user `runner.Runner`, and
|
||||||
|
calls `RunOnce` for each registered user in sequence.
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
sequenceDiagram
|
||||||
|
participant S as Scheduler goroutine
|
||||||
|
participant DB as Postgres (RLS)
|
||||||
|
participant YT as YouTube timedtext
|
||||||
|
participant LLM as LiteLLM gateway
|
||||||
|
|
||||||
|
loop every TAPIR_DISCOVERY_INTERVAL
|
||||||
|
S->>DB: ListAllUsers() [un-RLS'd]
|
||||||
|
loop per user
|
||||||
|
S->>DB: GetAutoSummarize(userID)
|
||||||
|
S->>YT: ListSubscriptions + NewVideos
|
||||||
|
Note over S,DB: auto: skip videos published before<br/>TAPIR_AUTO_SUMMARIZE_WINDOW (ADR-020);<br/>older ones listed, await manual request
|
||||||
|
Note over S,YT: WaitFetchGate(ctx) throttles<br/>all fetches to TAPIR_FETCH_RATE
|
||||||
|
alt transcript available
|
||||||
|
S->>LLM: Summarize
|
||||||
|
S->>DB: Deliver(summary)
|
||||||
|
else 429
|
||||||
|
S->>DB: SetTranscriptStatus(rate_limited)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
```
|
||||||
|
|
||||||
|
**Single-replica constraint (load-bearing).** The scheduler lives in the web process;
|
||||||
|
`replicas: 1` in the k3s deployment manifest is not cosmetic — running `tapir serve` at
|
||||||
|
>1 replica makes every replica run the full discovery loop, causing every registered user
|
||||||
|
to be fetched in parallel from the same egress IP (429s + duplicate work). Do not scale
|
||||||
|
`serve` past 1 replica without first moving discovery to a k8s CronJob or adding leader
|
||||||
|
election. The process logs a `Warn` at startup when scheduled discovery is enabled as a
|
||||||
|
reminder.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Process-wide timedtext rate gate
|
||||||
|
|
||||||
|
**`internal/adapters/youtube/gate.go`** (ADR-014 item 2): a single `rate.Limiter`
|
||||||
|
(`golang.org/x/time/rate`) shared across **all** Adapter instances. Every `httpDo` call for
|
||||||
|
a caption fetch passes through `WaitFetchGate(ctx)` before hitting YouTube. This serialises
|
||||||
|
the scheduler loop AND the web click-path through the same per-egress-IP budget. Configured
|
||||||
|
via `TAPIR_FETCH_RATE` (Go duration, default `2s`). Setting it to `0` disables the gate
|
||||||
|
(dev/tests only).
|
||||||
|
|
||||||
|
This is the precondition that makes scheduled auto-summarize safe: without the gate, a
|
||||||
|
multi-user scheduler pass could fire many concurrent timedtext requests from the same IP
|
||||||
|
within seconds, triggering 429s for all users.
|
||||||
|
|
||||||
|
### Two-path summarisation model
|
||||||
|
|
||||||
|
Both paths share `globalFetchGate` — rate limiting is **respected in both**, not routed around.
|
||||||
|
|
||||||
|
| Path | Trigger | Order | Rationale |
|
||||||
|
|------|---------|-------|-----------|
|
||||||
|
| **Foreground** | User clicks "Summarize" on any non-summarized card (`POST /v/{id}/retry-now` for rate-limited; `POST /v/{id}/summarize` for pending) | Single chosen video | On-demand value: user picks a specific video to read now — bypasses the recency bound |
|
||||||
|
| **Background batch** | Scheduled discovery pass every `TAPIR_DISCOVERY_INTERVAL` | **Newest-first across all channels** (see below), **bounded to the recency window** (ADR-020) | Onboarding prioritisation within bounded load: recent videos auto-fill; the older back-catalogue stays on-demand |
|
||||||
|
|
||||||
|
The rationale for both paths is **onboarding prioritisation under an honest, bounded load** — a
|
||||||
|
new user gets summaries of their most recent videos automatically, while the older back-catalogue
|
||||||
|
is listed but summarised only on demand, so it never re-drives the shared rate gate every cycle.
|
||||||
|
|
||||||
|
### Newest-first batch ordering (ADR-018)
|
||||||
|
|
||||||
|
Within each scheduled pass, `RunOnce` uses a three-phase structure:
|
||||||
|
|
||||||
|
1. **Discover + persist**: walk all channels, `UpsertVideo` every candidate (so it appears in
|
||||||
|
the list), apply pre-filters (seen/manual/backoff/**recency**), collect surviving candidates.
|
||||||
|
The recency pre-filter (ADR-020) drops auto-mode videos published before
|
||||||
|
`now - TAPIR_AUTO_SUMMARIZE_WINDOW` unless they are explicitly requested; an undated video is
|
||||||
|
never aged out. They remain persisted/listed — only auto-summarisation is skipped.
|
||||||
|
2. **Sort**: order candidates `published_at DESC, NULLS LAST, discovery_pos ASC`. Videos with
|
||||||
|
no publish date (schema 001: nullable) sort after all dated content. The sort is in-memory
|
||||||
|
(`slices.SortStableFunc`) — at current scale this is fine.
|
||||||
|
3. **Process**: feed candidates to the engine in sorted order through `globalFetchGate`.
|
||||||
|
|
||||||
|
Before (per-channel inline): `[chanA-old, chanA-mid, chanB-new, chanB-null]`
|
||||||
|
After (newest-first): `[chanB-new, chanA-mid, chanA-old, chanB-null]`
|
||||||
|
|
||||||
|
The set of *processed* videos now also excludes auto-mode back-catalogue beyond the recency
|
||||||
|
window (those stay listed, summarised on demand); within the processed set, only order changes.
|
||||||
|
|
||||||
|
### Connect-time onboarding burst (ADR-018 → ADR-028)
|
||||||
|
|
||||||
|
On a successful YouTube connect, `ConnectHandler` enqueues a connect-triggered discovery pass;
|
||||||
|
the `discoveryTrigger` runs that pass and then fires the **onboarding burst** — a third entry path
|
||||||
|
that summarises up to `TAPIR_ONBOARD_SUMMARIZE_COUNT` (default 3, hard-capped) of the new user's
|
||||||
|
videos so the first session is not empty. The burst still flows through `globalFetchGate` (it is
|
||||||
|
not a throughput change); ADR-028 sharpened *which* videos and *which model*:
|
||||||
|
|
||||||
|
- **Selection** is `OnboardBurstVideoIDs`, not pure newest-first. It keeps newest-first order but
|
||||||
|
excludes a video whose **known** duration is outside `[TAPIR_MIN_VIDEO_SECONDS,
|
||||||
|
TAPIR_ONBOARD_MAX_VIDEO_SECONDS]` (drops Shorts and multi-hour livestream VODs). An unknown
|
||||||
|
(NULL) duration is degrade-open — kept, but ranked after known-good rows. The connect-triggered
|
||||||
|
discovery pass runs *before* the burst, and ADR-023's `videos.list` enrichment now **persists**
|
||||||
|
`duration_s` (instead of discarding it after the Shorts filter), so a fresh user's candidates
|
||||||
|
carry a duration in time for selection.
|
||||||
|
- **Model**: the burst runs through a dedicated summarizer chain led by
|
||||||
|
`TAPIR_ONBOARD_SUMMARIZER_MODEL` (default `iguana/gemma4-26b`, the stronger local model), with
|
||||||
|
the standard ADR-022 chain following as fallback. This is a wiring choice — a second
|
||||||
|
`engineProcessor` over the same store / transcript cache / sink; the engine and ports are
|
||||||
|
unchanged. Empty / equal-to-primary collapses it back onto the shared processor.
|
||||||
|
|
||||||
|
`has-captions` is deliberately **not** a selection signal — it is only knowable after a gate fetch
|
||||||
|
(or a ~0-probability cache hit at pilot scale), so the burst can avoid known-junk but cannot
|
||||||
|
promise captions. Cached-transcript-first selection was investigated and rejected (ADR-028:
|
||||||
|
~3% cross-user overlap).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
## Sequence — core use case: new video summarized
|
## Sequence — core use case: new video summarized
|
||||||
|
|
||||||
```mermaid
|
```mermaid
|
||||||
@@ -191,6 +374,8 @@ Gherkin features in `docs/use-cases/`).
|
|||||||
|
|
||||||
- Audio-download + speech-to-text resolver (ADR-007) — would be an additional `VideoSource`
|
- Audio-download + speech-to-text resolver (ADR-007) — would be an additional `VideoSource`
|
||||||
fallback path, drawn when built.
|
fallback path, drawn when built.
|
||||||
- Multi-tenant isolation primitives (per-tenant Postgres role, NetworkPolicy, tenant label)
|
- Per-user isolation is **live, not deferred**: Postgres RLS `FORCE`d on every user-owned table
|
||||||
— activate at Stage 1 (ADR-002); single-user Stage 0 doesn't exercise them.
|
(ADR-012, migration 003), realising ADR-002's per-tenant intent at the DB layer. The coarser
|
||||||
|
multi-tenant primitives (per-namespace NetworkPolicy, Kyverno, tenant label) remain a
|
||||||
|
Stage-2 hardening item, not exercised yet.
|
||||||
- Public SaaS surface (sign-up, billing) — Future C, not built (ADR-008).
|
- Public SaaS surface (sign-up, billing) — Future C, not built (ADR-008).
|
||||||
|
|||||||
+168
-36
@@ -10,12 +10,20 @@ only opaque references to them; the secret material lives in ESO/1Password (ADR-
|
|||||||
|
|
||||||
## Design decisions baked into this model
|
## Design decisions baked into this model
|
||||||
|
|
||||||
- **Per-user isolation, not a shared global video table.** The earlier draft proposed a
|
- **Per-user isolation for everything except transcripts.** The earlier draft proposed a
|
||||||
global `videos`/`transcripts` table deduped across tenants. Rejected for Future B: it
|
global `videos`/`transcripts` table deduped across tenants. **Videos** stay per-user and
|
||||||
reintroduces exactly the cross-domain coupling the homelab architecture review is
|
RLS-scoped — a shared video table reintroduces exactly the cross-domain coupling the homelab
|
||||||
removing, and at 1–5 users the cost of occasionally re-summarizing the same video is
|
architecture review is removing. **Transcripts**, however, are now shared (ADR-021): keyed by
|
||||||
trivial compared to the isolation it would cost. Each user's data is self-contained.
|
`(provider, provider_video_id)`, no `user_id`, **not** RLS-scoped. The cost avoided there is
|
||||||
(Revisit only if Future C makes GPU/transcription cost dominate — a new ADR, not a default.)
|
not LLM re-summarization but a rate-gated, reputation-risky caption fetch (ADR-010/014), which
|
||||||
|
is paid per re-fetch regardless of user count — so persisting public caption content once and
|
||||||
|
sharing it strictly beats the coupling it removes. Everything else each user owns is
|
||||||
|
self-contained; `rls_test.go` proves transcripts is the single exception for *content*.
|
||||||
|
**Uploaded-audio transcripts are the opposite of public caption content** (private, owned by
|
||||||
|
one user) and therefore live in their own RLS-scoped `uploaded_transcripts` table, never in
|
||||||
|
`transcripts` (ADR-033). The isolation test now also pins the **exact set** of tables with row
|
||||||
|
security off (`transcripts`, `user_identities`, `invitations`, `schema_migrations`), so a new
|
||||||
|
table added without RLS fails CI.
|
||||||
- **Secrets by reference only.** Tables hold a `secret_ref` (opaque string/UUID resolved via
|
- **Secrets by reference only.** Tables hold a `secret_ref` (opaque string/UUID resolved via
|
||||||
the `SecretStore` port), never tokens or keys.
|
the `SecretStore` port), never tokens or keys.
|
||||||
- **The brain sink is just a delivery target.** No brain-specific tables. Whether a summary
|
- **The brain sink is just a delivery target.** No brain-specific tables. Whether a summary
|
||||||
@@ -23,27 +31,40 @@ only opaque references to them; the secret material lives in ESO/1Password (ADR-
|
|||||||
|
|
||||||
## Entities
|
## Entities
|
||||||
|
|
||||||
|
Solid entities below are **persisted today** (migrations 001–013). `AI_CREDENTIAL` and
|
||||||
|
`SUBSCRIPTION` are **planned, not yet a table** — kept in the model for intent; see the notes.
|
||||||
|
|
||||||
```mermaid
|
```mermaid
|
||||||
erDiagram
|
erDiagram
|
||||||
|
USER ||--|| USER_IDENTITY : "logs in via (Dex subject)"
|
||||||
USER ||--o{ VIDEO_CONNECTION : has
|
USER ||--o{ VIDEO_CONNECTION : has
|
||||||
USER ||--o{ AI_CREDENTIAL : has
|
USER ||--o{ SUMMARY_ACTION : records
|
||||||
VIDEO_CONNECTION ||--o{ SUBSCRIPTION : exposes
|
USER ||--o{ AI_CREDENTIAL : "has (planned)"
|
||||||
|
VIDEO_CONNECTION ||--o{ SUBSCRIPTION : "exposes (planned)"
|
||||||
SUBSCRIPTION ||--o{ VIDEO : "produces (per user)"
|
SUBSCRIPTION ||--o{ VIDEO : "produces (per user)"
|
||||||
VIDEO ||--o| TRANSCRIPT : "has at most one"
|
VIDEO }o--o| TRANSCRIPT : "shares one by (provider, provider_video_id) — not FK (ADR-021)"
|
||||||
VIDEO ||--o| SUMMARY : "has at most one"
|
VIDEO ||--o| SUMMARY : "has at most one"
|
||||||
SUMMARY ||--o{ SINK_DELIVERY : "delivered via"
|
SUMMARY ||--o{ SINK_DELIVERY : "delivered via"
|
||||||
|
USER ||--o{ CHANNEL_ERROR : "reports unavailable channels"
|
||||||
|
USER ||--o{ UPLOADED_TRANSCRIPT : "uploads (private, ADR-033)"
|
||||||
|
|
||||||
USER {
|
USER {
|
||||||
uuid id PK
|
uuid id PK
|
||||||
text display_name
|
text display_name
|
||||||
|
bool auto_summarize "default true for new users (migration 011, ADR-018)"
|
||||||
|
timestamptz created_at
|
||||||
|
}
|
||||||
|
USER_IDENTITY {
|
||||||
|
text dex_subject PK
|
||||||
|
uuid user_id FK "UNIQUE -> USER, ON DELETE CASCADE; NOT RLS-enabled"
|
||||||
timestamptz created_at
|
timestamptz created_at
|
||||||
}
|
}
|
||||||
VIDEO_CONNECTION {
|
VIDEO_CONNECTION {
|
||||||
uuid id PK
|
uuid id PK
|
||||||
uuid user_id FK
|
uuid user_id FK "-> USER, ON DELETE CASCADE"
|
||||||
text provider "youtube | vimeo"
|
text provider "youtube | vimeo"
|
||||||
text provider_account
|
text provider_account "nullable"
|
||||||
text token_secret_ref "-> SecretStore, never the token"
|
text token_ref "-> SecretStore, never the token"
|
||||||
text status "active | revoked | error"
|
text status "active | revoked | error"
|
||||||
timestamptz connected_at
|
timestamptz connected_at
|
||||||
}
|
}
|
||||||
@@ -66,28 +87,31 @@ erDiagram
|
|||||||
}
|
}
|
||||||
VIDEO {
|
VIDEO {
|
||||||
uuid id PK
|
uuid id PK
|
||||||
uuid user_id FK
|
uuid user_id FK "-> USER, ON DELETE CASCADE"
|
||||||
uuid subscription_id FK
|
uuid subscription_id "nullable; no FK at Stage 0"
|
||||||
text provider
|
text provider
|
||||||
text provider_video_id
|
text provider_video_id
|
||||||
text title
|
text title
|
||||||
int duration_s
|
int duration_s
|
||||||
timestamptz published_at
|
timestamptz published_at
|
||||||
text url
|
text url
|
||||||
|
bool summarize_requested "default false -> manual-mode queue flag (migration 006)"
|
||||||
timestamptz seen_at
|
timestamptz seen_at
|
||||||
|
text transcript_status "none|rate_limited|fetched (migration 007)"
|
||||||
|
timestamptz rate_limited_at "backoff clock for 429 retries (migration 007)"
|
||||||
}
|
}
|
||||||
TRANSCRIPT {
|
TRANSCRIPT {
|
||||||
uuid video_id PK_FK
|
text provider PK "part of shared key (ADR-021)"
|
||||||
uuid user_id FK
|
text provider_video_id PK "part of shared key — the cross-user dedup key"
|
||||||
text source "captions | none"
|
text source "captions | none"
|
||||||
text language
|
text language
|
||||||
text content "null when source = none"
|
text content "null when source = none"
|
||||||
timestamptz resolved_at
|
timestamptz fetched_at
|
||||||
}
|
}
|
||||||
SUMMARY {
|
SUMMARY {
|
||||||
uuid id PK
|
uuid id PK
|
||||||
uuid user_id FK
|
uuid user_id FK
|
||||||
uuid video_id FK
|
uuid video_id "no FK to videos; (user_id, video_id) UNIQUE is the dedup key"
|
||||||
text summary
|
text summary
|
||||||
jsonb highlights
|
jsonb highlights
|
||||||
jsonb takeaways
|
jsonb takeaways
|
||||||
@@ -98,41 +122,148 @@ erDiagram
|
|||||||
}
|
}
|
||||||
SINK_DELIVERY {
|
SINK_DELIVERY {
|
||||||
uuid id PK
|
uuid id PK
|
||||||
uuid summary_id FK
|
uuid summary_id FK "-> SUMMARY, ON DELETE CASCADE; ownership derived via this FK"
|
||||||
text sink "store | brain"
|
text sink "store | brain"
|
||||||
text status "pending | delivered | error"
|
text status "pending | delivered | error"
|
||||||
text detail "nullable; error message etc"
|
text detail "nullable; error message etc"
|
||||||
timestamptz updated_at
|
timestamptz updated_at
|
||||||
}
|
}
|
||||||
|
SUMMARY_ACTION {
|
||||||
|
uuid id PK
|
||||||
|
uuid user_id FK "-> USER"
|
||||||
|
text video_id "TEXT, not FK (mirrors summaries' standalone key)"
|
||||||
|
text action "watched | skipped | saved"
|
||||||
|
timestamptz acted_at
|
||||||
|
}
|
||||||
|
CHANNEL_ERROR {
|
||||||
|
uuid user_id FK
|
||||||
|
text channel_id
|
||||||
|
text channel_name
|
||||||
|
timestamptz first_seen
|
||||||
|
timestamptz last_seen
|
||||||
|
}
|
||||||
|
UPLOADED_TRANSCRIPT {
|
||||||
|
uuid id PK
|
||||||
|
uuid user_id FK "-> USER, ON DELETE CASCADE, NOT NULL; RLS ENABLE+FORCE (migration 017)"
|
||||||
|
text title
|
||||||
|
text status "queued | transcribing | done | failed"
|
||||||
|
text content "nullable until done; the transcript text"
|
||||||
|
jsonb segments "nullable until done; [{start,end,text}] in seconds"
|
||||||
|
text endpoint "which backend served it: iguana/... | berget/... (berget = audio left the homelab)"
|
||||||
|
float audio_seconds
|
||||||
|
text error_kind "typed: interrupted | transcription_failed — never a provider string"
|
||||||
|
bool had_video "the file carried a video track; only its audio was used (migration 018)"
|
||||||
|
jsonb reconciliation "NULL for a single engine; else status, exact counts and a bounded list of every judged spot (migration 019, ADR-034; all spots stored, ADR-035)"
|
||||||
|
jsonb progress "the job's ordered phases: key, label, state, times, done/total; NULL until the job starts (migration 020, ADR-035)"
|
||||||
|
timestamptz progress_at "when a phase last really changed or advanced (migration 020)"
|
||||||
|
timestamptz alive_at "heartbeat from the running job, every ~10 s; not progress (migration 020)"
|
||||||
|
timestamptz created_at
|
||||||
|
timestamptz finished_at
|
||||||
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
`SUMMARY_ACTION` has `UNIQUE (user_id, video_id, action)`; `VIDEO_CONNECTION` has
|
||||||
|
`UNIQUE (user_id, provider)` (one connection per provider — reconnect upserts in place).
|
||||||
|
RLS (`ENABLE` + `FORCE`) is on **every solid user-owned table above** — `users`, `videos`,
|
||||||
|
`summaries`, `summary_actions`, `video_connections`, `channel_errors`, `uploaded_transcripts`.
|
||||||
|
(`transcripts` is deliberately **not** in this list: since ADR-021 it is shared public caption
|
||||||
|
content with no `user_id`.)
|
||||||
|
`sink_deliveries` is RLS'd via an `EXISTS` on its parent summary; `user_identities` is
|
||||||
|
intentionally **not** RLS'd (auth plumbing). See the *Isolation invariant* section for the
|
||||||
|
mechanism.
|
||||||
|
|
||||||
## Notes per entity
|
## Notes per entity
|
||||||
|
|
||||||
- **USER** — at Stage 0 there is exactly one row. At Stage 1, identity comes via Dex; this
|
- **USER** — one row per registered user (Stage 1, ADR-012; no longer single-row). The Tapir-side
|
||||||
table holds the Tapir-side profile keyed to the Dex subject.
|
profile; the Dex identity is held separately in `USER_IDENTITY`, not on this row. `auto_summarize`
|
||||||
- **VIDEO_CONNECTION** — a connected YouTube/Vimeo account. `token_secret_ref` resolves to
|
(migration 006) is the per-user mode flag: `TRUE` = auto-summarize new videos **published within
|
||||||
the OAuth refresh token via `SecretStore`. Revocation flips `status`, doesn't delete history.
|
the recency window** (`TAPIR_AUTO_SUMMARIZE_WINDOW`, default ~7d, ADR-020); older videos are
|
||||||
- **AI_CREDENTIAL** — optional, per provider, per user (ADR-004's Fallback). Absent for users
|
listed but summarised on demand. Default is **true** for new users (migration 011, ADR-018);
|
||||||
who only use the local stack. One row per provider max.
|
existing rows were back-filled via migration 012 with RLS bypass.
|
||||||
- **SUBSCRIPTION** — a watched channel. `websub_expires` tracks the YouTube push lease so the
|
- **USER_IDENTITY** (migration 004) — the `dex_subject → user_id` map. `dex_subject` is the PK,
|
||||||
watcher knows when to re-subscribe; null for poll-based (Vimeo).
|
`user_id` a `UNIQUE` FK to `users` with `ON DELETE CASCADE`. This is the bridge resolved at login
|
||||||
|
*before* a `user_id` is known, so it is **deliberately not RLS-enabled** (it holds no user data;
|
||||||
|
RLS here would deadlock the lookup that yields the id used for scoping). Account deletion cascades
|
||||||
|
the mapping away (ADR-013).
|
||||||
|
- **VIDEO_CONNECTION** (migration 005) — a connected YouTube/Vimeo account. `token_ref` resolves to
|
||||||
|
the OAuth refresh token via `SecretStore` (per-user scheme `youtube/<userID>/refresh_token`).
|
||||||
|
`UNIQUE (user_id, provider)`: one connection per provider, reconnect upserts. Revocation/disconnect
|
||||||
|
flips `status`, doesn't delete history. FORCE RLS'd.
|
||||||
|
- **AI_CREDENTIAL** — *planned, no table yet.* Optional, per provider, per user (ADR-004's Fallback).
|
||||||
|
BYO keys are currently resolved via `SecretStore` refs without a dedicated table; this entity is
|
||||||
|
modelled for when per-credential metadata is needed.
|
||||||
|
- **SUBSCRIPTION** — *planned, no table yet.* A watched channel; `websub_expires` would track the
|
||||||
|
YouTube push lease. At Stage 0/1 `videos.subscription_id` is a nullable column with **no FK** (the
|
||||||
|
subscriptions table is not part of the shipped store-sink slice — migration 001).
|
||||||
- **VIDEO** — one row per (user, video) — note `user_id`, reflecting the per-user-isolation
|
- **VIDEO** — one row per (user, video) — note `user_id`, reflecting the per-user-isolation
|
||||||
decision. The same video seen by two users is two rows. `seen_at` is when Tapir detected it.
|
decision. The same video seen by two users is two rows. `seen_at` is when Tapir detected it.
|
||||||
- **TRANSCRIPT** — at most one per video. `source = none` records "checked, no usable
|
`summarize_requested` (migration 006) is the manual-mode queue flag: the web "Summarize" button
|
||||||
transcript" so the watcher doesn't reprocess (ADR-007). `content` null in that case.
|
sets it `TRUE`; the next `tapir run` picks it up, summarizes, and clears it back to `FALSE`.
|
||||||
|
`transcript_status` and `rate_limited_at` (migration 007) track caption-fetch outcomes for
|
||||||
|
rate-limit backoff: `NULL` = not attempted; `rate_limited` = 429 seen, skip until
|
||||||
|
`NOW() - rate_limited_at > TAPIR_FETCH_BACKOFF`; `fetched` = resolved; `none` = no transcript.
|
||||||
|
- **TRANSCRIPT** — shared public caption content, one row per `(provider, provider_video_id)`,
|
||||||
|
**not** RLS-scoped and carrying no `user_id` (ADR-021). Two users who watch the same video
|
||||||
|
share the one row; the summarize path reads it before any caption fetch, so re-analysis never
|
||||||
|
re-touches YouTube (ADR-010/014). `source = none` records "checked, no usable transcript" so
|
||||||
|
no one reprocesses (ADR-007); `content` null in that case. A transient 429 is never stored
|
||||||
|
here — it stays a per-user retry via `VIDEO.transcript_status`.
|
||||||
- **SUMMARY** — at most one per video. `fallback_used` + `ai_provider`/`ai_model` make the
|
- **SUMMARY** — at most one per video. `fallback_used` + `ai_provider`/`ai_model` make the
|
||||||
"is local good enough?" question queryable (the Stage 0 quality signal). `highlights`/
|
"is local good enough?" question queryable (the Stage 0 quality signal). `highlights`/
|
||||||
`takeaways` as jsonb to stay schema-flexible while the output format settles.
|
`takeaways` as jsonb to stay schema-flexible while the output format settles.
|
||||||
- **SINK_DELIVERY** — one row per (summary, sink) attempt. This is where "also sent to brain"
|
- **SINK_DELIVERY** — one row per (summary, sink) attempt. This is where "also sent to brain"
|
||||||
lives — no brain tables, just a delivery row with `sink = brain`. Sinks fail independently;
|
lives — no brain tables, just a delivery row with `sink = brain`. Sinks fail independently;
|
||||||
a failed brain delivery doesn't fail the store delivery.
|
a failed brain delivery doesn't fail the store delivery. No own `user_id`; RLS ownership is
|
||||||
|
derived from the parent summary via `EXISTS` (migration 003).
|
||||||
|
- **SUMMARY_ACTION** (migration 002) — records the maintainer's act on a summary (watch / skip /
|
||||||
|
save) — the column that makes the Stage-0 headline metric ("acts on ≥1 summary") queryable
|
||||||
|
(ui-spec.md §5, ADR-011). `video_id` is `TEXT` and **not** FK-constrained, mirroring summaries'
|
||||||
|
standalone `(user_id, video_id)` key. `UNIQUE (user_id, video_id, action)`. FORCE RLS'd.
|
||||||
|
- **CHANNEL_ERRORS** (migration 013) — channels that returned HTTP 404 (deleted or private) on
|
||||||
|
the most recent discovery pass. Upserted per scheduler pass (`last_seen` refreshed each run);
|
||||||
|
surfaced on the account page as a warning. Cascades on user deletion. Primary key is
|
||||||
|
`(user_id, channel_id)`. FORCE RLS'd.
|
||||||
|
- **UPLOADED_TRANSCRIPTS** (migration 017, ADR-033) — the **private** transcript of audio a user
|
||||||
|
uploaded. Deliberately a separate table from the shared `transcripts`: that one is non-RLS on the
|
||||||
|
grounds that it holds only public caption content, and a recorded meeting is the opposite.
|
||||||
|
`user_id NOT NULL ... ON DELETE CASCADE`, FORCE RLS'd. `status` runs `queued → transcribing →
|
||||||
|
done | failed`; `done` requires `content`, `failed` requires a typed `error_kind` (CHECK
|
||||||
|
constraints). `reconciliation` (migration 019, ADR-034) is NULL when one engine produced the text, so the page can never claim a second opinion that was not there; otherwise it holds the status (`reconciled` / `single_engine` / `skipped`), exact counts (differences, settled automatically, decided by the model, changed, uncertain, model errors) and up to 2,000 judged spots (ADR-035: every one the model decided, kept or changed or unsure, each a position in seconds, both readings, which was used, and the few words before it; the page lists only the changed and uncertain ones). `progress`, `progress_at` and `alive_at` (migration 020, ADR-035) let the page show the job's phases and tell a running job from a quiet one from a lost service: `alive_at` is a heartbeat from a goroutine independent of any engine call, `progress_at` moves only when work really moved; the saved `segments` carry the reconciled words with the base engine's timing. The spots hold fragments of the private transcript, which is why they live in this RLS-scoped row. `had_video` (migration 018) records that the file carried a video track — a Teams meeting with the camera off is still a video file — so the page can say only its audio was used and the video was discarded. `endpoint` records which backend served it — `berget/...` means the audio left the
|
||||||
|
homelab. The audio itself is never stored. `segments` keeps the timestamps (re-transcribing 90
|
||||||
|
minutes costs ~7). Unlike `summary_actions`/`login_events` it **does** cascade on user deletion.
|
||||||
|
- **LOGIN_EVENTS** (migration 010) — throttled one-row-per-(user, date) login stamp. Used by the
|
||||||
|
Stage-0 gate query (VISION §Stage 0: "returned and used in ≥2 distinct weeks").
|
||||||
|
|
||||||
## Isolation invariant (Stage 1+)
|
## Isolation invariant (Stage 1+) — LIVE
|
||||||
|
|
||||||
Every user-owned table carries `user_id`. At Stage 1, this is enforced at the DB layer via a
|
Every user-owned table carries `user_id`, and isolation is **enforced at the DB layer**, not
|
||||||
per-tenant Postgres role + row grants (architecture review SC7), not only in application code.
|
only in application code. ADR-011 shipped this surface single-user (one allowlisted subject,
|
||||||
At Stage 0 (single user) the column exists but the enforcement is dormant. The isolation test
|
enforcement dormant); **ADR-012 opened Stage 1 and turned enforcement on in the same slice.**
|
||||||
in VISION Stage 2 asserts user A cannot read user B's rows.
|
|
||||||
|
Enforcement is **Postgres Row-Level Security** (migration `003_rls.up.sql`):
|
||||||
|
|
||||||
|
- RLS is `ENABLE`d **and** `FORCE`d on every user-owned table — `users`, `videos`,
|
||||||
|
`summaries`, `summary_actions`, `video_connections`, `channel_errors`, and (migration 017)
|
||||||
|
`uploaded_transcripts`. `transcripts` is **not** one of them: ADR-021 made it shared public
|
||||||
|
content, and migration 003's original per-user `transcripts` policy was dropped with that
|
||||||
|
table in migration 015.
|
||||||
|
`FORCE` is load-bearing: the app connects as the table **owner** (`tapir` role), and owners
|
||||||
|
bypass RLS unless forced.
|
||||||
|
- Each policy keys off the per-request GUC `tapir.current_user_id`, set transaction-locally by
|
||||||
|
the store's `withUser` helper via `set_config('tapir.current_user_id', $1, true)` — it
|
||||||
|
auto-resets on commit/rollback, so it never leaks across a pooled connection.
|
||||||
|
- `current_setting('tapir.current_user_id', true)` uses `missing_ok = true`: an **unset** GUC
|
||||||
|
yields `NULL`, the predicate matches no rows, and access **denies by default**.
|
||||||
|
- `sink_deliveries` has no `user_id`; its policy derives ownership from the parent summary via
|
||||||
|
`EXISTS (SELECT 1 FROM summaries …)`.
|
||||||
|
- `user_identities` (the Dex-subject → user_id map) is **deliberately not RLS-enabled** — it is
|
||||||
|
auth plumbing read *before* a user_id is known; putting RLS there would deadlock. It holds no
|
||||||
|
user data.
|
||||||
|
|
||||||
|
The Stage-2 isolation bar is **pulled forward, not deferred**: `internal/adapters/store/rls_test.go`
|
||||||
|
runs two users against a non-superuser, non-`BYPASSRLS` role and asserts user A reads/writes zero
|
||||||
|
of user B's rows across every table. It ships green with the multi-user features (ADR-012); no
|
||||||
|
multi-user feature merges ahead of it passing.
|
||||||
|
|
||||||
## Job / processing state
|
## Job / processing state
|
||||||
|
|
||||||
@@ -144,7 +275,8 @@ queue, doesn't replace it). Deferred until there's a reason.
|
|||||||
|
|
||||||
## Explicitly out of scope (Future C)
|
## Explicitly out of scope (Future C)
|
||||||
|
|
||||||
- Global cross-tenant video/transcript dedup (rejected above).
|
- Global cross-tenant *video* dedup (rejected above). Note: cross-tenant *transcript* sharing
|
||||||
|
is now in scope and shipped (ADR-021); only the videos half stays per-user.
|
||||||
- Sharding / per-tenant physical databases.
|
- Sharding / per-tenant physical databases.
|
||||||
- Soft-delete + full audit trail on connections/credentials (a Stage 2 hardening item; add
|
- Soft-delete + full audit trail on connections/credentials (a Stage 2 hardening item; add
|
||||||
via ADR when Stage 2 work starts).
|
via ADR when Stage 2 work starts).
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
The concrete endpoints, conventions, and identifiers Tapir depends on, so an independent
|
The concrete endpoints, conventions, and identifiers Tapir depends on, so an independent
|
||||||
session doesn't have to rediscover them. **Verify anything marked "confirm" before relying on
|
session doesn't have to rediscover them. **Verify anything marked "confirm" before relying on
|
||||||
it** — endpoints and aliases drift, and this file is a snapshot (2026-06-02), not a live source.
|
it** — endpoints and aliases drift, and this file is a snapshot (2026-06-06), not a live source.
|
||||||
|
|
||||||
## Local AI (the Primary in `llm.Router`)
|
## Local AI (the Primary in `llm.Router`)
|
||||||
|
|
||||||
@@ -12,6 +12,29 @@ it** — endpoints and aliases drift, and this file is a snapshot (2026-06-02),
|
|||||||
- **Off-cluster (host / NodePort):** `http://koala:30401/v1/` — service `litellm-nodeport` (`4000:30401`).
|
- **Off-cluster (host / NodePort):** `http://koala:30401/v1/` — service `litellm-nodeport` (`4000:30401`).
|
||||||
- NOTE: the earlier `31234` was **wrong** — `31234` is the `llama-swap` NodePort (`8080:31234`),
|
- NOTE: the earlier `31234` was **wrong** — `31234` is the `llama-swap` NodePort (`8080:31234`),
|
||||||
not LiteLLM. Use `30401` for off-cluster LiteLLM. `confirm` resolved.
|
not LiteLLM. Use `30401` for off-cluster LiteLLM. `confirm` resolved.
|
||||||
|
- **Speech-to-text (ADR-033):** the same gateway serves `POST /v1/audio/transcriptions` for
|
||||||
|
`iguana/kb-whisper` (primary, local) and `berget/kb-whisper` (fallback, EU cloud — private audio
|
||||||
|
leaves the homelab if it is used: when the primary fails **or returns an empty transcript**,
|
||||||
|
e.g. a silent or music-only file; the transcript page then says so, #52). **Call it via the NodePort / in-cluster address above, NOT
|
||||||
|
`https://llm-api.d-ma.be`:** that public ingress (openresty) returns `413` for bodies over
|
||||||
|
roughly 8–10 MB (~5 minutes of 16 kHz WAV), while the NodePort took 174 MB (90 minutes) in one
|
||||||
|
request (measured 2026-10-01, `docs/research/audio-length-ceiling-2026-10-01.md`). Send multipart
|
||||||
|
field `file`, `response_format=verbose_json`, no `Expect: 100-continue`. iguana's whisper-server
|
||||||
|
can wedge permanently after a Metal OOM (infra#273); never branch on its
|
||||||
|
`{"error":"failed to process audio"}` string — it has had at least five causes. Tapir's knobs:
|
||||||
|
`TAPIR_UPLOADS_ENABLED` (off by default), `TAPIR_UPLOAD_MAX_MB` (200), `TAPIR_UPLOAD_MAX_MINUTES`
|
||||||
|
(90), `TAPIR_TRANSCRIBE_MODEL`, `TAPIR_TRANSCRIBE_FALLBACK_MODEL` (empty disables the cloud
|
||||||
|
fallback), `TAPIR_TRANSCRIBE_TIMEOUT` (60m, the whole job), `TAPIR_TRANSCRIBE_ENGINE_TIMEOUT` (20m, one engine call: a wedged engine then fails while the job lives and the fallback can run), `TAPIR_UPLOAD_TMP_DIR`, `TAPIR_FFMPEG_BIN`,
|
||||||
|
`TAPIR_FFPROBE_BIN`, `TAPIR_MAX_UPLOADED_TRANSCRIPT_CHARS` (90000 — the measured ceiling of
|
||||||
|
`iguana/gemma4-26b`, which silently drops the head above it), `TAPIR_UPLOAD_CHAT_MODEL`
|
||||||
|
(`iguana/gemma4-26b`; must be a local `koala/` or `iguana/` alias, since the whole private transcript is sent to it; `koala/phi4-mini`'s 8,192-token context 400s on an hour of speech). `ffmpeg` + `ffprobe` are in the runtime image (static, pinned by sha256; #34 decided A on 2026-10-02); Tapir's own ingress body limit for a 200 MB upload is unverified (infra).
|
||||||
|
**Two-engine mode (ADR-034), off unless `TAPIR_RECONCILE_BASE_MODEL` is set:** it runs
|
||||||
|
`iguana/pianissimo` (the base text) and `TAPIR_TRANSCRIBE_MODEL` (kb-whisper) on the same audio, then a
|
||||||
|
LOCAL chat model (`TAPIR_RECONCILE_MODEL`) chooses between their disagreements. **Deployment choice (2026-10-02): `iguana/gemma4-31b`** (97% against a speaker's blind answers; 92% for `iguana/gemma4-26b`). With iguana's macOS **Low Power Mode off** the 31B decodes at 22 tok/s and reconciled a 63.6-minute recording in 21 min 49 s end to end; with it on it ran at 8 tok/s. Check `pmset -g | grep lowpowermode` before blaming a model or an engine for slowness.
|
||||||
|
`TAPIR_RECONCILE_CHECK_ORDER=true` asks twice per dispute with the order swapped (doubles the time).
|
||||||
|
Every model must start `koala/` or `iguana/`: a cloud alias is a start-up error. The default job timeout is
|
||||||
|
60 minutes. `iguana/pianissimo` is KlangAI's Swedish ASR (a Parakeet/FastConformer on MLX, served by a small
|
||||||
|
FastAPI shell on iguana:8102) and speaks the same `/v1/audio/transcriptions` API.
|
||||||
- **Auth key:** treat as a secret via the SecretStore port; do not hardcode in committed code.
|
- **Auth key:** treat as a secret via the SecretStore port; do not hardcode in committed code.
|
||||||
NOTE: the documented `sk-local-123` returned `401` on `30401/v1/models` (2026-06-02) — it is
|
NOTE: the documented `sk-local-123` returned `401` on `30401/v1/models` (2026-06-02) — it is
|
||||||
stale or the gateway now enforces the `LITELLM_MASTER_KEY` (1Password `op://HomeLab/LITELLM_MASTER_KEY`).
|
stale or the gateway now enforces the `LITELLM_MASTER_KEY` (1Password `op://HomeLab/LITELLM_MASTER_KEY`).
|
||||||
@@ -27,6 +50,30 @@ it** — endpoints and aliases drift, and this file is a snapshot (2026-06-02),
|
|||||||
`iguana/deepseek-r1-14b`) is preferred for summary quality if its latency/output is acceptable.
|
`iguana/deepseek-r1-14b`) is preferred for summary quality if its latency/output is acceptable.
|
||||||
The `max_tokens` fix below means thinking models no longer return empty content, so they are now
|
The `max_tokens` fix below means thinking models no longer return empty content, so they are now
|
||||||
viable choices, not blocked ones. Do not assume a coder alias is right for prose.
|
viable choices, not blocked ones. Do not assume a coder alias is right for prose.
|
||||||
|
- **Summarizer fallback chain (ADR-022).** The primary alias is the *first* of an ordered chain;
|
||||||
|
on failure or unparseable output the summarizer advances to the next model. All reached through
|
||||||
|
the same gateway by alias.
|
||||||
|
- `TAPIR_FALLBACK_MODEL` — local fallback. **Default `iguana/gemma4-26b`** — on iguana, NOT
|
||||||
|
koala, so the fallback does not compete with koala's other GPU loads (and runs from a different
|
||||||
|
egress IP). Empty disables it.
|
||||||
|
- `TAPIR_CLOUD_FALLBACK_MODEL` — worst-case EXTERNAL fallback. **Default `berget/mistral-small`.**
|
||||||
|
**Set this empty (`""`) for any client/NDA deployment** so content never leaves the local
|
||||||
|
stack — the chain then contains only local endpoints.
|
||||||
|
- `TAPIR_SUMMARY_MAX_TOKENS` — per-summary completion budget. **Default `1500`.** Small on
|
||||||
|
purpose: with the old 8192 budget, prompt + completion overflowed `phi4-mini`'s 8k window.
|
||||||
|
- `TAPIR_MAX_TRANSCRIPT_CHARS` — transcript truncation budget sent to the model. **Default
|
||||||
|
`18000`** (~fits an 8k-context model). `0` disables truncation. Prevents the context-overflow
|
||||||
|
HTTP 400 a long transcript caused on `phi4-mini`.
|
||||||
|
- **Discovery low-value filter (ADR-023).** `TAPIR_MIN_VIDEO_SECONDS` — **default `60`**. At
|
||||||
|
discovery, `NewVideos` enriches candidates with one cheap `videos.list` call (quota API, NOT
|
||||||
|
the timedtext 429 path) and drops videos shorter than this plus any live/upcoming broadcast,
|
||||||
|
so the scarce caption-fetch budget isn't spent on Shorts. `0` disables the filter. The
|
||||||
|
paste-a-URL path is never filtered.
|
||||||
|
- **Per-channel caption memory (ADR-024).** `TAPIR_CHANNEL_CAPTIONLESS_THRESHOLD` — **default
|
||||||
|
`5`** consecutive no-caption results before a channel is suppressed (its videos listed but not
|
||||||
|
caption-fetched). `TAPIR_CHANNEL_CAPTIONLESS_WINDOW` — **default `336h`** (14d) suppression
|
||||||
|
before one video is re-probed. `THRESHOLD=0` disables. A successful fetch resets the channel;
|
||||||
|
a 429 does not count; an explicit manual request bypasses suppression.
|
||||||
- **Thinking models need an explicit `max_tokens`.** qwen3 / deepseek-r1 spend the budget on
|
- **Thinking models need an explicit `max_tokens`.** qwen3 / deepseek-r1 spend the budget on
|
||||||
reasoning and return **empty content** if `max_tokens` is too low (or unset). The summarizer's
|
reasoning and return **empty content** if `max_tokens` is too low (or unset). The summarizer's
|
||||||
parser treats an empty summary as an error for exactly this reason. **Done (2026-06-02, Worker F):**
|
parser treats an empty summary as an error for exactly this reason. **Done (2026-06-02, Worker F):**
|
||||||
@@ -160,7 +207,7 @@ allow per-provider when a user connects one.
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
_Snapshot date 2026-06-02. Items marked **confirm** were not verified to a pinned source at
|
_Snapshot date 2026-06-06. Items marked **confirm** were not verified to a pinned source at
|
||||||
snapshot time — check brain or the live cluster before depending on them._
|
snapshot time — check brain or the live cluster before depending on them._
|
||||||
|
|
||||||
## Stage 1 — multi-user facts (verified 2026-06-03)
|
## Stage 1 — multi-user facts (verified 2026-06-03)
|
||||||
@@ -191,3 +238,37 @@ snapshot time — check brain or the live cluster before depending on them._
|
|||||||
- `user_identities(dex_subject → user_id)` table is **intentionally NOT RLS-enabled**
|
- `user_identities(dex_subject → user_id)` table is **intentionally NOT RLS-enabled**
|
||||||
(it's auth plumbing, holds no user data; data isolation is on the user-owned tables).
|
(it's auth plumbing, holds no user data; data isolation is on the user-owned tables).
|
||||||
All data access after subject resolution goes through `withUser`.
|
All data access after subject resolution goes through `withUser`.
|
||||||
|
|
||||||
|
## Scheduled discovery (ADR-018, verified 2026-06-05)
|
||||||
|
|
||||||
|
`tapir serve` runs discovery for **all users** in-process on a timer (no CronJob). Three env
|
||||||
|
knobs plus one load-bearing deployment constraint:
|
||||||
|
|
||||||
|
- `TAPIR_DISCOVERY_INTERVAL` — Go duration, e.g. `2h`. The cadence the serve process runs a
|
||||||
|
discovery pass for every registered user (run-once-on-startup, then every interval).
|
||||||
|
**Unset or `0` = disabled** (dev/tests never auto-fetch).
|
||||||
|
- `TAPIR_USAGE_GATE_START` — `YYYY-MM-DD`, default **`2026-06-11`** (the morning the pilot was
|
||||||
|
unblocked and summaries started flowing). `tapir report` counts return-usage (distinct active
|
||||||
|
weeks, ADR-016) only from this date, so pre-launch testing and the blocked period are excluded.
|
||||||
|
- `TAPIR_METRICS_ADDR` — listen address for the Prometheus `/metrics` endpoint (ADR-030).
|
||||||
|
**Default `:9090`** — a SEPARATE port from `TAPIR_HTTP_ADDR` so metrics are never on the public
|
||||||
|
app; scraped in-cluster only (PodMonitor). Empty disables the metrics server. Key series:
|
||||||
|
`tapir_summarize_duration_seconds{model,outcome,fallback}`, `tapir_caption_fetch_duration_seconds{outcome}`,
|
||||||
|
`tapir_chat_duration_seconds{model}`, `tapir_llm_tokens_total{model,kind}`,
|
||||||
|
`tapir_http_request_duration_seconds{method,route}`, `tapir_logins_total`.
|
||||||
|
- `TAPIR_FETCH_RATE` — Go duration, default `2s`. The **process-wide per-egress-IP caption-fetch
|
||||||
|
rate gate** (ADR-014 item 2). Every caption fetch — scheduler runners *and* the web "Summarize"
|
||||||
|
click-path — serialises through this one limiter so the pod cannot collectively trip 429s. `0`
|
||||||
|
= unlimited (dev/tests). This is the precondition that makes auto-summarize-on-a-schedule safe;
|
||||||
|
do not raise it aggressively without watching for 429s.
|
||||||
|
- `TAPIR_FETCH_BACKOFF=4h` — per-video rate-limit retry window; default `1h`. A video that
|
||||||
|
returns HTTP 429 on a caption fetch is skipped for this duration before being retried. The
|
||||||
|
scheduler checks `NOW() - rate_limited_at > TAPIR_FETCH_BACKOFF` before attempting to fetch
|
||||||
|
a video marked `transcript_status = rate_limited`. Longer values reduce 429 pressure at the
|
||||||
|
cost of slower recovery after a throttling episode.
|
||||||
|
- **SINGLE-REPLICA WARNING (load-bearing).** The scheduler lives in the web process, so
|
||||||
|
`replicas: 1` in the deployment manifest is load-bearing: running `tapir serve` at >1 replica
|
||||||
|
makes **every** replica run the discovery loop → every user fetched in parallel from the same
|
||||||
|
egress IP (429s + duplicate work). Do **not** scale `serve` past 1 replica without first moving
|
||||||
|
discovery to a k8s CronJob or adding leader election. The process logs a `Warn` at startup when
|
||||||
|
scheduled discovery is enabled, as a reminder.
|
||||||
|
|||||||
@@ -0,0 +1,81 @@
|
|||||||
|
# kb-whisper vs pianissimo on one real hour of Swedish speech (tapir#33, 2026-10-01)
|
||||||
|
|
||||||
|
One recording, one speaker: a 63.7-minute Teams session of the maintainer dictating project
|
||||||
|
notes (domain-heavy, English loanwords and product names mixed into Swedish). **No transcript
|
||||||
|
text and no domain terms are recorded here**: this repo is public and the content is private.
|
||||||
|
Only aggregate numbers.
|
||||||
|
|
||||||
|
Backends, both through the LiteLLM gateway and Tapir's own adapter, unchanged:
|
||||||
|
`iguana/kb-whisper` (whisper.cpp, KB's Swedish fine-tune) and `iguana/pianissimo` (KlangAI
|
||||||
|
Swedish Parakeet/FastConformer-TDT, MLX 8-bit, served by a small FastAPI shell on iguana:8102).
|
||||||
|
|
||||||
|
## Result: a near tie on accuracy, a large difference in speed
|
||||||
|
|
||||||
|
| | kb-whisper | pianissimo |
|
||||||
|
|---|---|---|
|
||||||
|
| Wall time for 63.7 min | 4m08s (~15x realtime) | **40 s (~95x realtime)** |
|
||||||
|
| Segments | 1,302 (timed pieces, wrapped at ~60 chars) | 434 (sentences) |
|
||||||
|
| Words (normalised) | 6,606 | 6,510 |
|
||||||
|
| Punctuation | light, every segment ends in a full stop | heavier (138 vs 82 commas) |
|
||||||
|
| Repetition loops | none | none |
|
||||||
|
|
||||||
|
Word-level agreement between them: **92.4%** as first computed, **93.1%** after the segment
|
||||||
|
rejoin fix below. 380 aligned disagreement spans, median one word each.
|
||||||
|
|
||||||
|
**Blind A/B by the speaker** (the only ground truth available), on the 30 spans a model
|
||||||
|
(Claude Opus 5.5) had picked as changing meaning, shuffled and unlabelled: **pianissimo 17,
|
||||||
|
kb-whisper 13**, no "neither", no "can't tell".
|
||||||
|
|
||||||
|
| Category of the 30 | pianissimo | kb-whisper |
|
||||||
|
|---|---|---|
|
||||||
|
| Technical term / English loanword | 8 | 4 |
|
||||||
|
| Number / date / unit | 4 | 1 |
|
||||||
|
| Name / proper noun | 3 | 3 |
|
||||||
|
| Function word / small word | 1 | 2 |
|
||||||
|
| Other (everyday words) | 1 | 3 |
|
||||||
|
|
||||||
|
The reading: pianissimo is better on terminology and numbers, kb-whisper on ordinary Swedish
|
||||||
|
words; names are even. Their errors differ in kind: kb-whisper's tend to be garbled words, a
|
||||||
|
couple were wrong numbers; pianissimo's tend to be fluent substitutions that read as plausible
|
||||||
|
(a wrong institution, an invented person, a dropped sign), which a reader or an LLM is less
|
||||||
|
likely to catch.
|
||||||
|
|
||||||
|
Opus's own lean agreed with the speaker's blind answer on **30 of 30**. That is not a measure
|
||||||
|
of Opus over all 380 spans: it chose spans it could decide, and every one of the 30 carried a
|
||||||
|
lean. Its full classification of the 380 (counts only): function/small word 81, filler or
|
||||||
|
false start 65, technical term 58, compound split/join 46, other 39, name 28, casing or
|
||||||
|
punctuation residue 26, inflection 23, number/date/unit 10, content omission or insertion
|
||||||
|
(>= 3 words) 4.
|
||||||
|
|
||||||
|
## A bug this comparison found in Tapir (fixed)
|
||||||
|
|
||||||
|
kb-whisper's output is wrapped at ~60 characters and **a wrap can fall inside a word**; the
|
||||||
|
continuation segment has no leading space, the response's `text` field joins segments with
|
||||||
|
newlines, and a segment can be a lone `.`. Tapir stored that joined text, so chat received
|
||||||
|
broken words. Measured on real gateway output: 7% of segments (96/1,332) in a synthetic hour,
|
||||||
|
42/1,302 lines in the real one. berget's segments are whole sentences with no leading space, so
|
||||||
|
the two need different joins. `transcribe.joinSegments` now rebuilds the text from the segments.
|
||||||
|
Checked against the real transcript with pianissimo's vocabulary as an independent yardstick:
|
||||||
|
36 spans changed, 31 now entirely known words, words unknown to pianissimo 224 -> 177.
|
||||||
|
Five rejoined spans still contain an unknown word (a name or an unlucky glue): small residual
|
||||||
|
risk. Earlier tests used short sentences and could not see this.
|
||||||
|
|
||||||
|
## What this does not show
|
||||||
|
|
||||||
|
- **One recording, one speaker, one domain.** No claim about other voices, accents, crosstalk,
|
||||||
|
or room noise. 17 vs 13 is not a significant difference.
|
||||||
|
- **The other ~350 spans** were not judged by the speaker. They are mostly spelling, compounds,
|
||||||
|
fillers and small words; Opus's counts there are a model's judgement, not ground truth.
|
||||||
|
- Errors **both** models share in text they agree on are invisible to this method.
|
||||||
|
- Speed was measured once each, on an otherwise idle iguana; pianissimo's 40 s includes the
|
||||||
|
gateway hop and a 122 MB WAV upload.
|
||||||
|
- Whether pianissimo survives the infra#273 Metal-OOM contention the way whisper does not was
|
||||||
|
not tested; both run on iguana.
|
||||||
|
|
||||||
|
## What it suggests for Tapir (not decided, filed as an issue)
|
||||||
|
|
||||||
|
Pianissimo is a **second local backend on the same host, through the same adapter**. That
|
||||||
|
makes it a natural middle tier ahead of the cloud fallback (kb-whisper -> pianissimo ->
|
||||||
|
berget), so a wedged whisper no longer sends private audio off the homelab. At 40 s an hour it
|
||||||
|
is also cheap enough to run **both** on every upload and give chat both texts, or flag where
|
||||||
|
they disagree; the maintainer's blind answers show they err in different places.
|
||||||
@@ -0,0 +1,110 @@
|
|||||||
|
# Audio length ceiling — measured before designing chunking (tapir#33, 2026-10-01)
|
||||||
|
|
||||||
|
Question (#33, "the main technical risk"): at what recording length does Swedish
|
||||||
|
transcription through the LiteLLM gateway first fail, and with what failure mode?
|
||||||
|
Measured **before** any chunking design, with iguana not holding a large ollama model,
|
||||||
|
so the result is the ceiling and not the Metal OOM of infra#273.
|
||||||
|
|
||||||
|
## Answer
|
||||||
|
|
||||||
|
**No transcription ceiling was found at or below 90 minutes.** A single request carrying
|
||||||
|
90 minutes of 16 kHz mono PCM (174 MB) transcribed in one pass in 440 s. The first thing
|
||||||
|
that actually fails is **not whisper**: it is the public gateway ingress, which rejects the
|
||||||
|
upload with `413` at roughly 5.5 minutes of WAV.
|
||||||
|
|
||||||
|
| Route | First failure | Failure mode |
|
||||||
|
|---|---|---|
|
||||||
|
| `https://llm-api.d-ma.be` (openresty ingress) | between 8.0 MB and 10.5 MB body (~4–5.5 min of WAV) | `413 Request Entity Too Large`, HTML body from `openresty`, returned instantly |
|
||||||
|
| `http://koala:30401` (LiteLLM NodePort) → `iguana/kb-whisper` | **none up to 90 min / 174 MB** | — |
|
||||||
|
| `http://koala:30401` → `berget/kb-whisper` | **none up to 22 min / 42 MB** (largest tried) | — |
|
||||||
|
|
||||||
|
The ingress limit is the reason the 8:20 / 15 MB recording in #33 "failed repeatedly" and a
|
||||||
|
30-second slice succeeded. That failure was attributed to a Metal OOM; at least in the
|
||||||
|
size-dependent form described, a body-size cap explains it without one. (The OOM
|
||||||
|
mechanism in infra#273 is separately real; this run did not provoke it.)
|
||||||
|
|
||||||
|
The exact ingress cap was bracketed (8.0 MB passed, 10.5 MB failed), not read from the
|
||||||
|
ingress configuration, which lives in `infra`. "About 10 MB" is an inference.
|
||||||
|
|
||||||
|
## Method
|
||||||
|
|
||||||
|
- **Audio.** No known-good Swedish recording is available that is safe to use: the only real
|
||||||
|
one is a private recording (#31 custody still open) and was
|
||||||
|
deliberately not touched. Instead: macOS `say -v Alva` (Swedish) on iguana, a 1,171-char
|
||||||
|
five-paragraph meeting script → 82.5 s of 16 kHz mono PCM WAV (`ffmpeg -ar 16000 -ac 1`).
|
||||||
|
Concatenated ×4, ×8, ×16, ×32, ×44, ×66 with Python `wave` (the issue's own recipe).
|
||||||
|
Nothing here is a real transcript; none of it is committed.
|
||||||
|
- **Request.** `POST /v1/audio/transcriptions`, multipart `file=`, `model=iguana/kb-whisper`,
|
||||||
|
`response_format=verbose_json`, `-H 'Expect:'` (per #33's traps), master key supplied only
|
||||||
|
through `op run` and never on argv.
|
||||||
|
- **Confound control.** `ollama /api/ps` on iguana was read before every step. Resident:
|
||||||
|
nothing, or `nomic-embed-text` (0.37 GB). Never a large model.
|
||||||
|
|
||||||
|
## Results (`iguana/kb-whisper`, NodePort)
|
||||||
|
|
||||||
|
| Audio | WAV size | HTTP | Wall | Realtime factor | Segments | Chars |
|
||||||
|
|---|---|---|---|---|---|---|
|
||||||
|
| 0:30 | 0.96 MB | 200 | 2.8 s | 11× | — | — |
|
||||||
|
| 2:30 / 3:20 / 4:10 | 4.8 / 6.4 / 8.0 MB | 200 (ingress) | 12.9 / 16.0 / 19.9 s | 12× | — | — |
|
||||||
|
| 5:30 (×4) | 10.5 MB | **413 ingress**, 200 NodePort in 27 s | 27 s | 12× | 121 | 4,557 |
|
||||||
|
| 11:00 (×8) | 21.1 MB | 200 | 53 s | 12× | 241 | 9,094 |
|
||||||
|
| 22:00 (×16) | 42.3 MB | 200 | 105 s | 13× | 482 | 18,201 |
|
||||||
|
| 44:01 (×32) | 84.5 MB | 200 | 209 s | 13× | 966 | 36,438 |
|
||||||
|
| 60:32 (×44) | 116.2 MB | 200 | 291 s | 12× | 1,332 | 50,145 |
|
||||||
|
| 90:48 (×66) | 174.3 MB | 200 | 440 s | 12× | 2,005 | 75,165 |
|
||||||
|
|
||||||
|
- Throughput is flat at ~12× realtime. 90 minutes costs ~7.5 minutes of wall time. This is
|
||||||
|
a minutes-long job, not a request/response one (see ADR-033 on ADR-018).
|
||||||
|
- Segments per source repetition stay at 30.1–30.4 from ×4 to ×66, and segment times are
|
||||||
|
monotonic with `last_end` within 0.2 s of `duration`. No drift and no repetition-loop
|
||||||
|
hallucination appeared across 90 minutes. This is a weak check — the input is a repeated
|
||||||
|
loop and synthetic voice — not evidence of accuracy.
|
||||||
|
- **Only `iguana/kb-whisper`'s `x4` run reported `last_end` 360.0 s against `duration`
|
||||||
|
330.2 s.** Whisper's final segment can end past the audio; clamp to `duration`.
|
||||||
|
|
||||||
|
## Findings that change the design
|
||||||
|
|
||||||
|
1. **Use the NodePort route, not the public hostname.** Tapir runs in-cluster beside the
|
||||||
|
gateway; `llm-api.d-ma.be` is the external door and has a body cap. #33 said "use the
|
||||||
|
gateway, never the host" and named the public URL; the gateway is right, that URL is not.
|
||||||
|
`TAPIR_GATEWAY_URL` already points at `koala:30401/v1`.
|
||||||
|
2. **`response_format=srt` survives the LiteLLM hop, but wrapped.** The response is JSON
|
||||||
|
`{"text": "<the SRT>", "usage": …}`, not a bare SRT body — a client that parses the body
|
||||||
|
as SRT gets a parse error on `{`. #33 question 2 is answered: it passes through, inside
|
||||||
|
an envelope.
|
||||||
|
3. **The two backends return different timestamp granularity.** `iguana/kb-whisper`
|
||||||
|
segments are **whole seconds** (`0→1`, `1→4`, `4→7`); `berget/kb-whisper` returns real
|
||||||
|
sub-second times (`0.051→1.192`) and a different `language` token (`swedish` vs `sv`).
|
||||||
|
`verbose_json` and the default JSON were byte-identical on iguana. An adapter that
|
||||||
|
falls back between the two must normalise both, and "which endpoint served this" is
|
||||||
|
a quality fact, not just a log line.
|
||||||
|
4. **The existing chat path cannot answer about most of a long recording.** At
|
||||||
|
828 chars/min (this fixture; real speech is plausibly a little denser),
|
||||||
|
`TAPIR_MAX_TRANSCRIPT_CHARS`'s default of **18,000** is ~21.7 minutes. A 60-minute
|
||||||
|
transcript is 50 k chars, a 90-minute one 75 k. `chat.Service` truncates by keeping
|
||||||
|
the head (`internal/adapters/chat/chat.go`), so questions about the last two-thirds of
|
||||||
|
a one-hour meeting get an answer from a transcript that does not contain it. #33 says
|
||||||
|
chat "retrieves rather than generates" and therefore has no ceiling; it **stuffs and
|
||||||
|
truncates**, and its ceiling is the model context. See ADR-033.
|
||||||
|
5. **Chunking is not needed for v1 on length.** Nothing failed up to 90 minutes. The
|
||||||
|
chunking-and-stitching recipe in #33 (offset timestamps, renumber cues, strip `\r`) is
|
||||||
|
not built. It stays the answer *if* a longer cap is wanted or the OOM mechanism forces
|
||||||
|
smaller requests — see ADR-033's resolution triggers.
|
||||||
|
|
||||||
|
## What this did not measure
|
||||||
|
|
||||||
|
- **Metal OOM under contention.** Deliberately avoided. A big resident ollama model
|
||||||
|
during a long transcription is still the known way to wedge iguana permanently.
|
||||||
|
- **Real speech.** TTS is clean, single-voice, no crosstalk or room noise. WER and
|
||||||
|
speaker overlap in a Teams export are untested. Speed is probably representative;
|
||||||
|
accuracy is not claimed.
|
||||||
|
- **Compressed input.** Only WAV was sent. Whether `m4a`/`mp3`/`opus` can reach whisper
|
||||||
|
without a server-side transcode is not established here (ADR-033 treats it as needing one).
|
||||||
|
- **Concurrency.** One request at a time. Two simultaneous 90-minute jobs on iguana were
|
||||||
|
not tried.
|
||||||
|
- **`berget/kb-whisper` beyond 22 min / 42 MB.** Enough to know the fallback is not
|
||||||
|
size-capped at meeting length for 22 minutes; not proven at 90.
|
||||||
|
- **Time-of-day / other tenants.** One afternoon, one run per length.
|
||||||
|
|
||||||
|
Scripts were throwaway shell/Python in a session scratchpad and are not committed; the
|
||||||
|
method above reproduces them.
|
||||||
@@ -0,0 +1,66 @@
|
|||||||
|
# Chat over a long transcript — what the offered models really hold (tapir#33, 2026-10-01)
|
||||||
|
|
||||||
|
Question: ADR-033 reuses `chat.Service`, which stuffs the transcript into the prompt and
|
||||||
|
cuts the tail. #33 assumed chat "retrieves rather than generates" and so has no length
|
||||||
|
ceiling. It does not retrieve, so its ceiling is the model's context. What is it, for the
|
||||||
|
models chat actually offers?
|
||||||
|
|
||||||
|
## Answer
|
||||||
|
|
||||||
|
| Model (via the NodePort gateway) | Result | Limit found |
|
||||||
|
|---|---|---|
|
||||||
|
| `koala/phi4-mini` | correct to 30,000 chars; **HTTP 400 at 50,000** (13,332 tokens vs an 8,192-token context) | ~22,000 chars in production, where the client adds `max_tokens: 1500` |
|
||||||
|
| `iguana/gemma4-26b` | correct at 18k, 50k and **90,000 chars** (26.5 k tokens); **at 120,000 chars it silently loses the head** | **90,000 chars** |
|
||||||
|
| `berget/mistral-small` | answered "the transcript does not contain it" for a fact **at the end** of an 18,000-char and a 50,000-char transcript | unreliable here, one run each — see caveats |
|
||||||
|
|
||||||
|
The dangerous one is gemma at 120,000 chars: **no error, no truncation flag** — the reported
|
||||||
|
prompt tokens fell from 26,548 (90 k chars) to 16,387 (120 k chars), and the model answered
|
||||||
|
"it does not say who the chair is" about a sentence at the very start of the transcript. The
|
||||||
|
serving layer truncated the prompt from the front. `chat.Service` reports `Truncated: false`
|
||||||
|
because *it* did not cut anything.
|
||||||
|
|
||||||
|
## What this changes
|
||||||
|
|
||||||
|
1. **The budget for an uploaded transcript is 90,000 chars, not a round 120,000.** That is the
|
||||||
|
best offered model's measured limit, ~100 minutes of speech at the fixture's density.
|
||||||
|
`TAPIR_MAX_UPLOADED_TRANSCRIPT_CHARS` defaults to it.
|
||||||
|
2. **Chat over an upload must not open on the summarizer default.** The offered list is
|
||||||
|
`[SummarizerModel, FallbackModel, CloudFallbackModel]`, so `koala/phi4-mini` is first — and
|
||||||
|
400s on an hour of speech. `TAPIR_UPLOAD_CHAT_MODEL` (default `iguana/gemma4-26b`) is what
|
||||||
|
the upload chat opens with; the switcher still lists every model.
|
||||||
|
3. **A per-model budget does not exist yet.** Someone who switches the dropdown to
|
||||||
|
`phi4-mini` on a long recording gets a loud HTTP 400, shown inline as "couldn't answer".
|
||||||
|
That fails honestly but unhelpfully. Filed as a follow-up.
|
||||||
|
4. **The "retrieval chat" trigger in ADR-033 is now concrete:** any recording past 90,000 chars
|
||||||
|
(≈100 min) cannot be answered by stuffing, for any offered model.
|
||||||
|
|
||||||
|
## Method
|
||||||
|
|
||||||
|
`POST /v1/chat/completions` through `http://koala:30401/v1`, `temperature: 0`,
|
||||||
|
`max_tokens: 200` (production sends 1500 for chat via the summary token cap), the same system
|
||||||
|
framing as `chat.Service`. The transcript is the 60-minute synthetic fixture from
|
||||||
|
`audio-length-ceiling-2026-10-01.md` truncated/repeated to N chars, with one unique sentence
|
||||||
|
planted either at the **end** ("lagret på Kvarngatan ska tömmas … av Pelle", question: who/when)
|
||||||
|
or at the **head** ("kvällens ordförande är Ingrid", question: who chairs). Needle at the
|
||||||
|
end alone would have missed the gemma failure — a serving layer that truncates from the front
|
||||||
|
keeps the tail, so the end-needle passed at 120,000 chars; only the head-needle exposed it.
|
||||||
|
Grounded = the answer contains the planted fact.
|
||||||
|
|
||||||
|
## What this did not measure
|
||||||
|
|
||||||
|
- **Real speech**: the fixture is a repeated 82-second script, so retrieval "difficulty" is
|
||||||
|
artificially easy except for the planted needle. One question per run, one run per cell.
|
||||||
|
- **`berget/mistral-small`'s failure**: it missed an end-needle at 18,000 chars where the
|
||||||
|
others succeeded. One run; could be the model, the prompt framing, or position bias. Not
|
||||||
|
investigated; do not read it as a ranking.
|
||||||
|
- **Latency under load**, **concurrent** chat while whisper is transcribing, and in particular
|
||||||
|
the hazard below.
|
||||||
|
- **`koala/qwen36-35b-a3b` or any frontier model** — not in the offered list.
|
||||||
|
|
||||||
|
## A hazard this surfaces and does not solve
|
||||||
|
|
||||||
|
`iguana/gemma4-26b` is a large ollama model on the same host as `iguana/kb-whisper`. infra#273
|
||||||
|
is precisely that a large resident ollama model can Metal-OOM whisper-server permanently. Making
|
||||||
|
gemma the default *chat* model for uploads means chatting with a transcript can load the model
|
||||||
|
that wedges the transcriber. The primary-fails-over-to-berget design limits the blast radius but
|
||||||
|
does not remove it. Recorded in ADR-033 and filed.
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
# Can a local model adjudicate two ASR transcripts? (tapir#33, 2026-10-01)
|
||||||
|
|
||||||
|
Question: when `iguana/kb-whisper` and `iguana/pianissimo` disagree about a word or phrase,
|
||||||
|
can a **local** model say which reading is right, well enough to merge the two transcripts?
|
||||||
|
Ground truth: the speaker's own blind answers on 30 disagreements of one real recording
|
||||||
|
(see `asr-kb-whisper-vs-pianissimo-2026-10-01.md`; no transcript text is recorded here,
|
||||||
|
this repo is public). Reference: Claude Opus 5.5 agreed with the speaker on all 30.
|
||||||
|
|
||||||
|
## Method
|
||||||
|
|
||||||
|
- **Items:** 30 disagreement spans that Opus had picked as changing meaning, the speaker's
|
||||||
|
verdict as truth (pianissimo right 17, kb-whisper right 13).
|
||||||
|
- **Prompt:** one fixed English instruction (two transcriptions of Swedish speech differ at
|
||||||
|
ONE marked spot; say which reading the speaker most likely said; reply `ANSWER: 1|2|UNSURE`
|
||||||
|
plus one sentence). Each item was asked in **both reading orders** and in **two contexts**:
|
||||||
|
narrow (+-8 words) and wide (+-100 words). 120 calls per model, temperature 0.
|
||||||
|
- **Models:** 11, all local (`koala/*`, `iguana/*`): nothing left the homelab. Run one iguana
|
||||||
|
model at a time, each unloaded afterwards, to avoid the memory pressure behind infra#273;
|
||||||
|
whisper was confirmed healthy afterwards.
|
||||||
|
- **Pass criteria, fixed before the run:** wide context, strict accuracy >= 80% (an
|
||||||
|
unparseable or "unsure" answer counts as wrong), order-swap consistency >= 85%,
|
||||||
|
unparseable <= 10%.
|
||||||
|
|
||||||
|
## Results (wide context, 60 calls per model)
|
||||||
|
|
||||||
|
| Model | Correct | 95% CI | Same pick when order swapped | Unparseable | s/call | Pass |
|
||||||
|
|---|---|---|---|---|---|---|
|
||||||
|
| `iguana/gemma4-31b` | 58 = 97% | 89-99% | 93% | 0 | 10.4 | **yes** |
|
||||||
|
| `iguana/gemma4-26b` | 55 = 92% | 82-96% | 93% | 0 | 2.0 | **yes** |
|
||||||
|
| `iguana/ornith-35b` | 51 = 85% | 74-92% | 77% | 0 | 2.5 | no (consistency) |
|
||||||
|
| `koala/phi4-14b` | 50 = 83% | 72-91% | 80% | 0 | 1.0 | no (consistency) |
|
||||||
|
| `koala/qwen36-35b-a3b` | 49 = 82% | 70-89% | 83% | 0 | 3.3 | no (consistency) |
|
||||||
|
| `iguana/granite4` | 47 = 78% | 66-87% | 83% | 0 | 4.6 | no |
|
||||||
|
| `iguana/qwen3-14b` | 44 = 73% | 61-83% | 87% | 0 | 4.4 | no (accuracy) |
|
||||||
|
| `iguana/gpt-oss-20b` | 40 = 67% | 54-77% | 100% | 17 | 11.2 | no (unparseable) |
|
||||||
|
| `koala/phi4-mini` | 37 = 62% | 49-73% | 24% | 1 | 0.5 | no |
|
||||||
|
| `iguana/gemma4-26b-think` | 23 = 38% | 27-51% | 100% | 37 | 15.3 | no (unparseable) |
|
||||||
|
| `koala/qwen35-9b-fast` | 0 | n/a | n/a | 60 | 7.2 | no (no output) |
|
||||||
|
|
||||||
|
Narrow context only (+-8 words) costs 6-12 points for the models that pass: gemma4-31b 92%,
|
||||||
|
gemma4-26b 83%, ornith-35b 80%. Context is not decoration: Opus also used evidence from
|
||||||
|
elsewhere in the recording (an amount stated earlier, a term that recurs).
|
||||||
|
|
||||||
|
Position bias: the passing models pick "Reading 1" 47-51% of the time. `phi4-mini` picks it 15%
|
||||||
|
and flips its answer when the order is swapped 76% of the time: it carries almost no signal.
|
||||||
|
|
||||||
|
## Reading it
|
||||||
|
|
||||||
|
- **Two models pass:** the gemma4 pair. Both clear the bar and neither shows position bias.
|
||||||
|
26b is ~5x faster; their intervals overlap, so 30 items cannot separate 92% from 97%.
|
||||||
|
- **The thinking models were not tested fairly.** `gpt-oss-20b` and `gemma4-26b-think` spend
|
||||||
|
the 600-token budget reasoning and often never reach the answer line. When gemma4-26b-think
|
||||||
|
did answer, it was right 23 of 23; gpt-oss-20b 93% of the answered. A larger budget would
|
||||||
|
make them eligible; at 15 s a call it would not make them practical.
|
||||||
|
|
||||||
|
## What this does not show
|
||||||
|
|
||||||
|
- **Thirty items, one recording, one speaker, one domain.** The 30 were chosen by Opus as
|
||||||
|
decidable, so they overstate how easy the typical disagreement is; ~350 others are mostly
|
||||||
|
spelling, compounds and fillers, which no ground truth covers.
|
||||||
|
- Opus agreeing 30/30 is not a measure of Opus over all 380 disagreements.
|
||||||
|
- Accuracy was measured at temperature 0 on one prompt; no prompt tuning was tried.
|
||||||
|
- The speaker's answers are the only truth; they are one person's recollection of what they said.
|
||||||
|
|
||||||
|
## Why gemma4-31b is slow, and what was ruled out
|
||||||
|
|
||||||
|
On iguana (M2 Ultra, 64 GB) the dense 31B runs at **103 tok/s prefill, 7.9 tok/s decode**; the
|
||||||
|
26B MoE at 641 and 39.6. Ruled out as the cause: memory pressure (no swap, 45% free, model 100%
|
||||||
|
on GPU), context length (32k vs 4k made no difference), and the **inference engine** — MLX 4-bit
|
||||||
|
of the same model measured 76 tok/s prefill and 8.6 decode, the same within noise. (Terse
|
||||||
|
output saves only ~3.6 s of ~9.3 s because prefill dominates.) Two unrelated engines at the same
|
||||||
|
speed points at the machine: ~140 GB/s effective memory bandwidth on a part rated for 800, and
|
||||||
|
**iguana had macOS Low Power Mode on** (`pmset` `lowpowermode 1`). A first MLX benchmark was
|
||||||
|
wrong because `mlx_lm.server` caches prompts; the figures above are from the corrected run.
|
||||||
|
|
||||||
|
**Confirmed 2026-10-02.** The owner turned Low Power Mode off (`lowpowermode 0`, read back over
|
||||||
|
ssh) and `gemma4:31b` was re-measured with nothing else resident: **217 tok/s prefill, 22.3
|
||||||
|
tok/s decode** (4 runs, spread 22.2-22.6), against 103 and 7.9 with it on, so 2.1x and 2.8x. The
|
||||||
|
slowness was the power mode, not the engine or the model. Caveats on the comparison: the new run
|
||||||
|
used a different, synthetic Swedish prompt (~700 words, the original items held real transcript
|
||||||
|
text and were deleted) and the model answered in 26-43 tokens, not the ~200 of the first run, so
|
||||||
|
it is a same-machine before/after, not a controlled A/B. `gemma4:26b` and the MLX figures were
|
||||||
|
not re-measured. The 31B's 97% judge accuracy is therefore usable at roughly a third of the
|
||||||
|
latency the sweep implied; the 26B-vs-31B choice in ADR-034 was made on the slow numbers and is
|
||||||
|
reopened by this.
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
# Two-engine reconcile, first check on a real recording (tapir#33, ADR-034, 2026-10-02)
|
||||||
|
|
||||||
|
Aggregate numbers only. This repo is public and the recording is private (#31 custody is open), so no
|
||||||
|
transcript text, no disputed words and no per-item reading appears here.
|
||||||
|
|
||||||
|
## What was run
|
||||||
|
|
||||||
|
One real 63.6-minute Teams recording of one speaker dictating project notes (the video track, camera
|
||||||
|
off, was discarded). Build: PR #39 head `7f8e367`. Everything local: `iguana/pianissimo` as base,
|
||||||
|
`iguana/kb-whisper` as the second engine, `iguana/gemma4-31b` as reconciler (iguana Low Power Mode
|
||||||
|
off), cloud fallbacks off, a throwaway Postgres.
|
||||||
|
|
||||||
|
| Measure | Value |
|
||||||
|
|---|---|
|
||||||
|
| Job time, upload to saved | 21 min 49 s for 63.6 min of audio (20.6 min per hour of audio) |
|
||||||
|
| Differences between the engines | 362 |
|
||||||
|
| Settled automatically (compound splits, up to two filler words) | 80 |
|
||||||
|
| Decided by the model | 282 |
|
||||||
|
| Of those: alternative reading used / base kept / uncertain | 130 / 151 / 1 |
|
||||||
|
| Model call errors | 0 |
|
||||||
|
|
||||||
|
20.6 min per hour is inside the 25-minute resolution trigger in ADR-034, so no smaller adjudicator is
|
||||||
|
needed on speed grounds. The earlier estimate was about 10 min per hour on gemma4-26b; it was not
|
||||||
|
measured and this run used the 31B.
|
||||||
|
|
||||||
|
## Blind check
|
||||||
|
|
||||||
|
30 spots drawn at random (fixed seed) from the 89 replacement spots that could be placed in the base
|
||||||
|
text, across the whole recording (0:44 to 60:57). The two readings were shown as x and y in random
|
||||||
|
order, with the key kept apart from the judge. Judge: Claude Opus, text only, no audio, at the
|
||||||
|
speaker's request ("I trust Opus"). The speaker did not judge these items themselves.
|
||||||
|
|
||||||
|
| Verdict | Items |
|
||||||
|
|---|---|
|
||||||
|
| The reconciler's reading (the alternative) is right | 21 |
|
||||||
|
| The base reading (pianissimo) is right, so the change made it worse | 1 |
|
||||||
|
| Neither reading is right | 3 |
|
||||||
|
| Cannot tell from text (compound, hyphen and number-format differences) | 5 |
|
||||||
|
|
||||||
|
- Right of the items with a verdict: 21 of 22, 95% (95% interval 78% to 99%).
|
||||||
|
- Right of all 30, counting "neither" and "cannot tell" as not right: 21 of 30, 70% (52% to 83%).
|
||||||
|
- Right of the 25 that were not "cannot tell": 21 of 25, 84% (65% to 94%).
|
||||||
|
- Net effect against the base on these 30: 21 better, 1 worse, 8 unchanged in quality.
|
||||||
|
|
||||||
|
## Reading the result
|
||||||
|
|
||||||
|
ADR-034's bar was "right at least 85% of the time". It did not say how ties count, and that was a
|
||||||
|
flaw in the bar, not in the data. The literal reading (all 30) gives 70% and **does not meet** the bar;
|
||||||
|
counting only decided items gives 95% and meets it. Neither is a hidden pass or a hidden failure; both
|
||||||
|
are reported. What is not in doubt: on a random sample the reconciler improved the text far more often
|
||||||
|
than it harmed it (21 to 1), and the 8 unchanged cases are mostly spellings the text cannot settle.
|
||||||
|
|
||||||
|
## What this did not measure
|
||||||
|
|
||||||
|
- **Better single engine.** The second ADR-034 condition, "no worse than the better single engine", is
|
||||||
|
**not answered**. Only changed spots are stored, not the 151 where the model kept pianissimo, so the
|
||||||
|
reconciler cannot be compared with kb-whisper alone. The 21-to-1 result is against pianissimo.
|
||||||
|
- **The speaker as judge.** Opus judged from text. It cannot hear the audio, so it can be wrong where
|
||||||
|
the right word is audible but not inferable (items such as a mis-heard number or a name). Earlier,
|
||||||
|
Opus and the speaker agreed 30 of 30, but on items chosen as decidable; that does not transfer to a
|
||||||
|
random sample.
|
||||||
|
- **Insertions and deletions.** 42 of the 131 stored spots were not sampled: 39 are an added or
|
||||||
|
removed word, which cannot be placed in the base text, and 3 are the uncertain spot or could not be
|
||||||
|
located. The 39 are mostly small words ("och", "det"), so this sample says little about them.
|
||||||
|
- **One recording, one speaker, 30 items.** Wide intervals, and neighbouring spots (three pairs within
|
||||||
|
seconds of each other) are not independent.
|
||||||
|
- **Segments keep the base text.** The saved timed segments hold pianissimo's wording, not the merged
|
||||||
|
text; only the stored full text is merged. Found while building this sample; not yet an issue.
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
# ADR-034 second blind check: protocol (ready to run; tapir#33, D9, D10)
|
||||||
|
|
||||||
|
This repo is public and the recordings are private, so this file holds the method only. The
|
||||||
|
sheet, the key and the filled-in verdicts stay outside the repo, and the trial data is deleted
|
||||||
|
after the check. Results go back here as aggregate numbers, as in
|
||||||
|
`reconcile-blind-check-2026-10-02.md`.
|
||||||
|
|
||||||
|
## What the check decides
|
||||||
|
|
||||||
|
ADR-034 (two engines, a local model reconciles their disagreements) is `Proposed` and off in
|
||||||
|
production. Two conditions were set for keeping it. The first run answered the first one on one
|
||||||
|
reading only and did not answer the second:
|
||||||
|
|
||||||
|
1. **The reconciled reading is right at least 85% of the time.** Open flaw in the bar: it did not
|
||||||
|
say how ties count. The first run gave 95% (decided items only) or 70% (all 30), so it was met on
|
||||||
|
one reading and missed on the other.
|
||||||
|
2. **It is no worse than the better single engine.** Not answered: before ADR-035 the spots the
|
||||||
|
model *kept* as the base were not stored, so kb-whisper alone could not be compared.
|
||||||
|
|
||||||
|
ADR-035 now stores every judged spot (`reconciliation.contested`: both readings, which one was
|
||||||
|
used, unsure or not, a few words of left context, a time). That makes condition 2 measurable.
|
||||||
|
|
||||||
|
## Rules to fix BEFORE looking at any result (the owner confirms or changes them)
|
||||||
|
|
||||||
|
| Rule | Proposed |
|
||||||
|
|---|---|
|
||||||
|
| Who judges | The speaker, hearing the audio where the text cannot settle it (D9). Not Opus: it cannot hear, and its agreement with the speaker was only shown on items picked as decidable. |
|
||||||
|
| Sample | 30 spots drawn at random with a fixed seed, from ALL judged spots (changed and kept-base alike), not only the changed ones. Insertions and deletions that cannot be placed are listed separately and not sampled. |
|
||||||
|
| Verdict per spot | `x`, `y` (the two readings, shown in random order, key kept apart), `neither`, or `cannot tell`. |
|
||||||
|
| Condition 1 | Reconciled is right on at least 85% of the spots where the judge could tell (`cannot tell` excluded, reported, and `neither` counts as not right). Also report all-30. Pass needs the decided-only figure at 85% or more AND the all-30 figure above 70%. |
|
||||||
|
| Condition 2 | Over the same spots, the reconciled text is right at least as often as the better single engine (base engine alone is right where the base reading is right; the other engine alone where the alt reading is right). Ties favour the single engine. |
|
||||||
|
| If it fails | Switch reconcile off by configuration; the single-engine path is unchanged (ADR-034 reversibility). |
|
||||||
|
|
||||||
|
Why a fixed all-30 floor: the literal 70% last time sat well under the bar, and "decided only" can
|
||||||
|
hide a model that mostly lands on spelling differences.
|
||||||
|
|
||||||
|
## What a run needs (D10: the owner is present)
|
||||||
|
|
||||||
|
- The same real recording as before (about 60 minutes). Only the owner has it.
|
||||||
|
- A **throwaway** tapir instance with its own Postgres, not the production pod, so production's
|
||||||
|
key pin and deployment are untouched. Settings: `TAPIR_RECONCILE_BASE_MODEL=iguana/pianissimo`,
|
||||||
|
`TAPIR_TRANSCRIBE_MODEL=iguana/kb-whisper`, `TAPIR_RECONCILE_MODEL=iguana/gemma4-31b`, cloud
|
||||||
|
fallbacks empty. Job time was 21 min 49 s per 63.6 min last run.
|
||||||
|
- iguana with Low Power Mode off and gemma4-31b able to load (the production-iguana model load
|
||||||
|
that needs a quiet window; D6).
|
||||||
|
|
||||||
|
## Steps
|
||||||
|
|
||||||
|
1. Start the throwaway instance, upload the recording, wait for `done` (about 22 minutes).
|
||||||
|
2. Read `reconciliation.contested` from the throwaway database. From it build, **outside the repo**:
|
||||||
|
`sheet` (spot number, time, left context, reading x, reading y, random order) and `key` (which
|
||||||
|
of x/y is the base, which is the other engine, which one the reconciler used).
|
||||||
|
3. The speaker fills in the sheet (about 20 to 30 minutes for 30 spots, audio open at the times).
|
||||||
|
The agent does not see the key until the sheet is returned.
|
||||||
|
4. Score with the rules above. Write the aggregate numbers into a dated note next to this one,
|
||||||
|
and amend ADR-034's validation section (kept, or switched off, and the tie rule used).
|
||||||
|
5. Delete the throwaway instance, its database and the sheet, as last time.
|
||||||
|
6. Only if the check passes and the owner wants it in production: enabling reconcile there is
|
||||||
|
the separate D10 step (it needs `iguana/pianissimo` and the reconciler added to the tapir key
|
||||||
|
pin in the infra repo plus the env vars on the deployment, `risk/medium`, owner present).
|
||||||
|
|
||||||
|
## Open and owner-only
|
||||||
|
|
||||||
|
- Confirm or change the rules table above before the run.
|
||||||
|
- The **tool that builds the sheet and key** is the one piece not in the repo (last time it was a
|
||||||
|
one-off script). It can be built test-first on request; until then the agent builds it ad hoc
|
||||||
|
outside the repo.
|
||||||
|
- A Berget adjudicator (D8, tapir#45) is a different question with its own measurement and is
|
||||||
|
not part of this check: this check judges the local setup that ADR-034 describes.
|
||||||
@@ -0,0 +1,63 @@
|
|||||||
|
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>tapir — A · TUI panels</title>
|
||||||
|
<style>
|
||||||
|
:root{--bg:#0d0d12;--panel:#14141c;--line:#2a2a3a;--mint:#0EF9B6;--pink:#F740A0;--purple:#7653fc;--cream:#F5E9D6;--dim:#6b6b86;--text:#d9d9e6}
|
||||||
|
*{box-sizing:border-box}
|
||||||
|
body{margin:0;background:var(--bg);color:var(--text);font:14px/1.5 ui-monospace,SFMono-Regular,Menlo,"Cascadia Code",monospace}
|
||||||
|
.wrap{max-width:760px;margin:0 auto;padding:18px 14px 60px}
|
||||||
|
header{display:flex;align-items:center;gap:10px;border-bottom:1px solid var(--line);padding-bottom:12px;margin-bottom:18px}
|
||||||
|
.brand{color:var(--mint);font-weight:700;letter-spacing:.5px}
|
||||||
|
.brand b{color:var(--cream)}
|
||||||
|
.tip{margin-left:auto;color:var(--dim);font-size:12px}
|
||||||
|
.note{color:var(--dim);font-size:12.5px;border-left:2px solid var(--purple);padding:6px 10px;margin:0 0 16px;background:#11111a}
|
||||||
|
/* lipgloss-style bordered panel */
|
||||||
|
.card{border:1px solid var(--line);border-radius:8px;background:var(--panel);padding:12px 14px;margin:0 0 12px;position:relative}
|
||||||
|
.card::before{content:"";position:absolute;left:0;top:10px;bottom:10px;width:3px;border-radius:3px;background:var(--purple)}
|
||||||
|
.card.ready::before{background:var(--mint)}
|
||||||
|
.title{color:var(--cream);font-size:15px;font-weight:600;margin:0 0 4px}
|
||||||
|
.meta{color:var(--dim);font-size:12px}
|
||||||
|
.chip{display:inline-block;border:1px solid var(--mint);color:var(--mint);border-radius:4px;padding:0 6px;font-size:11px;margin-left:6px}
|
||||||
|
.chip.q{border-color:var(--pink);color:var(--pink)}
|
||||||
|
.preview{color:var(--dim);margin:6px 0 0;font-size:13px}
|
||||||
|
/* expanded */
|
||||||
|
.exp{border-color:var(--purple)}
|
||||||
|
.exp .head{display:flex;justify-content:space-between;align-items:baseline}
|
||||||
|
.collapse{color:var(--mint);font-size:12px;text-decoration:none;border:1px solid var(--line);border-radius:4px;padding:1px 7px}
|
||||||
|
.sec h2{color:var(--mint);font-size:12px;text-transform:uppercase;letter-spacing:1px;margin:16px 0 6px;border-bottom:1px dashed var(--line);padding-bottom:3px}
|
||||||
|
.sec ul{margin:0;padding-left:18px}.sec li{margin:3px 0}
|
||||||
|
.body{color:var(--text)}
|
||||||
|
.dock{margin-top:16px;border-top:1px solid var(--line);padding-top:12px}
|
||||||
|
.ask{display:inline-block;background:linear-gradient(90deg,var(--purple),var(--pink));color:#fff;border:0;border-radius:6px;padding:7px 12px;font:inherit;font-size:13px;cursor:pointer}
|
||||||
|
pre.tapir{margin:0;color:var(--mint);font-size:10px;line-height:1.05}
|
||||||
|
</style></head><body><div class="wrap">
|
||||||
|
<header>
|
||||||
|
<pre class="tapir"> ▄█▓▓█▄ ∩
|
||||||
|
█▓( ◕ ◕ )▓█──┘</pre>
|
||||||
|
<span class="brand">◆ <b>tapir</b> · watch less, know more</span>
|
||||||
|
<span class="tip">261 in queue</span>
|
||||||
|
</header>
|
||||||
|
<p class="note">Tapir fetches captions slowly on purpose, to respect YouTube's limits — new summaries land gradually.</p>
|
||||||
|
|
||||||
|
<div class="card ready"><div class="title">How the Attention Economy Rewires Your Brain</div>
|
||||||
|
<div class="meta">youtube · 2026-06-11 · 18 min <span class="chip">ready</span></div>
|
||||||
|
<div class="preview">A tour of the incentive loops behind infinite feeds and three concrete ways to claw back focus…</div></div>
|
||||||
|
|
||||||
|
<article class="card exp ready">
|
||||||
|
<div class="head"><div class="title">Postgres 18 — What's Actually New</div><a class="collapse" href="#">collapse ↑</a></div>
|
||||||
|
<div class="meta">youtube · 2026-06-10 · 42 min · phi4-mini</div>
|
||||||
|
<div class="sec"><h2>Takeaways</h2><ul>
|
||||||
|
<li>Async I/O cuts cold-cache read latency materially on NVMe.</li>
|
||||||
|
<li>Skip-scan makes more multicolumn indexes usable without rewrites.</li>
|
||||||
|
<li>Upgrade path is smooth; test the new planner stats first.</li></ul></div>
|
||||||
|
<div class="sec"><h2>Highlights</h2><ul>
|
||||||
|
<li>Async I/O subsystem (effective_io_concurrency now matters more).</li>
|
||||||
|
<li>B-tree skip scan for leading-column gaps.</li>
|
||||||
|
<li>Better partition-wise joins.</li></ul></div>
|
||||||
|
<div class="sec"><h2>Summary</h2><p class="body">Postgres 18 is an incremental but meaningful release: the headline is the new asynchronous I/O path, with skip-scan and planner improvements close behind. For most homelabs the upgrade is low-risk and worth it for the read-latency wins.</p></div>
|
||||||
|
<div class="dock"><button class="ask">Dig deeper — ask about this video →</button></div>
|
||||||
|
</article>
|
||||||
|
|
||||||
|
<div class="card"><div class="title">RAG is dead, right?? — Conference Talk</div>
|
||||||
|
<div class="meta">youtube · 2026-06-09 · 31 min <span class="chip q">queued</span></div></div>
|
||||||
|
</div></body></html>
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>tapir — B · reader + charm accents</title>
|
||||||
|
<style>
|
||||||
|
:root{--bg:#faf7f2;--card:#fff;--ink:#1c1b22;--soft:#6a6878;--line:#e7e2d8;--mint:#0bbf8c;--purple:#6a4cf0;--pink:#e0379a}
|
||||||
|
*{box-sizing:border-box}
|
||||||
|
body{margin:0;background:var(--bg);color:var(--ink);font:16px/1.6 -apple-system,BlinkMacSystemFont,"Segoe UI",Inter,sans-serif}
|
||||||
|
.mono{font-family:ui-monospace,SFMono-Regular,Menlo,monospace}
|
||||||
|
.wrap{max-width:680px;margin:0 auto;padding:20px 16px 60px}
|
||||||
|
header{display:flex;align-items:center;gap:10px;margin-bottom:18px}
|
||||||
|
.brand{font-weight:800;font-size:18px;letter-spacing:-.3px}
|
||||||
|
.brand .dot{color:var(--mint)}
|
||||||
|
.tag{color:var(--soft);font-size:13px}
|
||||||
|
.tip{margin-left:auto;color:var(--soft);font-size:12px}
|
||||||
|
.note{color:var(--soft);font-size:13px;background:#fff;border:1px solid var(--line);border-left:3px solid var(--mint);border-radius:8px;padding:8px 12px;margin:0 0 18px}
|
||||||
|
.card{background:var(--card);border:1px solid var(--line);border-radius:12px;padding:16px 18px;margin:0 0 14px;box-shadow:0 1px 2px rgba(20,18,40,.04)}
|
||||||
|
.title{font-size:18px;font-weight:700;letter-spacing:-.2px;margin:0 0 4px;line-height:1.3}
|
||||||
|
.meta{color:var(--soft);font-size:13px}
|
||||||
|
.meta .mono{font-size:12.5px}
|
||||||
|
.chip{display:inline-block;background:rgba(11,191,140,.12);color:var(--mint);border-radius:999px;padding:1px 9px;font-size:12px;font-weight:600;margin-left:6px}
|
||||||
|
.chip.q{background:rgba(224,55,154,.12);color:var(--pink)}
|
||||||
|
.preview{color:var(--soft);margin:8px 0 0}
|
||||||
|
.exp{border-color:#d9d0ee;box-shadow:0 6px 24px rgba(106,76,240,.10)}
|
||||||
|
.head{display:flex;justify-content:space-between;align-items:baseline;gap:10px}
|
||||||
|
.collapse{color:var(--purple);font-size:13px;font-weight:600;text-decoration:none;white-space:nowrap}
|
||||||
|
.sec h2{font-size:12px;text-transform:uppercase;letter-spacing:1.2px;color:var(--purple);margin:18px 0 6px}
|
||||||
|
.sec ul{margin:0;padding-left:20px}.sec li{margin:5px 0}
|
||||||
|
.body{line-height:1.7}
|
||||||
|
.dock{margin-top:18px;border-top:1px solid var(--line);padding-top:14px}
|
||||||
|
.ask{display:inline-flex;align-items:center;gap:6px;background:var(--ink);color:#fff;border:0;border-radius:999px;padding:9px 16px;font:inherit;font-size:14px;font-weight:600;cursor:pointer}
|
||||||
|
pre.tapir{margin:0;color:var(--mint);font-size:10px;line-height:1.05}
|
||||||
|
</style></head><body><div class="wrap">
|
||||||
|
<header>
|
||||||
|
<pre class="tapir"> ▄█▓▓█▄ ∩
|
||||||
|
█▓( ◕ ◕ )▓█──┘</pre>
|
||||||
|
<span class="brand">tapir<span class="dot">.</span></span><span class="tag">watch less, know more</span>
|
||||||
|
<span class="tip mono">261 in queue</span>
|
||||||
|
</header>
|
||||||
|
<p class="note">Tapir fetches captions slowly on purpose, to respect YouTube's limits — new summaries land gradually.</p>
|
||||||
|
|
||||||
|
<div class="card"><div class="title">How the Attention Economy Rewires Your Brain</div>
|
||||||
|
<div class="meta mono">youtube · 2026-06-11 · 18 min <span class="chip">ready</span></div>
|
||||||
|
<div class="preview">A tour of the incentive loops behind infinite feeds and three concrete ways to claw back focus…</div></div>
|
||||||
|
|
||||||
|
<article class="card exp">
|
||||||
|
<div class="head"><div class="title">Postgres 18 — What's Actually New</div><a class="collapse" href="#">collapse ↑</a></div>
|
||||||
|
<div class="meta mono">youtube · 2026-06-10 · 42 min · phi4-mini</div>
|
||||||
|
<div class="sec"><h2>Takeaways</h2><ul>
|
||||||
|
<li>Async I/O cuts cold-cache read latency materially on NVMe.</li>
|
||||||
|
<li>Skip-scan makes more multicolumn indexes usable without rewrites.</li>
|
||||||
|
<li>Upgrade path is smooth; test the new planner stats first.</li></ul></div>
|
||||||
|
<div class="sec"><h2>Highlights</h2><ul>
|
||||||
|
<li>Async I/O subsystem (effective_io_concurrency now matters more).</li>
|
||||||
|
<li>B-tree skip scan for leading-column gaps.</li>
|
||||||
|
<li>Better partition-wise joins.</li></ul></div>
|
||||||
|
<div class="sec"><h2>Summary</h2><p class="body">Postgres 18 is an incremental but meaningful release: the headline is the new asynchronous I/O path, with skip-scan and planner improvements close behind. For most homelabs the upgrade is low-risk and worth it for the read-latency wins.</p></div>
|
||||||
|
<div class="dock"><button class="ask">✦ Ask about this video</button></div>
|
||||||
|
</article>
|
||||||
|
|
||||||
|
<div class="card"><div class="title">RAG is dead, right?? — Conference Talk</div>
|
||||||
|
<div class="meta mono">youtube · 2026-06-09 · 31 min <span class="chip q">queued</span></div></div>
|
||||||
|
</div></body></html>
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>tapir — C · cozy terminal</title>
|
||||||
|
<style>
|
||||||
|
:root{--bg:#16131d;--card:#1e1a28;--ink:#ece7f5;--soft:#9a92b4;--line:#322b44;--mint:#2ee6b6;--purple:#9d7bff;--pink:#ff6bdb;--cream:#f3ead8}
|
||||||
|
*{box-sizing:border-box}
|
||||||
|
body{margin:0;background:radial-gradient(1200px 600px at 70% -10%,#231b33,transparent),var(--bg);color:var(--ink);font:15px/1.6 -apple-system,BlinkMacSystemFont,"Segoe UI",Inter,sans-serif}
|
||||||
|
.mono{font-family:ui-monospace,SFMono-Regular,Menlo,monospace}
|
||||||
|
.wrap{max-width:700px;margin:0 auto;padding:20px 16px 60px}
|
||||||
|
header{display:flex;align-items:center;gap:12px;margin-bottom:18px}
|
||||||
|
.brand{font-weight:800;font-size:18px}.brand .c{color:var(--mint)}
|
||||||
|
.tag{color:var(--soft);font-size:13px}
|
||||||
|
.tip{margin-left:auto;color:var(--soft);font-size:12px}
|
||||||
|
.note{color:var(--soft);font-size:13px;background:#1b1726;border:1px solid var(--line);border-radius:10px;padding:9px 12px;margin:0 0 18px}
|
||||||
|
.note b{color:var(--mint);font-weight:600}
|
||||||
|
.card{position:relative;background:var(--card);border:1px solid var(--line);border-radius:12px;padding:14px 16px 14px 18px;margin:0 0 13px;overflow:hidden}
|
||||||
|
.card::before{content:"";position:absolute;left:0;top:0;bottom:0;width:4px;background:var(--purple)}
|
||||||
|
.card.ready::before{background:linear-gradient(var(--mint),var(--purple))}
|
||||||
|
.title{font-size:16px;font-weight:700;margin:0 0 4px;color:var(--cream)}
|
||||||
|
.meta{color:var(--soft);font-size:12.5px}
|
||||||
|
.chip{display:inline-block;border-radius:999px;padding:1px 9px;font-size:11px;font-weight:600;margin-left:6px;background:rgba(46,230,182,.14);color:var(--mint)}
|
||||||
|
.chip.q{background:rgba(157,123,255,.18);color:var(--purple)}
|
||||||
|
.preview{color:var(--soft);margin:7px 0 0;font-size:14px}
|
||||||
|
.exp{border-color:#473a63;box-shadow:0 10px 40px rgba(0,0,0,.35)}
|
||||||
|
.head{display:flex;justify-content:space-between;align-items:baseline;gap:10px}
|
||||||
|
.collapse{color:var(--mint);font-size:12px;text-decoration:none;border:1px solid var(--line);border-radius:6px;padding:2px 8px;white-space:nowrap}
|
||||||
|
.sec h2{font-size:11px;text-transform:uppercase;letter-spacing:1.3px;color:var(--mint);margin:18px 0 7px;display:flex;align-items:center;gap:8px}
|
||||||
|
.sec h2::after{content:"";flex:1;height:1px;background:var(--line)}
|
||||||
|
.sec ul{margin:0;padding-left:18px}.sec li{margin:5px 0}
|
||||||
|
.body{line-height:1.7;color:var(--ink)}
|
||||||
|
.dock{margin-top:18px;border-top:1px dashed var(--line);padding-top:14px;display:flex;align-items:center;gap:10px}
|
||||||
|
.ask{display:inline-flex;align-items:center;gap:7px;background:linear-gradient(90deg,var(--purple),var(--mint));color:#10101a;border:0;border-radius:999px;padding:9px 16px;font:inherit;font-weight:700;font-size:14px;cursor:pointer}
|
||||||
|
.dockhint{color:var(--soft);font-size:12px}
|
||||||
|
pre.tapir{margin:0;color:var(--mint);font-size:11px;line-height:1.05}
|
||||||
|
</style></head><body><div class="wrap">
|
||||||
|
<header>
|
||||||
|
<pre class="tapir"> ▄█▓▓█▄ ∩
|
||||||
|
█▓( ◕ ◕ )▓█──┘</pre>
|
||||||
|
<span class="brand">tapir<span class="c">_</span></span><span class="tag">watch less, know more</span>
|
||||||
|
<span class="tip mono">261 in queue</span>
|
||||||
|
</header>
|
||||||
|
<p class="note">Tapir fetches captions slowly on purpose, to respect YouTube's limits — <b>new summaries land gradually</b>.</p>
|
||||||
|
|
||||||
|
<div class="card ready"><div class="title">How the Attention Economy Rewires Your Brain</div>
|
||||||
|
<div class="meta mono">youtube · 2026-06-11 · 18 min <span class="chip">ready</span></div>
|
||||||
|
<div class="preview">A tour of the incentive loops behind infinite feeds and three concrete ways to claw back focus…</div></div>
|
||||||
|
|
||||||
|
<article class="card exp ready">
|
||||||
|
<div class="head"><div class="title">Postgres 18 — What's Actually New</div><a class="collapse" href="#">collapse ↑</a></div>
|
||||||
|
<div class="meta mono">youtube · 2026-06-10 · 42 min · phi4-mini</div>
|
||||||
|
<div class="sec"><h2>Takeaways</h2><ul>
|
||||||
|
<li>Async I/O cuts cold-cache read latency materially on NVMe.</li>
|
||||||
|
<li>Skip-scan makes more multicolumn indexes usable without rewrites.</li>
|
||||||
|
<li>Upgrade path is smooth; test the new planner stats first.</li></ul></div>
|
||||||
|
<div class="sec"><h2>Highlights</h2><ul>
|
||||||
|
<li>Async I/O subsystem (effective_io_concurrency now matters more).</li>
|
||||||
|
<li>B-tree skip scan for leading-column gaps.</li>
|
||||||
|
<li>Better partition-wise joins.</li></ul></div>
|
||||||
|
<div class="sec"><h2>Summary</h2><p class="body">Postgres 18 is an incremental but meaningful release: the headline is the new asynchronous I/O path, with skip-scan and planner improvements close behind. For most homelabs the upgrade is low-risk and worth it for the read-latency wins.</p></div>
|
||||||
|
<div class="dock"><button class="ask">◆ Ask about this video</button><span class="dockhint mono">answers only from this video's transcript</span></div>
|
||||||
|
</article>
|
||||||
|
|
||||||
|
<div class="card"><div class="title">RAG is dead, right?? — Conference Talk</div>
|
||||||
|
<div class="meta mono">youtube · 2026-06-09 · 31 min <span class="chip q">queued</span></div></div>
|
||||||
|
</div></body></html>
|
||||||
@@ -0,0 +1,89 @@
|
|||||||
|
# Spec — Chat with a video's stored transcript (ADR-027)
|
||||||
|
|
||||||
|
**Repo:** tapir · **Size:** medium · **Solo session.** Implements ADR-027. Read CLAUDE.md,
|
||||||
|
DECISIONS.md (ADR-021 transcript store, ADR-022 model chain, ADR-012 isolation, ADR-027), and
|
||||||
|
`docs/ui-spec.md` first. TBD, conventional commits, `task check` green per commit, `templ
|
||||||
|
generate` after view changes.
|
||||||
|
|
||||||
|
## FIRST: append ADR-027 to DECISIONS.md
|
||||||
|
ADR-027 text is provided separately (planning thread). Insert immediately before the
|
||||||
|
`## Rejected alternatives` heading, as the first commit, so the decision precedes the build.
|
||||||
|
|
||||||
|
## What this is
|
||||||
|
A per-video chat letting the user ask questions against a video's **already-stored** transcript,
|
||||||
|
entered from the summary view. Born from observed demand: the maintainer read real summaries and
|
||||||
|
some made him want to dig deeper — this gives that "I want more" reaction somewhere to go, without
|
||||||
|
watching the video.
|
||||||
|
|
||||||
|
## HARD CONSTRAINT — stored-transcript-only (the safety property)
|
||||||
|
Chat is available **ONLY** for videos that already have a stored transcript (ADR-021). It must
|
||||||
|
**never** trigger a caption fetch, never touch the rate gate, never reach YouTube. Entry being
|
||||||
|
"from a summarized video" guarantees the transcript exists. If somehow invoked on a video with no
|
||||||
|
stored transcript → show "transcript not available for chat", NO fetch. This is what makes the
|
||||||
|
feature safe by construction; do not add an on-demand-fetch path (explicitly deferred).
|
||||||
|
|
||||||
|
## 1. Entry point
|
||||||
|
- A "Dig deeper" / "Ask about this" affordance on the **summary view** of a summarized video
|
||||||
|
(not the list cards — the detail/summary page). Quiet, consistent with the existing card-state
|
||||||
|
styling.
|
||||||
|
- Opens a chat panel/view scoped to that one video, with its stored transcript as context.
|
||||||
|
|
||||||
|
## 2. The chat
|
||||||
|
- Read the stored transcript for the video (via the ADR-021 `TranscriptStore`, keyed by
|
||||||
|
`(provider, provider_video_id)`). No fetch.
|
||||||
|
- Send transcript + the user's question + minimal system framing to the chosen model via the
|
||||||
|
**existing LiteLLM gateway** (the same client the summarizer uses — a chat is a different
|
||||||
|
call, not a new integration).
|
||||||
|
- Stream or return the answer; render in the chat panel. HTMX/no-JS ethos — match the existing
|
||||||
|
app (the summarize status uses HTMX polling; chat can use a simple POST-and-render or HTMX
|
||||||
|
streaming if clean).
|
||||||
|
- **Transcript truncation:** reuse/respect `TAPIR_MAX_TRANSCRIPT_CHARS` (ADR-022) so a long
|
||||||
|
transcript fits the model context. If truncated, the chat should be honest that it's working
|
||||||
|
from a bounded portion (a quiet note), since answers about the tail of a long video may be
|
||||||
|
incomplete.
|
||||||
|
|
||||||
|
## 3. Model selection (the instrumentation win)
|
||||||
|
- **Default model = the model that produced this video's summary.** (Store/lookup which chain
|
||||||
|
model summarized it — if not already recorded, this is a small addition; if recording it is
|
||||||
|
non-trivial, default to the chain primary and note the gap.)
|
||||||
|
- **User can switch** among the ADR-022 chain models (`phi4-mini`, `gemma4-26b`,
|
||||||
|
`mistral-small` to start) via a simple selector in the chat panel. Switching re-runs against
|
||||||
|
the same transcript — this is deliberate model-comparison instrumentation.
|
||||||
|
- Respect the local-first / NDA posture: if `TAPIR_CLOUD_FALLBACK_MODEL=""` (cloud disabled),
|
||||||
|
the external model is NOT offered in the switcher — only local models. Chat must honor the same
|
||||||
|
"content stays local" guarantee as ADR-022.
|
||||||
|
|
||||||
|
## 4. Ephemeral (v1)
|
||||||
|
- No persisted chat history. Conversation lives for the session/page. No new table, no migration.
|
||||||
|
- (Multi-turn within a session is fine — keep the running messages in the request/page state —
|
||||||
|
but nothing is written to the DB.)
|
||||||
|
|
||||||
|
## 5. Isolation
|
||||||
|
- The transcript is shared/non-RLS (ADR-021) — fine, it's public content. But the chat is invoked
|
||||||
|
by a user about a video **in their feed**; confirm the entry path is reachable only for the
|
||||||
|
requesting user's own videos (the summary view is already RLS-scoped). Chat adds no new
|
||||||
|
user-data surface (ephemeral), so there's nothing new to RLS — but the test should confirm a
|
||||||
|
user can only open chat from their own summary view, not arbitrary video ids.
|
||||||
|
|
||||||
|
## Tests
|
||||||
|
- Chat on a video with a stored transcript → answer returned; assert NO caption-fetch / no
|
||||||
|
YouTube call occurs (the safety property — this is the key assertion).
|
||||||
|
- Chat invoked on a video with no stored transcript → honest "not available", NO fetch.
|
||||||
|
- Model switch → re-runs against the same transcript with the selected model; cloud model absent
|
||||||
|
from the switcher when `TAPIR_CLOUD_FALLBACK_MODEL=""`.
|
||||||
|
- Truncation honored for a long transcript; the bounded-context note shows.
|
||||||
|
- Entry is reachable only from the user's own summary view (isolation).
|
||||||
|
|
||||||
|
## Out of scope / deferred (record, don't build)
|
||||||
|
- **Persisted chat history** (per-user, RLS-scoped) — deferred until evidence anyone revisits a
|
||||||
|
conversation.
|
||||||
|
- **Show-source / transcript-verification UI** — the natural v2 (ADR-027 records it); v1 is
|
||||||
|
chat-only/trust-the-model. ADR-021's stored transcript makes v2 cheap when wanted.
|
||||||
|
- **On-demand fetch** for un-stored videos — would reintroduce the caption-fetch surface the
|
||||||
|
stored-only constraint removes. Not now.
|
||||||
|
- Anything that nudges the user to return (ADR-020 — gate contamination).
|
||||||
|
|
||||||
|
## Boundaries
|
||||||
|
Stored-transcript-only (HARD). No rate-gate/fetch surface. No auth changes. No new persisted
|
||||||
|
state in v1. Reuse the existing gateway client + truncation config; don't build a new model
|
||||||
|
integration.
|
||||||
@@ -0,0 +1,153 @@
|
|||||||
|
# Spec — Landing page + documentation reconciliation
|
||||||
|
|
||||||
|
**Date:** 2026-06-03
|
||||||
|
**Status:** Ready to build
|
||||||
|
**Scope:** Two parallel workstreams — (A) a public landing page; (B) reconciling the
|
||||||
|
requirements / use-case / architecture / data-model docs against the deployed reality
|
||||||
|
(v0.4.0). These are separate concerns; do not let one worker do both, or the audit gets
|
||||||
|
done cursorily.
|
||||||
|
|
||||||
|
All work: read `CLAUDE.md` + `DECISIONS.md` first. TBD — commit directly to `main`, one
|
||||||
|
logical change per commit, conventional commits, `task check` green before every commit.
|
||||||
|
After editing any `.templ`, run `templ generate` (the repo commits both `views.templ` and the
|
||||||
|
generated `views_templ.go`).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Workstream A — Public landing page
|
||||||
|
|
||||||
|
### Goal
|
||||||
|
A public landing page at `/welcome`, in the established bubbletea aesthetic, that lets a
|
||||||
|
visitor sign in (one Dex flow) and, if already logged in, jump to their Tapir page or log out.
|
||||||
|
New public transport surface only — no engine/core change (ADR-003).
|
||||||
|
|
||||||
|
### Verified facts (read from `internal/web/oidc/oidc.go` @ main — do not re-guess)
|
||||||
|
- Auth endpoints are exactly `/auth/login`, `/auth/callback`, `/auth/logout`.
|
||||||
|
- `isPublicPath(p)` = `p == "/healthz" || strings.HasPrefix(p, "/auth/")` — the single
|
||||||
|
public-route chokepoint inside `DexAuth.Middleware`.
|
||||||
|
- `DexAuth.CurrentUser(r) (web.User, bool)` reads the session cookie and does NOT redirect —
|
||||||
|
this is the "peek" the landing page uses to branch logged-in vs logged-out.
|
||||||
|
- `handleCallback` redirects to `/` on success (correct — leave as-is).
|
||||||
|
- `handleLogout` currently redirects to `loginPath` (`/auth/login`) — this is wrong for this
|
||||||
|
feature (see A3).
|
||||||
|
- There is NO separate "sign up" against Dex/OIDC: one authorization flow. Registration is
|
||||||
|
Tapir's own `/register` step (ADR-012), reached after first login for an unknown subject.
|
||||||
|
|
||||||
|
### Tasks
|
||||||
|
**A1 — make `/welcome` public.** In `oidc.go`, extend `isPublicPath`:
|
||||||
|
```go
|
||||||
|
func isPublicPath(p string) bool {
|
||||||
|
return p == "/healthz" || p == "/welcome" || strings.HasPrefix(p, "/auth/")
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**A2 — unauthenticated bare-`/` → `/welcome`; deep links unchanged.** In `DexAuth.Middleware`,
|
||||||
|
the unauthenticated branch currently always calls `redirectToLogin`. Change it so that when
|
||||||
|
`r.URL.Path == "/"` an unauthenticated visitor is redirected to `/welcome`; for any other
|
||||||
|
guarded path keep `redirectToLogin` (so a shared `/v/{id}` deep link still bounces through Dex
|
||||||
|
and returns to the destination). Keep the `isPublicPath` check first (redirect-loop guard).
|
||||||
|
|
||||||
|
**A3 — logout lands on `/welcome`, not login.** In `handleLogout`, change the final redirect
|
||||||
|
from `loginPath` to `/welcome`. As written it sends the user to `/auth/login`, which
|
||||||
|
immediately starts a fresh Dex login — visibly failing to log out. This intentionally breaks
|
||||||
|
the existing logout test (oidc_test.go) which asserts redirect to `/auth/login`; update that
|
||||||
|
test to expect `/welcome`. That break is expected, not a regression.
|
||||||
|
|
||||||
|
**A4 — mount the landing handler** in `internal/web/handlers.go` `Router()`, on `root`,
|
||||||
|
OUTSIDE `Auth.Middleware`, alongside `/healthz`:
|
||||||
|
```go
|
||||||
|
root.HandleFunc("GET /welcome", a.handleWelcome)
|
||||||
|
```
|
||||||
|
`handleWelcome` peeks `a.Auth.CurrentUser(r)` and renders `WelcomePage(user, ok)`. Not behind
|
||||||
|
`Auth.Middleware` or `registrationGate`.
|
||||||
|
|
||||||
|
**A5 — `WelcomePage` templ component** in `views.templ`. Reuse the existing shared
|
||||||
|
layout/header partial and the established aesthetic (#7653FC purple rounded ╭─╮╰─╯ box, pink
|
||||||
|
tapir mascot, #0EF9B6 mint accents) — match the existing pages, do not reinvent styling.
|
||||||
|
- Logged out (`ok == false`): tapir mascot + tagline; one primary CTA **"Get Started"** →
|
||||||
|
`/auth/login`; honest sub-text: "New here? You'll set up your account right after signing in
|
||||||
|
— returning users go straight through." One button only (see verified facts: no separate
|
||||||
|
Dex sign-up; two buttons to the same URL would mislead).
|
||||||
|
- Logged in (`ok == true`): "Go to my Tapir" → `/`; "Log Out" → `/auth/logout`. May greet via
|
||||||
|
`user.Email`.
|
||||||
|
|
||||||
|
**A6 — tests** (extend `handlers_test.go` patterns). Note `StubAuth.CurrentUser` always returns
|
||||||
|
true; for the logged-out case use a fake Auth returning `(web.User{}, false)`.
|
||||||
|
- `GET /welcome`, no session → "Get Started" → `/auth/login`.
|
||||||
|
- `GET /welcome`, with session → "Go to my Tapir" + "Log Out".
|
||||||
|
- Unauthenticated `GET /` → 302 `/welcome`.
|
||||||
|
- Unauthenticated `GET /v/{id}` → still 302 `/auth/login` (deep link preserved).
|
||||||
|
- Authenticated `GET /` → still serves the list, unchanged.
|
||||||
|
- oidc: `handleLogout` → 302 `/welcome` (update the existing test).
|
||||||
|
|
||||||
|
**A out of scope:** no Dex config change, no new auth/session logic, no sign-up backend.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Workstream B — Documentation reconciliation
|
||||||
|
|
||||||
|
### Why
|
||||||
|
The guardrail docs were written before Stage 1 and the web surface. Several now describe the
|
||||||
|
opposite of the deployed reality (v0.4.0). Stale guardrail docs are worse than none — a future
|
||||||
|
cold session (human or agent) trusts them. This workstream brings requirements, use cases,
|
||||||
|
architecture, and data-model back in sync with `main`. Each fix is one commit; cite the ADR or
|
||||||
|
migration that is the source of truth.
|
||||||
|
|
||||||
|
### Known drift to fix (verified this session — not exhaustive; the worker confirms against code)
|
||||||
|
**B1 — `internal/web/auth.go` comments.** The `User.Subject` doc and package doc still say
|
||||||
|
"single-user allowlist (ADR-011)" / "Stage-0". Code is multi-user (ADR-012). Update the
|
||||||
|
comments to describe the current multi-user reality; reference ADR-012.
|
||||||
|
|
||||||
|
**B2 — `docs/data-model.md` isolation status.** It says isolation enforcement is "dormant at
|
||||||
|
Stage 0". It is now LIVE: Postgres RLS, `FORCE`d on all user-owned tables, with a passing
|
||||||
|
two-user isolation test (ADR-012, migration 003). Rewrite that section to describe enforced
|
||||||
|
RLS as the current state; keep the history honest (was dormant at Stage 0, enforced from
|
||||||
|
Stage 1).
|
||||||
|
|
||||||
|
**B3 — `docs/data-model.md` schema completeness.** The doc predates migrations 002–006. Add
|
||||||
|
the entities/columns that now exist: `summary_actions` (002), RLS (003), `user_identities`
|
||||||
|
(004, dex_subject→user_id), `video_connections` (005), `users.auto_summarize` +
|
||||||
|
`videos.summarize_requested` (006). The ER section should match the live schema. Cross-check
|
||||||
|
against `internal/adapters/store/migrations/*.up.sql` — those are ground truth.
|
||||||
|
|
||||||
|
**B4 — `docs/architecture/architecture.md`.** Predates the entire web surface. Update the C4
|
||||||
|
container diagram and text to include: `tapir serve` (HTMX+Templ web reader/writer), the Dex
|
||||||
|
OIDC session layer (`internal/web/oidc`), registration gate, web-initiated YouTube connect,
|
||||||
|
account management, and the immediate-processing path (web "Summarize" button → background
|
||||||
|
goroutine → status poll). The engine/ports/sinks core is unchanged (ADR-003) — show the web
|
||||||
|
surface as a new transport over the same core, not a core change.
|
||||||
|
|
||||||
|
**B5 — `docs/use-cases/*.feature`.** Add scenarios for the behaviours now live and unspecced:
|
||||||
|
register (new subject → registration → user row; returning user straight through), connect
|
||||||
|
YouTube (web OAuth), disconnect, delete-account (cascade + secret purge, Dex untouched —
|
||||||
|
ADR-013), manual-vs-auto summarize mode + the Summarize button, and the landing page
|
||||||
|
(logged-out CTA; logged-in shortcuts). Keep them as executable-style Gherkin consistent with
|
||||||
|
the existing files.
|
||||||
|
|
||||||
|
**B6 — `DECISIONS.md` ADR ordering (cosmetic).** ADR-010 sits before ADR-009/011 (append
|
||||||
|
order). Reorder to numeric while you're in the file. Pure tidy, no content change.
|
||||||
|
|
||||||
|
**B7 — requirements check.** If a requirements doc exists (e.g. `docs/ui-spec.md`, referenced
|
||||||
|
by ADR-011), reconcile it with what shipped: note where the build deviated (e.g. the spinner /
|
||||||
|
immediate processing / summarize mode were beyond the original spec) so the spec reflects
|
||||||
|
reality or explicitly records the deviation. Do not silently rewrite history — record
|
||||||
|
deviations as deviations.
|
||||||
|
|
||||||
|
### B working method
|
||||||
|
- Source of truth order: migrations + code > ADRs > prose docs. When a prose doc disagrees
|
||||||
|
with code, the code wins and the doc is corrected (unless the code is the bug — then flag it,
|
||||||
|
don't quietly doc around it).
|
||||||
|
- One logical doc per commit. Cite the ADR/migration that justifies each change in the commit
|
||||||
|
body.
|
||||||
|
- This is an audit, not a rewrite: preserve the docs' structure and the "rejected alternatives
|
||||||
|
/ history" honesty. The goal is *current and trustworthy*, not *pretty*.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Coordination
|
||||||
|
A and B touch mostly different files (A: oidc.go, handlers.go, views.templ, tests; B: docs/* +
|
||||||
|
auth.go comments). The one overlap is `auth.go` (B1 edits comments) vs A (reads it) — no
|
||||||
|
conflict. Run A and B in parallel; commit independently to `main`.
|
||||||
|
|
||||||
|
If anything in B reveals that code, not docs, is wrong (e.g. an isolation gap, a migration that
|
||||||
|
doesn't match the data-model intent), STOP and surface it — that's a finding, not a doc edit.
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
# Spec — Newest-first batch ordering + honest "Try now" / prioritisation docs
|
||||||
|
|
||||||
|
> **Extended by ADR-020 (2026-06-08).** This spec covers the *batch processing* order within a
|
||||||
|
> pass. ADR-020 adds (a) a recency pre-filter — auto mode skips videos published before
|
||||||
|
> `TAPIR_AUTO_SUMMARIZE_WINDOW`, listed but summarised on demand — and (b) the same
|
||||||
|
> `published_at DESC NULLS LAST` ordering on the **list read** (`ListVideos`), which previously
|
||||||
|
> sorted by `seen_at`. See `DECISIONS.md` ADR-020.
|
||||||
|
|
||||||
|
**Repo:** tapir · **Size:** small · **Solo session.**
|
||||||
|
|
||||||
|
**Why.** Product intent (maintainer, 2026-06-06): a new user should get summaries of their
|
||||||
|
**newest** videos quickly, while the older back-catalogue fills in behind — all within the one
|
||||||
|
shared rate gate. Today the foreground path ("Try now" button) lets a user hand-pick a video,
|
||||||
|
but the **background batch processes in subscription/channel order, not newest-first** — so a new
|
||||||
|
user with a large candidate set sees the batch summarise whatever channel is first in their
|
||||||
|
subscription list, not their newest videos. This slice makes the batch agree with the intent, and
|
||||||
|
fixes the docs to describe the real rationale (onboarding prioritisation), not the
|
||||||
|
traffic-disguising framing a prior session wrote.
|
||||||
|
|
||||||
|
Read `CLAUDE.md` + `DECISIONS.md` (ADR-014, ADR-018) first. TBD, conventional commits,
|
||||||
|
`task check` green per commit, `templ generate` if views change.
|
||||||
|
|
||||||
|
## 1. Newest-first batch ordering (the build)
|
||||||
|
In `internal/runner/runner.go` `RunOnce`: today the loop processes each video inline while
|
||||||
|
walking subscriptions channel-by-channel (`for sub → NewVideos → for v → process`). Change so
|
||||||
|
that, within a pass, **candidates are processed newest-first across ALL channels**:
|
||||||
|
- Collect the candidate videos across channels first (after dedup/seen/manual/rate-limit
|
||||||
|
filtering as today), then **sort by `published_at` descending before processing**, then process
|
||||||
|
in that order through the engine + shared `globalFetchGate`.
|
||||||
|
- **`published_at` is nullable** (schema 001). Sort **NULLS LAST** — videos with no publish date
|
||||||
|
must not jump ahead of dated newest videos. Decide a stable tiebreak (e.g. `seen_at DESC`) for
|
||||||
|
equal/again-null dates.
|
||||||
|
- Keep all existing behaviour: per-item failure isolation, the rate-limit backoff skip, manual
|
||||||
|
mode, channel-unavailable handling, stats. Ordering is the only change — not what gets
|
||||||
|
processed, just the order.
|
||||||
|
- At 868 candidates a collect-then-sort in memory is fine; do **not** build a streaming/external
|
||||||
|
sort. Keep it simple.
|
||||||
|
- The shared rate gate (`globalFetchGate`) is unchanged and still governs fetch pacing — ordering
|
||||||
|
does not bypass or weaken it.
|
||||||
|
|
||||||
|
**Optional (only if cheap and clearly correct):** a soft cap so the *first* pass for a brand-new
|
||||||
|
user summarises the newest N (e.g. 20) quickly and defers the long tail to subsequent passes — so
|
||||||
|
onboarding value lands fast without waiting for the whole sorted set. If this adds real
|
||||||
|
complexity, SKIP it and just do the newest-first ordering; the ordering alone delivers the intent.
|
||||||
|
|
||||||
|
## 2. Tests
|
||||||
|
- Given candidates across multiple channels with mixed `published_at` (incl. some NULL), assert
|
||||||
|
the processing order is newest-first, NULLS LAST, with the chosen tiebreak. Use the existing
|
||||||
|
fake VideoStore/Processor pattern in `runner_test.go`.
|
||||||
|
- Assert ordering does not change *which* videos are processed vs. today (same set, new order).
|
||||||
|
- Rate-gate / backoff / manual-mode behaviour unchanged (existing tests stay green).
|
||||||
|
|
||||||
|
## 3. Docs — describe the REAL rationale (replace prior framing)
|
||||||
|
The "Try now" button and the discovery batch together implement **onboarding prioritisation**:
|
||||||
|
foreground (user-clicked "Try now") summarises a specific video on demand; background batch
|
||||||
|
summarises newest-first; both honour the shared rate gate. **Update the docs to state this intent
|
||||||
|
— and explicitly REMOVE/replace any framing that describes "Try now" as making traffic "look
|
||||||
|
organic to YouTube" or evading rate limits.** That is not the rationale. The rationale is: *get
|
||||||
|
the user a few summaries of their newest, most relevant videos fast; process the back-catalogue in
|
||||||
|
the background; always within the honest shared rate limit.* Rate limiting is **respected**, not
|
||||||
|
evaded.
|
||||||
|
- `docs/ui-spec.md`: "Try now" = on-demand foreground summarisation of a chosen (typically newer)
|
||||||
|
video; rationale = fast onboarding value, not traffic shaping.
|
||||||
|
- `docs/architecture/architecture.md`: document the two-path model — foreground on-demand vs.
|
||||||
|
background newest-first batch, both through `globalFetchGate` — and the newest-first ordering.
|
||||||
|
- Any requirements/use-case doc mentioning discovery order: state newest-first.
|
||||||
|
- If a brain note or `wiki` entry captured the "looks organic" rationale, correct it there too.
|
||||||
|
|
||||||
|
## Boundaries
|
||||||
|
- Do NOT increase fetch rate or weaken the rate gate. Account-safety constraint stands: the
|
||||||
|
caption endpoint is unofficial (ADR-010) and must be treated with honest backoff, never evasion.
|
||||||
|
- Do NOT touch RLS, credentials, or the Dex surface.
|
||||||
|
- Ordering change is within a pass only — no persisted priority queue, no new table.
|
||||||
|
|
||||||
|
## Out of scope
|
||||||
|
Per-user configurable ordering; priority weighting beyond newest-first; the soft-cap if it proves
|
||||||
|
non-trivial.
|
||||||
@@ -0,0 +1,128 @@
|
|||||||
|
# Spec — Onboarding "wow" burst: better picks, stronger model
|
||||||
|
|
||||||
|
**Repo:** tapir · **Size:** medium · **Solo session** (not a swarm).
|
||||||
|
|
||||||
|
> **Status: built (v0.25.0, ADR-028).** This supersedes the original investigate-first brief
|
||||||
|
> (committed as the prior version of this file): Phase 1 was run against the live pilot DB and its
|
||||||
|
> findings are folded into "Why this exists" below; Phase 2 was built as described here. The one
|
||||||
|
> brief lever NOT built — the honest "the rest fill in over the coming days" framing copy — is
|
||||||
|
> listed under *Explicitly NOT in this slice*.
|
||||||
|
|
||||||
|
**Why this exists.** A new user's first session decides whether they return (the Stage-0 gate,
|
||||||
|
VISION.md). On connect, Tapir fires a capped burst (≤`TAPIR_ONBOARD_SUMMARIZE_COUNT`, default 3)
|
||||||
|
that summarizes the user's newest unsummarized videos so the feed isn't empty (the burst itself
|
||||||
|
works — wired in `cmd/tapir/discovery.go` → `cmd/tapir/main.go` `onboard`). A Phase-1
|
||||||
|
investigation of the live pilot DB found the burst *fires* but delivers a **weak first
|
||||||
|
impression** for two concrete reasons, and ruled out a third idea:
|
||||||
|
|
||||||
|
1. **Picks are junk.** Selection is pure newest-first (`videos.NewestUnsummarizedVideoIDs`,
|
||||||
|
`ORDER BY published_at DESC`) with **zero quality signal**. Pilot user "Jonte"'s live burst-3
|
||||||
|
were a stock-ticker **livestream** + two regional news clips — the newest, not the best.
|
||||||
|
2. **Weakest model on the first impression.** All of Jonte's summaries ran on
|
||||||
|
`koala/phi4-mini` (the documented weak link — ADR-022 was born from its failures). The
|
||||||
|
stronger, brain-validated `iguana/gemma4-26b` was never used for the burst.
|
||||||
|
3. **Cached-first is empty at pilot scale — REJECTED.** The idea (summarize already-cached
|
||||||
|
transcripts instantly, zero fetch) dies on the numbers: only **11 videos** overlap between the
|
||||||
|
two pilot users (~3% of each library), **0** cached-and-unsummarized, and a new user's
|
||||||
|
newest-20 unsummarized are **20/20 NOT cached** — newest-first and cached-first are
|
||||||
|
structurally incompatible (fresh uploads are exactly what nobody has fetched yet). Not built.
|
||||||
|
|
||||||
|
This is a **curation/latency problem for ~3 videos, NOT a throughput/429 problem** — fetching 3
|
||||||
|
captions is nowhere near the rate limit. Nothing here fetches harder or pressures the rate gate;
|
||||||
|
it picks the right few videos and runs a better model on them.
|
||||||
|
|
||||||
|
Read `CLAUDE.md`, `DECISIONS.md` (esp. ADR-014, ADR-018, ADR-020, ADR-021, ADR-022, ADR-023,
|
||||||
|
and the new **ADR-028**), and `VISION.md` (the Stage-0 gate) first. TBD — commit directly to
|
||||||
|
`main`, one logical change per commit, conventional commits, `task check` green before each
|
||||||
|
commit, `templ generate` if any view changes (none expected).
|
||||||
|
|
||||||
|
## Decisions already made (do not reopen)
|
||||||
|
- **Not a throughput change.** The caption rate gate (ADR-014) is untouched — same pacing, same
|
||||||
|
priority lane (ADR-026). This slice changes *which* ≤3 videos the burst spends its fetches on
|
||||||
|
and *which model* summarizes them, never how fast or how many.
|
||||||
|
- **Cached-first is dropped** (ADR-028, the 3% overlap). The engine's existing read-stored-first
|
||||||
|
(ADR-021, `resolveTranscript`) stays — it already gives a free instant summary on the rare
|
||||||
|
cache hit, transparently. We do not *select* for cache hits.
|
||||||
|
- **has-captions is not a pre-fetch signal.** It is only knowable after a gate fetch (or a cache
|
||||||
|
hit, ~0 for new videos). Selection can only *avoid known-junk* (Shorts/live/over-long) — it
|
||||||
|
cannot *guarantee* captions. The spec is honest about this: better odds, not a promise.
|
||||||
|
- **No credentialed caption fetch** (ADR-010/ADR-026 dead end). **No client extension.**
|
||||||
|
|
||||||
|
## 1. Persist `duration_s` at discovery (the enabling change)
|
||||||
|
The `videos.duration_s` column exists (migration 001) but is **never written** — ADR-023's
|
||||||
|
`filterLowValue` (`internal/adapters/youtube/youtube.go`) already fetches each candidate's
|
||||||
|
duration via the cheap quota `videos.list` call, uses it to drop Shorts/live, then **discards
|
||||||
|
it**. Stop discarding:
|
||||||
|
- Add `DurationSeconds int` to `domain.Video`.
|
||||||
|
- In `filterLowValue`, set `DurationSeconds` on each kept video from the `videos.list` `meta`.
|
||||||
|
- `UpsertVideo` writes `duration_s`, **COALESCE-preserving** a known value (never overwrite a
|
||||||
|
real duration with 0/unknown), mirroring the `channel_title` backfill stance (migration 014).
|
||||||
|
- No new migration — the column is already there.
|
||||||
|
|
||||||
|
Consequence: a fresh user's connect-triggered discovery pass runs **before** the onboard burst
|
||||||
|
(`Enqueue`: `run()` then `onboard()`), so duration is populated for the burst's candidates at
|
||||||
|
connect. Existing rows backfill on their next discovery pass; until then their `duration_s` is
|
||||||
|
NULL and treated as "unknown" (§2).
|
||||||
|
|
||||||
|
## 2. Junk-avoiding burst selection
|
||||||
|
New store method, RLS-scoped via `withUser`:
|
||||||
|
|
||||||
|
```
|
||||||
|
OnboardBurstVideoIDs(ctx, userID string, limit, minSeconds, maxSeconds int) ([]string, error)
|
||||||
|
```
|
||||||
|
|
||||||
|
- Same base as the old `NewestUnsummarizedVideoIDs`: the user's videos with no summary yet,
|
||||||
|
`ORDER BY published_at DESC NULLS LAST, seen_at DESC`, `LIMIT limit`.
|
||||||
|
- **Exclude known-junk**: a row is dropped only when `duration_s IS NOT NULL` **and**
|
||||||
|
(`duration_s < minSeconds` OR `duration_s > maxSeconds`). A NULL duration is **unknown** — kept
|
||||||
|
(degrade-open: never starve the burst because metadata is missing), but ordered *after* rows
|
||||||
|
with a known-good duration so a freshly-enriched good pick wins when both exist.
|
||||||
|
- `minSeconds` reuses `TAPIR_MIN_VIDEO_SECONDS` (default 60 — the Shorts floor, ADR-023).
|
||||||
|
`maxSeconds` is new: `TAPIR_ONBOARD_MAX_VIDEO_SECONDS` (default 14400 = 4h) — drops the
|
||||||
|
multi-hour livestream VODs that pass the live filter once ended.
|
||||||
|
- `minSeconds<=0` and `maxSeconds<=0` each disable that bound (so `0/0` == the old
|
||||||
|
newest-first behaviour, the reversibility lever).
|
||||||
|
- The burst switches to this method; `NewestUnsummarizedVideoIDs` is removed (fully superseded —
|
||||||
|
`OnboardBurstVideoIDs(., 0, 0)` is identical pure-newest behaviour).
|
||||||
|
|
||||||
|
## 3. Stronger model for the burst
|
||||||
|
The burst summarizes only ≤3 videos, so a slower, stronger model is affordable exactly here.
|
||||||
|
- New config `TAPIR_ONBOARD_SUMMARIZER_MODEL` (default `iguana/gemma4-26b` — the brain-validated
|
||||||
|
homelab general-purpose model, already the ADR-022 fallback).
|
||||||
|
- Build a **burst-specific summarizer chain** that puts the onboard model **first**, then the
|
||||||
|
standard chain (primary → local fallback → cloud) as resilience, deduped. Wrap it in a
|
||||||
|
burst-specific `engineProcessor` reusing the same store/transcript-cache/sink — a pure wiring
|
||||||
|
choice, engine and ports unchanged (Clean Architecture, ADR-003).
|
||||||
|
- The `onboard` closure uses the burst processor instead of `app.Processor`.
|
||||||
|
- **Collapse cleanly**: when `OnboardSummarizerModel` is empty or equals `SummarizerModel`, the
|
||||||
|
onboard path reuses `app.Processor` (no separate chain) — the reversibility lever.
|
||||||
|
- Local-first preserved: the onboard model is a local alias; the cloud endpoint stays last in the
|
||||||
|
chain, so a client/NDA deployment with `TAPIR_CLOUD_FALLBACK_MODEL=""` keeps burst content
|
||||||
|
local too.
|
||||||
|
|
||||||
|
## 4. Behaviour spec + docs
|
||||||
|
- Add scenarios to `docs/use-cases/connect_account.feature` (the connect → burst flow): burst
|
||||||
|
skips a too-long/live video in favour of a reasonable-length one; burst summarizes with the
|
||||||
|
stronger model first. Map them in `scenarioCoverage` so `TestScenarioCoverage` stays green.
|
||||||
|
- Update `docs/architecture/architecture.md` (the onboarding-burst section) to describe the
|
||||||
|
junk-avoiding selection + the burst model override.
|
||||||
|
- ADR-028 in `DECISIONS.md` records the rationale (incl. the rejected cached-first lever).
|
||||||
|
|
||||||
|
## Success criteria
|
||||||
|
- `task check` green (fmt, vet, lint, `go test -p 1 ./...`).
|
||||||
|
- A unit test proves `OnboardBurstVideoIDs` drops a known too-long / sub-min video and keeps a
|
||||||
|
good one, newest-first, RLS-scoped, unsummarized-only.
|
||||||
|
- A test proves discovery persists `duration_s` and does not clobber it on re-upsert.
|
||||||
|
- A test proves the burst chain leads with the onboard model (then the standard chain).
|
||||||
|
- Config defaults + bounds tested (`OnboardMaxVideoSeconds`, `OnboardSummarizerModel`).
|
||||||
|
- No change to the rate gate, fetch pacing, or burst cap. `0/0` + empty model == prior behaviour.
|
||||||
|
|
||||||
|
## Explicitly NOT in this slice
|
||||||
|
- Cached-first selection (rejected, ADR-028).
|
||||||
|
- Any caption-availability *guarantee* (impossible pre-fetch).
|
||||||
|
- **Honest "taster" framing copy** ("summaries of a few of your videos to get you started — the
|
||||||
|
rest fill in over the coming days"). A good lever from the original brief, but it's a UI/copy
|
||||||
|
change with no backend dependency; deferred to a UI pass, tracked as an issue.
|
||||||
|
- Backfilling `duration_s` for existing rows via a migration (it backfills lazily on discovery).
|
||||||
|
- Return-nudges / digests (ADR-020: poisons the unprompted-return signal).
|
||||||
|
- Raising fetch throughput, multi-IP, or Whisper (out of scope; the gate is deliberate).
|
||||||
@@ -0,0 +1,92 @@
|
|||||||
|
# Spec — In-process scheduled discovery + auto-summarize + rate-gate finish
|
||||||
|
|
||||||
|
> **Extended by ADR-020 (2026-06-08).** Auto-summarize is no longer "every unseen video": the
|
||||||
|
> scheduler now skips videos published before `TAPIR_AUTO_SUMMARIZE_WINDOW` (default ~7d) unless
|
||||||
|
> explicitly requested, so a back-catalogue does not re-drive the rate gate every cycle. See
|
||||||
|
> `DECISIONS.md` ADR-020.
|
||||||
|
|
||||||
|
**Repo:** tapir · **Size:** medium · **Solo session** (not a swarm).
|
||||||
|
|
||||||
|
**Why this exists.** The Stage-0 gate ("me or a friend returns and reads/acts in ≥2 separate
|
||||||
|
weeks") cannot be met because the system is not usable *unprompted*: discovery (`tapir run`) is
|
||||||
|
host-side manual, so a newly onboarded user sees an empty list and never comes back. This slice
|
||||||
|
makes Tapir watch on its own — the thing that makes the gate experiment actually runnable.
|
||||||
|
|
||||||
|
Read `CLAUDE.md` + `DECISIONS.md` (esp. ADR-012, ADR-014, and the new ADR-018) first. TBD —
|
||||||
|
commit directly to `main`, one logical change per commit, conventional commits, `task check`
|
||||||
|
green before each commit. `templ generate` if any view changes.
|
||||||
|
|
||||||
|
## Decisions already made (do not reopen)
|
||||||
|
- **In-process scheduler**, NOT a k8s CronJob (maintainer's call: simpler deploy, acceptable
|
||||||
|
coupling at 3 users). The known cost — discovery shares the web process's lifetime and egress
|
||||||
|
— is accepted and recorded in ADR-018.
|
||||||
|
- **Auto-summarize ON** for the maintainer + onboarded friends (zero-friction: the list fills
|
||||||
|
and summarizes itself).
|
||||||
|
- **Gate clock resets** to when this ships (ADR-018) — until unprompted use is possible, the
|
||||||
|
prior window measured nothing.
|
||||||
|
|
||||||
|
## 1. In-process scheduled discovery (core)
|
||||||
|
- In `tapir serve` startup, launch a background goroutine that runs discovery for ALL users on
|
||||||
|
an interval: env `TAPIR_DISCOVERY_INTERVAL` (Go duration, e.g. `2h`). **Unset or 0 = disabled**
|
||||||
|
(so dev/tests never auto-fetch).
|
||||||
|
- **Reuse the existing `runner.Runner` + `Loop`/`RunOnce`. Do NOT write a new scheduler.** The
|
||||||
|
per-user Runner already exists; the new work is **iterating users** and running one pass each
|
||||||
|
per tick. Enumerate users from the un-RLS'd `user_identities` (the same enumerate-then-act
|
||||||
|
pattern the login_events gate query established), then run each user's pass **inside that
|
||||||
|
user's RLS scope** (`withUser`).
|
||||||
|
- **Stateless timing:** run-once-on-startup, then every interval — exactly the existing `Loop`
|
||||||
|
shape. Do NOT persist schedule state; a pod restart just restarts the cycle. Acceptable at this
|
||||||
|
scale. Do not build cron-in-Go.
|
||||||
|
- **Graceful shutdown:** the goroutine respects `ctx` cancellation so a pod term doesn't wedge.
|
||||||
|
- **Failure isolation in the loop:** one user's pass failing (or one channel/video) must not
|
||||||
|
abort the other users or crash `serve` — log and continue. (RunOnce already collects per-item
|
||||||
|
errors; preserve that at the per-user level too.)
|
||||||
|
|
||||||
|
## 2. Auto-summarize default ON for Future-B users
|
||||||
|
- New registrations default `auto_summarize = true` (so onboarded friends get zero-friction);
|
||||||
|
keep the account-page toggle so a user can switch to manual. One-off update existing user rows
|
||||||
|
to `true` as well (maintainer + any current users).
|
||||||
|
- Consequence (intended): scheduled discovery both discovers AND summarizes new videos — which
|
||||||
|
is the point, and is why §3 is mandatory in the same slice.
|
||||||
|
|
||||||
|
## 3. Finish/confirm the ADR-014 shared per-egress-IP rate gate (NOW load-bearing)
|
||||||
|
- In-process scheduling + auto-summarize + multiple users = all caption fetches leave the **one
|
||||||
|
web pod's egress IP**, concurrently with any live "Summarize" button clicks. The timedtext
|
||||||
|
endpoint rate-limits per IP (ADR-010/014). Without a shared gate this self-inflicts 429s every
|
||||||
|
cycle.
|
||||||
|
- **Confirm in code whether ADR-014 item 2 (a single PROCESS-WIDE rate gate) exists.**
|
||||||
|
Reconciliation flagged it as possibly built only as per-*video* backoff. If it is not a
|
||||||
|
process-wide gate, **build it now**: ONE shared limiter (token-bucket / min-interval) that
|
||||||
|
every timedtext/caption fetch passes through — scheduler loop AND click-path alike. Per-process,
|
||||||
|
not per-user, not per-video.
|
||||||
|
- Keep the existing per-video 429 backoff (`rate_limited_at` + retry window) — complementary: the
|
||||||
|
gate prevents tripping 429; the backoff handles it if one still happens.
|
||||||
|
- Honest UX (ADR-014 item 3) still applies: a fetch waiting on the gate shows "queued/waiting",
|
||||||
|
never a stuck spinner.
|
||||||
|
|
||||||
|
## 4. Gate-clock reset — already recorded in ADR-018; verify VISION reflects it
|
||||||
|
- ADR-018 (committed) resets the Stage-0 3–4 week window to start when this ships, and revises the
|
||||||
|
check-in date. VISION Stage 0 carries a pointer to it. The build doesn't re-decide this; just
|
||||||
|
ensure nothing in docs still implies the clock started earlier.
|
||||||
|
|
||||||
|
## Tests
|
||||||
|
- **Scheduler:** fake clock + fake Runner → N users each get one pass per tick; one user's failure
|
||||||
|
doesn't stop the others; `ctx` cancel stops the loop; interval=0 disables it entirely.
|
||||||
|
- **Rate gate:** concurrent fetches (scheduler + simulated click) are serialized/limited through
|
||||||
|
the ONE gate — assert max-in-flight / min-interval honored regardless of caller.
|
||||||
|
- **Auto-summarize default:** new registration → `auto_summarize = true`; account-page toggle
|
||||||
|
still flips it.
|
||||||
|
|
||||||
|
## Out of scope / known constraints
|
||||||
|
- No CronJob / k8s objects (in-process chosen).
|
||||||
|
- **SINGLE-REPLICA ASSUMPTION (load-bearing).** In-process scheduling means if `tapir serve` ever
|
||||||
|
runs >1 replica, every replica runs the discovery loop → every user fetched in parallel (429s +
|
||||||
|
duplicate work). At 3 users this is single-replica, fine — but the build MUST note this
|
||||||
|
constraint in ADR-018 / deploy docs so a future scale-up doesn't silently double-run.
|
||||||
|
- No persisted schedules, no multi-pod coordination.
|
||||||
|
|
||||||
|
## Fallback if the session runs short
|
||||||
|
Ship the scheduler with **auto-summarize OFF** (discovery only; manual Summarize button) until the
|
||||||
|
process-wide rate gate (§3) is confirmed/built. NEVER ship auto-summarize-on-a-schedule without the
|
||||||
|
gate — that combination self-inflicts 429s for every user every cycle. Auto-summarize ON is gated
|
||||||
|
on §3 being done.
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
# Spec — Stage 0 usage measurement (login events)
|
||||||
|
|
||||||
|
**Date:** 2026-06-03
|
||||||
|
**Status:** Ready to build · **Repo:** tapir · **Size:** small (one migration + middleware + query)
|
||||||
|
**Why:** The Stage 0 gate (VISION, ADR-016) is *return usage in ≥2 separate weeks*. `summary_actions`
|
||||||
|
captures *acts* (watch/skip/save) but not *reads* — a friend who logs in weekly and reads summaries
|
||||||
|
without clicking anything is invisible. For a **reading** product that is the most important signal.
|
||||||
|
This adds the missing data so the gate is measurable as written. Solo session, not a swarm.
|
||||||
|
|
||||||
|
Read `CLAUDE.md` + ADR-016 first. TBD, conventional commits, `task check` green before each commit.
|
||||||
|
|
||||||
|
## Scope (resist sprawl — this is NOT analytics)
|
||||||
|
|
||||||
|
A lightweight, append-only record of *when each user was active*, enough to answer
|
||||||
|
"returned/read in ≥N distinct weeks". Not page-level events, not click tracking, not a funnel.
|
||||||
|
|
||||||
|
### 1. Migration — `login_events` (append-only)
|
||||||
|
```
|
||||||
|
login_events (
|
||||||
|
id UUID PK default gen_random_uuid(),
|
||||||
|
user_id UUID NOT NULL, -- per-user; RLS like every user-owned table
|
||||||
|
seen_at TIMESTAMPTZ NOT NULL default NOW()
|
||||||
|
)
|
||||||
|
INDEX (user_id, seen_at)
|
||||||
|
```
|
||||||
|
- **RLS:** `FORCE ROW LEVEL SECURITY`, same policy/pattern as the other user-owned tables (the
|
||||||
|
`tapir.current_user_id` GUC via the `withUser` seam — match migration 003). A reporting query that
|
||||||
|
needs cross-user counts runs as the owner/maintainer outside the per-user scope, or via a dedicated
|
||||||
|
read — decide consistently with how existing admin-ish reads are done.
|
||||||
|
- Append-only: no updates, no deletes except the user-delete cascade. **Add to the delete-account
|
||||||
|
cascade** (ADR-013) — `login_events` has no FK (mirrors `summary_actions`), so `DeleteUser` needs an
|
||||||
|
explicit delete for it, and the delete test must assert it's covered. *Do not forget this* — it's the
|
||||||
|
exact footgun the last delete work caught.
|
||||||
|
|
||||||
|
### 2. Middleware — throttled stamp
|
||||||
|
- In the authenticated request path (after `CurrentUserID` resolves, inside the registration-gated
|
||||||
|
app — NOT on `/welcome`/`/healthz`/`/auth`), record one `login_events` row **per user per day**
|
||||||
|
(throttle: skip if a row exists for this user with `seen_at` ≥ start-of-today). One insert per active
|
||||||
|
day, not per request — keeps the table small and the signal clean.
|
||||||
|
- Throttle check must itself be RLS-scoped (`withUser`). Keep it cheap (indexed lookup).
|
||||||
|
|
||||||
|
### 3. Query — the gate report
|
||||||
|
Provide a query (and optionally a tiny `tapir report` CLI subcommand or an admin page — your call,
|
||||||
|
CLI is fine) answering, per user:
|
||||||
|
```sql
|
||||||
|
-- distinct active weeks from reads (login_events) AND acts (summary_actions), unioned
|
||||||
|
WITH weeks AS (
|
||||||
|
SELECT user_id, date_trunc('week', seen_at) AS wk FROM login_events
|
||||||
|
UNION
|
||||||
|
SELECT user_id, date_trunc('week', acted_at) FROM summary_actions
|
||||||
|
)
|
||||||
|
SELECT user_id, COUNT(DISTINCT wk) AS active_weeks
|
||||||
|
FROM weeks GROUP BY user_id
|
||||||
|
ORDER BY active_weeks DESC;
|
||||||
|
```
|
||||||
|
Gate passes when any user_id (maintainer or friend) reaches `active_weeks >= 2` within the window.
|
||||||
|
|
||||||
|
## Honesty caveats to carry (from VISION/ADR-016)
|
||||||
|
- **"Unprompted" is not measurable here.** login_events records *that* a user returned, not *why*. A
|
||||||
|
nudged return looks identical to an organic one. This build does not close that gap and must not
|
||||||
|
claim to — the VISION measurement note stands: count returns, read a nudged return as weaker signal.
|
||||||
|
(If prompt-tracking is ever wanted, that's a separate decision, not this build.)
|
||||||
|
- **Data accrues from deploy onward.** The gate window's read-data starts when this ships — so ship
|
||||||
|
soon (maintainer's call) rather than batching with the infra tooling session.
|
||||||
|
- **`date_trunc('week')` is ISO/timezone-sensitive** and noisy at low volume (N=3). Two visits days
|
||||||
|
apart can fall in the same or different weeks. Acceptable, but don't over-read a single-week-margin
|
||||||
|
pass/fail.
|
||||||
|
|
||||||
|
## Out of scope
|
||||||
|
Page/event analytics; prompt-vs-organic tracking; dashboards beyond the one gate query; anything
|
||||||
|
touching the engine or sinks (this is web/store only — ADR-003 holds).
|
||||||
|
|
||||||
|
## Tests
|
||||||
|
- Migration up/down; RLS on `login_events` (extend the two-user isolation test to cover it).
|
||||||
|
- Throttle: N requests same day → 1 row; next day → 2nd row.
|
||||||
|
- `DeleteUser` removes the user's `login_events` and leaves others' intact (extend the delete test).
|
||||||
|
- The gate query returns correct distinct-week counts across a seeded reads+acts fixture.
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
# Spec — Unify video-card states: one "Summarize now" verb, honest no-captions state
|
||||||
|
|
||||||
|
> **Superseded in part by ADR-020 (2026-06-08).** The five card states still hold, but the copy
|
||||||
|
> changed: the nudge verb is now **"Summarize"** (not "Summarize now"), the rate-limited state
|
||||||
|
> reads **"In queue"** (not "Fetching soon…"), and the queued state reads **"summarizing
|
||||||
|
> shortly"** (not "waiting for the next run"). The list also now collapses older un-summarized
|
||||||
|
> and caption-less videos. See `DECISIONS.md` ADR-020 and `views.templ` (`VideoCard`) for the
|
||||||
|
> current copy; this doc is kept as the original design record.
|
||||||
|
|
||||||
|
**Repo:** tapir · **Size:** small, **view-layer only** (`views.templ` + a little CSS in
|
||||||
|
`view.go`; regenerate `views_templ.go`). No handler, store, or DB change. The two existing
|
||||||
|
handlers (`/summarize`, `/retry-now`) stay exactly as they are — only what the card *shows*
|
||||||
|
changes.
|
||||||
|
|
||||||
|
**Why.** The video card today presents two different verbs — "Try now" (`.btn-retry`, on
|
||||||
|
rate-limited videos) and "Summarize" (`.btn-secondary`, on pending videos) — for what the user
|
||||||
|
experiences as one intent: *"summarize this video now."* The user doesn't know or care about the
|
||||||
|
internal pipeline state (rate-limited vs. manual-queue); two differently-labelled, differently-
|
||||||
|
styled buttons leak that state machine into the UI as a choice. Also: a **no-captions video
|
||||||
|
(`TranscriptStatus == "none"`) currently falls into the `else` branch and wrongly shows a
|
||||||
|
"Summarize" button** that, if clicked, tries and fails — there are no captions to fetch. That's
|
||||||
|
the confusing dead-end to remove.
|
||||||
|
|
||||||
|
In normal use the maintainer is in **auto mode**, so these buttons are *exceptions*, not the main
|
||||||
|
path — videos summarize themselves. So the card should be **status-first**: the state is what the
|
||||||
|
user reads constantly; the manual nudge is a small, quiet affordance for impatience, not a
|
||||||
|
prominent call-to-action.
|
||||||
|
|
||||||
|
## The honest per-state card model (footer of `VideoCard`)
|
||||||
|
|
||||||
|
Restructure the footer branch in `VideoCard` (in `internal/web/views.templ`) to these states.
|
||||||
|
The branch ORDER matters (summarized first, then terminal/no-action states, then actionable):
|
||||||
|
|
||||||
|
1. **Summarized** — preview + provider chip + fallback badge + actions. **No button.** (unchanged)
|
||||||
|
2. **No captions** (`r.TranscriptStatus == "none"`) — **NEW branch.** Quiet status text, e.g.
|
||||||
|
`No transcript available` (use a muted `.card-state`/`.chip-retry`-style treatment, NOT a
|
||||||
|
button). This is a terminal honest dead-end — the user can do nothing, so offer nothing.
|
||||||
|
3. **Queued / requested** (`r.SummarizeRequested`) — "Queued · waiting for the next run". **No
|
||||||
|
button.** (unchanged)
|
||||||
|
4. **Rate-limited** (`r.TranscriptStatus == "rate_limited"`) — quiet status (keep the
|
||||||
|
"fetching soon" sense) + a **quiet "Summarize now"** button POSTing to `retryNowURL` (clears
|
||||||
|
backoff then processes). Same quiet style as state 5.
|
||||||
|
5. **Pending** (else — discovered, not yet attempted) — quiet "Not summarized" + a **quiet
|
||||||
|
"Summarize now"** button POSTing to `summarizeURL` (flips the queue flag then processes).
|
||||||
|
|
||||||
|
## Unify the verb and the style
|
||||||
|
- **One label everywhere a manual nudge is offered: "Summarize now"** (states 4 and 5). Drop the
|
||||||
|
"Try now" wording entirely.
|
||||||
|
- **One quiet style** for both: use the understated `.btn-retry` pattern (small, pill, outline,
|
||||||
|
transparent bg) — NOT `.btn-secondary`/`.btn` (heavier). Rename the CSS class to something
|
||||||
|
state-neutral (e.g. `.btn-quiet` or `.btn-summarize-now`) so it no longer reads as
|
||||||
|
"retry"-specific; keep the same visual. The point: the nudge is subtle, status is primary.
|
||||||
|
- Keep both `<form>`s posting to their respective existing handler URLs
|
||||||
|
(`retryNowURL` for rate-limited, `summarizeURL` for pending) with the existing HTMX
|
||||||
|
attributes (`hx-post`, `hx-target=#video-{id}`, `hx-swap=outerHTML`) — only the button
|
||||||
|
label/class change. The backend side-effect difference (clear-backoff vs. set-flag) stays
|
||||||
|
invisible to the user, which is correct.
|
||||||
|
- Drop the engineer-facing `title="Fetch transcript now through the shared rate gate"` tooltip;
|
||||||
|
if a hint is wanted, make it user-facing ("Summarize this one now").
|
||||||
|
|
||||||
|
## Quietness check (the design intent)
|
||||||
|
The summary content and the per-state *status* are the card's primary information. The "Summarize
|
||||||
|
now" button is a minor affordance. Do not make it a prominent solid-accent CTA — it must read as
|
||||||
|
"you can nudge this if you're impatient", not "action required". Status text uses muted styling;
|
||||||
|
the button uses the quiet outline style.
|
||||||
|
|
||||||
|
## Tests
|
||||||
|
- `videocard_internal_test.go` (exists): assert each of the 5 states renders the expected
|
||||||
|
footer — summarized (no button), no-captions (status, NO button, no `summarize`/`retry-now`
|
||||||
|
URL present), queued (no button), rate-limited ("Summarize now" → retry-now URL), pending
|
||||||
|
("Summarize now" → summarize URL). The key new assertion: **a `none`-status video renders no
|
||||||
|
action button and no POST URL.**
|
||||||
|
- Assert the label string "Try now" no longer appears anywhere in rendered output.
|
||||||
|
|
||||||
|
## Out of scope
|
||||||
|
Handler/DB changes; the detail-page action buttons (watched/skipped/saved — unrelated); the
|
||||||
|
pipeline stats bar wording; auto/manual mode behaviour. Verb/label/style/no-captions-state only.
|
||||||
@@ -0,0 +1,115 @@
|
|||||||
|
# Sprint: audio upload follow-ups (Sweep Sprint, 2026-10)
|
||||||
|
|
||||||
|
Method: Sweep Sprint (phase 1 parallel read-only research per issue: **done**; phase 2 sequential
|
||||||
|
execution, one issue per commit: **v0.31.1 and v0.32.0 shipped 2026-10-03/04, the owner-held items
|
||||||
|
below remain open**). Closing oath below.
|
||||||
|
|
||||||
|
Milestones: [v0.31.1 - security and robustness](https://git.d-ma.be/mathias/tapir/milestone/10),
|
||||||
|
[v0.32.0 - audio upload follow-ups](https://git.d-ma.be/mathias/tapir/milestone/11).
|
||||||
|
|
||||||
|
Context: v0.31.0 shipped audio upload (ADR-033/034/035) and is live in production behind auth at
|
||||||
|
`/uploads`, with the Berget cloud fallback on for audio (infra#522). A real 63.7 min upload
|
||||||
|
transcribed in 2m53s with no errors. The phase-1 investigation found two pre-existing or new
|
||||||
|
weaknesses an authenticated user can reach, so the first release is a hotfix.
|
||||||
|
|
||||||
|
## Oath (closing gates, every clause must hold; blocked = descoped and tracked, never "done")
|
||||||
|
|
||||||
|
| # | Clause | Evidence |
|
||||||
|
|---|--------|----------|
|
||||||
|
| SG1 | Every issue in the sprint is closed with a linked commit, or moved out with a stated reason | issue list of both milestones |
|
||||||
|
| SG2 | Each fix has a test that was seen red on main first, named after the mechanism it observes | red/green output in the issue |
|
||||||
|
| SG3 | `task check` AND `golangci-lint run ./...` exit 0 (checked by `$?`, not by grep); `task check` does not gate lint until #41 | gate output |
|
||||||
|
| SG4 | Control code (#48, #49, the upload cap in #47) has a recorded adversarial review verified by execution, verdict in the issue | gatekeeper verdict comment |
|
||||||
|
| SG5 | Docs current in the same commit: ADR-033/035 text, feature file, use-case coverage test | diff |
|
||||||
|
| SG6 | Each release tagged (v0.31.1, v0.32.0), pushed, deployed, and observed live (pod running the new image, no restart loop, a real upload still works) | `kubectl`/log + one upload |
|
||||||
|
| SG7 | Learnings written to the brain; deferred work filed as issues; decisions that are the owner's recorded as DECISION NEEDED | brain ids + issue numbers |
|
||||||
|
|
||||||
|
## Work order
|
||||||
|
|
||||||
|
### v0.31.1 (security and robustness; hotfix, do first)
|
||||||
|
|
||||||
|
1. **#41 lint gate.** `task check` swallows lint findings (`|| echo`). Make the gate fail on findings,
|
||||||
|
fix what it surfaces. First, so every later gate means something. (Decision D1.)
|
||||||
|
2. **#48 cross-origin POST protection.** `http.NewCrossOriginProtection()` around `Router()`.
|
||||||
|
Red: foreign-origin POST to `/account/delete`, `/uploads`, `/uploads/{id}/chat` is refused.
|
||||||
|
3. **#49 ffprobe/ffmpeg limits.** `-max_streams 32` + narrower probe output; probe stdout cap;
|
||||||
|
probe deadline. Three commits.
|
||||||
|
4. **#47 concurrent-upload cap (remaining item).** Bound the spool: reservation per accepted upload
|
||||||
|
against the 6Gi emptyDir so concurrent uploads cannot evict the pod.
|
||||||
|
5. **#50 SIGTERM interim.** Handle SIGTERM; record the interrupted job at once; fix the false
|
||||||
|
comment. Full drain recorded as a limitation. (Decision D2.)
|
||||||
|
|
||||||
|
Release v0.31.1: one adversarial review round over 2-5, then tag, push, observe.
|
||||||
|
|
||||||
|
### v0.32.0 (follow-ups)
|
||||||
|
|
||||||
|
6. **#52** label a cloud-served transcript in plain words; document the empty-transcript trigger.
|
||||||
|
7. **#53** bound the LLM client's response body read.
|
||||||
|
8. **#51** tracker: save timeout, then app-clock heartbeat.
|
||||||
|
9. **#37** nav link to uploads and per-recording delete (queued/transcribing: refuse until it
|
||||||
|
ends). (Decision D3.)
|
||||||
|
10. **#36** chat default model after measurement. (Decision D4.)
|
||||||
|
11. **#38** measurements (Berget length limit, large models on production iguana). (D5, D6.)
|
||||||
|
12. **#46** remainder of the hardening list after the items above are split out.
|
||||||
|
13. **#45** evaluate Berget as the reconciler: a question, not work. (D8.)
|
||||||
|
|
||||||
|
## DECISION NEEDED (owner's call; recommendation first)
|
||||||
|
|
||||||
|
| # | Question | Recommendation | Blocks |
|
||||||
|
|---|----------|----------------|--------|
|
||||||
|
| D1 | #41: should any linter be allowed to opt out of the gate? | No opt-out; fix findings | #41 |
|
||||||
|
| D2 | #50: full SIGTERM drain, or record as a limitation and ship the interim fix? | Limitation + interim; a drain costs 4-6 h and downtime on every deploy at current volume | #50 |
|
||||||
|
| D3 | #37: delete of a queued/transcribing recording | Refuse until it ends | #37 |
|
||||||
|
| D4 | #36: make a 31B model the chat default? | Only after measurement, conditional | #36 |
|
||||||
|
| D5 | #38: Berget length measurement needs a throwaway key | Yes, throwaway key | #38 |
|
||||||
|
| D6 | #38: load large models on production iguana | Yes, in a quiet window with the owner present | #38 |
|
||||||
|
| D7 | #46: where the ffmpeg tarball lives | Do nothing now; file an issue | none |
|
||||||
|
| D8 | #45: evaluate Berget as the reconciler | **Decided 2026-10-04 by the owner: yes, Berget may be used, and each user must be able to keep a recording on local models only** (reverses the recommendation). Opt-out first: #60; #45 stays as the model measurement | #60, #45 |
|
||||||
|
| D9 | Who judges ADR-034's second blind check | The owner; do not reuse the one-off Opus authorisation. Protocol: `docs/research/reconcile-second-check-protocol.md` | ADR-034 |
|
||||||
|
| D10 | When to enable reconcile in production | Needs `iguana/pianissimo` in the key pin and env vars; owner present | ADR-034 |
|
||||||
|
|
||||||
|
## Dropped or reduced, with reasons
|
||||||
|
|
||||||
|
- **#46 item 1 (`alis`/`dref` external reference):** checked, not a problem. Under the application's
|
||||||
|
flags ffmpeg skips the external track and writes a 78-byte WAV.
|
||||||
|
- **#46 item 2 / #47 item 2:** merged into #49.
|
||||||
|
- **#47 item 3 (full drain):** reduced to #50 plus a recorded limitation.
|
||||||
|
- **#46 `Beat` taking the tracker mutex:** incorrect in #46; only `logStill` does. Corrected in #51.
|
||||||
|
|
||||||
|
## Human-present items (do not run unattended)
|
||||||
|
|
||||||
|
- Production iguana model loads (#38, D6).
|
||||||
|
- Enabling reconcile in production (D10).
|
||||||
|
- Any key change or `apply` in infra (infra#523, infra#524 are open and not part of this sprint).
|
||||||
|
|
||||||
|
## Release plan
|
||||||
|
|
||||||
|
SemVer: v0.31.1 patch (fixes, no new behaviour); v0.32.0 minor (nav link, delete, cloud labelling).
|
||||||
|
Every push to tapir main redeploys the pod and fails any job in flight: push when no upload is
|
||||||
|
running (check the pod log first).
|
||||||
|
|
||||||
|
## Status (2026-10-04)
|
||||||
|
|
||||||
|
**Shipped and deployed:** v0.31.1 (PR #55, merge e037c6b) and v0.32.0 (PR #58, merge 85f4cc3; pod
|
||||||
|
tapir-597dbfccc-ng78q 1/1, 0 restarts, image 85f4cc3). Each went through an adversarial review
|
||||||
|
round by agent with the verdicts recorded on the PR (v0.31.1: BLOCK then APPROVE; v0.32.0: BLOCK,
|
||||||
|
BLOCK, APPROVE, APPROVE; every block was a real defect, fixed test-first).
|
||||||
|
|
||||||
|
**Live evidence:** a foreign-origin `POST` to `/account/delete` and to `/uploads/{id}/delete` is
|
||||||
|
refused (403) on production, a same-origin one reaches the login redirect (302).
|
||||||
|
|
||||||
|
**Not observed live (needs an authenticated session, so it is the owner's look; closed on word or screenshot):**
|
||||||
|
|
||||||
|
- #52, cloud side: a transcript served by the Berget fallback names `berget`; a reconciled all-local
|
||||||
|
one shows no note. The local side was observed on 2026-10-04 (a new clip served by iguana: chip
|
||||||
|
`iguana/…`, no cloud note). The cloud side is carried as an acceptance item in #60.
|
||||||
|
|
||||||
|
**Observed live since:** #37 (owner, 2026-10-04: link, confirm-then-delete, no control while transcribing; closed). #50: at the 2026-10-04 rollout the old pod went `Terminating` then `Completed`
|
||||||
|
(not `Error`), 0 restarts; #50 closed. No job was in flight, so a drained job is still unobserved
|
||||||
|
(a recorded limitation, D2).
|
||||||
|
|
||||||
|
**Still open (owner's call, milestone 11):** #36, #38, #45 (now the Berget adjudicator measurement), #60 (per-recording local-only choice, 4 sub-decisions), #46 remainder, #56 (slow-drip bodies hold
|
||||||
|
a receive slot), #57 (shutdown logging, second signal). Decisions D4-D6, D9, D10 above are unchanged; D8 is decided (above).
|
||||||
|
|
||||||
|
**Deferred and filed from the reviews:** #56, #57, #59 (start-up exits when the identity provider's OIDC
|
||||||
|
discovery returns 503 (seen once at 19:40 on 2026-10-03; the pod restarted itself and recovered)).
|
||||||
+36
-2
@@ -72,7 +72,14 @@ summary_actions
|
|||||||
and join into the existing `SummaryRow` reads so list/detail show current state.
|
and join into the existing `SummaryRow` reads so list/detail show current state.
|
||||||
- This column is what makes the Stage-0 metric ("did I act on a summary?") queryable.
|
- This column is what makes the Stage-0 metric ("did I act on a summary?") queryable.
|
||||||
|
|
||||||
## 6. Auth (Dex OIDC, single-user authz)
|
## 6. Auth (Dex OIDC)
|
||||||
|
|
||||||
|
Authentication is delegated to the homelab OIDC provider at `TAPIR_OIDC_ISSUER` —
|
||||||
|
**Authentik** since the Dex→Authentik migration (infra ADR-0001; ADR-019). It offers a
|
||||||
|
Google upstream and Authentik-managed accounts (incl. its invite flow); Tapir no longer
|
||||||
|
provisions accounts itself. Any authenticated subject can register a Tapir account
|
||||||
|
(ADR-012: allowlist removed). The `oidc`/`DexAuth` package keeps its name for now (rename
|
||||||
|
deferred, ADR-019).
|
||||||
|
|
||||||
- **Flow:** standard Authorization Code. Use `coreos/go-oidc` + `golang.org/x/oauth2`
|
- **Flow:** standard Authorization Code. Use `coreos/go-oidc` + `golang.org/x/oauth2`
|
||||||
(justify the deps in the commit; both are the homelab-standard OIDC libs and small).
|
(justify the deps in the commit; both are the homelab-standard OIDC libs and small).
|
||||||
@@ -92,7 +99,7 @@ summary_actions
|
|||||||
`TAPIR_OIDC_ISSUER` (`https://auth.d-ma.be`), `TAPIR_DEX_CLIENT_ID`, `TAPIR_DEX_CLIENT_SECRET`,
|
`TAPIR_OIDC_ISSUER` (`https://auth.d-ma.be`), `TAPIR_DEX_CLIENT_ID`, `TAPIR_DEX_CLIENT_SECRET`,
|
||||||
`TAPIR_OIDC_REDIRECT_URL` (`https://tapir.d-ma.be/auth/callback`), `TAPIR_SESSION_SECRET`.
|
`TAPIR_OIDC_REDIRECT_URL` (`https://tapir.d-ma.be/auth/callback`), `TAPIR_SESSION_SECRET`.
|
||||||
Reuses existing `TAPIR_DB_DSN`, `TAPIR_USER_ID` (the StubAuth dev subject only). No secrets
|
Reuses existing `TAPIR_DB_DSN`, `TAPIR_USER_ID` (the StubAuth dev subject only). No secrets
|
||||||
committed. (`TAPIR_ALLOWED_SUBJECT` was removed by ADR-012.)
|
committed. (`TAPIR_ALLOWED_SUBJECT` was removed by ADR-012; use the keys above.)
|
||||||
|
|
||||||
## 8. Deployment — k3s + Flux GitOps
|
## 8. Deployment — k3s + Flux GitOps
|
||||||
|
|
||||||
@@ -147,3 +154,30 @@ Gate (lane A) commits first; B/C/D follow.
|
|||||||
|
|
||||||
`task check` green per lane; B/C/D rebase on A. Deploy (D) lands last, after the binary serves
|
`task check` green per lane; B/C/D rebase on A. Deploy (D) lands last, after the binary serves
|
||||||
locally.
|
locally.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Deviations and additions (as-built)
|
||||||
|
|
||||||
|
This spec describes the **Stage-0 single-user reader** (ADR-011). What actually shipped through
|
||||||
|
v0.4.0 went further — Stage 1 (ADR-012) opened multi-user, and several UX features were added on
|
||||||
|
top. Recorded here (append-only; the spec above is left intact) so intent and reality stay
|
||||||
|
distinguishable.
|
||||||
|
|
||||||
|
| As-built feature | What it is | Why | Covered by |
|
||||||
|
|------------------|-----------|-----|------------|
|
||||||
|
| **Multi-user + RLS isolation** | Several Dex users per deployment; isolation enforced by Postgres RLS, not the single-subject allowlist of §6. | Maintainer opened Stage 1 ahead of the formal Stage-0 gate, with DB-enforced isolation as the guardrail that keeps it safe. | ADR-012; migration 003 (`6775e5f`, `f28fdc0`, `2ae66da`) |
|
||||||
|
| **Registration gate** | A Dex subject with no `users` row is routed to `/register`, which creates the `users` row + a `user_identities` mapping. (§2 listed "sign-up / user CRUD" as a non-goal.) | Explicit registration is how a multi-user surface stays honest — no just-in-time row creation. | ADR-012; `f396e01` |
|
||||||
|
| **Per-user YouTube web connect** | `/oauth/youtube/connect` → `/oauth/youtube/callback` stores a per-user refresh-token ref + a `video_connections` row. (The spec assumed a host-side `tapir auth` only.) | Multi-user means each user connects their own account from the browser. | ADR-006, ADR-012; migration 005 (`0c9531a`, `2aad79b`) |
|
||||||
|
| **Account management** | `/account` page with **disconnect** and **delete account**; delete removes only Tapir-side state and leaves the Dex identity intact. (§2 listed isolation/CRUD as non-goals.) | A real account needs a way out; deletion semantics are deliberately Tapir-side only. | ADR-013; `22eafcf`, `c7624d9`, `17d5e8c` |
|
||||||
|
| **Immediate web summarization** | A quiet "Summarize now" button on non-summarized video cards. Pending cards POST to `/v/{id}/summarize` (queues + triggers engine); rate-limited cards POST to `/v/{id}/retry-now` (clears backoff + triggers engine). Both use the same `.btn-quiet` style and label — the internal pipeline distinction is invisible to the user. The page HTMX-polls `GET /v/{id}/status` while processing. Videos with `TranscriptStatus == "none"` show "No transcript available" with no button — this is a terminal honest state. (§2 said "triggering runs from the browser … do NOT build".) | Reading a list you can't act on is half a product; on-demand summarize closes the loop without waiting for a batch `tapir run`. | ADR-012, ADR-014; `25215cb`, `8c6c7ca` |
|
||||||
|
| **Charmbracelet tapir spinner** | An animated in-flight indicator (charm palette) shown while a summarize is processing; an honest "queued/waiting" state under rate-limiting rather than a stuck spinner. | The spinner must tell the truth when the timedtext endpoint rate-limits (429), not imply imminence. | ADR-014; `25215cb`, `a4aeb5e` |
|
||||||
|
| **Auto/manual summarization mode** | Per-user `auto_summarize`; manual lists new videos unsummarized and queues via `summarize_requested`; a mode toggle at `/account/summarize-mode`. Default is **true** for new users (migration 011, ADR-018); existing rows back-filled via migration 012. | Control over compute/noise — only summarize what the user cares about. | migration 006 (`748d5eb`, `bdbdce7`, `3014ee0`, `a269d4a`); migration 011/012 |
|
||||||
|
| **Public landing page** | `/welcome` mounted **outside** the auth guard; unauthenticated `/` redirects there; logout returns there (not `/auth/login`). (The spec guarded everything except `/healthz` and `/auth/*`.) | A first-time visitor needs a public "what is this / get started" page before the login wall. | `d83943c`, `0fdf2f7`, `3a27bf1`, `d208110`, `8ca374e`, `f15f57f` |
|
||||||
|
| **Invite onboarding** | **Removed from Tapir (ADR-019).** Invites are owned by the IdP (Authentik) now, not Tapir — the Dex local-password provisioning path (`tapir invite` CLI, `/invite/{token}` web flow, `internal/adapters/dex`) was deleted when the homelab migrated Dex→Authentik (infra ADR-0001). A new user is invited via Authentik's invite flow, logs into Tapir via OIDC, and is captured by the existing `/register` (display-name) gate. | Onboarding belongs to the identity provider; keeps Tapir out of the shared identity provider's write path. | ADR-019; infra ADR-0001 |
|
||||||
|
| **Summarized-only filter** | `?summarized=1` query param on the list view. When set, only videos with a completed summary (`SummaryRow.Summarized = true`) are shown. Rendered as a "Summarized only" checkbox in the filter form. Summarized videos also sort to the top of the unfiltered list (`ORDER BY (s.id IS NOT NULL) DESC, seen_at DESC`). | Lets users focus on videos that are ready to read; newly landing summaries are visible at the top without filtering. | `internal/web/view.go` (`Filter.OnlySummarized`, `ListVideos` ORDER BY) |
|
||||||
|
| **"Summarize now" foreground path** | Unified quiet nudge button on actionable non-summarized cards. Five explicit card states — (1) summarized: chip + no button; (2) no captions (`transcript_status = 'none'`): "No transcript available", no button; (3) queued: "Queued" chip, no button; (4) rate-limited: "Fetching soon…" + "Summarize now" → `POST /v/{id}/retry-now` (clears `rate_limited_at`, triggers engine); (5) pending: "Not summarized" + "Summarize now" → `POST /v/{id}/summarize` (queues + triggers engine). One verb, one style (`.btn-quiet`); backend difference invisible to user. Both handlers call `ProcessVideo` through `globalFetchGate`. Rate gate respected, not bypassed — this is onboarding prioritisation. | Fast onboarding value; honest dead-end for no-captions videos (no button that fails). | `internal/web/handlers.go` (`handleRetryNow`, `handleRequestSummarize`); `internal/web/views.templ` (`VideoCard`) |
|
||||||
|
| **Pipeline stats bar** | A one-line status bar above the video list: `N summarized · M fetching soon · K no captions`. Computed from the unfiltered row set; hidden when all videos are summarized. Gives the user a clear read on pipeline state without any interaction. | Replaces the "why is nothing happening?" confusion when most videos are pending or rate-limited. | `internal/web/view.go` (`PipelineStats`, `pipelineStats`) |
|
||||||
|
| **Unavailable channels (account page)** | The `/account` page shows a "Unavailable channels" section when any channels returned HTTP 404 on the last discovery pass. Lists channel name, an "unavailable" badge, and the first-seen date. Data sourced from the `channel_errors` table (migration 013). | Surfaces silent failures so users know why some subscribed channels produce no new videos. | migration 013; `internal/web/account.go`; `internal/adapters/youtube/youtube.go` (`domain.ErrChannelUnavailable`) |
|
||||||
|
| **Visual refresh — charm-reader theme + light/dark toggle (ADR-032)** | One layout in two palettes expressed as CSS custom properties: a warm "reader" light theme (sketch B) and a "cozy terminal" dark theme (sketch C). Palette is chosen in cascade order — `:root` light default, an OS-preference dark block scoped to `:root:not([data-theme])` so it only applies absent an explicit choice, and `:root[data-theme="dark"\|"light"]` set by a header toggle that outranks the media query and persists in `localStorage` (guarded; degrades to OS default). An init script in `<head>` applies the stored choice before paint (no flash). Charm touches: monospace meta lines, accent uppercase section dividers with a trailing rule, pill buttons, a lifted/accent-edged expanded card. Error/danger shades became `--err-*` tokens so they follow the theme without per-block dark overrides. Sketches kept as the design record under `docs/sketches/`. | The UI read "flat and boring"; the charm/TUI aesthetic makes it distinctive and gives a real light/dark choice rather than OS-only. | ADR-032; `docs/use-cases/visual_theme.feature`; `internal/web/visual_theme_test.go`; `internal/web/view.go` (`stylesheet`, `themeScript`), `internal/web/views.templ` (Layout/PublicLayout) |
|
||||||
|
| **Recency window + sparse-state honesty (ADR-020)** | Supersedes the copy/sort in the rows above. Auto-summarize is bounded to videos published within `TAPIR_AUTO_SUMMARIZE_WINDOW` (~7d); older un-summarized videos collapse behind a single "Show N older videos — summarize on demand" disclosure, and caption-less videos collapse to a one-line count (not N cards). List order is now `summarized-first, published_at DESC NULLS LAST`. Copy reframed for honest scarcity: pipeline bar reads "N ready · M in queue · K no captions" (no "fetching soon"); a gradual-fill note explains the rate limit; the nudge verb is "Summarize" (not "Summarize now"); the queued card says "summarizing shortly"; the empty-connected state drops the impossible `tapir run` instruction. Detail leads with Takeaways. Filters slimmed (no date pickers; hidden when empty); watched/skipped segmented; back link on detail; empty terms checkbox removed. | Make the sparse reality legible and honest instead of implying abundance/imminence; bound auto load so the back-catalogue doesn't re-drive the caption gate. Never fetch harder — scarcity is surfaced, not engineered around. | ADR-020; `2384c47`, `3df0459`, `40b703e`, `a1a5217`, `4a0a56e`, `9bf1c31`, `980638d`, `12fb031`, `f775441`, `51aa5d9` |
|
||||||
|
|||||||
@@ -31,5 +31,17 @@ Feature: Local-first AI with optional BYO fallback
|
|||||||
When any transcript is summarized
|
When any transcript is summarized
|
||||||
Then my content is only ever sent to the local AI stack
|
Then my content is only ever sent to the local AI stack
|
||||||
|
|
||||||
# "Reliably" is operationalized as: Primary returned without error within timeout.
|
Scenario: A model returns unparseable output and the next endpoint succeeds
|
||||||
|
Given the local AI stack is available
|
||||||
|
But the primary model returns output that cannot be parsed into a summary
|
||||||
|
And a fallback model is configured
|
||||||
|
When a transcript is summarized
|
||||||
|
Then Tapir falls back to the next model in the chain
|
||||||
|
And the summary records fallback_used as true
|
||||||
|
|
||||||
|
# "Reliably" is operationalized as: an endpoint returned a PARSEABLE summary
|
||||||
|
# within timeout. A 200 with malformed JSON (or highlights emitted as a bare
|
||||||
|
# string) counts as a failure and advances the chain (ADR-022). Endpoints are
|
||||||
|
# tried in order, locals first, so the external worst-case model only ever sees
|
||||||
|
# content after every local endpoint has failed.
|
||||||
# Quality scoring may be added later without changing these scenarios.
|
# Quality scoring may be added later without changing these scenarios.
|
||||||
|
|||||||
@@ -0,0 +1,293 @@
|
|||||||
|
Feature: Upload an audio recording, get a private Swedish transcript, discuss it
|
||||||
|
As someone with a recorded meeting on my computer
|
||||||
|
I want to upload it and then ask questions about what was said
|
||||||
|
So that I can find out what was decided, what is open and who owes what
|
||||||
|
|
||||||
|
# ADR-033, tapir#33. The load-bearing constraint: an uploaded transcript is
|
||||||
|
# PRIVATE content, so it lives in its own RLS-scoped store and never in the
|
||||||
|
# shared public-caption `transcripts` table (ADR-021). Rejection is synchronous,
|
||||||
|
# the audio is never retained, and a failed job ends in a terminal state with a
|
||||||
|
# message — never a spinner (ADR-025). A file with a video track (a Teams meeting
|
||||||
|
# with the camera off is one) is accepted: only its audio is used and the user is
|
||||||
|
# told. Video playback, keyframes and analysis of what is shown remain #28.
|
||||||
|
|
||||||
|
Background:
|
||||||
|
Given I am a registered user with audio upload enabled
|
||||||
|
|
||||||
|
Scenario: The upload form states the limits up front
|
||||||
|
When I open the upload page
|
||||||
|
Then I see the size limit and the length limit before I choose a file
|
||||||
|
|
||||||
|
Scenario: An accepted recording is queued and I am taken to its page
|
||||||
|
When I upload an audio file within the limits
|
||||||
|
Then it is recorded as queued for me
|
||||||
|
And I am redirected to its page
|
||||||
|
|
||||||
|
Scenario: A recording with no title is named after its file
|
||||||
|
When I upload an audio file and give no title
|
||||||
|
Then its title is the filename without its extension
|
||||||
|
|
||||||
|
Scenario: A file with a video track is accepted and I am told the video was ignored
|
||||||
|
When I upload a recording that also has a video track
|
||||||
|
Then it is accepted and queued like any other
|
||||||
|
And its page says only its audio was used and the video was discarded
|
||||||
|
And it says so while queued, while transcribing, when done and when failed
|
||||||
|
|
||||||
|
Scenario: A video file with no audio is rejected for having no audio
|
||||||
|
When I upload a video that has no audio track
|
||||||
|
Then it is rejected with a message that it has no audio
|
||||||
|
And nothing is recorded
|
||||||
|
|
||||||
|
Scenario: A recording over the length limit is rejected with both lengths
|
||||||
|
When I upload a recording longer than the limit
|
||||||
|
Then it is rejected stating the recording's length and the limit
|
||||||
|
And nothing is recorded
|
||||||
|
|
||||||
|
Scenario: A file over the size limit is rejected as too large
|
||||||
|
When I upload a file larger than the size limit
|
||||||
|
Then it is rejected with 413 and a message about the limit
|
||||||
|
And nothing is recorded
|
||||||
|
|
||||||
|
Scenario: Cover art on an audio file is not a video track
|
||||||
|
When I upload an mp3 whose only video stream is attached cover art
|
||||||
|
Then it is treated as audio and no video notice is shown
|
||||||
|
|
||||||
|
Scenario: A file that is not media is rejected as unreadable
|
||||||
|
When I upload a file ffprobe cannot read
|
||||||
|
Then it is rejected as the file's fault
|
||||||
|
|
||||||
|
Scenario: A broken ffprobe is reported as our fault, not as a bad file
|
||||||
|
When the prober itself fails
|
||||||
|
Then the upload is an error rather than a rejection of my file
|
||||||
|
|
||||||
|
Scenario: A recording becomes a transcript and the audio is deleted
|
||||||
|
Given I have uploaded a recording
|
||||||
|
When it has been transcribed
|
||||||
|
Then the transcript is stored privately for me
|
||||||
|
And the audio is deleted
|
||||||
|
|
||||||
|
Scenario: The endpoint that served a transcript is recorded and shown
|
||||||
|
Given a recording was transcribed by the fallback endpoint
|
||||||
|
When I view its page
|
||||||
|
Then it names the fallback endpoint, because that means the audio left the homelab
|
||||||
|
|
||||||
|
Scenario: The local endpoint is tried first and the fallback only on failure
|
||||||
|
When the primary transcription endpoint fails for any reason
|
||||||
|
Then the fallback endpoint transcribes it
|
||||||
|
But when the primary succeeds the fallback is never contacted
|
||||||
|
|
||||||
|
Scenario: A failed transcription ends in a message, never a spinner
|
||||||
|
When every transcription endpoint fails
|
||||||
|
Then the upload is failed with a typed reason
|
||||||
|
And its page shows that reason and stops polling
|
||||||
|
|
||||||
|
Scenario: A restart fails the jobs it interrupted
|
||||||
|
Given the service restarted while a recording was queued or transcribing
|
||||||
|
When it starts again
|
||||||
|
Then those uploads are failed as interrupted
|
||||||
|
And any audio left in the spool is deleted
|
||||||
|
|
||||||
|
Scenario: Only one recording is transcribed at a time
|
||||||
|
When several recordings are queued
|
||||||
|
Then they are transcribed one after another
|
||||||
|
|
||||||
|
Scenario: My uploads are invisible to other users
|
||||||
|
Given another user has uploaded a recording
|
||||||
|
When I ask for it by its id
|
||||||
|
Then I get 404 and no part of its title or text
|
||||||
|
And the chat for it calls no model
|
||||||
|
|
||||||
|
Scenario: I can ask about my recording and get a grounded answer
|
||||||
|
Given my recording has been transcribed
|
||||||
|
When I ask a question about it
|
||||||
|
Then the answer comes from my transcript through the same chat as a video's
|
||||||
|
|
||||||
|
Scenario: A long recording is not cut to its first third
|
||||||
|
Given my recording's transcript is longer than the caption default
|
||||||
|
When I ask a question about it
|
||||||
|
Then the transcript is sent with the upload budget, not the 18,000-character caption default
|
||||||
|
|
||||||
|
# ADR-034, two engines and a local reconciler (off unless TAPIR_RECONCILE_BASE_MODEL is set)
|
||||||
|
|
||||||
|
Scenario: Two engines transcribe the audio and their disagreements are reconciled
|
||||||
|
Given two-engine transcription is enabled
|
||||||
|
When a recording is transcribed
|
||||||
|
Then both engines run on the same audio, the fast one first
|
||||||
|
And a local model chooses between their disagreements
|
||||||
|
And the endpoint names every engine and the model that took part
|
||||||
|
|
||||||
|
Scenario: Trivial disagreements never reach the model
|
||||||
|
When the engines differ only by a compound written as one word or two, or by a filler word
|
||||||
|
Then the base reading is kept without a model call
|
||||||
|
|
||||||
|
Scenario: Differences close together are one decision
|
||||||
|
When two differences are separated by a single shared word
|
||||||
|
Then they are asked about as one dispute
|
||||||
|
|
||||||
|
Scenario: An unsure or changed reading is flagged for me
|
||||||
|
When the model changes a reading or cannot decide
|
||||||
|
Then the page lists the spot with a minute mark, both readings and which was used
|
||||||
|
And an unsure spot keeps the base reading and is marked uncertain
|
||||||
|
|
||||||
|
Scenario: The model cannot be a cloud model
|
||||||
|
When the reconciliation model or base engine is not a koala/ or iguana/ alias
|
||||||
|
Then the service refuses to start
|
||||||
|
And a cloud alias that merely contains a local host name is refused too
|
||||||
|
|
||||||
|
Scenario: One engine failing does not fail the upload
|
||||||
|
When one of the two engines fails
|
||||||
|
Then the other engine's transcript is used
|
||||||
|
And the page says only one engine produced it and why
|
||||||
|
And the cloud fallback is not used
|
||||||
|
|
||||||
|
Scenario: Only when both local engines fail does the cloud fallback run
|
||||||
|
When both local engines fail and a fallback is configured
|
||||||
|
Then the fallback transcribes it and the row records that it did
|
||||||
|
|
||||||
|
Scenario: A model that is down gives an unreconciled transcript, not a failed job
|
||||||
|
When the reconciliation model fails three times in a row
|
||||||
|
Then the base transcript is kept unreconciled with the reason
|
||||||
|
And the page says it could not be reconciled
|
||||||
|
|
||||||
|
Scenario: A cancelled job never reaches the cloud
|
||||||
|
When the job is cancelled during either engine
|
||||||
|
Then neither the second engine nor the cloud fallback is called
|
||||||
|
|
||||||
|
Scenario: A single-engine transcript claims no second opinion
|
||||||
|
When only one engine was configured
|
||||||
|
Then the page does not mention reconciliation
|
||||||
|
|
||||||
|
# ADR-035: progress, the log, and the judged spots
|
||||||
|
|
||||||
|
Scenario: I can see which phase my upload is in
|
||||||
|
Given a recording is being transcribed
|
||||||
|
When I open its page
|
||||||
|
Then it lists every phase in order, the finished ones with how long each took
|
||||||
|
And the running phase is marked, with how long it has run
|
||||||
|
And the page updates in place every two seconds until the job ends
|
||||||
|
|
||||||
|
Scenario: A bar appears only where the count is real
|
||||||
|
When the deciding phase has decided 142 of 282 differences
|
||||||
|
Then the page shows a bar with "142 of 282"
|
||||||
|
And an engine call, which is one request with nothing to count, never gets a bar or a percentage
|
||||||
|
|
||||||
|
Scenario: A slow engine call is not mistaken for a dead service
|
||||||
|
When an engine call has run for 25 minutes and the service is still reporting
|
||||||
|
Then the page shows the elapsed time and says nothing is wrong
|
||||||
|
|
||||||
|
Scenario: A service that stopped reporting is called lost
|
||||||
|
When nothing has been heard from the service for longer than the heartbeat allows
|
||||||
|
Then the page says the service stopped reporting and may have restarted or crashed
|
||||||
|
And the running phase carries a warning mark and no spinner
|
||||||
|
And the page keeps checking
|
||||||
|
|
||||||
|
Scenario: A model that stops advancing is called quiet
|
||||||
|
When the deciding phase has not advanced for five minutes and the service is still reporting
|
||||||
|
Then the page says nothing new has happened for that long and that the service is still running
|
||||||
|
And it does not say the service is lost
|
||||||
|
|
||||||
|
Scenario: A failed job shows where it stopped
|
||||||
|
When a job fails
|
||||||
|
Then the failed phase is marked and what never started stays pending
|
||||||
|
And the page stops checking
|
||||||
|
|
||||||
|
Scenario: A finished job keeps its record of what was done
|
||||||
|
When a job is done
|
||||||
|
Then the page keeps a collapsed list of the phases with how long each took
|
||||||
|
|
||||||
|
Scenario: The heartbeat runs on its own while an engine call is stuck
|
||||||
|
When an engine call does not return
|
||||||
|
Then the row is still stamped alive every few seconds
|
||||||
|
And no heartbeat outlives its job
|
||||||
|
|
||||||
|
Scenario: The log names every phase and never the transcript
|
||||||
|
When a job runs
|
||||||
|
Then the log has a line per phase in the order it ran, each naming the upload
|
||||||
|
And no line contains transcript text or the spool path
|
||||||
|
|
||||||
|
Scenario: Every judged spot is stored but only the ones to check are listed
|
||||||
|
When the model has changed, kept and been unsure about spots
|
||||||
|
Then every one is stored with its verdict and the words before it
|
||||||
|
And the page lists only the changed and uncertain ones, with the words before each
|
||||||
|
|
||||||
|
Scenario: The saved segments carry the merged words
|
||||||
|
When the model changes a word
|
||||||
|
Then the segment containing it carries the reconciled word
|
||||||
|
And the segment's timing is unchanged
|
||||||
|
|
||||||
|
# ADR-035 decision 8: hardening from the adversarial review
|
||||||
|
|
||||||
|
Scenario: Chat about a recording is offered with local models only
|
||||||
|
When I open the chat for my recording
|
||||||
|
Then the model list holds only models that run on the homelab
|
||||||
|
And a cloud model the shared chat service knows is not in it
|
||||||
|
|
||||||
|
Scenario: A forged cloud model is never sent the transcript
|
||||||
|
When a chat request names a cloud model
|
||||||
|
Then a local model answers instead, and nothing was sent to the cloud one
|
||||||
|
|
||||||
|
Scenario: A wedged engine does not use up the whole job
|
||||||
|
When the primary engine never answers
|
||||||
|
Then it fails at its own deadline while the job is still alive
|
||||||
|
And the fallback runs, unless the job itself was cancelled or timed out
|
||||||
|
|
||||||
|
Scenario: A file that lies about its length is refused
|
||||||
|
When a file claims to be short but converts to more than the limit
|
||||||
|
Then the job fails with a plain explanation and no engine receives the audio
|
||||||
|
|
||||||
|
Scenario: A recording much shorter than its file claimed is transcribed and noted
|
||||||
|
When a file converts to under 80 percent of the length it claimed and more than two minutes short
|
||||||
|
Then it is transcribed anyway and its audio step says how long the audio really is
|
||||||
|
And a complete recording whose header over-estimated its length is not refused
|
||||||
|
|
||||||
|
Scenario: A transcript made by the cloud fallback says the audio left the homelab
|
||||||
|
When the transcript was produced by an endpoint that is not local
|
||||||
|
Then its page says in plain words that the audio left the homelab and names the provider
|
||||||
|
And a locally transcribed recording carries no such note
|
||||||
|
|
||||||
|
Scenario: I can delete a finished recording
|
||||||
|
When I confirm deleting a done or failed recording
|
||||||
|
Then the recording and its transcript are gone and I am back on my list
|
||||||
|
And another user's recording is a 404 and is untouched
|
||||||
|
|
||||||
|
Scenario: A recording still being transcribed cannot be deleted
|
||||||
|
When I try to delete a recording that is queued or transcribing
|
||||||
|
Then I am told it is still being transcribed and nothing is removed
|
||||||
|
|
||||||
|
Scenario: The home list links to my recordings when uploads are on
|
||||||
|
Given audio upload is enabled
|
||||||
|
Then the home list links to the recordings page
|
||||||
|
|
||||||
|
Scenario: A full queue is a retry, not an error
|
||||||
|
When too many recordings are already waiting
|
||||||
|
Then the upload is answered as busy, its row is failed honestly and its audio is deleted
|
||||||
|
|
||||||
|
Scenario: One user cannot fill the queue for everyone
|
||||||
|
When one user has several recordings waiting
|
||||||
|
Then further uploads from that user are refused and other users are not affected
|
||||||
|
|
||||||
|
Scenario: Errors are shown as kinds, never their text
|
||||||
|
When a model or an engine fails
|
||||||
|
Then the log, the stored reason and the page carry a kind such as timeout or http_502
|
||||||
|
And never the provider's response, a file path or a word from the transcript
|
||||||
|
|
||||||
|
Scenario: Input that cannot be right is a client error
|
||||||
|
When the id is not an id, or a header or title holds control bytes
|
||||||
|
Then the answer is a 4xx or the title is cleaned, never a server error
|
||||||
|
|
||||||
|
Scenario: I choose the language of a recording
|
||||||
|
When I upload a recording and choose Swedish or Other on the form
|
||||||
|
Then the recording is transcribed as that language and its page says which
|
||||||
|
|
||||||
|
Scenario: Swedish stays the default
|
||||||
|
When an upload names no language
|
||||||
|
Then it is transcribed as Swedish, as before the choice existed
|
||||||
|
|
||||||
|
Scenario: A language outside the list is refused
|
||||||
|
When an upload names a language the form does not offer
|
||||||
|
Then it is answered 400 before anything is stored or sent to an engine
|
||||||
|
|
||||||
|
Scenario: A recording in another language never reaches the Swedish-only engine
|
||||||
|
When I choose Other for a recording
|
||||||
|
Then the language is left for the engine to detect and the Swedish-only base engine and the reconcile are not used
|
||||||
|
And the engine that does run is the configured one, which is not tuned for other languages
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
Feature: Chat with a video's stored transcript
|
||||||
|
As a reader whose summary made me want to dig deeper
|
||||||
|
I want to ask questions about the video without watching it
|
||||||
|
So that I can go further on the ones worth it, without leaving the reader
|
||||||
|
|
||||||
|
# ADR-027. The load-bearing constraint is safety-by-construction: chat runs
|
||||||
|
# ONLY against an already-stored transcript (ADR-021) and never fetches captions,
|
||||||
|
# never touches the rate gate, never reaches YouTube. Entry is from the summary
|
||||||
|
# view of one's OWN video; the conversation is ephemeral (no persisted history).
|
||||||
|
|
||||||
|
Background:
|
||||||
|
Given I have a summarized video with a stored transcript
|
||||||
|
|
||||||
|
Scenario: A summary view offers a deeper-dive into the video
|
||||||
|
When I view the summary
|
||||||
|
Then I see a "dig deeper" affordance that opens a chat about this video
|
||||||
|
And it opens the chat in place, below the summary, without leaving the page
|
||||||
|
|
||||||
|
Scenario: The summary and the chat are on one page
|
||||||
|
When I open the chat
|
||||||
|
Then the summary stays visible alongside the chat
|
||||||
|
And I can read the summary while I ask questions
|
||||||
|
|
||||||
|
Scenario: Ask a question answered from the stored transcript
|
||||||
|
When I ask a question in the chat
|
||||||
|
Then the answer is produced from the stored transcript
|
||||||
|
And no caption fetch and no YouTube call occurs
|
||||||
|
|
||||||
|
Scenario: Chat never fetches captions or reaches YouTube
|
||||||
|
When I ask a question in the chat
|
||||||
|
Then Tapir reads only the stored transcript
|
||||||
|
And the caption-fetch and video-fetch paths are never invoked
|
||||||
|
|
||||||
|
Scenario: A video with no stored transcript offers no chat
|
||||||
|
Given a video that has no stored transcript
|
||||||
|
When I open the chat for it
|
||||||
|
Then I am told chat is not available
|
||||||
|
And no fetch is attempted and no model is called
|
||||||
|
|
||||||
|
Scenario: The default model is the summary's model and is switchable
|
||||||
|
When I open the chat
|
||||||
|
Then the model defaults to the model that produced the summary
|
||||||
|
And I can switch among the offered chain models
|
||||||
|
|
||||||
|
Scenario: Switching models re-runs against the same transcript
|
||||||
|
When I ask a question with a different chain model selected
|
||||||
|
Then the chosen model answers
|
||||||
|
And it answers against the same stored transcript
|
||||||
|
|
||||||
|
Scenario: The cloud model is hidden when cloud is disabled
|
||||||
|
Given the cloud fallback model is disabled
|
||||||
|
Then the chat switcher offers only local models
|
||||||
|
|
||||||
|
Scenario: A long transcript is bounded and the chat says so
|
||||||
|
Given the stored transcript is longer than the model budget
|
||||||
|
When I ask a question
|
||||||
|
Then the answer is produced from a bounded portion
|
||||||
|
And the chat notes that it worked from a bounded portion
|
||||||
|
|
||||||
|
Scenario: A multi-turn conversation is ephemeral
|
||||||
|
When I ask a follow-up question
|
||||||
|
Then the prior turn is carried into the answer
|
||||||
|
And nothing about the conversation is written to the database
|
||||||
|
|
||||||
|
Scenario: Chat is reachable only from my own summary view
|
||||||
|
Given another user has a summarized video with a stored transcript
|
||||||
|
When I try to open the chat for their video
|
||||||
|
Then I get a not-found response
|
||||||
|
And no model is called
|
||||||
@@ -10,12 +10,33 @@ Feature: Connect and manage video accounts
|
|||||||
And my refresh token is stored only as a secret reference
|
And my refresh token is stored only as a secret reference
|
||||||
And my subscriptions are synced
|
And my subscriptions are synced
|
||||||
|
|
||||||
|
@pending
|
||||||
|
# Vimeo connect is not built yet (provider label exists; no connect flow or test).
|
||||||
Scenario: Connect a Vimeo account
|
Scenario: Connect a Vimeo account
|
||||||
Given I have no connected video accounts
|
Given I have no connected video accounts
|
||||||
When I connect my Vimeo account
|
When I connect my Vimeo account
|
||||||
Then the connection is stored with status "active"
|
Then the connection is stored with status "active"
|
||||||
And my subscriptions are synced
|
And my subscriptions are synced
|
||||||
|
|
||||||
|
Scenario: Connecting an account discovers videos immediately
|
||||||
|
Given I have no connected video accounts
|
||||||
|
When I connect my YouTube account
|
||||||
|
Then a discovery pass for my account is triggered right away
|
||||||
|
And I do not have to wait for the next scheduled pass to see my videos
|
||||||
|
|
||||||
|
Scenario: Connecting summarizes my best recent videos right away
|
||||||
|
Given I have no connected video accounts
|
||||||
|
When I connect my YouTube account
|
||||||
|
Then up to the onboarding cap of my newest likely-good videos are summarized through the rate gate
|
||||||
|
And videos whose known duration is too short or too long are skipped
|
||||||
|
And the rest are left to the scheduled recency-bounded pass
|
||||||
|
|
||||||
|
Scenario: The onboarding burst summarizes with a stronger model
|
||||||
|
Given I have no connected video accounts
|
||||||
|
When I connect my YouTube account
|
||||||
|
Then the burst summarizes with the stronger onboarding model first
|
||||||
|
And the standard summarizer chain still follows as a fallback
|
||||||
|
|
||||||
Scenario: Tokens are never stored in the clear
|
Scenario: Tokens are never stored in the clear
|
||||||
When I connect any video account
|
When I connect any video account
|
||||||
Then no OAuth token value is stored in the database
|
Then no OAuth token value is stored in the database
|
||||||
@@ -28,6 +49,9 @@ Feature: Connect and manage video accounts
|
|||||||
And no new videos are watched for that connection
|
And no new videos are watched for that connection
|
||||||
And my existing summaries remain readable
|
And my existing summaries remain readable
|
||||||
|
|
||||||
|
@pending
|
||||||
|
# Per-provider BYO credential config is not built as a web flow yet (the summarizer
|
||||||
|
# supports a fallback endpoint, but there is no user-facing BYO setup + its test).
|
||||||
Scenario Outline: BYO AI credential is optional and per-provider
|
Scenario Outline: BYO AI credential is optional and per-provider
|
||||||
When I configure a BYO provider "<provider>"
|
When I configure a BYO provider "<provider>"
|
||||||
Then the credential is stored only as a secret reference
|
Then the credential is stored only as a secret reference
|
||||||
|
|||||||
@@ -0,0 +1,37 @@
|
|||||||
|
Feature: Inline-expand summary + Q&A in the list (ADR-031, #16)
|
||||||
|
As a reader skimming my summaries
|
||||||
|
I want to open a summary and its Q&A in place in the list
|
||||||
|
So that I get the full read and follow-up without leaving the list (SPA-like, no page hop)
|
||||||
|
|
||||||
|
# HTMX inline-expand, no SPA framework (ADR-031). Each scenario maps to a Go test
|
||||||
|
# in scenario_coverage_test.go (the BDD name-coverage gate).
|
||||||
|
|
||||||
|
Scenario: A summarized card expands to the full summary in place
|
||||||
|
Given a summarized video in my list
|
||||||
|
When I expand its card
|
||||||
|
Then the full summary, highlights, and takeaways are returned as an in-place card fragment, not a full page
|
||||||
|
|
||||||
|
Scenario: An expanded card collapses back to the compact card
|
||||||
|
Given an expanded card
|
||||||
|
When I collapse it
|
||||||
|
Then the compact card fragment is returned in its place
|
||||||
|
|
||||||
|
Scenario: The expanded card offers the Q&A dock
|
||||||
|
Given chat is enabled
|
||||||
|
When a summarized card is expanded
|
||||||
|
Then the expanded card includes the deeper-dive chat affordance for that video
|
||||||
|
|
||||||
|
Scenario: Only a summarized card offers expand
|
||||||
|
Given a discovered but not-yet-summarized card
|
||||||
|
When the card is rendered
|
||||||
|
Then it shows its summarize/queue footer and no expand affordance
|
||||||
|
|
||||||
|
Scenario: With JS off the card still reaches the full summary
|
||||||
|
Given a summarized card
|
||||||
|
When it is rendered
|
||||||
|
Then its expand affordance carries an href to the detail page as a no-JS fallback
|
||||||
|
|
||||||
|
Scenario: The detail page and the expanded card show the same summary
|
||||||
|
Given a summarized video
|
||||||
|
When I view it on the detail page and as an expanded card
|
||||||
|
Then both render the same summary body (one shared fragment, no drift)
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
Feature: Public landing page
|
||||||
|
As a first-time visitor
|
||||||
|
I want a public welcome page before I log in
|
||||||
|
So that I understand what Tapir is and how to get started without hitting a login wall
|
||||||
|
|
||||||
|
Scenario: An unauthenticated visit to the root is sent to the welcome page
|
||||||
|
Given I am not logged in
|
||||||
|
When I open the root path "/"
|
||||||
|
Then I am redirected to "/welcome"
|
||||||
|
|
||||||
|
Scenario: The welcome page invites an unauthenticated visitor to start
|
||||||
|
Given I am not logged in
|
||||||
|
When I open "/welcome"
|
||||||
|
Then I see a "Get Started" call to action
|
||||||
|
|
||||||
|
Scenario: An authenticated user on the welcome page sees their way in and out
|
||||||
|
Given I am logged in
|
||||||
|
When I open "/welcome"
|
||||||
|
Then I see a link to my summaries
|
||||||
|
And I see a way to log out
|
||||||
|
|
||||||
|
@pending
|
||||||
|
# Behaviour ships (logout redirects to /welcome) but is not unit-tested: logout lives in
|
||||||
|
# the OIDC Auth impl and StubAuth has no routes to exercise it cheaply.
|
||||||
|
Scenario: Logging out returns to the welcome page
|
||||||
|
Given I am logged in
|
||||||
|
When I log out
|
||||||
|
Then I am returned to "/welcome"
|
||||||
|
|
||||||
|
# /welcome is mounted outside the auth guard so it is reachable without a session;
|
||||||
|
# the root and all data routes stay behind it (commits around the WelcomePage work).
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
Feature: Observability — timing and metrics for performance and UX (ADR-030, #15)
|
||||||
|
As the maintainer running Tapir for pilot users
|
||||||
|
I want timing and Prometheus metrics for the activities that drive performance and UX
|
||||||
|
So that I can see latency, model behaviour, and usage — and feed the Stage-0 eval gate
|
||||||
|
|
||||||
|
# AI metrics are the priority (ADR-030 R3). Each scenario maps to a Go test in
|
||||||
|
# test/acceptance/scenario_coverage_test.go (the BDD name-coverage gate).
|
||||||
|
|
||||||
|
Scenario: Summarization latency is recorded per endpoint
|
||||||
|
Given the summarizer runs a transcript through its endpoint chain
|
||||||
|
When an endpoint returns a parseable summary
|
||||||
|
Then the summarize latency is recorded with the model, outcome "success", and whether it was a fallback
|
||||||
|
|
||||||
|
Scenario: A failing summarizer endpoint records its failure outcome
|
||||||
|
Given the summarizer runs a transcript through its endpoint chain
|
||||||
|
When an endpoint errors or returns unparseable output
|
||||||
|
Then the summarize latency is recorded with outcome "error" or "parse_error" before the chain advances
|
||||||
|
|
||||||
|
Scenario: Caption fetch latency is recorded by outcome
|
||||||
|
Given a caption fetch is attempted for a video
|
||||||
|
When it resolves to captions, no captions, or a rate limit
|
||||||
|
Then the caption-fetch latency is recorded labelled by that outcome
|
||||||
|
|
||||||
|
Scenario: LLM token usage is recorded from the completion
|
||||||
|
Given an LLM completion returns a usage block with prompt and completion tokens
|
||||||
|
When the client finishes the call
|
||||||
|
Then the prompt and completion tokens are recorded for that model
|
||||||
|
|
||||||
|
Scenario: Q&A answer latency is recorded
|
||||||
|
Given a user asks a question about a video
|
||||||
|
When the answer is produced from the stored transcript
|
||||||
|
Then the chat answer latency is recorded for the answering model
|
||||||
|
|
||||||
|
Scenario: HTTP requests are counted by route, method, and status
|
||||||
|
Given the metrics HTTP middleware wraps the app
|
||||||
|
When a request is served against a registered route
|
||||||
|
Then it is counted and timed under the bounded route pattern, not the raw path
|
||||||
|
|
||||||
|
Scenario: A successful login is counted
|
||||||
|
Given a user completes the OIDC callback and a session is established
|
||||||
|
Then the login counter is incremented
|
||||||
|
|
||||||
|
Scenario: The metrics endpoint is not on the public app port
|
||||||
|
Given the service is running
|
||||||
|
When the public app mux is inspected
|
||||||
|
Then it exposes no /metrics route — metrics are served on the dedicated metrics port only
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
Feature: Paste a YouTube URL to summarize any video
|
||||||
|
As a user
|
||||||
|
I want to paste a YouTube link and get a summary
|
||||||
|
So that I can pull the specific video I want now, even from channels I don't follow
|
||||||
|
|
||||||
|
Scenario: Paste a valid YouTube URL
|
||||||
|
Given I am connected
|
||||||
|
When I paste a valid YouTube video URL
|
||||||
|
Then the video is added to my feed scoped to me
|
||||||
|
And it is queued for summarization through the shared rate gate
|
||||||
|
|
||||||
|
Scenario: Pasting an invalid link is rejected
|
||||||
|
When I paste something that is not a YouTube video URL
|
||||||
|
Then I get a clear error and nothing is added
|
||||||
|
|
||||||
|
Scenario: Pasting a video that cannot be found is honest
|
||||||
|
When I paste a URL whose video cannot be found
|
||||||
|
Then I am told it couldn't be found and nothing is added
|
||||||
|
|
||||||
|
Scenario: Pasting the same video twice does not duplicate it
|
||||||
|
Given I have pasted a video
|
||||||
|
When I paste the same video again
|
||||||
|
Then my feed still has exactly one entry for it
|
||||||
|
|
||||||
|
@pending
|
||||||
|
# Covered by the engine's ADR-010 no-transcript terminal state (degrade-never-error);
|
||||||
|
# there is no paste-specific test for it.
|
||||||
|
Scenario: A pasted video with no captions resolves honestly
|
||||||
|
When I paste a video that has no captions
|
||||||
|
Then it resolves to the "no transcript available" terminal state
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
Feature: Register and manage a multi-user account
|
||||||
|
As one of a handful of trusted users
|
||||||
|
I want my own account, isolated from everyone else's
|
||||||
|
So that Tapir can serve several people from one deployment without leaking data
|
||||||
|
|
||||||
|
# Stage 1 (ADR-012): Dex authenticates, Tapir authorizes per user. A Dex subject
|
||||||
|
# with no users row is a new user and must register before reaching any data.
|
||||||
|
|
||||||
|
Scenario: A new Dex subject is routed to registration
|
||||||
|
Given I am authenticated by Dex with a subject that has no Tapir account
|
||||||
|
When I open any page that requires an account
|
||||||
|
Then I am routed to the registration page
|
||||||
|
And no summaries are shown until I register
|
||||||
|
|
||||||
|
Scenario: Registering creates the account and its identity mapping
|
||||||
|
Given I am authenticated by Dex with a subject that has no Tapir account
|
||||||
|
When I complete registration
|
||||||
|
Then a user row is created for me
|
||||||
|
And a user_identities row maps my Dex subject to that user
|
||||||
|
And I am taken into the app as a registered user
|
||||||
|
|
||||||
|
Scenario: A returning subject passes straight through
|
||||||
|
Given I am authenticated by Dex with a subject that already has a Tapir account
|
||||||
|
When I open the app
|
||||||
|
Then I am not asked to register again
|
||||||
|
And I see my own summaries
|
||||||
|
|
||||||
|
Scenario: Deleting an account removes only my data and leaves other users untouched
|
||||||
|
Given I am a registered user with summaries, a connected account, and recorded actions
|
||||||
|
And another user exists with their own summaries
|
||||||
|
When I delete my account
|
||||||
|
Then all of my rows are removed across every user-owned table
|
||||||
|
And my stored secret references are removed
|
||||||
|
And the other user's data remains intact
|
||||||
|
And my Dex identity is left intact
|
||||||
|
|
||||||
|
@pending
|
||||||
|
# Re-registration after delete is supported by design (delete leaves the Dex identity,
|
||||||
|
# ADR-013) but has no dedicated end-to-end test yet.
|
||||||
|
Scenario: A deleted user can register again as a fresh account
|
||||||
|
Given I deleted my Tapir account but my Dex identity still exists
|
||||||
|
When I sign in again
|
||||||
|
Then I am routed to the registration page as a new user
|
||||||
|
And registering creates a fresh user row with none of my old data
|
||||||
|
|
||||||
|
# Isolation is DB-enforced (Postgres RLS, ADR-012, migration 003): a user can never
|
||||||
|
# read or write another user's rows even if an application WHERE clause is wrong.
|
||||||
|
# Deletion is Tapir-side only — the shared Dex directory is never modified (ADR-013).
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
Feature: Choose how new videos get summarized
|
||||||
|
As a user who wants control over compute and noise
|
||||||
|
I want to pick whether new videos are summarized automatically or on demand
|
||||||
|
So that I only spend summarization on the videos I actually care about
|
||||||
|
|
||||||
|
Background:
|
||||||
|
Given I am a registered user with a connected video account
|
||||||
|
|
||||||
|
Scenario: Auto mode summarizes recent new videos automatically
|
||||||
|
Given my summarization mode is "auto"
|
||||||
|
When a subscribed channel posts a new video with captions within the recency window
|
||||||
|
Then Tapir summarizes it without my asking
|
||||||
|
And the summary appears in my list
|
||||||
|
|
||||||
|
Scenario: Auto mode lists older videos without summarizing them
|
||||||
|
Given my summarization mode is "auto"
|
||||||
|
When discovery finds a video published before the recency window
|
||||||
|
Then the video appears in my list with no summary
|
||||||
|
And it is not summarized automatically
|
||||||
|
And I can still summarize it on demand with "Summarize"
|
||||||
|
|
||||||
|
Scenario: Automatic is the default for a new user
|
||||||
|
Given I have just registered
|
||||||
|
Then my summarization mode is "auto"
|
||||||
|
|
||||||
|
Scenario: Manual mode leaves new videos unsummarized
|
||||||
|
Given my summarization mode is "manual"
|
||||||
|
When a subscribed channel posts a new video with captions
|
||||||
|
Then the video appears in my list with no summary
|
||||||
|
And nothing is summarized until I request it
|
||||||
|
|
||||||
|
Scenario: Requesting a summary in manual mode queues it for the next run
|
||||||
|
Given my summarization mode is "manual"
|
||||||
|
And a new video is in my list with no summary
|
||||||
|
When I click "Summarize" on that video
|
||||||
|
Then the video is marked as requested
|
||||||
|
And the next run summarizes it
|
||||||
|
And the request flag is cleared after it is processed
|
||||||
|
|
||||||
|
# auto_summarize is a per-user setting and summarize_requested is a per-video queue
|
||||||
|
# flag (migration 006). The web button sets the flag; `tapir run` processes both the
|
||||||
|
# auto videos and the manually queued ones, then clears the flag.
|
||||||
|
#
|
||||||
|
# Recency bound (ADR-020): in auto mode the scheduler only summarizes videos published
|
||||||
|
# within TAPIR_AUTO_SUMMARIZE_WINDOW (default ~7d); older videos are discovered and
|
||||||
|
# listed but wait for an explicit "Summarize" — so a back-catalogue does not re-drive
|
||||||
|
# the per-IP caption gate (ADR-014) every cycle. A manual request bypasses the bound.
|
||||||
@@ -31,5 +31,16 @@ Feature: Summarize new videos from subscribed channels
|
|||||||
When the watcher sees "Designing for Attention" again
|
When the watcher sees "Designing for Attention" again
|
||||||
Then Tapir does not produce a second summary for it
|
Then Tapir does not produce a second summary for it
|
||||||
|
|
||||||
|
Scenario: Re-analyzing a stored video does not re-fetch its transcript
|
||||||
|
Given a transcript for "Designing for Attention" is already stored
|
||||||
|
When the video is summarized again
|
||||||
|
Then Tapir reads the stored transcript
|
||||||
|
And Tapir does not fetch captions from YouTube
|
||||||
|
|
||||||
# Captions-first is the core path (ADR-007). Audio-download + speech-to-text is
|
# Captions-first is the core path (ADR-007). Audio-download + speech-to-text is
|
||||||
# deferred and intentionally has no scenario here yet.
|
# deferred and intentionally has no scenario here yet.
|
||||||
|
#
|
||||||
|
# Transcript persistence (ADR-021): the stored transcript is shared, keyed by
|
||||||
|
# (provider, provider_video_id) and read before any caption fetch, so the
|
||||||
|
# re-analysis scenario above also covers paste-a-URL and the onboarding burst —
|
||||||
|
# both summarize through the same engine chokepoint.
|
||||||
|
|||||||
@@ -0,0 +1,29 @@
|
|||||||
|
Feature: Visual refresh — one charm-reader layout, light + dark themes (ADR-032, #17)
|
||||||
|
As a reader who likes a TUI/charm aesthetic
|
||||||
|
I want a fresh look with a light and a dark theme
|
||||||
|
So that the app feels distinctive and reads well in either mode
|
||||||
|
|
||||||
|
# Direction B (light) + C (dark) are one layout in two palettes (CSS variables),
|
||||||
|
# plus a persisted toggle and an OS-preference default. Colours are reviewed via
|
||||||
|
# the mockups in docs/sketches/, not unit-tested; these scenarios cover the
|
||||||
|
# testable structure. Each maps to a Go test in scenario_coverage_test.go.
|
||||||
|
|
||||||
|
Scenario: Light and dark themes share one layout via CSS variables
|
||||||
|
Given the app stylesheet
|
||||||
|
When it is rendered
|
||||||
|
Then it defines a light palette on the root and a dark palette under data-theme="dark", with no duplicate markup
|
||||||
|
|
||||||
|
Scenario: Without a stored choice the theme follows the OS preference
|
||||||
|
Given a visitor with no saved theme
|
||||||
|
When the page loads
|
||||||
|
Then a prefers-color-scheme dark block applies the dark palette automatically
|
||||||
|
|
||||||
|
Scenario: A persisted toggle switches light and dark
|
||||||
|
Given any page
|
||||||
|
When it is rendered
|
||||||
|
Then it includes a theme-toggle control and a small script that flips data-theme and persists the choice
|
||||||
|
|
||||||
|
Scenario: The expanded card embeds the video player
|
||||||
|
Given a summarized video with a valid provider id
|
||||||
|
When its card is expanded
|
||||||
|
Then the expanded card includes the embedded video player
|
||||||
@@ -0,0 +1,207 @@
|
|||||||
|
# Tapir — Heuristic Review (Stage-0, sparse + recency-bounded)
|
||||||
|
|
||||||
|
_Findings document, not a build spec. The maintainer filters; a spec follows separately.
|
||||||
|
Reviewed against `VISION.md` (Stage-0 gate), `docs/ui-spec.md`, `internal/web/views.templ` +
|
||||||
|
`view.go`, the prior `UX-REVIEW.md` pass, and the current screenshots. Written for the product
|
||||||
|
**as it actually is**: ~283 discovered, ~15 summarized, ~256 in queue behind a respected per-IP
|
||||||
|
caption rate limit, ~12 no-captions; single-user (maintainer) with friends pending; recency-
|
||||||
|
bounded auto-summarize about to ship (auto = recent ~7d, older browsable + manual on demand)._
|
||||||
|
|
||||||
|
## Reviewer stance
|
||||||
|
|
||||||
|
The Stage-0 gate is **return usage**. So every finding is judged by one question: does this make
|
||||||
|
the maintainer (or a friend) come back to an honestly-sparse feed? The visual layer is already
|
||||||
|
decent — dark mode, cards, the constrained reader were all fixed in the prior pass. The open
|
||||||
|
problems are **expectation-setting, honesty-of-scale, and the recency feed** — not pixels.
|
||||||
|
|
||||||
|
**The single biggest risk:** a new user connects, sees `15 summarized · 256 fetching soon`,
|
||||||
|
nothing visibly moves, and never returns. The whole gate dies at that moment. Most P0s below
|
||||||
|
attack that one moment.
|
||||||
|
|
||||||
|
## Tagging
|
||||||
|
|
||||||
|
- **`[NOW]`** — improves the product as it is today (sparse, recency-bounded, single-user). Ships
|
||||||
|
in the upcoming bundle.
|
||||||
|
- **`[LATER]`** — improves the product we hope it becomes (abundance, engagement, multiple users).
|
||||||
|
Valuable but premature until real usage validates the core loop.
|
||||||
|
|
||||||
|
Severity: 🔴 breaks the core loop · 🟠 hurts it · 🟡 noticeable · 🔵 polish.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## P0 — fix before/with the recency ship
|
||||||
|
|
||||||
|
### 1. 🔴 Empty-connected state tells the user to run a CLI command they can't run — `[NOW]`
|
||||||
|
**Problem.** After connecting YouTube, the empty list says: *"Run `tapir run` to discover your
|
||||||
|
subscriptions."* A friend on the web has no shell. And post-ADR-018 discovery is an in-process
|
||||||
|
scheduled loop — so the instruction is wrong *even for the maintainer*. It is the first thing a
|
||||||
|
newly-onboarded user sees, and it is an impossible, stale instruction.
|
||||||
|
**Principle.** Match between system and the real world; help users recognize, not be blocked
|
||||||
|
(Nielsen #1, #2, #9).
|
||||||
|
**Proposal.** Replace with a passive, honest "we're working" state: *"Your account is connected.
|
||||||
|
Tapir is finding your subscriptions and fetching captions — summaries appear here gradually. Check
|
||||||
|
back later."* No command. No imperative the user can't satisfy.
|
||||||
|
**Evidence.** `views.templ` `summaryList` → `.empty-connected`.
|
||||||
|
|
||||||
|
### 2. 🔴 No expectation set for *gradual* fill — the return loop breaks at the cliff — `[NOW]`
|
||||||
|
**Problem.** Captions are rate-limited by design; the backlog trickles over days. Nothing tells
|
||||||
|
the user this. A first visit shows few/no summaries and no "come back" framing. The Stage-0 gate
|
||||||
|
is literally about returns, and the product never asks for one or explains why patience is
|
||||||
|
warranted.
|
||||||
|
**Principle.** Visibility of system status (#1). And: the gate can't be cleared if the UX doesn't
|
||||||
|
survive first contact.
|
||||||
|
**Proposal.** One honest sentence near the pipeline bar / empty state: *"Tapir fetches captions
|
||||||
|
slowly on purpose, to respect YouTube's limits. New summaries land gradually — usually best to
|
||||||
|
check back tomorrow."* Turns confusing emptiness into intentional design. Highest-leverage change
|
||||||
|
in the review.
|
||||||
|
|
||||||
|
### 3. 🟠 "256 fetching soon" overstates imminence — a lie of scale — `[NOW]`
|
||||||
|
**Problem.** The pipeline bar and rate-limited cards both say "fetching soon." For 256 items
|
||||||
|
behind a per-IP throttle, "soon" is false — they trickle over days/weeks. Exactly the abundance-
|
||||||
|
implying language the mandate forbids, inverted: it makes the *queue* look imminent.
|
||||||
|
**Principle.** Honesty of scarcity (project mandate); #1.
|
||||||
|
**Proposal.** Relabel by scale. Bar: `15 ready · 256 in queue · 12 no captions`. Card state:
|
||||||
|
"In queue" / "Waiting its turn", not "Fetching soon…". Reserve "soon" for items actually next.
|
||||||
|
**Evidence.** `views.templ` `pipelineBar`; card State 4.
|
||||||
|
|
||||||
|
### 4. 🟠 Recency boundary is invisible in the feed — `[NOW, ships with recency]`
|
||||||
|
**Problem.** Once auto-summarize is bounded to ~7d, a 6-month-old pending video and a 2-day-old
|
||||||
|
pending video render identically ("Not summarized" + button). But only one is in the auto path;
|
||||||
|
the other will *never* process unless clicked. The user can't tell "be patient, this is coming"
|
||||||
|
from "this is yours to trigger or ignore."
|
||||||
|
**Principle.** Visibility of system status; predictability (#1).
|
||||||
|
**Proposal.** Bucket the list into two sections: **Recent** (auto, will fill itself) and
|
||||||
|
**Older — browse / summarize on demand**. Card status language should encode *which side of the
|
||||||
|
line it's on*, not the pipeline internals. Central IA decision of the recency change.
|
||||||
|
|
||||||
|
### 5. 🟠 The un-summarized mass buries the ~15 readable summaries — `[NOW]`
|
||||||
|
**Problem.** ~268 of 283 cards are not readable (pending / queued / no-captions). Summarized-first
|
||||||
|
ordering helps, but the page is still 95% noise below the fold. "Attention is the scarce resource"
|
||||||
|
is the product's own principle — and the default view violates it.
|
||||||
|
**Principle.** Aesthetic/minimalist design; signal-to-noise (#8).
|
||||||
|
**Proposal.** Default view = readable summaries + the Recent bucket. Collapse the older
|
||||||
|
un-summarized mass behind *"Show 256 older un-summarized videos."* Collapse the 12 no-caption
|
||||||
|
videos into a single line: *"12 videos have no captions"* (terminal, never readable — they don't
|
||||||
|
deserve 12 full cards).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## P1 — high, near-term
|
||||||
|
|
||||||
|
### 6. 🟠 "Summarize now" overpromises against the rate gate — `[NOW]`
|
||||||
|
**Problem.** Button says "Summarize now" (title: "Summarize this one now"). Backend queues it
|
||||||
|
behind the shared per-IP gate. Click 10 older videos and they all sit at "Fetching soon…". The
|
||||||
|
verb sells immediacy the system can't honor.
|
||||||
|
**Principle.** Honesty; match system/reality (#1).
|
||||||
|
**Proposal.** Drop "now" → "Summarize". On click the card should honestly become "Queued" (it
|
||||||
|
already can). Optionally show queue position once the queue is real. Don't engineer the gate
|
||||||
|
harder — just stop the verb from lying.
|
||||||
|
**Evidence.** `views.templ` card States 4 & 5; `handlers.go` `handleRetryNow`,
|
||||||
|
`handleRequestSummarize`.
|
||||||
|
|
||||||
|
### 7. 🟠 Welcome-page copy is stale and misleading post-ADR-019 — `[NOW]`
|
||||||
|
**Problem.** Landing sub-copy: *"If you have an invite link, it will set up your account
|
||||||
|
automatically."* Invites moved to Authentik (ADR-019); Tapir no longer handles invite links. The
|
||||||
|
"Get Started" button goes straight to OIDC. The copy promises a flow that no longer exists.
|
||||||
|
**Principle.** Match between system and reality (#2); honesty.
|
||||||
|
**Proposal.** Rewrite: *"Tapir is invite-only right now. If you've been invited, sign in below."*
|
||||||
|
Single CTA. Also set the gradual-fill expectation here (ties to #2) so it lands before the wall,
|
||||||
|
not after.
|
||||||
|
**Evidence.** `views.templ` `WelcomePage` → `.welcome-sub`.
|
||||||
|
|
||||||
|
### 8. 🟡 "Queued · waiting for the next run" leaks system jargon — `[NOW]`
|
||||||
|
**Problem.** "the next run" exposes the discovery-loop concept; a user doesn't know what a "run"
|
||||||
|
is.
|
||||||
|
**Principle.** Speak the user's language (#2).
|
||||||
|
**Proposal.** "Queued — summarizing shortly." Hide the scheduler.
|
||||||
|
**Evidence.** `views.templ` card State 3.
|
||||||
|
|
||||||
|
### 9. 🟡 Filters render before there's anything to filter — `[NOW]`
|
||||||
|
**Problem.** The fresh/empty state shows the full Channel/From/To/Filter bar *above* "Connect
|
||||||
|
YouTube." Power tooling stacked on top of the one action that matters.
|
||||||
|
**Principle.** Progressive disclosure; minimalist design (#8).
|
||||||
|
**Proposal.** Hide the filter bar when there are 0 rows (and arguably below ~20). Show the connect
|
||||||
|
CTA alone.
|
||||||
|
**Evidence.** `fixes/03-empty-fresh.png`; `ListPage` renders `filterForm` unconditionally.
|
||||||
|
|
||||||
|
### 10. 🟡 Date-range filters are dead weight at this scale — `[NOW]` demote / `[LATER]` rebuild
|
||||||
|
**Problem.** From/To date pickers + a free-text exact-match Channel field are corpus-scale tools.
|
||||||
|
With 15 summaries they're noise; channel-as-freetext is unguessable. (Confirm the prior review's
|
||||||
|
#7 is resolved — that `Channel` shows a real channel name, not the `provider` string; seeded
|
||||||
|
screenshots suggest it is, live data may differ.)
|
||||||
|
**Principle.** Match tool to task; minimalist design.
|
||||||
|
**Proposal.** `[NOW]`: reduce to the "Summarized only" toggle (+ maybe channel chips derived from
|
||||||
|
present rows). Drop date pickers until the corpus justifies them. `[LATER]`: real channel facets +
|
||||||
|
search when there's volume.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## P2 — reading experience & polish
|
||||||
|
|
||||||
|
### 11. 🟡 Detail leads with Summary; the attention-saving payload (Takeaways) is last — `[NOW]`
|
||||||
|
**Problem.** Product promise is "decide what's worth your time." The element that answers that —
|
||||||
|
Takeaways / verdict — sits at the bottom. The user reads a full summary to reach the point.
|
||||||
|
**Principle.** Lead with the user's actual job-to-be-done.
|
||||||
|
**Proposal.** Reorder or add a one-line TL;DR/verdict at top. Takeaways → Highlights → Summary, or
|
||||||
|
a "Worth watching?" lede. Data already exists; a reorder, not new machinery.
|
||||||
|
**Evidence.** `views.templ` `DetailPage`.
|
||||||
|
|
||||||
|
### 12. 🔵 No "back to Summaries" on detail — `[NOW]`
|
||||||
|
**Problem.** Only the brand returns home, losing any filter context.
|
||||||
|
**Proposal.** Explicit "← Summaries" link on the detail page.
|
||||||
|
|
||||||
|
### 13. 🔵 Auto/manual toggle copy will be wrong after recency — `[NOW, with recency]`
|
||||||
|
**Problem.** Account copy: *"Automatic summarizes every new video as it is discovered."* Becomes
|
||||||
|
false once auto is bounded to ~7d.
|
||||||
|
**Proposal.** *"Automatic summarizes new videos from the last ~7 days. Older videos stay
|
||||||
|
browsable — summarize them on demand."*
|
||||||
|
**Evidence.** `views.templ` `AccountPage` summarization section.
|
||||||
|
|
||||||
|
### 14. 🔵 Register step asks acceptance of nonexistent terms — `[NOW]`
|
||||||
|
**Problem.** "I accept the terms of use" — no terms linked. For a friends-only tool, ceremony
|
||||||
|
accepting nothing.
|
||||||
|
**Proposal.** Either link real terms or drop the checkbox at this stage.
|
||||||
|
**Evidence.** `views.templ` `RegisterPage`.
|
||||||
|
|
||||||
|
### 15. 🔵 watched/skipped mutual exclusivity unsignaled — `[NOW]` low
|
||||||
|
**Problem.** Three independent-looking buttons; watched↔skipped are exclusive (carryover from
|
||||||
|
prior review #14).
|
||||||
|
**Proposal.** Segmented control for watched/skipped; keep Saved separate.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## `[LATER]` — premature until the core loop is validated
|
||||||
|
|
||||||
|
_(abundance / engagement / multiple users)_
|
||||||
|
|
||||||
|
- **Return-nudges (digest email / push).** 🔴 **Caution, not just defer.** A notification that
|
||||||
|
drives returns *contaminates the exact signal the gate measures* — VISION wants *unprompted*
|
||||||
|
returns and admits it can't distinguish prompted from organic. Building a nudge now poisons the
|
||||||
|
experiment. Defer until after the gate reads. `[LATER]`
|
||||||
|
- **Full-text search across summaries** — needs volume to matter. `[LATER]`
|
||||||
|
- **Channel facets / saved filters / sorting** — corpus-scale tooling (#10). `[LATER]`
|
||||||
|
- **Read/unread + "new since last visit"** — genuinely helps returns, but only meaningful once
|
||||||
|
there is throughput to be "new." `[LATER]`
|
||||||
|
- **Saved/queue view, collections** — engagement surface; no payoff at 15 items. `[LATER]`
|
||||||
|
- **Richer card previews (top-takeaway as preview, 2–3 lines)** — triage aid that only pays off
|
||||||
|
with many cards to triage. `[LATER]`
|
||||||
|
- **Backlog progress heartbeat ("N summarized this week", queue burn-down)** — rewards returning,
|
||||||
|
but needs real throughput to show motion; a static "last updated X ago" is the only `[NOW]`-worthy
|
||||||
|
slice. `[LATER]`
|
||||||
|
- **Onboarding tour / multi-step welcome** — over-built for one user + a few friends. `[LATER]`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## What's already right (don't regress)
|
||||||
|
|
||||||
|
Pipeline-bar concept, the honest "No transcript available" terminal state, the queued/waiting
|
||||||
|
spinner instead of a fake-imminent one, the Unavailable-channels surface, the constrained-width
|
||||||
|
reader, dark mode, the account danger-zone behind a disclosure. The honesty instincts are present
|
||||||
|
— the P0s are about making that honesty *legible and correctly-scaled*, not adding it.
|
||||||
|
|
||||||
|
## Two judgment calls to settle first
|
||||||
|
|
||||||
|
1. **#4 (Recent/Older split) and #5 (collapse the older mass) are one decision viewed twice.**
|
||||||
|
Settle the recency-feed IA once and both fall out.
|
||||||
|
2. **The return-nudge caution (`[LATER]` list, item 1) is the one to put in writing now** — before
|
||||||
|
someone "helpfully" ships an email digest to juice the gate and destroys the signal.
|
||||||
Binary file not shown.
|
After Width: | Height: | Size: 116 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 73 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 58 KiB |
@@ -1,4 +1,4 @@
|
|||||||
module gitea.d-ma.be/mathias/tapir
|
module git.d-ma.be/mathias/tapir
|
||||||
|
|
||||||
go 1.26.1
|
go 1.26.1
|
||||||
|
|
||||||
@@ -9,21 +9,33 @@ require (
|
|||||||
github.com/go-jose/go-jose/v4 v4.1.4
|
github.com/go-jose/go-jose/v4 v4.1.4
|
||||||
github.com/golang-migrate/migrate/v4 v4.19.1
|
github.com/golang-migrate/migrate/v4 v4.19.1
|
||||||
github.com/jackc/pgx/v5 v5.9.2
|
github.com/jackc/pgx/v5 v5.9.2
|
||||||
|
github.com/prometheus/client_golang v1.23.2
|
||||||
|
github.com/prometheus/client_model v0.6.2
|
||||||
github.com/stretchr/testify v1.11.1
|
github.com/stretchr/testify v1.11.1
|
||||||
golang.org/x/oauth2 v0.36.0
|
golang.org/x/oauth2 v0.36.0
|
||||||
|
golang.org/x/time v0.15.0
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
github.com/beorn7/perks v1.0.1 // indirect
|
||||||
|
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||||
github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa // indirect
|
github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa // indirect
|
||||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||||
|
github.com/kylelemons/godebug v1.1.0 // indirect
|
||||||
github.com/lib/pq v1.10.9 // indirect
|
github.com/lib/pq v1.10.9 // indirect
|
||||||
|
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||||
|
github.com/prometheus/common v0.66.1 // indirect
|
||||||
|
github.com/prometheus/procfs v0.16.1 // indirect
|
||||||
github.com/rogpeppe/go-internal v1.15.0 // indirect
|
github.com/rogpeppe/go-internal v1.15.0 // indirect
|
||||||
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
|
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
|
||||||
|
go.yaml.in/yaml/v2 v2.4.2 // indirect
|
||||||
golang.org/x/sync v0.18.0 // indirect
|
golang.org/x/sync v0.18.0 // indirect
|
||||||
|
golang.org/x/sys v0.41.0 // indirect
|
||||||
golang.org/x/text v0.31.0 // indirect
|
golang.org/x/text v0.31.0 // indirect
|
||||||
|
google.golang.org/protobuf v1.36.8 // indirect
|
||||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -4,6 +4,10 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo
|
|||||||
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
|
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
|
||||||
github.com/a-h/templ v0.3.1020 h1:ypAT/L5ySWEnZ6Zft/5yfoWXYYkhFNvEFOeeqecg4tw=
|
github.com/a-h/templ v0.3.1020 h1:ypAT/L5ySWEnZ6Zft/5yfoWXYYkhFNvEFOeeqecg4tw=
|
||||||
github.com/a-h/templ v0.3.1020/go.mod h1:A2DlK61v+K+NRoGnhmYbNYVmtYHcFO5/AisMvBdDxTM=
|
github.com/a-h/templ v0.3.1020/go.mod h1:A2DlK61v+K+NRoGnhmYbNYVmtYHcFO5/AisMvBdDxTM=
|
||||||
|
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||||
|
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||||
|
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||||
|
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||||
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
|
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
|
||||||
github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
|
github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
|
||||||
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
|
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
|
||||||
@@ -37,8 +41,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
|||||||
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||||
github.com/golang-migrate/migrate/v4 v4.19.1 h1:OCyb44lFuQfYXYLx1SCxPZQGU7mcaZ7gH9yH4jSFbBA=
|
github.com/golang-migrate/migrate/v4 v4.19.1 h1:OCyb44lFuQfYXYLx1SCxPZQGU7mcaZ7gH9yH4jSFbBA=
|
||||||
github.com/golang-migrate/migrate/v4 v4.19.1/go.mod h1:CTcgfjxhaUtsLipnLoQRWCrjYXycRz/g5+RWDuYgPrE=
|
github.com/golang-migrate/migrate/v4 v4.19.1/go.mod h1:CTcgfjxhaUtsLipnLoQRWCrjYXycRz/g5+RWDuYgPrE=
|
||||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||||
github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa h1:s+4MhCQ6YrzisK6hFJUX53drDT4UsSW3DEhKn0ifuHw=
|
github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa h1:s+4MhCQ6YrzisK6hFJUX53drDT4UsSW3DEhKn0ifuHw=
|
||||||
github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds=
|
github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds=
|
||||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||||
@@ -49,10 +53,14 @@ github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw=
|
|||||||
github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
|
github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
|
||||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||||
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
|
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
|
||||||
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
|
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
|
||||||
|
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||||
|
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||||
|
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||||
|
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||||
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
|
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
|
||||||
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||||
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
|
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
|
||||||
@@ -61,6 +69,8 @@ github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
|
|||||||
github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
|
github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
|
||||||
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
|
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
|
||||||
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
|
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
|
||||||
|
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||||
|
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||||
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
||||||
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
|
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
|
||||||
github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
|
github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
|
||||||
@@ -70,6 +80,14 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
|
|||||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
|
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
|
||||||
|
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
|
||||||
|
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
|
||||||
|
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
|
||||||
|
github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
|
||||||
|
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
|
||||||
|
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
|
||||||
|
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
|
||||||
github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc=
|
github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc=
|
||||||
github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs=
|
github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs=
|
||||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
@@ -91,6 +109,8 @@ go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mx
|
|||||||
go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
|
go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
|
||||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||||
|
go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
|
||||||
|
go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
|
||||||
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
|
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
|
||||||
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
|
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
|
||||||
golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
|
golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
|
||||||
@@ -99,6 +119,10 @@ golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
|
|||||||
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||||
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
|
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
|
||||||
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
|
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
|
||||||
|
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||||
|
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||||
|
google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
|
||||||
|
google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||||
|
|||||||
@@ -0,0 +1,66 @@
|
|||||||
|
package audio
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// FFmpeg converts any audio container to the strict format whisper wants: 16 kHz
|
||||||
|
// mono signed 16-bit PCM WAV (ADR-033). It ALWAYS transcodes, even when the input
|
||||||
|
// already is that format — a Teams export often is, and the transcode is then
|
||||||
|
// nearly free; assuming it would be the bug. This replaces whisper-server's
|
||||||
|
// --convert, whose --tmp-dir is created at process start and rots under macOS
|
||||||
|
// periodic cleanup.
|
||||||
|
type FFmpeg struct {
|
||||||
|
// Bin is the ffmpeg executable ("ffmpeg" resolves via PATH).
|
||||||
|
Bin string
|
||||||
|
// MaxSeconds caps the converted length when positive. The duration limit is a claim
|
||||||
|
// in the file's header and can be lied about (a 2 h file claiming 10 s converted to a
|
||||||
|
// 120-minute WAV); this bounds what a lie can cost. A minute of slack is added so a
|
||||||
|
// file at the limit is never cut, and the worker measures the result.
|
||||||
|
MaxSeconds int
|
||||||
|
}
|
||||||
|
|
||||||
|
// lengthSlackSeconds is added to MaxSeconds so ffmpeg's rounding never clips a file that
|
||||||
|
// is really at the limit.
|
||||||
|
const lengthSlackSeconds = 60
|
||||||
|
|
||||||
|
// ToWAV writes src, converted, to dst. Video streams are dropped (-vn): by now the
|
||||||
|
// file has already been probed as audio-only, but a cover-art stream must not end
|
||||||
|
// up in the WAV.
|
||||||
|
func (f *FFmpeg) ToWAV(ctx context.Context, src, dst string) error {
|
||||||
|
bin := f.Bin
|
||||||
|
if bin == "" {
|
||||||
|
bin = "ffmpeg"
|
||||||
|
}
|
||||||
|
// Input restrictions go BEFORE -i: placed after it they would apply to the output.
|
||||||
|
args := append([]string{"-y", "-v", "error"}, inputRestrictions...)
|
||||||
|
args = append(args, "-i", src, "-vn", "-ac", "1", "-ar", "16000", "-c:a", "pcm_s16le")
|
||||||
|
if f.MaxSeconds > 0 {
|
||||||
|
// After -i: an OUTPUT cap. Before it, it would limit what is read, not written.
|
||||||
|
args = append(args, "-t", strconv.Itoa(f.MaxSeconds+lengthSlackSeconds))
|
||||||
|
}
|
||||||
|
args = append(args, "-f", "wav", dst)
|
||||||
|
cmd := exec.CommandContext(ctx, bin, args...)
|
||||||
|
cmd.WaitDelay = killGrace
|
||||||
|
// Only a bounded head of ffmpeg's output is kept: a hostile file made it print ~0.4
|
||||||
|
// bytes per input byte, all of which CombinedOutput would buffer and copy (#49).
|
||||||
|
out := &headWriter{max: maxStderr}
|
||||||
|
cmd.Stdout, cmd.Stderr = out, out
|
||||||
|
if err := cmd.Run(); err != nil {
|
||||||
|
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||||
|
return fmt.Errorf("audio: ffmpeg: %w", ctxErr)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("audio: ffmpeg: %w: %s", err, strings.TrimSpace(out.buf.String()))
|
||||||
|
}
|
||||||
|
// A zero exit that wrote nothing is still a failure: never hand whisper a
|
||||||
|
// missing file and let the error surface three layers later.
|
||||||
|
if info, err := os.Stat(dst); err != nil || info.Size() == 0 {
|
||||||
|
return fmt.Errorf("audio: ffmpeg wrote no output at %s", dst)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,140 @@
|
|||||||
|
package audio_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/adapters/audio"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestFFmpegToWAV_AsksForSixteenKilohertzMonoPCMAlways(t *testing.T) {
|
||||||
|
// Always transcode, even when the input already is 16 kHz mono: a Teams export
|
||||||
|
// often is, and the point is to not ASSUME (ADR-033). Records the argv.
|
||||||
|
rec := filepath.Join(t.TempDir(), "argv")
|
||||||
|
bin := stub(t, `printf '%s\n' "$@" > `+rec+`
|
||||||
|
for a; do last="$a"; done
|
||||||
|
printf data > "$last"`)
|
||||||
|
|
||||||
|
err := (&audio.FFmpeg{Bin: bin}).ToWAV(context.Background(), "/spool/in", filepath.Join(t.TempDir(), "out.wav"))
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
raw, err := os.ReadFile(rec)
|
||||||
|
require.NoError(t, err)
|
||||||
|
argv := strings.Split(strings.TrimSpace(string(raw)), "\n")
|
||||||
|
joined := " " + strings.Join(argv, " ") + " "
|
||||||
|
for _, want := range []string{" -ac 1 ", " -ar 16000 ", " -c:a pcm_s16le ", " -vn ", " -i /spool/in "} {
|
||||||
|
assert.Contains(t, joined, want)
|
||||||
|
}
|
||||||
|
assert.True(t, strings.HasSuffix(strings.TrimSpace(joined), "out.wav"), "the output path is the last argument")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFFmpegToWAV_NonzeroExitIsAnErrorCarryingStderr(t *testing.T) {
|
||||||
|
bin := stub(t, `echo "moov atom not found" >&2; exit 1`)
|
||||||
|
|
||||||
|
err := (&audio.FFmpeg{Bin: bin}).ToWAV(context.Background(), "/in", filepath.Join(t.TempDir(), "o.wav"))
|
||||||
|
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Contains(t, err.Error(), "moov atom not found")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFFmpegToWAV_ZeroExitWithNoOutputIsAnError(t *testing.T) {
|
||||||
|
bin := stub(t, `exit 0`)
|
||||||
|
|
||||||
|
err := (&audio.FFmpeg{Bin: bin}).ToWAV(context.Background(), "/in", filepath.Join(t.TempDir(), "never-written.wav"))
|
||||||
|
|
||||||
|
require.Error(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFFmpegToWAV_MissingBinaryIsAnError(t *testing.T) {
|
||||||
|
err := (&audio.FFmpeg{Bin: filepath.Join(t.TempDir(), "no-ffmpeg")}).ToWAV(context.Background(), "/in", "/out.wav")
|
||||||
|
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.False(t, errors.Is(err, context.Canceled))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFFmpegToWAV_RestrictsWhatTheFileMayMakeFFmpegOpen(t *testing.T) {
|
||||||
|
rec := filepath.Join(t.TempDir(), "argv")
|
||||||
|
bin := stub(t, `printf '%s\n' "$@" > `+rec+`
|
||||||
|
for a; do last="$a"; done
|
||||||
|
printf data > "$last"`)
|
||||||
|
|
||||||
|
err := (&audio.FFmpeg{Bin: bin}).ToWAV(context.Background(), "/spool/in", filepath.Join(t.TempDir(), "o.wav"))
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
args := argvOf(t, rec)
|
||||||
|
pw, fw, in := indexOf(args, "-protocol_whitelist"), indexOf(args, "-format_whitelist"), indexOf(args, "-i")
|
||||||
|
require.GreaterOrEqual(t, pw, 0, "argv: %v", args)
|
||||||
|
require.GreaterOrEqual(t, fw, 0, "argv: %v", args)
|
||||||
|
assert.Equal(t, "file,pipe", args[pw+1])
|
||||||
|
assert.Equal(t, audio.AllowedDemuxers, args[fw+1])
|
||||||
|
assert.Less(t, pw, in, "input options must come BEFORE -i or ffmpeg applies them to the output")
|
||||||
|
assert.Less(t, fw, in)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFFmpegToWAV_PassesAnOutputLengthCapAfterTheInputWhenAMaximumIsSet(t *testing.T) {
|
||||||
|
// The duration limit is a CLAIM in the file's header; a file that lies about it can
|
||||||
|
// convert to hours of audio (adversarial review F2: a 2 h file claiming 10 s became a
|
||||||
|
// 120-minute WAV). The cap is an OUTPUT option, after -i, with a minute of slack.
|
||||||
|
rec := filepath.Join(t.TempDir(), "argv")
|
||||||
|
bin := stub(t, `printf '%s\n' "$@" > `+rec+`
|
||||||
|
for a; do last="$a"; done
|
||||||
|
printf data > "$last"`)
|
||||||
|
|
||||||
|
err := (&audio.FFmpeg{Bin: bin, MaxSeconds: 5400}).ToWAV(context.Background(), "/spool/in", filepath.Join(t.TempDir(), "out.wav"))
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
raw, err := os.ReadFile(rec)
|
||||||
|
require.NoError(t, err)
|
||||||
|
joined := " " + strings.Join(strings.Split(strings.TrimSpace(string(raw)), "\n"), " ") + " "
|
||||||
|
assert.Contains(t, joined, " -t 5460 ", "limit plus a minute of slack")
|
||||||
|
assert.Less(t, strings.Index(joined, " -i /spool/in "), strings.Index(joined, " -t 5460 "), "-t after -i is an output cap; before it, an input one")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFFmpegToWAV_PassesNoLengthCapWhenNoMaximumIsSet(t *testing.T) {
|
||||||
|
rec := filepath.Join(t.TempDir(), "argv")
|
||||||
|
bin := stub(t, `printf '%s\n' "$@" > `+rec+`
|
||||||
|
for a; do last="$a"; done
|
||||||
|
printf data > "$last"`)
|
||||||
|
|
||||||
|
require.NoError(t, (&audio.FFmpeg{Bin: bin}).ToWAV(context.Background(), "/spool/in", filepath.Join(t.TempDir(), "out.wav")))
|
||||||
|
|
||||||
|
raw, err := os.ReadFile(rec)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.NotContains(t, " "+strings.ReplaceAll(string(raw), "\n", " ")+" ", " -t ")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFFmpegToWAV_CapsTheStreamCountBeforeTheInput(t *testing.T) {
|
||||||
|
// ffmpeg also allocated ~570 MB opening the 900-stream file (#49).
|
||||||
|
rec := filepath.Join(t.TempDir(), "argv")
|
||||||
|
bin := stub(t, `printf '%s\n' "$@" > `+rec+`
|
||||||
|
for a; do last="$a"; done
|
||||||
|
printf data > "$last"`)
|
||||||
|
|
||||||
|
err := (&audio.FFmpeg{Bin: bin}).ToWAV(context.Background(), "/spool/in", filepath.Join(t.TempDir(), "o.wav"))
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
args := argvOf(t, rec)
|
||||||
|
ms, in := indexOf(args, "-max_streams"), indexOf(args, "-i")
|
||||||
|
require.GreaterOrEqual(t, ms, 0, "argv: %v", args)
|
||||||
|
assert.Equal(t, "32", args[ms+1])
|
||||||
|
assert.Less(t, ms, in, "an input option placed after -i is applied to the output")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFFmpegToWAV_ABoundedPartOfAFloodingStderrIsKept(t *testing.T) {
|
||||||
|
// A hostile mp3 made ffmpeg print ~0.4 bytes of stderr per input byte (83 MB for a
|
||||||
|
// 209 MB file), all of it buffered, then copied twice into the error (#49 review).
|
||||||
|
bin := stub(t, `head -c 5000000 /dev/zero | tr '\0' 'e' >&2; exit 1`)
|
||||||
|
|
||||||
|
err := (&audio.FFmpeg{Bin: bin}).ToWAV(context.Background(), "/spool/in", filepath.Join(t.TempDir(), "o.wav"))
|
||||||
|
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Less(t, len(err.Error()), 16*1024, "the error carries a bounded head of stderr, not all of it")
|
||||||
|
assert.Contains(t, err.Error(), "eeee", "and still carries some of it")
|
||||||
|
}
|
||||||
@@ -0,0 +1,185 @@
|
|||||||
|
// Package audio holds the adapters that inspect and convert user-supplied audio
|
||||||
|
// with ffmpeg's tools (ADR-033). They sit behind small ports so the use case is
|
||||||
|
// tested with fakes and only this package knows a subprocess is involved.
|
||||||
|
package audio
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os/exec"
|
||||||
|
"strconv"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||||
|
)
|
||||||
|
|
||||||
|
// FFprobe implements uploads.Prober by running ffprobe on the spooled file.
|
||||||
|
type FFprobe struct {
|
||||||
|
// Bin is the ffprobe executable ("ffprobe" resolves via PATH).
|
||||||
|
Bin string
|
||||||
|
// MaxOutput bounds how much of ffprobe's stdout is read; zero means DefaultMaxOutput.
|
||||||
|
MaxOutput int
|
||||||
|
// Timeout bounds one probe; zero means DefaultProbeTimeout. A probe is milliseconds of
|
||||||
|
// work on a real file, so one that runs this long is the FILE's doing (#49).
|
||||||
|
Timeout time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// DefaultProbeTimeout is the probe deadline when FFprobe.Timeout is unset.
|
||||||
|
const DefaultProbeTimeout = 45 * time.Second
|
||||||
|
|
||||||
|
// DefaultMaxOutput is far above the narrowed probe output of a legitimate file (a few
|
||||||
|
// hundred bytes) and far below what a hostile one can make ffprobe print.
|
||||||
|
const DefaultMaxOutput = 1 << 20
|
||||||
|
|
||||||
|
// maxStderr is how much of ffprobe's stderr is kept for the error message.
|
||||||
|
const maxStderr = 4096
|
||||||
|
|
||||||
|
// errOutputCap is the write error that stops ffprobe once it prints past the cap.
|
||||||
|
var errOutputCap = errors.New("audio: ffprobe output exceeds the cap")
|
||||||
|
|
||||||
|
// capWriter collects up to max bytes and then refuses further writes, which closes the
|
||||||
|
// pipe and ends the process instead of buffering whatever it prints.
|
||||||
|
type capWriter struct {
|
||||||
|
buf bytes.Buffer
|
||||||
|
max int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *capWriter) Write(p []byte) (int, error) {
|
||||||
|
if w.buf.Len()+len(p) > w.max {
|
||||||
|
return 0, errOutputCap
|
||||||
|
}
|
||||||
|
return w.buf.Write(p)
|
||||||
|
}
|
||||||
|
|
||||||
|
// headWriter keeps the first max bytes and silently drops the rest.
|
||||||
|
type headWriter struct {
|
||||||
|
buf bytes.Buffer
|
||||||
|
max int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *headWriter) Write(p []byte) (int, error) {
|
||||||
|
if room := w.max - w.buf.Len(); room > 0 {
|
||||||
|
w.buf.Write(p[:min(room, len(p))])
|
||||||
|
}
|
||||||
|
return len(p), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// AllowedDemuxers are the input formats an upload may be. The input is
|
||||||
|
// attacker-supplied, and a file's CONTENT can tell ffmpeg to open other files or
|
||||||
|
// URLs (HLS and concat playlists, image2 sequences, network demuxers). Recent
|
||||||
|
// ffmpeg refuses the ones tried (7.0.2: HLS without a .m3u8 name is not sniffed,
|
||||||
|
// concat rejects unsafe paths), but that is version luck; this list is not. It
|
||||||
|
// holds the containers real recordings use and none of the playlist or network
|
||||||
|
// formats. "mov" covers .mp4/.m4a/.3gp; "matroska" covers .webm.
|
||||||
|
const AllowedDemuxers = "wav,mov,mp4,m4a,3gp,mp3,ogg,matroska,webm,flac,aac,opus,aiff,amr,asf"
|
||||||
|
|
||||||
|
// MaxStreams bounds how many streams a file may declare. A 1.3 MB file declaring 900 of
|
||||||
|
// them cost ~570 MB of RSS in each of ffprobe and ffmpeg (#49), against a 512 Mi pod;
|
||||||
|
// real recordings have a handful.
|
||||||
|
const MaxStreams = "32"
|
||||||
|
|
||||||
|
// inputRestrictions are ffmpeg/ffprobe INPUT options and must precede the input.
|
||||||
|
// Local files and pipes only: no network, no other protocols, and a bounded stream count.
|
||||||
|
var inputRestrictions = []string{
|
||||||
|
"-protocol_whitelist", "file,pipe",
|
||||||
|
"-format_whitelist", AllowedDemuxers,
|
||||||
|
"-max_streams", MaxStreams,
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeEntries is exactly what ParseProbe reads. -show_format/-show_streams print every
|
||||||
|
// tag too, and a 120 MB WAV made 491 MB of JSON that Output() buffers.
|
||||||
|
const probeEntries = "format=duration:stream=codec_type:stream_disposition=attached_pic"
|
||||||
|
|
||||||
|
// killGrace bounds how long Probe waits for a cancelled ffprobe's pipes to close.
|
||||||
|
const killGrace = 2 * time.Second
|
||||||
|
|
||||||
|
// Probe runs ffprobe and classifies the outcome. A non-zero exit means ffprobe
|
||||||
|
// looked at the file and could not read it: that is the FILE's fault and returns
|
||||||
|
// uploads.ErrUnreadable. A missing binary, a cancelled context or unparsable
|
||||||
|
// output is OURS and is returned as an ordinary error, so it is never reported to
|
||||||
|
// the user as their file being bad.
|
||||||
|
func (f *FFprobe) Probe(ctx context.Context, path string) (uploads.Probe, error) {
|
||||||
|
bin := f.Bin
|
||||||
|
if bin == "" {
|
||||||
|
bin = "ffprobe"
|
||||||
|
}
|
||||||
|
args := append([]string{"-v", "error"}, inputRestrictions...)
|
||||||
|
args = append(args, "-print_format", "json", "-show_entries", probeEntries, path)
|
||||||
|
timeout := f.Timeout
|
||||||
|
if timeout <= 0 {
|
||||||
|
timeout = DefaultProbeTimeout
|
||||||
|
}
|
||||||
|
probeCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||||
|
defer cancel()
|
||||||
|
cmd := exec.CommandContext(probeCtx, bin, args...)
|
||||||
|
cmd.WaitDelay = killGrace
|
||||||
|
maxOut := f.MaxOutput
|
||||||
|
if maxOut <= 0 {
|
||||||
|
maxOut = DefaultMaxOutput
|
||||||
|
}
|
||||||
|
stdout, stderr := &capWriter{max: maxOut}, &headWriter{max: maxStderr}
|
||||||
|
cmd.Stdout, cmd.Stderr = stdout, stderr
|
||||||
|
if err := cmd.Run(); err != nil {
|
||||||
|
if ctxErr := ctx.Err(); ctxErr != nil { // the caller gave up: not a verdict on the file
|
||||||
|
return uploads.Probe{}, fmt.Errorf("audio: ffprobe: %w", ctxErr)
|
||||||
|
}
|
||||||
|
if probeCtx.Err() != nil { // our own deadline passed while the caller was still waiting
|
||||||
|
return uploads.Probe{}, fmt.Errorf("%w: ffprobe did not finish within %s", uploads.ErrUnreadable, timeout)
|
||||||
|
}
|
||||||
|
if errors.Is(err, errOutputCap) {
|
||||||
|
return uploads.Probe{}, fmt.Errorf("%w: ffprobe printed more than %d bytes", uploads.ErrUnreadable, maxOut)
|
||||||
|
}
|
||||||
|
var exit *exec.ExitError
|
||||||
|
if errors.As(err, &exit) {
|
||||||
|
return uploads.Probe{}, fmt.Errorf("%w: ffprobe: %s", uploads.ErrUnreadable, stderr.buf.String())
|
||||||
|
}
|
||||||
|
return uploads.Probe{}, fmt.Errorf("audio: run ffprobe: %w", err)
|
||||||
|
}
|
||||||
|
p, err := ParseProbe(stdout.buf.Bytes())
|
||||||
|
if err != nil {
|
||||||
|
return uploads.Probe{}, fmt.Errorf("audio: ffprobe output: %w", err)
|
||||||
|
}
|
||||||
|
return p, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeJSON is the subset of `ffprobe -print_format json` output that matters.
|
||||||
|
type probeJSON struct {
|
||||||
|
Streams []struct {
|
||||||
|
CodecType string `json:"codec_type"`
|
||||||
|
Disposition struct {
|
||||||
|
AttachedPic int `json:"attached_pic"`
|
||||||
|
} `json:"disposition"`
|
||||||
|
} `json:"streams"`
|
||||||
|
Format struct {
|
||||||
|
Duration string `json:"duration"`
|
||||||
|
} `json:"format"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseProbe turns ffprobe's JSON into a uploads.Probe. A video stream that is
|
||||||
|
// only attached cover art (an mp3/m4a with an album picture) is not video. A
|
||||||
|
// missing or "N/A" duration yields zero, which uploads.Validate rejects as
|
||||||
|
// unreadable rather than accepting an empty recording.
|
||||||
|
func ParseProbe(data []byte) (uploads.Probe, error) {
|
||||||
|
var j probeJSON
|
||||||
|
if err := json.Unmarshal(data, &j); err != nil {
|
||||||
|
return uploads.Probe{}, fmt.Errorf("decode: %w", err)
|
||||||
|
}
|
||||||
|
var p uploads.Probe
|
||||||
|
for _, s := range j.Streams {
|
||||||
|
switch s.CodecType {
|
||||||
|
case "audio":
|
||||||
|
p.HasAudio = true
|
||||||
|
case "video":
|
||||||
|
if s.Disposition.AttachedPic == 0 {
|
||||||
|
p.HasVideo = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if secs, err := strconv.ParseFloat(j.Format.Duration, 64); err == nil && secs > 0 {
|
||||||
|
p.Duration = time.Duration(secs * float64(time.Second))
|
||||||
|
}
|
||||||
|
return p, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,261 @@
|
|||||||
|
package audio_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/adapters/audio"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParseProbe(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
json string
|
||||||
|
want uploads.Probe
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "audio-only m4a",
|
||||||
|
json: `{"streams":[{"codec_type":"audio","disposition":{"attached_pic":0}}],"format":{"duration":"2701.5"}}`,
|
||||||
|
want: uploads.Probe{Duration: 2701*time.Second + 500*time.Millisecond, HasAudio: true},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "mp4 with a real video stream",
|
||||||
|
json: `{"streams":[{"codec_type":"video","disposition":{"attached_pic":0}},{"codec_type":"audio"}],"format":{"duration":"240.0"}}`,
|
||||||
|
want: uploads.Probe{Duration: 240 * time.Second, HasAudio: true, HasVideo: true},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "mp3 whose only video stream is attached cover art is NOT video",
|
||||||
|
json: `{"streams":[{"codec_type":"audio"},{"codec_type":"video","disposition":{"attached_pic":1}}],"format":{"duration":"60"}}`,
|
||||||
|
want: uploads.Probe{Duration: 60 * time.Second, HasAudio: true},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "no streams at all",
|
||||||
|
json: `{"streams":[],"format":{"duration":"10"}}`,
|
||||||
|
want: uploads.Probe{Duration: 10 * time.Second},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "duration absent means zero, which Validate rejects as unreadable",
|
||||||
|
json: `{"streams":[{"codec_type":"audio"}],"format":{}}`,
|
||||||
|
want: uploads.Probe{HasAudio: true},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "duration reported as N/A means zero",
|
||||||
|
json: `{"streams":[{"codec_type":"audio"}],"format":{"duration":"N/A"}}`,
|
||||||
|
want: uploads.Probe{HasAudio: true},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
got, err := audio.ParseProbe([]byte(tt.json))
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, tt.want, got)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseProbe_GarbageIsAnError(t *testing.T) {
|
||||||
|
_, err := audio.ParseProbe([]byte("not json"))
|
||||||
|
assert.Error(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// stub writes an executable shell script standing in for ffprobe. This is a fake
|
||||||
|
// at the process boundary: the real binary is not on every host (it is not on
|
||||||
|
// koala), and what is under test is how Probe drives and interprets a process.
|
||||||
|
func stub(t *testing.T, body string) string {
|
||||||
|
t.Helper()
|
||||||
|
p := filepath.Join(t.TempDir(), "ffprobe")
|
||||||
|
require.NoError(t, os.WriteFile(p, []byte("#!/bin/sh\n"+body+"\n"), 0o755))
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFFprobeProbe_ParsesWhatTheProcessPrints(t *testing.T) {
|
||||||
|
bin := stub(t, `echo '{"streams":[{"codec_type":"audio"}],"format":{"duration":"90"}}'`)
|
||||||
|
|
||||||
|
got, err := (&audio.FFprobe{Bin: bin}).Probe(context.Background(), "/ignored")
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, uploads.Probe{Duration: 90 * time.Second, HasAudio: true}, got)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFFprobeProbe_PassesTheFilePathAsTheLastArgument(t *testing.T) {
|
||||||
|
out := filepath.Join(t.TempDir(), "args")
|
||||||
|
bin := stub(t, `for a; do last="$a"; done; printf '%s' "$last" > `+out+`
|
||||||
|
echo '{"streams":[{"codec_type":"audio"}],"format":{"duration":"1"}}'`)
|
||||||
|
|
||||||
|
_, err := (&audio.FFprobe{Bin: bin}).Probe(context.Background(), "/spool/upload-123")
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
got, err := os.ReadFile(out)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "/spool/upload-123", string(got))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFFprobeProbe_NonzeroExitMeansTheFileIsUnreadable(t *testing.T) {
|
||||||
|
bin := stub(t, `echo "Invalid data found when processing input" >&2; exit 1`)
|
||||||
|
|
||||||
|
_, err := (&audio.FFprobe{Bin: bin}).Probe(context.Background(), "/x")
|
||||||
|
|
||||||
|
assert.ErrorIs(t, err, uploads.ErrUnreadable)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFFprobeProbe_MissingBinaryIsOurFaultNotAnUnreadableFile(t *testing.T) {
|
||||||
|
_, err := (&audio.FFprobe{Bin: filepath.Join(t.TempDir(), "no-such-ffprobe")}).Probe(context.Background(), "/x")
|
||||||
|
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.False(t, errors.Is(err, uploads.ErrUnreadable),
|
||||||
|
"a missing ffprobe must not be reported to the user as their file being bad")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFFprobeProbe_StopsWhenTheContextIsCancelled(t *testing.T) {
|
||||||
|
bin := stub(t, `sleep 30`)
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
start := time.Now()
|
||||||
|
_, err := (&audio.FFprobe{Bin: bin}).Probe(ctx, "/x")
|
||||||
|
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Less(t, time.Since(start), 5*time.Second)
|
||||||
|
assert.False(t, errors.Is(err, uploads.ErrUnreadable), "a timeout is not a verdict on the file")
|
||||||
|
}
|
||||||
|
|
||||||
|
// argvStub records the arguments it was called with into a file and then behaves as
|
||||||
|
// a successful ffprobe, so a test can assert exactly what the process was asked.
|
||||||
|
func argvStub(t *testing.T) (bin, argvFile string) {
|
||||||
|
t.Helper()
|
||||||
|
argvFile = filepath.Join(t.TempDir(), "argv")
|
||||||
|
bin = stub(t, `printf '%s\n' "$@" > `+argvFile+`
|
||||||
|
echo '{"streams":[{"codec_type":"audio"}],"format":{"duration":"1"}}'`)
|
||||||
|
return bin, argvFile
|
||||||
|
}
|
||||||
|
|
||||||
|
func argvOf(t *testing.T, file string) []string {
|
||||||
|
t.Helper()
|
||||||
|
raw, err := os.ReadFile(file)
|
||||||
|
require.NoError(t, err)
|
||||||
|
return strings.Split(strings.TrimSpace(string(raw)), "\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
func indexOf(args []string, want string) int {
|
||||||
|
for i, a := range args {
|
||||||
|
if a == want {
|
||||||
|
return i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFFprobeProbe_RestrictsWhatTheFileMayMakeFFmpegOpen(t *testing.T) {
|
||||||
|
// The input is attacker-supplied. ffmpeg can be told by a file's CONTENT to open
|
||||||
|
// other files or URLs (HLS and concat playlists); recent versions refuse by
|
||||||
|
// default, but that is version luck. Pin it: local files and pipes only, and
|
||||||
|
// only the demuxers real audio recordings use.
|
||||||
|
bin, argvFile := argvStub(t)
|
||||||
|
|
||||||
|
_, err := (&audio.FFprobe{Bin: bin}).Probe(context.Background(), "/spool/upload-1")
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
args := argvOf(t, argvFile)
|
||||||
|
pw := indexOf(args, "-protocol_whitelist")
|
||||||
|
require.GreaterOrEqual(t, pw, 0, "argv: %v", args)
|
||||||
|
assert.Equal(t, "file,pipe", args[pw+1])
|
||||||
|
fw := indexOf(args, "-format_whitelist")
|
||||||
|
require.GreaterOrEqual(t, fw, 0, "argv: %v", args)
|
||||||
|
assert.Equal(t, audio.AllowedDemuxers, args[fw+1])
|
||||||
|
assert.Equal(t, "/spool/upload-1", args[len(args)-1], "the path stays the last argument")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAllowedDemuxers_ExcludeThePlaylistFormats(t *testing.T) {
|
||||||
|
for _, bad := range []string{"concat", "hls", "applehttp", "dash", "rtsp", "sdp", "image2"} {
|
||||||
|
assert.NotContains(t, strings.Split(audio.AllowedDemuxers, ","), bad,
|
||||||
|
"%s can make ffmpeg open other files or URLs named inside the upload", bad)
|
||||||
|
}
|
||||||
|
for _, good := range []string{"wav", "mov", "mp4", "m4a", "mp3", "ogg", "matroska", "flac"} {
|
||||||
|
assert.Contains(t, strings.Split(audio.AllowedDemuxers, ","), good)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFFprobeProbe_CapsTheStreamCountBeforeTheInput(t *testing.T) {
|
||||||
|
// A 1.3 MB file declaring 900 streams cost ~570 MB of RSS in ffprobe (#49) against
|
||||||
|
// a 512 Mi pod. -max_streams is an input option: it must precede the path.
|
||||||
|
bin, argvFile := argvStub(t)
|
||||||
|
|
||||||
|
_, err := (&audio.FFprobe{Bin: bin}).Probe(context.Background(), "/spool/upload-1")
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
args := argvOf(t, argvFile)
|
||||||
|
ms := indexOf(args, "-max_streams")
|
||||||
|
require.GreaterOrEqual(t, ms, 0, "argv: %v", args)
|
||||||
|
assert.Equal(t, "32", args[ms+1])
|
||||||
|
assert.Less(t, ms, len(args)-1, "must come before the path, which stays last")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFFprobeProbe_AsksOnlyForTheFieldsParseProbeReads(t *testing.T) {
|
||||||
|
// -show_format/-show_streams print every tag: a 120 MB WAV produced 491 MB of JSON
|
||||||
|
// that cmd.Output() buffers in Go (#49).
|
||||||
|
bin, argvFile := argvStub(t)
|
||||||
|
|
||||||
|
_, err := (&audio.FFprobe{Bin: bin}).Probe(context.Background(), "/spool/upload-1")
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
args := argvOf(t, argvFile)
|
||||||
|
assert.Equal(t, -1, indexOf(args, "-show_format"), "argv: %v", args)
|
||||||
|
assert.Equal(t, -1, indexOf(args, "-show_streams"), "argv: %v", args)
|
||||||
|
se := indexOf(args, "-show_entries")
|
||||||
|
require.GreaterOrEqual(t, se, 0, "argv: %v", args)
|
||||||
|
assert.Equal(t, "format=duration:stream=codec_type:stream_disposition=attached_pic", args[se+1])
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFFprobeProbe_OutputPastTheCapIsUnreadableNotBuffered(t *testing.T) {
|
||||||
|
// ffprobe's stdout is attacker-influenced (tags, 491 MB observed). Past the cap the
|
||||||
|
// probe is stopped and the file is reported as unreadable (#49).
|
||||||
|
bin := stub(t, `head -c 200000 /dev/zero | tr '\0' 'a'`)
|
||||||
|
|
||||||
|
_, err := (&audio.FFprobe{Bin: bin, MaxOutput: 4096}).Probe(context.Background(), "/x")
|
||||||
|
|
||||||
|
assert.ErrorIs(t, err, uploads.ErrUnreadable)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFFprobeProbe_OutputWithinTheCapIsParsed(t *testing.T) {
|
||||||
|
bin := stub(t, `echo '{"streams":[{"codec_type":"audio"}],"format":{"duration":"90"}}'`)
|
||||||
|
|
||||||
|
got, err := (&audio.FFprobe{Bin: bin, MaxOutput: 4096}).Probe(context.Background(), "/x")
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.True(t, got.HasAudio)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFFprobeProbe_AHungProbeTimesOutOnItsOwnDeadlineAsUnreadableNotAsOurFault(t *testing.T) {
|
||||||
|
// Accept runs the probe on the request context, which has no deadline of its own;
|
||||||
|
// a probe that never returns would pin a request (#49). `exec` so the sleep is the
|
||||||
|
// process itself and not a child holding the pipe open.
|
||||||
|
bin := stub(t, `exec sleep 30`)
|
||||||
|
|
||||||
|
start := time.Now()
|
||||||
|
_, err := (&audio.FFprobe{Bin: bin, Timeout: 200 * time.Millisecond}).Probe(context.Background(), "/x")
|
||||||
|
|
||||||
|
assert.ErrorIs(t, err, uploads.ErrUnreadable)
|
||||||
|
assert.Less(t, time.Since(start), 3*time.Second)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFFprobeProbe_ParentCancellationIsStillNotUnreadableWhenATimeoutIsSet(t *testing.T) {
|
||||||
|
bin := stub(t, `exec sleep 30`)
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
_, err := (&audio.FFprobe{Bin: bin, Timeout: time.Minute}).Probe(ctx, "/x")
|
||||||
|
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.False(t, errors.Is(err, uploads.ErrUnreadable), "the caller giving up is not a verdict on the file")
|
||||||
|
assert.ErrorIs(t, err, context.DeadlineExceeded)
|
||||||
|
}
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
package audio_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/adapters/audio"
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/uploads"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestFFprobeLive_HostilePlaylistsAreRefusedAndRealAudioIsNot runs the REAL ffprobe
|
||||||
|
// (opt-in: TAPIR_LIVE_FFPROBE) on crafted concat/HLS files and on a real m4a.
|
||||||
|
func TestFFprobeLive_HostilePlaylistsAreRefusedAndRealAudioIsNot(t *testing.T) {
|
||||||
|
bin := os.Getenv("TAPIR_LIVE_FFPROBE")
|
||||||
|
if bin == "" {
|
||||||
|
t.Skip("set TAPIR_LIVE_FFPROBE (and TAPIR_LIVE_HOSTILE_DIR, TAPIR_LIVE_M4A) to run against the real ffprobe")
|
||||||
|
}
|
||||||
|
p := &audio.FFprobe{Bin: bin}
|
||||||
|
for _, name := range []string{"up-concat", "up-hls"} {
|
||||||
|
_, err := p.Probe(context.Background(), os.Getenv("TAPIR_LIVE_HOSTILE_DIR")+"/"+name)
|
||||||
|
assert.ErrorIs(t, err, uploads.ErrUnreadable, name)
|
||||||
|
}
|
||||||
|
got, err := p.Probe(context.Background(), os.Getenv("TAPIR_LIVE_M4A"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.True(t, got.HasAudio)
|
||||||
|
assert.False(t, got.HasVideo)
|
||||||
|
assert.InDelta(t, 82.5, got.Duration.Seconds(), 1)
|
||||||
|
}
|
||||||
@@ -0,0 +1,188 @@
|
|||||||
|
// Package chat implements the per-video deeper-dive chat (ADR-027): a read-only
|
||||||
|
// QA over a video's ALREADY-STORED transcript (ADR-021). It is the enforcement
|
||||||
|
// point for the feature's load-bearing safety property — stored-transcript-only:
|
||||||
|
// the Service has NO VideoSource and NO caption-fetch dependency, only a
|
||||||
|
// Completer factory, so it CANNOT reach YouTube or the rate gate by construction.
|
||||||
|
// The caller supplies the stored transcript text; chat never fetches.
|
||||||
|
//
|
||||||
|
// It reuses the same LiteLLM gateway as the summarizer (a chat is a different
|
||||||
|
// call, not a new integration) and the same transcript-truncation discipline
|
||||||
|
// (TAPIR_MAX_TRANSCRIPT_CHARS) so a long transcript fits a small-context model.
|
||||||
|
package chat
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
"unicode/utf8"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/metrics"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Completer is the minimal LLM chat surface the Service needs. *llm.Client
|
||||||
|
// satisfies it; tests use a fake. It is the SAME surface the summarizer uses.
|
||||||
|
type Completer interface {
|
||||||
|
Complete(ctx context.Context, system, user string) (string, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Turn is one completed exchange in an ephemeral, session-only conversation
|
||||||
|
// (ADR-027 v1: nothing is persisted).
|
||||||
|
type Turn struct {
|
||||||
|
Question string
|
||||||
|
Answer string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Request is one chat turn: the chosen model, the stored transcript text, the
|
||||||
|
// prior turns (for multi-turn context within the session), and the new question.
|
||||||
|
type Request struct {
|
||||||
|
Model string
|
||||||
|
Transcript string
|
||||||
|
History []Turn
|
||||||
|
Question string
|
||||||
|
// MaxChars, when positive, overrides the Service's transcript budget for this
|
||||||
|
// request. The Service default is sized for YouTube captions (~22 minutes of
|
||||||
|
// speech); an uploaded recording is much longer, and truncation keeps only the
|
||||||
|
// head (ADR-033). Zero or negative means "use the Service default".
|
||||||
|
MaxChars int
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reply is the model's answer plus whether the transcript was bounded to fit the
|
||||||
|
// model context (so the UI can be honest that an answer about the tail of a long
|
||||||
|
// video may be incomplete).
|
||||||
|
type Reply struct {
|
||||||
|
Answer string
|
||||||
|
Truncated bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// Service answers questions against a stored transcript via a switchable set of
|
||||||
|
// models. models is the ordered, local-first list offered to the user (the cloud
|
||||||
|
// model is simply absent when disabled — see cmd wiring); maxChars bounds the
|
||||||
|
// transcript sent to any model (0 = unbounded). newClient builds a Completer for
|
||||||
|
// a chosen model alias (the same gateway, a different alias).
|
||||||
|
type Service struct {
|
||||||
|
newClient func(model string) Completer
|
||||||
|
models []string
|
||||||
|
maxChars int
|
||||||
|
}
|
||||||
|
|
||||||
|
// New constructs a Service. models must be non-empty and already filtered to the
|
||||||
|
// offerable set (cloud excluded when disabled) and de-duplicated by the caller.
|
||||||
|
func New(newClient func(model string) Completer, models []string, maxChars int) *Service {
|
||||||
|
return &Service{newClient: newClient, models: models, maxChars: maxChars}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Models returns a copy of the offerable model list (local-first order).
|
||||||
|
func (s *Service) Models() []string {
|
||||||
|
out := make([]string, len(s.models))
|
||||||
|
copy(out, s.models)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// offers reports whether model is in the offerable set — the guard that keeps an
|
||||||
|
// arbitrary, un-offered alias (e.g. a forged form value) from reaching the gateway.
|
||||||
|
func (s *Service) offers(model string) bool {
|
||||||
|
for _, m := range s.models {
|
||||||
|
if m == model {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// DefaultModel resolves the model a fresh chat opens with: the summary's own
|
||||||
|
// model when it is still an offered option (the ADR-027 default — chat continues
|
||||||
|
// in the model that produced the summary), otherwise the first offered model.
|
||||||
|
// Returns "" only when no models are configured.
|
||||||
|
func (s *Service) DefaultModel(summaryModel string) string {
|
||||||
|
if summaryModel != "" && s.offers(summaryModel) {
|
||||||
|
return summaryModel
|
||||||
|
}
|
||||||
|
if len(s.models) > 0 {
|
||||||
|
return s.models[0]
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// Answer runs one chat turn. The model is forced back to a default if the request
|
||||||
|
// names an un-offered alias, so chat can never call the gateway with an arbitrary
|
||||||
|
// model. The transcript is truncated up front (reporting whether it was cut) and
|
||||||
|
// passed as system context; the running conversation is the user message.
|
||||||
|
func (s *Service) Answer(ctx context.Context, req Request) (Reply, error) {
|
||||||
|
if len(s.models) == 0 {
|
||||||
|
return Reply{}, fmt.Errorf("chat: no models configured")
|
||||||
|
}
|
||||||
|
model := req.Model
|
||||||
|
if !s.offers(model) {
|
||||||
|
model = s.DefaultModel("")
|
||||||
|
}
|
||||||
|
|
||||||
|
budget := s.maxChars
|
||||||
|
if req.MaxChars > 0 {
|
||||||
|
budget = req.MaxChars
|
||||||
|
}
|
||||||
|
transcript, truncated := truncate(req.Transcript, budget)
|
||||||
|
system := buildSystem(transcript, truncated)
|
||||||
|
user := buildUser(req.History, req.Question)
|
||||||
|
|
||||||
|
start := time.Now()
|
||||||
|
out, err := s.newClient(model).Complete(ctx, system, user)
|
||||||
|
if err != nil {
|
||||||
|
return Reply{}, fmt.Errorf("chat: %s: %w", model, err)
|
||||||
|
}
|
||||||
|
dur := time.Since(start)
|
||||||
|
metrics.ObserveChat(model, dur)
|
||||||
|
slog.Default().Info("chat answer", "model", model, "elapsed_ms", dur.Milliseconds())
|
||||||
|
answer := strings.TrimSpace(out)
|
||||||
|
if answer == "" {
|
||||||
|
return Reply{}, fmt.Errorf("chat: %s returned an empty answer", model)
|
||||||
|
}
|
||||||
|
return Reply{Answer: answer, Truncated: truncated}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const systemPreamble = `You are Tapir, answering questions about ONE video using ONLY the transcript below.
|
||||||
|
Ground every answer in the transcript. If the transcript does not contain the answer, say so plainly rather than guessing.`
|
||||||
|
|
||||||
|
const truncatedNote = `
|
||||||
|
The transcript below is truncated to fit the model — if a question seems to concern something missing, note it may be beyond the available portion.`
|
||||||
|
|
||||||
|
// buildSystem frames the model as a transcript-grounded QA assistant and embeds
|
||||||
|
// the (possibly truncated) transcript as context.
|
||||||
|
func buildSystem(transcript string, truncated bool) string {
|
||||||
|
var b strings.Builder
|
||||||
|
b.WriteString(systemPreamble)
|
||||||
|
if truncated {
|
||||||
|
b.WriteString(truncatedNote)
|
||||||
|
}
|
||||||
|
b.WriteString("\n\nTranscript:\n")
|
||||||
|
b.WriteString(transcript)
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildUser renders the running conversation as the user message: prior turns as
|
||||||
|
// Q/A pairs followed by the new question. Folding history into one message keeps
|
||||||
|
// the Completer surface (a single system+user call) unchanged — no new llm method.
|
||||||
|
func buildUser(history []Turn, question string) string {
|
||||||
|
var b strings.Builder
|
||||||
|
for _, t := range history {
|
||||||
|
fmt.Fprintf(&b, "Q: %s\nA: %s\n\n", t.Question, t.Answer)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(&b, "Q: %s", question)
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// truncate caps content to max bytes on a UTF-8 rune boundary, reporting whether
|
||||||
|
// it cut. It mirrors the summarizer's truncation discipline (ADR-022) but returns
|
||||||
|
// the cut flag so the chat UI can be honest about a bounded transcript. A
|
||||||
|
// non-positive max (or content already within budget) returns content unchanged.
|
||||||
|
func truncate(content string, max int) (string, bool) {
|
||||||
|
if max <= 0 || len(content) <= max {
|
||||||
|
return content, false
|
||||||
|
}
|
||||||
|
cut := max
|
||||||
|
for cut > 0 && !utf8.RuneStart(content[cut]) {
|
||||||
|
cut--
|
||||||
|
}
|
||||||
|
return content[:cut], true
|
||||||
|
}
|
||||||
@@ -0,0 +1,164 @@
|
|||||||
|
package chat
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// recordingCompleter captures the system+user it was asked with and returns a
|
||||||
|
// canned answer (or error). It also records which model alias built it.
|
||||||
|
type recordingCompleter struct {
|
||||||
|
model string
|
||||||
|
lastSystem string
|
||||||
|
lastUser string
|
||||||
|
answer string
|
||||||
|
err error
|
||||||
|
calls *int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *recordingCompleter) Complete(_ context.Context, system, user string) (string, error) {
|
||||||
|
*c.calls++
|
||||||
|
c.lastSystem = system
|
||||||
|
c.lastUser = user
|
||||||
|
if c.err != nil {
|
||||||
|
return "", c.err
|
||||||
|
}
|
||||||
|
return c.answer, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// factory builds a recordingCompleter per model and records the last one built so
|
||||||
|
// the test can assert which model alias was actually used for the gateway call.
|
||||||
|
type factory struct {
|
||||||
|
answer string
|
||||||
|
err error
|
||||||
|
calls int
|
||||||
|
used *recordingCompleter
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *factory) make(model string) Completer {
|
||||||
|
c := &recordingCompleter{model: model, answer: f.answer, err: f.err, calls: &f.calls}
|
||||||
|
f.used = c
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestModelsAreOfferedLocalFirstAndCopied(t *testing.T) {
|
||||||
|
f := &factory{answer: "ok"}
|
||||||
|
s := New(f.make, []string{"koala/phi4-mini", "iguana/gemma4-26b"}, 0)
|
||||||
|
|
||||||
|
got := s.Models()
|
||||||
|
want := []string{"koala/phi4-mini", "iguana/gemma4-26b"}
|
||||||
|
if len(got) != len(want) || got[0] != want[0] || got[1] != want[1] {
|
||||||
|
t.Fatalf("Models() = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
// Mutating the returned slice must not corrupt the Service's list.
|
||||||
|
got[0] = "tampered"
|
||||||
|
if s.Models()[0] != "koala/phi4-mini" {
|
||||||
|
t.Fatal("Models() leaked its backing slice")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDefaultModelIsTheSummarysModelWhenOffered(t *testing.T) {
|
||||||
|
f := &factory{answer: "ok"}
|
||||||
|
s := New(f.make, []string{"koala/phi4-mini", "iguana/gemma4-26b", "berget/mistral-small"}, 0)
|
||||||
|
|
||||||
|
if got := s.DefaultModel("iguana/gemma4-26b"); got != "iguana/gemma4-26b" {
|
||||||
|
t.Fatalf("DefaultModel(summary) = %q, want the summary's model", got)
|
||||||
|
}
|
||||||
|
// A summary model no longer offered (e.g. cloud disabled) falls back to first.
|
||||||
|
if got := s.DefaultModel("berget/old-model"); got != "koala/phi4-mini" {
|
||||||
|
t.Fatalf("DefaultModel(un-offered) = %q, want the first offered model", got)
|
||||||
|
}
|
||||||
|
// No summary model recorded → first offered.
|
||||||
|
if got := s.DefaultModel(""); got != "koala/phi4-mini" {
|
||||||
|
t.Fatalf("DefaultModel(\"\") = %q, want the first offered model", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnswerGroundsOnTranscriptAndCarriesHistory(t *testing.T) {
|
||||||
|
f := &factory{answer: " The video is about attention. "}
|
||||||
|
s := New(f.make, []string{"koala/phi4-mini"}, 0)
|
||||||
|
|
||||||
|
reply, err := s.Answer(context.Background(), Request{
|
||||||
|
Model: "koala/phi4-mini",
|
||||||
|
Transcript: "ATTENTION-TRANSCRIPT-MARKER",
|
||||||
|
History: []Turn{{Question: "who", Answer: "the host"}},
|
||||||
|
Question: "what is it about",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Answer: %v", err)
|
||||||
|
}
|
||||||
|
if reply.Answer != "The video is about attention." {
|
||||||
|
t.Fatalf("answer not trimmed: %q", reply.Answer)
|
||||||
|
}
|
||||||
|
if reply.Truncated {
|
||||||
|
t.Fatal("short transcript must not report truncated")
|
||||||
|
}
|
||||||
|
// The transcript rides in the system prompt; the conversation in the user msg.
|
||||||
|
if !strings.Contains(f.used.lastSystem, "ATTENTION-TRANSCRIPT-MARKER") {
|
||||||
|
t.Fatal("transcript not grounded into the system prompt")
|
||||||
|
}
|
||||||
|
if !strings.Contains(f.used.lastUser, "Q: who") || !strings.Contains(f.used.lastUser, "A: the host") {
|
||||||
|
t.Fatalf("history not carried into the user message: %q", f.used.lastUser)
|
||||||
|
}
|
||||||
|
if !strings.Contains(f.used.lastUser, "what is it about") {
|
||||||
|
t.Fatal("new question missing from the user message")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnswerTruncatesLongTranscriptAndReportsIt(t *testing.T) {
|
||||||
|
f := &factory{answer: "answer"}
|
||||||
|
s := New(f.make, []string{"koala/phi4-mini"}, 10)
|
||||||
|
|
||||||
|
reply, err := s.Answer(context.Background(), Request{
|
||||||
|
Model: "koala/phi4-mini",
|
||||||
|
Transcript: strings.Repeat("x", 500),
|
||||||
|
Question: "summarize",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Answer: %v", err)
|
||||||
|
}
|
||||||
|
if !reply.Truncated {
|
||||||
|
t.Fatal("a transcript past maxChars must report Truncated")
|
||||||
|
}
|
||||||
|
if strings.Count(f.used.lastSystem, "x") != 10 {
|
||||||
|
t.Fatalf("transcript not bounded to maxChars: got %d x's", strings.Count(f.used.lastSystem, "x"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnswerForcesAnUnofferedModelBackToDefault(t *testing.T) {
|
||||||
|
f := &factory{answer: "answer"}
|
||||||
|
s := New(f.make, []string{"koala/phi4-mini", "iguana/gemma4-26b"}, 0)
|
||||||
|
|
||||||
|
// A forged/un-offered model must never reach the gateway as-is — it is forced
|
||||||
|
// to the default offered model (the cloud-absent guarantee depends on this).
|
||||||
|
_, err := s.Answer(context.Background(), Request{
|
||||||
|
Model: "berget/secret-cloud-model",
|
||||||
|
Transcript: "t",
|
||||||
|
Question: "q",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Answer: %v", err)
|
||||||
|
}
|
||||||
|
if f.used.model != "koala/phi4-mini" {
|
||||||
|
t.Fatalf("un-offered model reached the gateway as %q, want the default", f.used.model)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnswerPropagatesCompleterError(t *testing.T) {
|
||||||
|
f := &factory{err: errors.New("gateway down")}
|
||||||
|
s := New(f.make, []string{"koala/phi4-mini"}, 0)
|
||||||
|
|
||||||
|
_, err := s.Answer(context.Background(), Request{Model: "koala/phi4-mini", Transcript: "t", Question: "q"})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected the gateway error to propagate")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnswerRejectsEmptyModelSet(t *testing.T) {
|
||||||
|
s := New(func(string) Completer { return nil }, nil, 0)
|
||||||
|
if _, err := s.Answer(context.Background(), Request{Question: "q"}); err == nil {
|
||||||
|
t.Fatal("expected an error when no models are configured")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
package chat
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A Request may carry its own transcript budget. The Service's cap (18,000 chars by
|
||||||
|
// default) is sized for YouTube captions; an uploaded recording is much longer
|
||||||
|
// (~50,000 chars an hour), and keeping only the head of it would answer questions
|
||||||
|
// about the last two-thirds of a meeting from text that does not contain them
|
||||||
|
// (ADR-033). The per-request value overrides; zero means "the service default".
|
||||||
|
|
||||||
|
func TestAnswerMaxCharsOverridesTheServiceCap(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
serviceCap int
|
||||||
|
requestCap int
|
||||||
|
transcriptLen int
|
||||||
|
wantKept int
|
||||||
|
wantTruncated bool
|
||||||
|
}{
|
||||||
|
{"request cap above the service cap keeps the whole transcript", 10, 100, 50, 50, false},
|
||||||
|
{"request cap below the service cap bounds tighter", 100, 10, 50, 10, true},
|
||||||
|
{"zero request cap falls back to the service cap", 20, 0, 50, 20, true},
|
||||||
|
{"negative request cap is treated as unset", 20, -1, 50, 20, true},
|
||||||
|
{"request cap exceeded is reported as truncated", 10, 30, 50, 30, true},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
f := &factory{answer: "answer"}
|
||||||
|
s := New(f.make, []string{"koala/phi4-mini"}, tt.serviceCap)
|
||||||
|
|
||||||
|
reply, err := s.Answer(context.Background(), Request{
|
||||||
|
Model: "koala/phi4-mini",
|
||||||
|
Transcript: strings.Repeat("x", tt.transcriptLen),
|
||||||
|
Question: "what was decided?",
|
||||||
|
MaxChars: tt.requestCap,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Answer: %v", err)
|
||||||
|
}
|
||||||
|
if got := strings.Count(f.used.lastSystem, "x"); got != tt.wantKept {
|
||||||
|
t.Errorf("transcript chars sent = %d, want %d", got, tt.wantKept)
|
||||||
|
}
|
||||||
|
if reply.Truncated != tt.wantTruncated {
|
||||||
|
t.Errorf("Truncated = %v, want %v", reply.Truncated, tt.wantTruncated)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,156 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
// Observes how much of a gateway response the client will read. The body is read before
|
||||||
|
// the status is looked at, so a misbehaving gateway or proxy that streams forever was
|
||||||
|
// bounded only by the client's overall timeout (#53).
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// endlessGateway answers with the given status and streams filler until the client hangs
|
||||||
|
// up; wrote reports how many bytes it managed to send.
|
||||||
|
func endlessGateway(t *testing.T, status int, wrote *atomic.Int64) *httptest.Server {
|
||||||
|
t.Helper()
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
w.WriteHeader(status)
|
||||||
|
chunk := []byte(strings.Repeat("a", 4096))
|
||||||
|
for {
|
||||||
|
n, err := w.Write(chunk)
|
||||||
|
wrote.Add(int64(n))
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if f, ok := w.(http.Flusher); ok {
|
||||||
|
f.Flush()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
t.Cleanup(srv.Close)
|
||||||
|
return srv
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClient_ASuccessBodyLargerThanTheCapIsAnErrorAndTheClientStopsReading(t *testing.T) {
|
||||||
|
var wrote atomic.Int64
|
||||||
|
srv := endlessGateway(t, http.StatusOK, &wrote)
|
||||||
|
c := New(srv.URL, "", "m", 30*time.Second, WithMaxResponseBytes(8192))
|
||||||
|
|
||||||
|
start := time.Now()
|
||||||
|
_, err := c.Complete(context.Background(), "s", "u")
|
||||||
|
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "exceeds") {
|
||||||
|
t.Fatalf("want a response-too-large error, got %v", err)
|
||||||
|
}
|
||||||
|
if time.Since(start) > 5*time.Second {
|
||||||
|
t.Fatalf("the client kept reading: took %s", time.Since(start))
|
||||||
|
}
|
||||||
|
// Hanging up lets the server's writes fail; allow what the transport had already buffered.
|
||||||
|
time.Sleep(100 * time.Millisecond)
|
||||||
|
if got := wrote.Load(); got > 8192+1<<20 {
|
||||||
|
t.Fatalf("server managed to send %d bytes after a cap of 8192", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClient_AFailingBodyIsReadOnlyUpToTheErrorBound(t *testing.T) {
|
||||||
|
var wrote atomic.Int64
|
||||||
|
srv := endlessGateway(t, http.StatusBadGateway, &wrote)
|
||||||
|
c := New(srv.URL, "", "m", 30*time.Second)
|
||||||
|
|
||||||
|
start := time.Now()
|
||||||
|
_, err := c.Complete(context.Background(), "s", "u")
|
||||||
|
|
||||||
|
he, ok := err.(*HTTPError)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("want *HTTPError, got %T: %v", err, err)
|
||||||
|
}
|
||||||
|
if he.Code != http.StatusBadGateway {
|
||||||
|
t.Fatalf("code %d", he.Code)
|
||||||
|
}
|
||||||
|
if len(he.Body) > maxErrorBody+1 {
|
||||||
|
t.Fatalf("kept %d bytes of an error body, bound is %d", len(he.Body), maxErrorBody)
|
||||||
|
}
|
||||||
|
if time.Since(start) > 5*time.Second {
|
||||||
|
t.Fatalf("the client kept reading an endless error body: took %s", time.Since(start))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// sizedGateway answers 200 with a valid completion padded with trailing whitespace to
|
||||||
|
// exactly size bytes, so the boundary of the cap can be probed to the byte.
|
||||||
|
func sizedGateway(t *testing.T, size int) *httptest.Server {
|
||||||
|
t.Helper()
|
||||||
|
const base = `{"choices":[{"message":{"content":"ok"}}]}`
|
||||||
|
payload := base + strings.Repeat(" ", size-len(base))
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = w.Write([]byte(payload))
|
||||||
|
}))
|
||||||
|
t.Cleanup(srv.Close)
|
||||||
|
return srv
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClient_ABodyOfExactlyTheCapIsReadAndOneByteMoreIsRefused(t *testing.T) {
|
||||||
|
const capBytes = 4096
|
||||||
|
for _, tt := range []struct {
|
||||||
|
name string
|
||||||
|
size int
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{"one under", capBytes - 1, false},
|
||||||
|
{"exactly the cap", capBytes, false},
|
||||||
|
{"one over", capBytes + 1, true},
|
||||||
|
} {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
c := New(sizedGateway(t, tt.size).URL, "", "m", 5*time.Second, WithMaxResponseBytes(capBytes))
|
||||||
|
|
||||||
|
got, err := c.Complete(context.Background(), "s", "u")
|
||||||
|
|
||||||
|
if tt.wantErr {
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "exceeds") {
|
||||||
|
t.Fatalf("want a response-too-large error, got %q, %v", got, err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil || got != "ok" {
|
||||||
|
t.Fatalf("want ok, got %q, %v", got, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// New is what production calls: no option. The default is therefore the live bound.
|
||||||
|
func TestClient_TheDefaultCapIsEightMiBWhenNoOptionIsGiven(t *testing.T) {
|
||||||
|
for _, tt := range []struct {
|
||||||
|
name string
|
||||||
|
size int
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{"exactly 8 MiB", 8 << 20, false},
|
||||||
|
{"8 MiB plus one", 8<<20 + 1, true},
|
||||||
|
} {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
c := New(sizedGateway(t, tt.size).URL, "", "m", 10*time.Second)
|
||||||
|
|
||||||
|
_, err := c.Complete(context.Background(), "s", "u")
|
||||||
|
|
||||||
|
if (err != nil) != tt.wantErr {
|
||||||
|
t.Fatalf("size %d: err = %v, wantErr %v", tt.size, err, tt.wantErr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClient_ANonPositiveCapOptionKeepsTheDefault(t *testing.T) {
|
||||||
|
for _, n := range []int64{0, -1} {
|
||||||
|
c := New(sizedGateway(t, 1024).URL, "", "m", 5*time.Second, WithMaxResponseBytes(n))
|
||||||
|
|
||||||
|
if got, err := c.Complete(context.Background(), "s", "u"); err != nil || got != "ok" {
|
||||||
|
t.Fatalf("WithMaxResponseBytes(%d): want ok from the default cap, got %q, %v", n, got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -25,24 +25,103 @@ import (
|
|||||||
// docs/homelab-integration.md.
|
// docs/homelab-integration.md.
|
||||||
const defaultMaxTokens = 8192
|
const defaultMaxTokens = 8192
|
||||||
|
|
||||||
|
// defaultTemperature is what every caller got before WithTemperature existed.
|
||||||
|
const defaultTemperature = 0.2
|
||||||
|
|
||||||
// Client calls an OpenAI-compatible chat completions endpoint.
|
// Client calls an OpenAI-compatible chat completions endpoint.
|
||||||
type Client struct {
|
type Client struct {
|
||||||
baseURL string
|
baseURL string
|
||||||
apiKey string
|
apiKey string
|
||||||
model string
|
model string
|
||||||
maxTokens int
|
maxTokens int
|
||||||
|
temp float64
|
||||||
httpClient *http.Client
|
httpClient *http.Client
|
||||||
|
usageHook func(model string, prompt, completion int)
|
||||||
|
maxBody int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// defaultMaxResponseBytes bounds a success body. A completion is a few KB of text; this
|
||||||
|
// leaves room for a long one and none for a gateway that streams forever (#53).
|
||||||
|
const defaultMaxResponseBytes = 8 << 20
|
||||||
|
|
||||||
|
// WithMaxResponseBytes overrides the bound on a success response body. A non-positive n is
|
||||||
|
// ignored (keeps the default).
|
||||||
|
func WithMaxResponseBytes(n int64) Option {
|
||||||
|
return func(c *Client) {
|
||||||
|
if n > 0 {
|
||||||
|
c.maxBody = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Option configures a Client at construction. Variadic so the existing 4-arg
|
||||||
|
// call sites stay valid as new knobs are added.
|
||||||
|
type Option func(*Client)
|
||||||
|
|
||||||
|
// WithMaxTokens overrides the per-request completion budget. The summarizer uses
|
||||||
|
// this to cap completion for small-context models (e.g. koala/phi4-mini, 8k):
|
||||||
|
// with the default 8192 budget, prompt + max_tokens overflows an 8k context and
|
||||||
|
// the gateway returns HTTP 400. A non-positive n is ignored (keeps the default).
|
||||||
|
func WithMaxTokens(n int) Option {
|
||||||
|
return func(c *Client) {
|
||||||
|
if n > 0 {
|
||||||
|
c.maxTokens = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// maxErrorBody bounds how much of a failing response ends up in an error message.
|
||||||
|
const maxErrorBody = 512
|
||||||
|
|
||||||
|
// HTTPError is a non-200 answer from the gateway. Its text keeps the form callers that
|
||||||
|
// log it already rely on, with the body bounded; its Code is what code may branch on and
|
||||||
|
// what errkind turns into a kind. The body is provider text and can echo the prompt, so
|
||||||
|
// nothing private may persist or display Error() (ADR-035 review F3).
|
||||||
|
type HTTPError struct {
|
||||||
|
Code int
|
||||||
|
Body string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *HTTPError) Error() string {
|
||||||
|
body := e.Body
|
||||||
|
if len(body) > maxErrorBody {
|
||||||
|
body = body[:maxErrorBody] + "…"
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("LLM returned %d: %s", e.Code, body)
|
||||||
|
}
|
||||||
|
|
||||||
|
// HTTPStatus implements errkind.HTTPStatuser.
|
||||||
|
func (e *HTTPError) HTTPStatus() int { return e.Code }
|
||||||
|
|
||||||
|
// WithTemperature overrides the sampling temperature. Zero IS a valid value and is sent
|
||||||
|
// as zero: the reconciler's accuracy was measured at temperature 0 and must run there.
|
||||||
|
func WithTemperature(t float64) Option {
|
||||||
|
return func(c *Client) { c.temp = t }
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithUsageHook registers a callback fired after a successful completion with the
|
||||||
|
// model and the prompt/completion token counts from the response usage block. It
|
||||||
|
// keeps this copied, stdlib-only package (ADR-004) decoupled from metrics: the
|
||||||
|
// caller wires it to internal/metrics, the client imports nothing. nil is ignored.
|
||||||
|
func WithUsageHook(fn func(model string, prompt, completion int)) Option {
|
||||||
|
return func(c *Client) { c.usageHook = fn }
|
||||||
}
|
}
|
||||||
|
|
||||||
// New constructs a Client.
|
// New constructs a Client.
|
||||||
func New(baseURL, apiKey, model string, timeout time.Duration) *Client {
|
func New(baseURL, apiKey, model string, timeout time.Duration, opts ...Option) *Client {
|
||||||
return &Client{
|
c := &Client{
|
||||||
baseURL: strings.TrimRight(baseURL, "/"),
|
baseURL: strings.TrimRight(baseURL, "/"),
|
||||||
apiKey: apiKey,
|
apiKey: apiKey,
|
||||||
model: model,
|
model: model,
|
||||||
maxTokens: defaultMaxTokens,
|
maxTokens: defaultMaxTokens,
|
||||||
|
temp: defaultTemperature,
|
||||||
httpClient: &http.Client{Timeout: timeout},
|
httpClient: &http.Client{Timeout: timeout},
|
||||||
|
maxBody: defaultMaxResponseBytes,
|
||||||
}
|
}
|
||||||
|
for _, opt := range opts {
|
||||||
|
opt(c)
|
||||||
|
}
|
||||||
|
return c
|
||||||
}
|
}
|
||||||
|
|
||||||
type chatRequest struct {
|
type chatRequest struct {
|
||||||
@@ -61,6 +140,10 @@ type chatResponse struct {
|
|||||||
Choices []struct {
|
Choices []struct {
|
||||||
Message message `json:"message"`
|
Message message `json:"message"`
|
||||||
} `json:"choices"`
|
} `json:"choices"`
|
||||||
|
Usage struct {
|
||||||
|
PromptTokens int `json:"prompt_tokens"`
|
||||||
|
CompletionTokens int `json:"completion_tokens"`
|
||||||
|
} `json:"usage"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Complete sends a system + user message and returns the assistant's reply.
|
// Complete sends a system + user message and returns the assistant's reply.
|
||||||
@@ -72,7 +155,7 @@ func (c *Client) Complete(ctx context.Context, system, user string) (string, err
|
|||||||
{Role: "system", Content: system},
|
{Role: "system", Content: system},
|
||||||
{Role: "user", Content: user},
|
{Role: "user", Content: user},
|
||||||
},
|
},
|
||||||
Temperature: 0.2,
|
Temperature: c.temp,
|
||||||
MaxTokens: c.maxTokens,
|
MaxTokens: c.maxTokens,
|
||||||
}
|
}
|
||||||
b, err := json.Marshal(body)
|
b, err := json.Marshal(body)
|
||||||
@@ -117,12 +200,18 @@ func (c *Client) Complete(ctx context.Context, system, user string) (string, err
|
|||||||
}
|
}
|
||||||
defer resp.Body.Close() //nolint:errcheck
|
defer resp.Body.Close() //nolint:errcheck
|
||||||
|
|
||||||
out, err := io.ReadAll(resp.Body)
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
// Only the bounded head is ever used (HTTPError.Error truncates to it), so read no more.
|
||||||
|
// A read error here only shortens a diagnostic; the status code below is what callers act on.
|
||||||
|
head, _ := io.ReadAll(io.LimitReader(resp.Body, maxErrorBody+1))
|
||||||
|
return "", &HTTPError{Code: resp.StatusCode, Body: string(head)}
|
||||||
|
}
|
||||||
|
out, err := io.ReadAll(io.LimitReader(resp.Body, c.maxBody+1))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", fmt.Errorf("read response: %w", err)
|
return "", fmt.Errorf("read response: %w", err)
|
||||||
}
|
}
|
||||||
if resp.StatusCode != http.StatusOK {
|
if int64(len(out)) > c.maxBody {
|
||||||
return "", fmt.Errorf("LLM returned %d: %s", resp.StatusCode, out)
|
return "", fmt.Errorf("LLM response exceeds %d bytes", c.maxBody)
|
||||||
}
|
}
|
||||||
|
|
||||||
var cr chatResponse
|
var cr chatResponse
|
||||||
@@ -132,5 +221,8 @@ func (c *Client) Complete(ctx context.Context, system, user string) (string, err
|
|||||||
if len(cr.Choices) == 0 {
|
if len(cr.Choices) == 0 {
|
||||||
return "", fmt.Errorf("LLM returned no choices")
|
return "", fmt.Errorf("LLM returned no choices")
|
||||||
}
|
}
|
||||||
|
if c.usageHook != nil {
|
||||||
|
c.usageHook(c.model, cr.Usage.PromptTokens, cr.Usage.CompletionTokens)
|
||||||
|
}
|
||||||
return cr.Choices[0].Message.Content, nil
|
return cr.Choices[0].Message.Content, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -64,6 +64,51 @@ func TestClient_SendsMaxTokens(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestClient_WithMaxTokens overrides the completion budget — the summarizer caps
|
||||||
|
// it small so prompt + max_tokens fits a small-context model's window (8k).
|
||||||
|
func TestClient_WithMaxTokens(t *testing.T) {
|
||||||
|
var body chatRequest
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
_ = json.NewDecoder(r.Body).Decode(&body)
|
||||||
|
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||||
|
"choices": []map[string]any{{"message": map[string]any{"content": "ok"}}},
|
||||||
|
})
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
c := New(srv.URL, "", "test-model", 10*time.Second, WithMaxTokens(1500))
|
||||||
|
if _, err := c.Complete(context.Background(), "sys", "user"); err != nil {
|
||||||
|
t.Fatalf("Complete: %v", err)
|
||||||
|
}
|
||||||
|
if body.MaxTokens != 1500 {
|
||||||
|
t.Errorf("max_tokens = %d, want 1500", body.MaxTokens)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestClient_UsageHookRecordsTokens: the usage hook fires with the model and the
|
||||||
|
// prompt/completion token counts parsed from the response usage block.
|
||||||
|
func TestClient_UsageHookRecordsTokens(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||||
|
"choices": []map[string]any{{"message": map[string]any{"content": "ok"}}},
|
||||||
|
"usage": map[string]any{"prompt_tokens": 123, "completion_tokens": 45},
|
||||||
|
})
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
var gotModel string
|
||||||
|
var gotPrompt, gotCompletion int
|
||||||
|
c := New(srv.URL, "", "test-model", 10*time.Second, WithUsageHook(func(model string, p, comp int) {
|
||||||
|
gotModel, gotPrompt, gotCompletion = model, p, comp
|
||||||
|
}))
|
||||||
|
if _, err := c.Complete(context.Background(), "sys", "user"); err != nil {
|
||||||
|
t.Fatalf("Complete: %v", err)
|
||||||
|
}
|
||||||
|
if gotModel != "test-model" || gotPrompt != 123 || gotCompletion != 45 {
|
||||||
|
t.Errorf("usage hook got (%q, %d, %d), want (test-model, 123, 45)", gotModel, gotPrompt, gotCompletion)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestClient_ReturnsErrorOnNon200(t *testing.T) {
|
func TestClient_ReturnsErrorOnNon200(t *testing.T) {
|
||||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
http.Error(w, "overloaded", http.StatusServiceUnavailable)
|
http.Error(w, "overloaded", http.StatusServiceUnavailable)
|
||||||
|
|||||||
@@ -0,0 +1,63 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.d-ma.be/mathias/tapir/internal/errkind"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A non-200 from the gateway is an HTTP status error: its kind can be logged and stored
|
||||||
|
// without its body, which can echo the prompt (adversarial review F3).
|
||||||
|
|
||||||
|
func failingGateway(t *testing.T, status int, body string) *httptest.Server {
|
||||||
|
t.Helper()
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
w.WriteHeader(status)
|
||||||
|
_, _ = w.Write([]byte(body))
|
||||||
|
}))
|
||||||
|
t.Cleanup(srv.Close)
|
||||||
|
return srv
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClient_ANon200IsAnHTTPStatusErrorWhoseKindCarriesNoBody(t *testing.T) {
|
||||||
|
srv := failingGateway(t, http.StatusBadRequest, `{"error":"prompt was: Anders ansvarar för virket"}`)
|
||||||
|
|
||||||
|
_, err := New(srv.URL, "", "m", 5*time.Second).Complete(context.Background(), "s", "u")
|
||||||
|
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected an error")
|
||||||
|
}
|
||||||
|
if got := errkind.Of(err); got != "http_400" {
|
||||||
|
t.Errorf("errkind.Of = %q, want http_400", got)
|
||||||
|
}
|
||||||
|
var he *HTTPError
|
||||||
|
if !errors.As(err, &he) || he.Code != http.StatusBadRequest {
|
||||||
|
t.Errorf("error is not an *HTTPError with the status code: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClient_TheErrorTextIsUnchangedForCallersThatLogIt(t *testing.T) {
|
||||||
|
srv := failingGateway(t, http.StatusBadGateway, "upstream down")
|
||||||
|
|
||||||
|
_, err := New(srv.URL, "", "m", 5*time.Second).Complete(context.Background(), "s", "u")
|
||||||
|
|
||||||
|
if err == nil || err.Error() != "LLM returned 502: upstream down" {
|
||||||
|
t.Errorf("error text = %v: callers that log it already rely on this form", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClient_AnEnormousErrorBodyIsBoundedInTheErrorText(t *testing.T) {
|
||||||
|
srv := failingGateway(t, http.StatusInternalServerError, strings.Repeat("x", 100_000))
|
||||||
|
|
||||||
|
_, err := New(srv.URL, "", "m", 5*time.Second).Complete(context.Background(), "s", "u")
|
||||||
|
|
||||||
|
if err == nil || len(err.Error()) > 1024 {
|
||||||
|
t.Errorf("error text is %d bytes: a gateway's whole body must not become an error message", len(err.Error()))
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,57 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// capturedTemperature runs one Complete against a fake gateway and returns the
|
||||||
|
// "temperature" field the request carried, and whether it was present at all.
|
||||||
|
func capturedTemperature(t *testing.T, opts ...Option) (float64, bool) {
|
||||||
|
t.Helper()
|
||||||
|
var body map[string]any
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
raw, _ := io.ReadAll(r.Body)
|
||||||
|
_ = json.Unmarshal(raw, &body)
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_ = json.NewEncoder(w).Encode(map[string]any{"choices": []map[string]any{{"message": map[string]any{"role": "assistant", "content": "ok"}}}})
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
if _, err := New(srv.URL, "", "m", 5*time.Second, opts...).Complete(context.Background(), "s", "u"); err != nil {
|
||||||
|
t.Fatalf("Complete: %v", err)
|
||||||
|
}
|
||||||
|
v, ok := body["temperature"].(float64)
|
||||||
|
return v, ok
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClient_DefaultTemperatureIsUnchanged(t *testing.T) {
|
||||||
|
got, ok := capturedTemperature(t)
|
||||||
|
|
||||||
|
if !ok || got != 0.2 {
|
||||||
|
t.Fatalf("default temperature = %v (present %v), want 0.2: existing callers must not change behaviour", got, ok)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClient_WithTemperatureZeroSendsExactlyZero(t *testing.T) {
|
||||||
|
// The reconciler's accuracy (92%/97%) was measured at temperature 0. Zero must be
|
||||||
|
// SENT, not treated as "unset" and replaced by the default.
|
||||||
|
got, ok := capturedTemperature(t, WithTemperature(0))
|
||||||
|
|
||||||
|
if !ok || got != 0 {
|
||||||
|
t.Fatalf("temperature = %v (present %v), want an explicit 0", got, ok)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClient_WithTemperatureSendsTheGivenValue(t *testing.T) {
|
||||||
|
got, ok := capturedTemperature(t, WithTemperature(0.7))
|
||||||
|
|
||||||
|
if !ok || got != 0.7 {
|
||||||
|
t.Fatalf("temperature = %v (present %v), want 0.7", got, ok)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -18,7 +18,7 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
"gitea.d-ma.be/mathias/tapir/internal/ports"
|
"git.d-ma.be/mathias/tapir/internal/ports"
|
||||||
)
|
)
|
||||||
|
|
||||||
// FileStore is a SecretStore backed by a single 0600 JSON file mapping opaque
|
// FileStore is a SecretStore backed by a single 0600 JSON file mapping opaque
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.d-ma.be/mathias/tapir/internal/adapters/secrets"
|
"git.d-ma.be/mathias/tapir/internal/adapters/secrets"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestPutThenGet(t *testing.T) {
|
func TestPutThenGet(t *testing.T) {
|
||||||
|
|||||||
@@ -10,13 +10,17 @@ import (
|
|||||||
// DeleteUser permanently removes a user and all of their data. It runs through
|
// DeleteUser permanently removes a user and all of their data. It runs through
|
||||||
// withUser so RLS confines every statement to the calling user's own rows.
|
// withUser so RLS confines every statement to the calling user's own rows.
|
||||||
//
|
//
|
||||||
// Deleting the users row cascades (ON DELETE CASCADE) to videos, transcripts,
|
// Deleting the users row cascades (ON DELETE CASCADE) to videos, summaries
|
||||||
// summaries (→ sink_deliveries), video_connections, and the user_identities map
|
// (→ sink_deliveries), video_connections, and the user_identities map —
|
||||||
// — referential-integrity cascades bypass RLS, so a user's child rows are removed
|
// referential-integrity cascades bypass RLS, so a user's child rows are removed
|
||||||
// even though the deleting connection is scoped. summary_actions is the exception:
|
// even though the deleting connection is scoped. Transcripts are NOT removed:
|
||||||
// it carries a user_id but has NO foreign key to users (migration 002), so the
|
// since ADR-021 they are shared public content keyed by (provider,
|
||||||
// cascade does not reach it; it is deleted explicitly in the same scoped
|
// provider_video_id) with no user_id, so another user may still reference the
|
||||||
// transaction. Deleting an absent user is a no-op (idempotent).
|
// same row — a user deletion must not strip shared caption content. summary_actions and login_events
|
||||||
|
// are the exceptions: each carries a user_id but has NO foreign key to users
|
||||||
|
// (migrations 002 and 010), so the cascade does not reach them; they are deleted
|
||||||
|
// explicitly in the same scoped transaction. Deleting an absent user is a no-op
|
||||||
|
// (idempotent).
|
||||||
//
|
//
|
||||||
// This is tapir-side only (decision 2026-06-03): it removes all tapir data; the
|
// This is tapir-side only (decision 2026-06-03): it removes all tapir data; the
|
||||||
// Dex login identity is left untouched — a later login simply re-enters
|
// Dex login identity is left untouched — a later login simply re-enters
|
||||||
@@ -28,6 +32,10 @@ func (s *Store) DeleteUser(ctx context.Context, userID string) error {
|
|||||||
`DELETE FROM summary_actions WHERE user_id = $1`, userID); err != nil {
|
`DELETE FROM summary_actions WHERE user_id = $1`, userID); err != nil {
|
||||||
return fmt.Errorf("store: delete summary_actions: %w", err)
|
return fmt.Errorf("store: delete summary_actions: %w", err)
|
||||||
}
|
}
|
||||||
|
if _, err := tx.Exec(ctx,
|
||||||
|
`DELETE FROM login_events WHERE user_id = $1`, userID); err != nil {
|
||||||
|
return fmt.Errorf("store: delete login_events: %w", err)
|
||||||
|
}
|
||||||
if _, err := tx.Exec(ctx,
|
if _, err := tx.Exec(ctx,
|
||||||
`DELETE FROM users WHERE id = $1`, userID); err != nil {
|
`DELETE FROM users WHERE id = $1`, userID); err != nil {
|
||||||
return fmt.Errorf("store: delete user: %w", err)
|
return fmt.Errorf("store: delete user: %w", err)
|
||||||
|
|||||||
@@ -0,0 +1,83 @@
|
|||||||
|
package store
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CaptionlessChannels returns the set of channel ids currently suppressed for the
|
||||||
|
// user — channels whose recent videos all yielded no captions, within their
|
||||||
|
// suppression window (ADR-024). The runner skips caption fetches for these
|
||||||
|
// channels' videos. A channel whose window has expired is not returned, so its
|
||||||
|
// next video is re-probed (auto-recovery).
|
||||||
|
func (s *Store) CaptionlessChannels(ctx context.Context, userID string) (map[string]bool, error) {
|
||||||
|
out := map[string]bool{}
|
||||||
|
err := s.withUser(ctx, userID, func(tx pgx.Tx) error {
|
||||||
|
rows, err := tx.Query(ctx, `
|
||||||
|
SELECT channel_id FROM channel_caption_state
|
||||||
|
WHERE user_id = $1 AND captionless_until IS NOT NULL AND captionless_until > now()`,
|
||||||
|
userID)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("store: caption-less channels: %w", err)
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
for rows.Next() {
|
||||||
|
var ch string
|
||||||
|
if err := rows.Scan(&ch); err != nil {
|
||||||
|
return fmt.Errorf("store: scan caption-less channel: %w", err)
|
||||||
|
}
|
||||||
|
out[ch] = true
|
||||||
|
}
|
||||||
|
return rows.Err()
|
||||||
|
})
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordChannelCaptionOutcome updates a channel's caption-availability memory
|
||||||
|
// after a fetch attempt (ADR-024). hadCaptions resets the channel (consecutive
|
||||||
|
// count to 0, suppression cleared). Otherwise the consecutive no-caption count is
|
||||||
|
// incremented; once it reaches threshold the channel is suppressed for window.
|
||||||
|
// threshold <= 0 is a no-op (feature disabled). An empty channelID is ignored
|
||||||
|
// (some sources may not carry one).
|
||||||
|
func (s *Store) RecordChannelCaptionOutcome(ctx context.Context, userID, channelID string, hadCaptions bool, threshold int, window time.Duration) error {
|
||||||
|
if channelID == "" || threshold <= 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return s.withUser(ctx, userID, func(tx pgx.Tx) error {
|
||||||
|
if hadCaptions {
|
||||||
|
_, err := tx.Exec(ctx, `
|
||||||
|
INSERT INTO channel_caption_state (user_id, channel_id, consecutive_none, captionless_until, updated_at)
|
||||||
|
VALUES ($1, $2, 0, NULL, now())
|
||||||
|
ON CONFLICT (user_id, channel_id)
|
||||||
|
DO UPDATE SET consecutive_none = 0, captionless_until = NULL, updated_at = now()`,
|
||||||
|
userID, channelID)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("store: reset channel caption state: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
// No captions: increment the streak; suppress once it reaches threshold.
|
||||||
|
// captionless_until is set from the NEW count inside the same statement so
|
||||||
|
// the decision is atomic with the increment.
|
||||||
|
until := time.Now().Add(window)
|
||||||
|
_, err := tx.Exec(ctx, `
|
||||||
|
INSERT INTO channel_caption_state (user_id, channel_id, consecutive_none, captionless_until, updated_at)
|
||||||
|
VALUES ($1, $2, 1, CASE WHEN 1 >= $3 THEN $4::timestamptz ELSE NULL END, now())
|
||||||
|
ON CONFLICT (user_id, channel_id)
|
||||||
|
DO UPDATE SET
|
||||||
|
consecutive_none = channel_caption_state.consecutive_none + 1,
|
||||||
|
captionless_until = CASE
|
||||||
|
WHEN channel_caption_state.consecutive_none + 1 >= $3 THEN $4::timestamptz
|
||||||
|
ELSE channel_caption_state.captionless_until
|
||||||
|
END,
|
||||||
|
updated_at = now()`,
|
||||||
|
userID, channelID, threshold, until)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("store: record channel no-caption: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
package store_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestChannelCaptionMemory_SuppressesAfterThreshold(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
s := newStore(t)
|
||||||
|
super := rawPool(t)
|
||||||
|
resetDB(t, super)
|
||||||
|
seedUser(t, super, userA)
|
||||||
|
|
||||||
|
const threshold = 3
|
||||||
|
window := time.Hour
|
||||||
|
|
||||||
|
// Below threshold: not yet suppressed.
|
||||||
|
require.NoError(t, s.RecordChannelCaptionOutcome(ctx, userA, "chanX", false, threshold, window))
|
||||||
|
require.NoError(t, s.RecordChannelCaptionOutcome(ctx, userA, "chanX", false, threshold, window))
|
||||||
|
got, err := s.CaptionlessChannels(ctx, userA)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotContains(t, got, "chanX", "2 < threshold 3: not suppressed yet")
|
||||||
|
|
||||||
|
// Crossing the threshold suppresses the channel.
|
||||||
|
require.NoError(t, s.RecordChannelCaptionOutcome(ctx, userA, "chanX", false, threshold, window))
|
||||||
|
got, err = s.CaptionlessChannels(ctx, userA)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Contains(t, got, "chanX", "3 consecutive no-caption results suppress the channel")
|
||||||
|
|
||||||
|
// A successful caption fetch resets it.
|
||||||
|
require.NoError(t, s.RecordChannelCaptionOutcome(ctx, userA, "chanX", true, threshold, window))
|
||||||
|
got, err = s.CaptionlessChannels(ctx, userA)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotContains(t, got, "chanX", "a captioned video clears suppression")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChannelCaptionMemory_WindowExpiryReProbes(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
s := newStore(t)
|
||||||
|
super := rawPool(t)
|
||||||
|
resetDB(t, super)
|
||||||
|
seedUser(t, super, userA)
|
||||||
|
|
||||||
|
// A negative window means captionless_until lands in the past — modelling an
|
||||||
|
// elapsed suppression window, which must make the channel eligible again.
|
||||||
|
require.NoError(t, s.RecordChannelCaptionOutcome(ctx, userA, "chanY", false, 1, -time.Hour))
|
||||||
|
got, err := s.CaptionlessChannels(ctx, userA)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotContains(t, got, "chanY", "an expired window re-enables the channel for a re-probe")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChannelCaptionMemory_DisabledThresholdIsNoOp(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
s := newStore(t)
|
||||||
|
super := rawPool(t)
|
||||||
|
resetDB(t, super)
|
||||||
|
seedUser(t, super, userA)
|
||||||
|
|
||||||
|
require.NoError(t, s.RecordChannelCaptionOutcome(ctx, userA, "chanZ", false, 0, time.Hour))
|
||||||
|
got, err := s.CaptionlessChannels(ctx, userA)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Empty(t, got, "threshold 0 disables the memory — nothing recorded")
|
||||||
|
}
|
||||||
@@ -0,0 +1,59 @@
|
|||||||
|
package store
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ChannelError is a channel that returned HTTP 404 on a discovery pass.
|
||||||
|
type ChannelError struct {
|
||||||
|
ChannelID string
|
||||||
|
ChannelName string
|
||||||
|
FirstSeen time.Time
|
||||||
|
LastSeen time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpsertChannelError records (or refreshes) a 404 channel for the current user.
|
||||||
|
// Called by the runner inside a withUser scope; RLS guards user isolation.
|
||||||
|
func (s *Store) UpsertChannelError(ctx context.Context, userID, channelID, channelName string) error {
|
||||||
|
return s.withUser(ctx, userID, func(tx pgx.Tx) error {
|
||||||
|
_, err := tx.Exec(ctx, `
|
||||||
|
INSERT INTO channel_errors (user_id, channel_id, channel_name)
|
||||||
|
VALUES ($1, $2, $3)
|
||||||
|
ON CONFLICT (user_id, channel_id)
|
||||||
|
DO UPDATE SET channel_name = EXCLUDED.channel_name, last_seen = now()`,
|
||||||
|
userID, channelID, channelName)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("store: upsert channel error: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListChannelErrors returns all 404-flagged channels for the user, newest first.
|
||||||
|
func (s *Store) ListChannelErrors(ctx context.Context, userID string) ([]ChannelError, error) {
|
||||||
|
var out []ChannelError
|
||||||
|
err := s.withUser(ctx, userID, func(tx pgx.Tx) error {
|
||||||
|
rows, err := tx.Query(ctx, `
|
||||||
|
SELECT channel_id, channel_name, first_seen, last_seen
|
||||||
|
FROM channel_errors
|
||||||
|
WHERE user_id = $1
|
||||||
|
ORDER BY last_seen DESC`, userID)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("store: list channel errors: %w", err)
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
for rows.Next() {
|
||||||
|
var ce ChannelError
|
||||||
|
if err := rows.Scan(&ce.ChannelID, &ce.ChannelName, &ce.FirstSeen, &ce.LastSeen); err != nil {
|
||||||
|
return fmt.Errorf("store: scan channel error: %w", err)
|
||||||
|
}
|
||||||
|
out = append(out, ce)
|
||||||
|
}
|
||||||
|
return rows.Err()
|
||||||
|
})
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
@@ -6,7 +6,7 @@ import (
|
|||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
"gitea.d-ma.be/mathias/tapir/internal/adapters/store"
|
"git.d-ma.be/mathias/tapir/internal/adapters/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
// seedUserRow inserts a bare users row (FK target for a connection) as the
|
// seedUserRow inserts a bare users row (FK target for a connection) as the
|
||||||
|
|||||||
@@ -31,6 +31,41 @@ func (s *Store) UserBySubject(ctx context.Context, subject string) (userID strin
|
|||||||
return userID, true, nil
|
return userID, true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// UserIdentity is one (userID, dexSubject) pair from the un-RLS'd
|
||||||
|
// user_identities map — the unit the scheduler enumerates to run a discovery
|
||||||
|
// pass per user (ADR-018).
|
||||||
|
type UserIdentity struct {
|
||||||
|
UserID string
|
||||||
|
DexSubject string
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListAllUsers returns every (userID, dexSubject) pair from user_identities. It
|
||||||
|
// runs as a plain pool query WITHOUT withUser — intentional and legitimate:
|
||||||
|
// user_identities is un-RLS'd auth plumbing (like UserBySubject), and the
|
||||||
|
// scheduler enumerating all users to run their discovery passes is an admin
|
||||||
|
// operation that cannot be scoped to any single user. Order is unspecified.
|
||||||
|
func (s *Store) ListAllUsers(ctx context.Context) ([]UserIdentity, error) {
|
||||||
|
rows, err := s.pool.Query(ctx,
|
||||||
|
`SELECT user_id, dex_subject FROM user_identities`)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("store: list all users: %w", err)
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
var users []UserIdentity
|
||||||
|
for rows.Next() {
|
||||||
|
var u UserIdentity
|
||||||
|
if err := rows.Scan(&u.UserID, &u.DexSubject); err != nil {
|
||||||
|
return nil, fmt.Errorf("store: scan user identity: %w", err)
|
||||||
|
}
|
||||||
|
users = append(users, u)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, fmt.Errorf("store: iterate user identities: %w", err)
|
||||||
|
}
|
||||||
|
return users, nil
|
||||||
|
}
|
||||||
|
|
||||||
// RegisterUser creates the tapir user for a Dex subject and the identity mapping
|
// RegisterUser creates the tapir user for a Dex subject and the identity mapping
|
||||||
// that points to it, returning the new user_id. It errors with
|
// that points to it, returning the new user_id. It errors with
|
||||||
// ErrSubjectRegistered if the subject already maps.
|
// ErrSubjectRegistered if the subject already maps.
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import (
|
|||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
"gitea.d-ma.be/mathias/tapir/internal/adapters/store"
|
"git.d-ma.be/mathias/tapir/internal/adapters/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -85,6 +85,39 @@ func TestRegisterUserRejectsDuplicateSubject(t *testing.T) {
|
|||||||
require.Equal(t, first, got)
|
require.Equal(t, first, got)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestListAllUsersReturnsEveryIdentity(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
s := newStore(t)
|
||||||
|
resetDB(t, rawPool(t))
|
||||||
|
|
||||||
|
empty, err := s.ListAllUsers(ctx)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Empty(t, empty, "no registrations yet → empty slice")
|
||||||
|
|
||||||
|
const subjectC = "dex|carol-789"
|
||||||
|
idA, err := s.RegisterUser(ctx, subjectA, "Alice")
|
||||||
|
require.NoError(t, err)
|
||||||
|
idB, err := s.RegisterUser(ctx, subjectB, "Bob")
|
||||||
|
require.NoError(t, err)
|
||||||
|
idC, err := s.RegisterUser(ctx, subjectC, "Carol")
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
users, err := s.ListAllUsers(ctx)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, users, 3)
|
||||||
|
|
||||||
|
// Order is unspecified; compare as a set of (userID, subject) pairs.
|
||||||
|
got := make(map[string]string, len(users))
|
||||||
|
for _, u := range users {
|
||||||
|
got[u.DexSubject] = u.UserID
|
||||||
|
}
|
||||||
|
require.Equal(t, map[string]string{
|
||||||
|
subjectA: idA,
|
||||||
|
subjectB: idB,
|
||||||
|
subjectC: idC,
|
||||||
|
}, got)
|
||||||
|
}
|
||||||
|
|
||||||
func TestRegisterUserDistinctSubjectsGetDistinctUsers(t *testing.T) {
|
func TestRegisterUserDistinctSubjectsGetDistinctUsers(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
s := newStore(t)
|
s := newStore(t)
|
||||||
|
|||||||
@@ -0,0 +1,41 @@
|
|||||||
|
package store
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5"
|
||||||
|
)
|
||||||
|
|
||||||
|
// StampLogin records that the user was active today, throttled to one row per
|
||||||
|
// user per day. It is the read-side counterpart to SetAction: the middleware
|
||||||
|
// calls it on every authenticated request, but the append happens at most once a
|
||||||
|
// day so login_events stays small and the signal clean (one row = one active
|
||||||
|
// day, not one request).
|
||||||
|
//
|
||||||
|
// The check-and-insert is a single atomic statement: the INSERT ... SELECT ...
|
||||||
|
// WHERE NOT EXISTS only writes when no row for this user has seen_at in today
|
||||||
|
// (date_trunc('day', NOW()), server timezone). It runs through withUser, so the
|
||||||
|
// NOT EXISTS probe is itself RLS-scoped to the calling user via the
|
||||||
|
// tapir.current_user_id GUC — one user's stamp can never be suppressed or
|
||||||
|
// triggered by another user's rows. The explicit user_id predicate also keeps the
|
||||||
|
// probe on the (user_id, seen_at) index.
|
||||||
|
//
|
||||||
|
// A unique constraint is deliberately not used: under concurrent same-day
|
||||||
|
// requests the worst case is two rows for one day, which the gate query collapses
|
||||||
|
// to a single week bucket anyway — not worth a write-blocking constraint.
|
||||||
|
func (s *Store) StampLogin(ctx context.Context, userID string) error {
|
||||||
|
if err := s.withUser(ctx, userID, func(tx pgx.Tx) error {
|
||||||
|
_, err := tx.Exec(ctx,
|
||||||
|
`INSERT INTO login_events (user_id)
|
||||||
|
SELECT $1
|
||||||
|
WHERE NOT EXISTS (
|
||||||
|
SELECT 1 FROM login_events
|
||||||
|
WHERE user_id = $1 AND seen_at >= date_trunc('day', NOW())
|
||||||
|
)`, userID)
|
||||||
|
return err
|
||||||
|
}); err != nil {
|
||||||
|
return fmt.Errorf("store: stamp login: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
package store_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
// countLoginEvents counts a user's login_events via the superuser pool, which
|
||||||
|
// bypasses RLS — so the assertion sees the true row count regardless of scope.
|
||||||
|
func countLoginEvents(t *testing.T, p *pgxpool.Pool, userID string) int {
|
||||||
|
t.Helper()
|
||||||
|
var n int
|
||||||
|
require.NoError(t, p.QueryRow(context.Background(),
|
||||||
|
`SELECT count(*) FROM login_events WHERE user_id = $1`, userID).Scan(&n))
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestStampLoginThrottlesToOnePerDay: repeated stamps within the same day insert
|
||||||
|
// exactly one row — the throttle that keeps login_events one-row-per-active-day.
|
||||||
|
func TestStampLoginThrottlesToOnePerDay(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
s := newStore(t)
|
||||||
|
super := rawPool(t)
|
||||||
|
resetDB(t, super)
|
||||||
|
_, err := super.Exec(ctx, `INSERT INTO users (id) VALUES ($1)`, userA)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
for i := 0; i < 3; i++ {
|
||||||
|
require.NoError(t, s.StampLogin(ctx, userA))
|
||||||
|
}
|
||||||
|
require.Equal(t, 1, countLoginEvents(t, super, userA),
|
||||||
|
"three same-day stamps must collapse to one row")
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestStampLoginRecordsOncePerNewDay: with yesterday's row already present, a
|
||||||
|
// stamp today is NOT throttled — it appends the day's row, so distinct active days
|
||||||
|
// accumulate (the substrate the gate's distinct-week count reads).
|
||||||
|
func TestStampLoginRecordsOncePerNewDay(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
s := newStore(t)
|
||||||
|
super := rawPool(t)
|
||||||
|
resetDB(t, super)
|
||||||
|
_, err := super.Exec(ctx, `INSERT INTO users (id) VALUES ($1)`, userA)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// Seed an event dated yesterday (before today's start), so the throttle's
|
||||||
|
// "row exists with seen_at >= start-of-today" probe finds nothing for today.
|
||||||
|
_, err = super.Exec(ctx,
|
||||||
|
`INSERT INTO login_events (user_id, seen_at) VALUES ($1, NOW() - INTERVAL '1 day')`, userA)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
require.NoError(t, s.StampLogin(ctx, userA))
|
||||||
|
require.Equal(t, 2, countLoginEvents(t, super, userA),
|
||||||
|
"a stamp on a new day must append a second row")
|
||||||
|
|
||||||
|
// A second stamp the same day is throttled again.
|
||||||
|
require.NoError(t, s.StampLogin(ctx, userA))
|
||||||
|
require.Equal(t, 2, countLoginEvents(t, super, userA),
|
||||||
|
"the same-day repeat must not add a third row")
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestStampLoginIsUserScoped: one user's stamp lands only on that user's rows —
|
||||||
|
// the throttle probe is RLS-scoped, so user B's existing same-day row neither
|
||||||
|
// suppresses nor is touched by user A's stamp.
|
||||||
|
func TestStampLoginIsUserScoped(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
s := newStore(t)
|
||||||
|
super := rawPool(t)
|
||||||
|
resetDB(t, super)
|
||||||
|
_, err := super.Exec(ctx, `INSERT INTO users (id) VALUES ($1), ($2)`, userA, userB)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// B already has a same-day row; it must not throttle A's first stamp.
|
||||||
|
_, err = super.Exec(ctx, `INSERT INTO login_events (user_id) VALUES ($1)`, userB)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
require.NoError(t, s.StampLogin(ctx, userA))
|
||||||
|
require.Equal(t, 1, countLoginEvents(t, super, userA), "A's stamp must record despite B's same-day row")
|
||||||
|
require.Equal(t, 1, countLoginEvents(t, super, userB), "A's stamp must not touch B's rows")
|
||||||
|
}
|
||||||
@@ -0,0 +1,165 @@
|
|||||||
|
package store_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/golang-migrate/migrate/v4"
|
||||||
|
migratepgx "github.com/golang-migrate/migrate/v4/database/pgx/v5"
|
||||||
|
"github.com/golang-migrate/migrate/v4/source/iofs"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
_ "github.com/jackc/pgx/v5/stdlib" // register the "pgx" database/sql driver
|
||||||
|
)
|
||||||
|
|
||||||
|
// fileMigrator builds a golang-migrate instance from the on-disk migration files
|
||||||
|
// (not the embedded FS the production Migrate uses), so a test can step the schema
|
||||||
|
// up and down. os.DirFS(".") is rooted at the package dir; the SQL lives under
|
||||||
|
// "migrations". Mirrors store.Migrate's construction otherwise.
|
||||||
|
func fileMigrator(t *testing.T) *migrate.Migrate {
|
||||||
|
t.Helper()
|
||||||
|
db, err := sql.Open("pgx", dsn)
|
||||||
|
require.NoError(t, err)
|
||||||
|
t.Cleanup(func() { _ = db.Close() })
|
||||||
|
|
||||||
|
drv, err := migratepgx.WithInstance(db, &migratepgx.Config{})
|
||||||
|
require.NoError(t, err)
|
||||||
|
src, err := iofs.New(os.DirFS("."), "migrations")
|
||||||
|
require.NoError(t, err)
|
||||||
|
m, err := migrate.NewWithInstance("iofs", src, "pgx", drv)
|
||||||
|
require.NoError(t, err)
|
||||||
|
t.Cleanup(func() { _, _ = m.Close() })
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
// headVersion reports the current (HEAD) schema version so a test can restore
|
||||||
|
// to it after stepping down, without hard-coding what HEAD is. Adding a
|
||||||
|
// migration on top changes HEAD but no test that uses this needs editing.
|
||||||
|
func headVersion(t *testing.T, m *migrate.Migrate) uint {
|
||||||
|
t.Helper()
|
||||||
|
v, dirty, err := m.Version()
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.False(t, dirty, "schema must not be dirty")
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
|
||||||
|
// migrateTo drives the schema to an exact version *by version number*, not by
|
||||||
|
// step count. This is the whole point of the migrate-test design: a migration
|
||||||
|
// added above the target does not shift any count here, so unrelated tests stay
|
||||||
|
// green (see issue #8). ErrNoChange (already at that version) is not a failure.
|
||||||
|
func migrateTo(t *testing.T, m *migrate.Migrate, version uint) {
|
||||||
|
t.Helper()
|
||||||
|
err := m.Migrate(version)
|
||||||
|
if errors.Is(err, migrate.ErrNoChange) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// loginEventsExists reports whether the login_events relation is present.
|
||||||
|
func loginEventsExists(t *testing.T) bool {
|
||||||
|
t.Helper()
|
||||||
|
var reg *string
|
||||||
|
require.NoError(t, rawPool(t).QueryRow(context.Background(),
|
||||||
|
`SELECT to_regclass('public.login_events')::text`).Scan(®))
|
||||||
|
return reg != nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMigration010LoginEventsUpDown proves migration 010 is reversible: the down
|
||||||
|
// migration drops login_events cleanly and the up migration recreates it. A rotten
|
||||||
|
// down migration (forgotten DROP, dangling policy) would fail here rather than in
|
||||||
|
// production during a rollback. The test restores the schema to latest before
|
||||||
|
// returning so the shared embedded-postgres stays at HEAD for sibling tests.
|
||||||
|
func TestMigration010LoginEventsUpDown(t *testing.T) {
|
||||||
|
newStore(t) // ensure the schema is migrated to latest (011 applied)
|
||||||
|
require.True(t, loginEventsExists(t), "login_events must exist at latest migration")
|
||||||
|
|
||||||
|
m := fileMigrator(t)
|
||||||
|
head := headVersion(t, m)
|
||||||
|
|
||||||
|
migrateTo(t, m, 9) // just below 010 — everything above steps down
|
||||||
|
require.False(t, loginEventsExists(t), "login_events must be gone after the down migration")
|
||||||
|
|
||||||
|
migrateTo(t, m, 10) // up 010
|
||||||
|
require.True(t, loginEventsExists(t), "login_events must be restored after the up migration")
|
||||||
|
|
||||||
|
migrateTo(t, m, head) // restore to HEAD for sibling tests
|
||||||
|
}
|
||||||
|
|
||||||
|
// autoSummarizeDefault reads the users.auto_summarize column default as text
|
||||||
|
// ("true"/"false"), so the migration's default flip is verifiable directly.
|
||||||
|
func autoSummarizeDefault(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
var def string
|
||||||
|
require.NoError(t, rawPool(t).QueryRow(context.Background(),
|
||||||
|
`SELECT column_default FROM information_schema.columns
|
||||||
|
WHERE table_name = 'users' AND column_name = 'auto_summarize'`).Scan(&def))
|
||||||
|
return def
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMigration011AutoSummarizeDefaultUpDown proves migration 011 is reversible:
|
||||||
|
// up sets the auto_summarize column default to TRUE (ADR-018), down restores
|
||||||
|
// FALSE. The down intentionally does not revert existing rows — only the default.
|
||||||
|
func TestMigration011AutoSummarizeDefaultUpDown(t *testing.T) {
|
||||||
|
newStore(t) // latest (013 applied)
|
||||||
|
require.Equal(t, "true", autoSummarizeDefault(t), "011 sets the default to TRUE")
|
||||||
|
|
||||||
|
m := fileMigrator(t)
|
||||||
|
head := headVersion(t, m)
|
||||||
|
|
||||||
|
migrateTo(t, m, 10) // just below 011 — reverts the column default
|
||||||
|
require.Equal(t, "false", autoSummarizeDefault(t), "default is FALSE after the down migration")
|
||||||
|
|
||||||
|
migrateTo(t, m, 11) // up 011 re-applies the TRUE default
|
||||||
|
require.Equal(t, "true", autoSummarizeDefault(t))
|
||||||
|
|
||||||
|
migrateTo(t, m, head) // restore to HEAD for sibling tests
|
||||||
|
}
|
||||||
|
|
||||||
|
// channelTitleExists reports whether videos.channel_title is present.
|
||||||
|
func channelTitleExists(t *testing.T) bool {
|
||||||
|
t.Helper()
|
||||||
|
var exists bool
|
||||||
|
require.NoError(t, rawPool(t).QueryRow(context.Background(),
|
||||||
|
`SELECT EXISTS (SELECT 1 FROM information_schema.columns
|
||||||
|
WHERE table_name = 'videos' AND column_name = 'channel_title')`).Scan(&exists))
|
||||||
|
return exists
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMigration014VideoChannelTitleUpDown proves 014 is reversible: down drops
|
||||||
|
// videos.channel_title, up recreates it.
|
||||||
|
func TestMigration014VideoChannelTitleUpDown(t *testing.T) {
|
||||||
|
newStore(t) // latest (014 applied)
|
||||||
|
require.True(t, channelTitleExists(t), "channel_title exists at latest migration")
|
||||||
|
|
||||||
|
m := fileMigrator(t)
|
||||||
|
head := headVersion(t, m)
|
||||||
|
|
||||||
|
migrateTo(t, m, 13) // just below 014 — drops channel_title
|
||||||
|
require.False(t, channelTitleExists(t), "channel_title must be gone after the down migration")
|
||||||
|
|
||||||
|
migrateTo(t, m, 14) // up 014 recreates channel_title
|
||||||
|
require.True(t, channelTitleExists(t), "channel_title must be restored after the up migration")
|
||||||
|
|
||||||
|
migrateTo(t, m, head) // restore to HEAD for sibling tests
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMigration012FixAutoSummarizeRLS proves 012 runs cleanly and flips any
|
||||||
|
// remaining auto_summarize=FALSE rows to TRUE (the back-fill blocked by RLS in 011).
|
||||||
|
func TestMigration012FixAutoSummarizeRLS(t *testing.T) {
|
||||||
|
newStore(t) // apply all migrations including 012
|
||||||
|
require.Equal(t, "true", autoSummarizeDefault(t), "column default is TRUE after 012")
|
||||||
|
|
||||||
|
// Round-trip: down 012, then up 012 — must be idempotent.
|
||||||
|
m := fileMigrator(t)
|
||||||
|
head := headVersion(t, m)
|
||||||
|
|
||||||
|
migrateTo(t, m, 11) // down 012 must not error
|
||||||
|
migrateTo(t, m, 12) // up 012 must re-apply cleanly
|
||||||
|
require.Equal(t, "true", autoSummarizeDefault(t), "default still TRUE after 012 re-applied")
|
||||||
|
|
||||||
|
migrateTo(t, m, head) // restore to HEAD for sibling tests
|
||||||
|
}
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
ALTER TABLE videos DROP COLUMN IF EXISTS rate_limited_at;
|
||||||
|
ALTER TABLE videos DROP COLUMN IF EXISTS transcript_status;
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user