Upload a recording (audio, or video with the camera off), get a private Swedish transcript, and chat about it. - ADR-033: RLS-scoped private transcript store; ffprobe/ffmpeg intake; gateway transcription (iguana kb-whisper, berget fallback); in-process single-replica worker; chat over the transcript (local models only). - ADR-034: optional two-engine mode (pianissimo + kb-whisper) reconciled by a LOCAL model; off unless TAPIR_RECONCILE_BASE_MODEL is set. - ADR-035: real progress (phases, running/quiet/lost), phase logging, every judged spot stored, merged segments. - Hardening from four adversarial reviews: errors logged and stored as kinds never text; per-engine deadlines; converted-length check; bounded queue; client hang-ups are not errors; strict local-alias check. - Dockerfile: static ffmpeg/ffprobe 7.0.2 pinned by sha256 (#34). Built and inspected by hand; CI does not build the image on a PR, so its first CI build is this merge. Uploads stay OFF until TAPIR_UPLOADS_ENABLED is set in the deployment. Closes #33, #34, #42, #43, #44. Co-authored-by: Mathias <mthbqv@gmail.com>
This commit was merged in pull request #39.
This commit is contained in:
@@ -0,0 +1,188 @@
|
||||
// Package transcribe turns a WAV file into a timed Swedish transcript by calling
|
||||
// the LiteLLM gateway's OpenAI-compatible /audio/transcriptions endpoint
|
||||
// (ADR-033). A Gateway is one model alias; a Chain tries a local one and falls
|
||||
// back to a second.
|
||||
package transcribe
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"git.d-ma.be/mathias/tapir/internal/domain"
|
||||
)
|
||||
|
||||
// ErrEmptyTranscript is returned when the endpoint answered 200 but with no text.
|
||||
// An empty transcript is not a success: chat over it could only answer "nothing".
|
||||
var ErrEmptyTranscript = errors.New("transcribe: endpoint returned an empty transcript")
|
||||
|
||||
// StatusError is a non-2xx answer from the gateway. Nothing may branch on the
|
||||
// body — {"error":"failed to process audio"} has had at least five distinct
|
||||
// causes — so only the status code is exposed for logic; Snippet is for logs.
|
||||
type StatusError struct {
|
||||
Code int
|
||||
Snippet string
|
||||
}
|
||||
|
||||
// HTTPStatus implements errkind.HTTPStatuser: the code, never the snippet.
|
||||
func (e *StatusError) HTTPStatus() int { return e.Code }
|
||||
|
||||
func (e *StatusError) Error() string {
|
||||
return fmt.Sprintf("transcribe: gateway returned HTTP %d: %s", e.Code, e.Snippet)
|
||||
}
|
||||
|
||||
// Gateway transcribes through one model alias on the gateway. baseURL is the
|
||||
// gateway's /v1 root — the in-cluster NodePort, not the public hostname, whose
|
||||
// ingress rejects bodies over ~8-10 MB with 413 (docs/research/audio-length-ceiling).
|
||||
type Gateway struct {
|
||||
baseURL string
|
||||
key string
|
||||
model string
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// NewGateway builds a Gateway. The client should carry no overall Timeout: a 90
|
||||
// minute recording takes minutes, so the deadline belongs to the caller's context.
|
||||
func NewGateway(baseURL, key, model string, client *http.Client) *Gateway {
|
||||
return &Gateway{baseURL: strings.TrimRight(baseURL, "/"), key: key, model: model, client: client}
|
||||
}
|
||||
|
||||
// Name is the model alias; it is what gets recorded as the serving endpoint.
|
||||
func (g *Gateway) Name() string { return g.model }
|
||||
|
||||
// Transcribe uploads the WAV at path and returns the normalised transcription.
|
||||
// The body is streamed from disk with a COMPUTED Content-Length: a 90 minute WAV
|
||||
// is ~170 MB, so it is neither buffered in memory nor sent chunked.
|
||||
func (g *Gateway) Transcribe(ctx context.Context, path string) (domain.Transcription, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return domain.Transcription{}, fmt.Errorf("transcribe: open audio: %w", err)
|
||||
}
|
||||
defer func() { _ = f.Close() }()
|
||||
info, err := f.Stat()
|
||||
if err != nil {
|
||||
return domain.Transcription{}, fmt.Errorf("transcribe: stat audio: %w", err)
|
||||
}
|
||||
|
||||
var prefix bytes.Buffer
|
||||
mw := multipart.NewWriter(&prefix)
|
||||
for _, kv := range [][2]string{{"model", g.model}, {"response_format", "verbose_json"}, {"language", "sv"}} {
|
||||
if err := mw.WriteField(kv[0], kv[1]); err != nil {
|
||||
return domain.Transcription{}, fmt.Errorf("transcribe: build request: %w", err)
|
||||
}
|
||||
}
|
||||
// The multipart field is `file`; `audio_file` is rejected as "Invalid request".
|
||||
if _, err := mw.CreateFormFile("file", "audio.wav"); err != nil {
|
||||
return domain.Transcription{}, fmt.Errorf("transcribe: build request: %w", err)
|
||||
}
|
||||
suffix := []byte("\r\n--" + mw.Boundary() + "--\r\n")
|
||||
|
||||
body := io.MultiReader(bytes.NewReader(prefix.Bytes()), f, bytes.NewReader(suffix))
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, g.baseURL+"/audio/transcriptions", body)
|
||||
if err != nil {
|
||||
return domain.Transcription{}, fmt.Errorf("transcribe: build request: %w", err)
|
||||
}
|
||||
req.ContentLength = int64(prefix.Len()) + info.Size() + int64(len(suffix))
|
||||
req.Header.Set("Content-Type", mw.FormDataContentType())
|
||||
req.Header.Set("Authorization", "Bearer "+g.key)
|
||||
// No "Expect: 100-continue": it trips the proxy for any body over 1 KB.
|
||||
|
||||
resp, err := g.client.Do(req)
|
||||
if err != nil {
|
||||
return domain.Transcription{}, fmt.Errorf("transcribe: %s: %w", g.model, err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode < 200 || resp.StatusCode > 299 {
|
||||
snip, _ := io.ReadAll(io.LimitReader(resp.Body, 200))
|
||||
return domain.Transcription{}, &StatusError{Code: resp.StatusCode, Snippet: strings.TrimSpace(string(snip))}
|
||||
}
|
||||
|
||||
t, err := parseVerbose(resp.Body)
|
||||
if err != nil {
|
||||
return domain.Transcription{}, fmt.Errorf("transcribe: %s: %w", g.model, err)
|
||||
}
|
||||
t.Endpoint = g.model
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// verboseJSON is the subset of the verbose_json response used. Both backends
|
||||
// return it; they differ in segment granularity (iguana: whole seconds and
|
||||
// sub-sentence pieces, berget: sub-second sentences), in the language token, and in
|
||||
// whether a segment's text starts with a space.
|
||||
type verboseJSON struct {
|
||||
Text string `json:"text"`
|
||||
Duration float64 `json:"duration"`
|
||||
Segments []struct {
|
||||
Start float64 `json:"start"`
|
||||
End float64 `json:"end"`
|
||||
Text string `json:"text"`
|
||||
} `json:"segments"`
|
||||
}
|
||||
|
||||
// parseVerbose normalises a verbose_json body. Whisper's final window can end past
|
||||
// the audio (9.0 s for 4.25 s of audio was observed), so segment ends are clamped
|
||||
// to the reported duration. Text is rebuilt from the segments, not taken from the
|
||||
// response's `text` field: see joinSegments.
|
||||
func parseVerbose(r io.Reader) (domain.Transcription, error) {
|
||||
var v verboseJSON
|
||||
if err := json.NewDecoder(r).Decode(&v); err != nil {
|
||||
return domain.Transcription{}, fmt.Errorf("decode response: %w", err)
|
||||
}
|
||||
raw := make([]string, 0, len(v.Segments))
|
||||
for _, s := range v.Segments {
|
||||
raw = append(raw, s.Text)
|
||||
}
|
||||
text := joinSegments(raw)
|
||||
if text == "" {
|
||||
text = collapseSpace(v.Text)
|
||||
}
|
||||
if text == "" {
|
||||
return domain.Transcription{}, ErrEmptyTranscript
|
||||
}
|
||||
t := domain.Transcription{Text: text, AudioSeconds: v.Duration}
|
||||
for _, s := range v.Segments {
|
||||
seg := domain.Segment{Start: s.Start, End: s.End, Text: strings.TrimSpace(s.Text)}
|
||||
if v.Duration > 0 {
|
||||
seg.End = min(seg.End, v.Duration)
|
||||
seg.Start = min(seg.Start, seg.End)
|
||||
}
|
||||
t.Segments = append(t.Segments, seg)
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// joinSegments rebuilds the running text from raw (untrimmed) segment texts.
|
||||
//
|
||||
// whisper.cpp wraps its output at ~60 characters and a wrap can land INSIDE a word
|
||||
// ("nyt" | "t godkännande"): a segment that begins with a space starts a new word,
|
||||
// one that does not continues the previous one, and the response's own `text`
|
||||
// field joins them with newlines, which breaks the word. So when any segment
|
||||
// carries a leading space the raw texts are concatenated as they are. Backends
|
||||
// that return whole sentences with no leading space (berget) are joined with a
|
||||
// space instead — concatenating those would glue sentences together.
|
||||
func joinSegments(raw []string) string {
|
||||
leadingSpace := false
|
||||
for _, s := range raw {
|
||||
if strings.HasPrefix(s, " ") {
|
||||
leadingSpace = true
|
||||
break
|
||||
}
|
||||
}
|
||||
sep := " "
|
||||
if leadingSpace {
|
||||
sep = ""
|
||||
}
|
||||
return collapseSpace(strings.Join(raw, sep))
|
||||
}
|
||||
|
||||
// collapseSpace turns every run of whitespace (including newlines) into one space.
|
||||
func collapseSpace(s string) string {
|
||||
return strings.Join(strings.Fields(s), " ")
|
||||
}
|
||||
Reference in New Issue
Block a user