mathias
a8b438db2d
- connect_account.feature: refine the burst scenario to "best recent" (likely-good selection, skips too-short/too-long) and add a scenario for the stronger model. - scenario_coverage: remap to TestOnboardBurstVideoIDs + add the model scenario (the old NewestUnsummarizedVideoIDs test was removed). - architecture.md: new "Connect-time onboarding burst" subsection — junk-avoiding selection over persisted duration, the burst-only stronger-model chain, and why has-captions/cached-first are not selection signals.
236 lines
10 KiB
Go
236 lines
10 KiB
Go
package acceptance
|
|
|
|
// This is the name-coverage gate for the BDD spec (see docs/use-cases/*.feature).
|
|
// There is no godog runner — the .feature files are design records, and the real
|
|
// behaviour is covered by the hand-written Go tests across the module. This test
|
|
// keeps the two from drifting in the cheapest honest way: every non-@pending
|
|
// Scenario must have an entry in scenarioCoverage pointing at a Go test that
|
|
// actually exists. It does NOT prove the test exercises the scenario (only godog
|
|
// could); it catches the common drift — "added a scenario, forgot the test", a
|
|
// renamed/deleted covering test, or a scenario removed without cleaning the map.
|
|
//
|
|
// When you add a Scenario: either map it here to its covering test, or tag it
|
|
// @pending in the .feature with a one-line reason for why it has no test yet.
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// scenarioCoverage maps each non-@pending Scenario name to the Go test that
|
|
// covers it. Keep it in sync with docs/use-cases/*.feature — the test below
|
|
// fails if a scenario is unmapped, a mapped test is missing, or an entry no
|
|
// longer matches a real non-pending scenario.
|
|
var scenarioCoverage = map[string]string{
|
|
// ai_routing.feature
|
|
"Local AI produces the summary": "TestSummarize_LocalSucceeds",
|
|
"Local AI fails and the user has a BYO provider configured": "TestSummarize_FallsBackToBYO",
|
|
"Local AI fails and the user has no BYO provider": "TestSummarize_LocalFailsNoBYO_NoExternalSend",
|
|
"A user without BYO never has content sent externally": "TestSummarize_NoBYO_ContentOnlyLocal",
|
|
"A model returns unparseable output and the next endpoint succeeds": "TestSummarize_FallsBackOnMalformedOutput",
|
|
|
|
// landing_page.feature
|
|
"An unauthenticated visit to the root is sent to the welcome page": "TestUnauthenticatedRootRedirectsToWelcome",
|
|
"The welcome page invites an unauthenticated visitor to start": "TestWelcomeLoggedOut",
|
|
"An authenticated user on the welcome page sees their way in and out": "TestWelcomeLoggedIn",
|
|
|
|
// connect_account.feature
|
|
"Connect a YouTube account": "TestCallbackExchangesAndRecordsConnection",
|
|
"Connecting an account discovers videos immediately": "TestCallbackTriggersDiscovery",
|
|
"Connecting summarizes my best recent videos right away": "TestOnboardBurstVideoIDs",
|
|
"The onboarding burst summarizes with a stronger model": "TestBurstChainModelsLeadsWithOnboardModel",
|
|
"Tokens are never stored in the clear": "TestCallbackExchangesAndRecordsConnection",
|
|
|
|
// paste_url.feature
|
|
"Paste a valid YouTube URL": "TestPasteValidURLAddsAndRequests",
|
|
"Pasting an invalid link is rejected": "TestPasteInvalidURLRejected",
|
|
"Pasting a video that cannot be found is honest": "TestPasteVideoNotFound",
|
|
"Pasting the same video twice does not duplicate it": "TestPasteDedupNoDuplicate",
|
|
"Revoking a connection stops watching but keeps history": "TestDisconnectRemovesTokenAndConnectionKeepsAccount",
|
|
|
|
// summarize_mode.feature
|
|
"Auto mode summarizes recent new videos automatically": "TestRunOnce_AutoMode_SkipsOldVideos",
|
|
"Auto mode lists older videos without summarizing them": "TestRunOnce_AutoMode_SkipsOldVideos",
|
|
"Automatic is the default for a new user": "TestRegisteredUserDefaultsAutoSummarizeOn",
|
|
"Manual mode leaves new videos unsummarized": "TestRunOnce_ManualMode_SkipsUnrequested",
|
|
"Requesting a summary in manual mode queues it for the next run": "TestRunOnce_ManualMode_ProcessesRequested",
|
|
|
|
// registration.feature
|
|
"A new Dex subject is routed to registration": "TestUnregisteredSubjectRedirectedToRegister",
|
|
"Registering creates the account and its identity mapping": "TestRegisterCreatesExactlyOneUserAndIdentity",
|
|
"A returning subject passes straight through": "TestRegisteredSubjectPassesThrough",
|
|
"Deleting an account removes only my data and leaves other users untouched": "TestDeleteAccountWipesDataAndSecretsAndLogsOut",
|
|
|
|
// chat_transcript.feature (ADR-027)
|
|
"A summary view offers a deeper-dive into the video": "TestChatEntryAffordanceOnSummaryView",
|
|
"The summary and the chat are on one page": "TestChatIntegratedWithSummaryOnSamePage",
|
|
"Ask a question answered from the stored transcript": "TestChatAnswersFromStoredTranscriptWithoutAnyFetch",
|
|
"Chat never fetches captions or reaches YouTube": "TestChatAnswersFromStoredTranscriptWithoutAnyFetch",
|
|
"A video with no stored transcript offers no chat": "TestChatUnavailableWhenNoStoredTranscript",
|
|
"The default model is the summary's model and is switchable": "TestChatModelSwitcherDefaultAndSwitch",
|
|
"Switching models re-runs against the same transcript": "TestChatModelSwitcherDefaultAndSwitch",
|
|
"The cloud model is hidden when cloud is disabled": "TestChatCloudModelAbsentWhenDisabled",
|
|
"A long transcript is bounded and the chat says so": "TestChatTruncationNoteShown",
|
|
"A multi-turn conversation is ephemeral": "TestChatMultiTurnHistoryIsEphemeral",
|
|
"Chat is reachable only from my own summary view": "TestChatOnlyReachableForOwnVideo",
|
|
|
|
// summarize_new_video.feature
|
|
"A subscribed channel posts a video that has captions": "TestSubscribedVideoWithCaptionsIsSummarizedAndDelivered",
|
|
"A subscribed channel posts a video with no usable transcript": "TestVideoWithNoTranscriptIsSkipped",
|
|
"A channel I am not subscribed to posts a video": "TestUnsubscribedChannelVideoIsNotProcessed",
|
|
"The same video is not summarized twice": "TestAlreadySummarizedVideoIsNotReprocessed",
|
|
"Re-analyzing a stored video does not re-fetch its transcript": "TestProcessNewVideo_SecondSummarizeDoesNotRefetch",
|
|
}
|
|
|
|
var (
|
|
scenarioRe = regexp.MustCompile(`^\s*Scenario(?: Outline)?:\s*(.+?)\s*$`)
|
|
testFuncRe = regexp.MustCompile(`^func (Test\w+)\(`)
|
|
)
|
|
|
|
// scenario is one parsed Gherkin scenario and whether it is @pending.
|
|
type scenario struct {
|
|
name string
|
|
pending bool
|
|
}
|
|
|
|
func TestScenarioCoverage(t *testing.T) {
|
|
root := moduleRoot(t)
|
|
|
|
scenarios := parseScenarios(t, filepath.Join(root, "docs", "use-cases"))
|
|
if len(scenarios) == 0 {
|
|
t.Fatal("no scenarios parsed from docs/use-cases — wrong path?")
|
|
}
|
|
tests := allTestFuncNames(t, root)
|
|
|
|
// Index scenario names for the reverse (stale-entry) check.
|
|
active := map[string]bool{} // non-pending scenario names
|
|
var pending []string
|
|
for _, s := range scenarios {
|
|
if s.pending {
|
|
pending = append(pending, s.name)
|
|
continue
|
|
}
|
|
active[s.name] = true
|
|
|
|
// 1. Every non-pending scenario must be mapped.
|
|
fn, ok := scenarioCoverage[s.name]
|
|
if !ok {
|
|
t.Errorf("scenario %q has no coverage entry — map it in scenarioCoverage to a covering test, or tag it @pending in the .feature", s.name)
|
|
continue
|
|
}
|
|
// 2. The mapped test must actually exist.
|
|
if !tests[fn] {
|
|
t.Errorf("scenario %q maps to %q, which is not a Test function anywhere in the module", s.name, fn)
|
|
}
|
|
}
|
|
|
|
// 3. No stale entries: every map key must be a real, non-pending scenario.
|
|
for name := range scenarioCoverage {
|
|
if !active[name] {
|
|
t.Errorf("scenarioCoverage has entry %q, which is not a current non-pending scenario (renamed, removed, or now @pending?)", name)
|
|
}
|
|
}
|
|
|
|
if len(pending) > 0 {
|
|
t.Logf("%d @pending scenario(s) without a test (tracked, not required): %s",
|
|
len(pending), strings.Join(pending, "; "))
|
|
}
|
|
}
|
|
|
|
// parseScenarios reads every *.feature under dir and returns its scenarios with
|
|
// their @pending status. A scenario is @pending when a `@pending` tag line
|
|
// precedes it (tags survive intervening comment lines, the layout these files
|
|
// use); the flag is consumed at the Scenario line and reset afterward.
|
|
func parseScenarios(t *testing.T, dir string) []scenario {
|
|
t.Helper()
|
|
entries, err := os.ReadDir(dir)
|
|
if err != nil {
|
|
t.Fatalf("read use-cases dir: %v", err)
|
|
}
|
|
var out []scenario
|
|
for _, e := range entries {
|
|
if e.IsDir() || !strings.HasSuffix(e.Name(), ".feature") {
|
|
continue
|
|
}
|
|
b, err := os.ReadFile(filepath.Join(dir, e.Name()))
|
|
if err != nil {
|
|
t.Fatalf("read %s: %v", e.Name(), err)
|
|
}
|
|
pending := false
|
|
for _, line := range strings.Split(string(b), "\n") {
|
|
trimmed := strings.TrimSpace(line)
|
|
if strings.HasPrefix(trimmed, "@") {
|
|
if strings.Contains(trimmed, "@pending") {
|
|
pending = true
|
|
}
|
|
continue
|
|
}
|
|
if m := scenarioRe.FindStringSubmatch(line); m != nil {
|
|
out = append(out, scenario{name: m[1], pending: pending})
|
|
pending = false
|
|
}
|
|
// comment (#) and step lines leave a set @pending intact until the
|
|
// scenario consumes it; a blank line between scenarios is harmless.
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// allTestFuncNames walks the module for `func TestXxx(` declarations, excluding
|
|
// this file (whose regex literal would otherwise look like a definition).
|
|
func allTestFuncNames(t *testing.T, root string) map[string]bool {
|
|
t.Helper()
|
|
self := "scenario_coverage_test.go"
|
|
names := map[string]bool{}
|
|
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if d.IsDir() {
|
|
if d.Name() == ".git" {
|
|
return filepath.SkipDir
|
|
}
|
|
return nil
|
|
}
|
|
if !strings.HasSuffix(path, "_test.go") || filepath.Base(path) == self {
|
|
return nil
|
|
}
|
|
b, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, line := range strings.Split(string(b), "\n") {
|
|
if m := testFuncRe.FindStringSubmatch(line); m != nil {
|
|
names[m[1]] = true
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("walk module: %v", err)
|
|
}
|
|
return names
|
|
}
|
|
|
|
// moduleRoot walks up from the working directory to the dir containing go.mod.
|
|
func moduleRoot(t *testing.T) string {
|
|
t.Helper()
|
|
dir, err := os.Getwd()
|
|
if err != nil {
|
|
t.Fatalf("getwd: %v", err)
|
|
}
|
|
for {
|
|
if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {
|
|
return dir
|
|
}
|
|
parent := filepath.Dir(dir)
|
|
if parent == dir {
|
|
t.Fatal("go.mod not found walking up from cwd")
|
|
}
|
|
dir = parent
|
|
}
|
|
}
|