tapir/test/acceptance/scenario_coverage_test.go

package acceptance

// This is the name-coverage gate for the BDD spec (see docs/use-cases/*.feature).
// There is no godog runner — the .feature files are design records, and the real
// behaviour is covered by the hand-written Go tests across the module. This test
// keeps the two from drifting in the cheapest honest way: every non-@pending
// Scenario must have an entry in scenarioCoverage pointing at a Go test that
// actually exists. It does NOT prove the test exercises the scenario (only godog
// could); it catches the common drift — "added a scenario, forgot the test", a
// renamed/deleted covering test, or a scenario removed without cleaning the map.
//
// When you add a Scenario: either map it here to its covering test, or tag it
// @pending in the .feature with a one-line reason for why it has no test yet.

import (
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"testing"
)

// scenarioCoverage maps each non-@pending Scenario name to the Go test that
// covers it. Keep it in sync with docs/use-cases/*.feature — the test below
// fails if a scenario is unmapped, a mapped test is missing, or an entry no
// longer matches a real non-pending scenario.
var scenarioCoverage = map[string]string{
	// ai_routing.feature
	"Local AI produces the summary":                                     "TestSummarize_LocalSucceeds",
	"Local AI fails and the user has a BYO provider configured":         "TestSummarize_FallsBackToBYO",
	"Local AI fails and the user has no BYO provider":                   "TestSummarize_LocalFailsNoBYO_NoExternalSend",
	"A user without BYO never has content sent externally":              "TestSummarize_NoBYO_ContentOnlyLocal",
	"A model returns unparseable output and the next endpoint succeeds": "TestSummarize_FallsBackOnMalformedOutput",

	// landing_page.feature
	"An unauthenticated visit to the root is sent to the welcome page":    "TestUnauthenticatedRootRedirectsToWelcome",
	"The welcome page invites an unauthenticated visitor to start":        "TestWelcomeLoggedOut",
	"An authenticated user on the welcome page sees their way in and out": "TestWelcomeLoggedIn",

	// connect_account.feature
	"Connect a YouTube account":                          "TestCallbackExchangesAndRecordsConnection",
	"Connecting an account discovers videos immediately": "TestCallbackTriggersDiscovery",
	"Connecting summarizes my newest videos right away":  "TestNewestUnsummarizedVideoIDs",
	"Tokens are never stored in the clear":               "TestCallbackExchangesAndRecordsConnection",

	// paste_url.feature
	"Paste a valid YouTube URL":                              "TestPasteValidURLAddsAndRequests",
	"Pasting an invalid link is rejected":                    "TestPasteInvalidURLRejected",
	"Pasting a video that cannot be found is honest":         "TestPasteVideoNotFound",
	"Pasting the same video twice does not duplicate it":     "TestPasteDedupNoDuplicate",
	"Revoking a connection stops watching but keeps history": "TestDisconnectRemovesTokenAndConnectionKeepsAccount",

	// summarize_mode.feature
	"Auto mode summarizes recent new videos automatically":           "TestRunOnce_AutoMode_SkipsOldVideos",
	"Auto mode lists older videos without summarizing them":          "TestRunOnce_AutoMode_SkipsOldVideos",
	"Automatic is the default for a new user":                        "TestRegisteredUserDefaultsAutoSummarizeOn",
	"Manual mode leaves new videos unsummarized":                     "TestRunOnce_ManualMode_SkipsUnrequested",
	"Requesting a summary in manual mode queues it for the next run": "TestRunOnce_ManualMode_ProcessesRequested",

	// registration.feature
	"A new Dex subject is routed to registration":                               "TestUnregisteredSubjectRedirectedToRegister",
	"Registering creates the account and its identity mapping":                  "TestRegisterCreatesExactlyOneUserAndIdentity",
	"A returning subject passes straight through":                               "TestRegisteredSubjectPassesThrough",
	"Deleting an account removes only my data and leaves other users untouched": "TestDeleteAccountWipesDataAndSecretsAndLogsOut",

	// chat_transcript.feature (ADR-027)
	"A summary view offers a deeper-dive into the video":         "TestChatEntryAffordanceOnSummaryView",
	"Ask a question answered from the stored transcript":         "TestChatAnswersFromStoredTranscriptWithoutAnyFetch",
	"Chat never fetches captions or reaches YouTube":             "TestChatAnswersFromStoredTranscriptWithoutAnyFetch",
	"A video with no stored transcript offers no chat":           "TestChatUnavailableWhenNoStoredTranscript",
	"The default model is the summary's model and is switchable": "TestChatModelSwitcherDefaultAndSwitch",
	"Switching models re-runs against the same transcript":       "TestChatModelSwitcherDefaultAndSwitch",
	"The cloud model is hidden when cloud is disabled":           "TestChatCloudModelAbsentWhenDisabled",
	"A long transcript is bounded and the chat says so":          "TestChatTruncationNoteShown",
	"A multi-turn conversation is ephemeral":                     "TestChatMultiTurnHistoryIsEphemeral",
	"Chat is reachable only from my own summary view":            "TestChatOnlyReachableForOwnVideo",

	// summarize_new_video.feature
	"A subscribed channel posts a video that has captions":         "TestSubscribedVideoWithCaptionsIsSummarizedAndDelivered",
	"A subscribed channel posts a video with no usable transcript": "TestVideoWithNoTranscriptIsSkipped",
	"A channel I am not subscribed to posts a video":               "TestUnsubscribedChannelVideoIsNotProcessed",
	"The same video is not summarized twice":                       "TestAlreadySummarizedVideoIsNotReprocessed",
	"Re-analyzing a stored video does not re-fetch its transcript": "TestProcessNewVideo_SecondSummarizeDoesNotRefetch",
}

var (
	scenarioRe = regexp.MustCompile(`^\s*Scenario(?: Outline)?:\s*(.+?)\s*$`)
	testFuncRe = regexp.MustCompile(`^func (Test\w+)\(`)
)

// scenario is one parsed Gherkin scenario and whether it is @pending.
type scenario struct {
	name    string
	pending bool
}

func TestScenarioCoverage(t *testing.T) {
	root := moduleRoot(t)

	scenarios := parseScenarios(t, filepath.Join(root, "docs", "use-cases"))
	if len(scenarios) == 0 {
		t.Fatal("no scenarios parsed from docs/use-cases — wrong path?")
	}
	tests := allTestFuncNames(t, root)

	// Index scenario names for the reverse (stale-entry) check.
	active := map[string]bool{} // non-pending scenario names
	var pending []string
	for _, s := range scenarios {
		if s.pending {
			pending = append(pending, s.name)
			continue
		}
		active[s.name] = true

		// 1. Every non-pending scenario must be mapped.
		fn, ok := scenarioCoverage[s.name]
		if !ok {
			t.Errorf("scenario %q has no coverage entry — map it in scenarioCoverage to a covering test, or tag it @pending in the .feature", s.name)
			continue
		}
		// 2. The mapped test must actually exist.
		if !tests[fn] {
			t.Errorf("scenario %q maps to %q, which is not a Test function anywhere in the module", s.name, fn)
		}
	}

	// 3. No stale entries: every map key must be a real, non-pending scenario.
	for name := range scenarioCoverage {
		if !active[name] {
			t.Errorf("scenarioCoverage has entry %q, which is not a current non-pending scenario (renamed, removed, or now @pending?)", name)
		}
	}

	if len(pending) > 0 {
		t.Logf("%d @pending scenario(s) without a test (tracked, not required): %s",
			len(pending), strings.Join(pending, "; "))
	}
}

// parseScenarios reads every *.feature under dir and returns its scenarios with
// their @pending status. A scenario is @pending when a `@pending` tag line
// precedes it (tags survive intervening comment lines, the layout these files
// use); the flag is consumed at the Scenario line and reset afterward.
func parseScenarios(t *testing.T, dir string) []scenario {
	t.Helper()
	entries, err := os.ReadDir(dir)
	if err != nil {
		t.Fatalf("read use-cases dir: %v", err)
	}
	var out []scenario
	for _, e := range entries {
		if e.IsDir() || !strings.HasSuffix(e.Name(), ".feature") {
			continue
		}
		b, err := os.ReadFile(filepath.Join(dir, e.Name()))
		if err != nil {
			t.Fatalf("read %s: %v", e.Name(), err)
		}
		pending := false
		for _, line := range strings.Split(string(b), "\n") {
			trimmed := strings.TrimSpace(line)
			if strings.HasPrefix(trimmed, "@") {
				if strings.Contains(trimmed, "@pending") {
					pending = true
				}
				continue
			}
			if m := scenarioRe.FindStringSubmatch(line); m != nil {
				out = append(out, scenario{name: m[1], pending: pending})
				pending = false
			}
			// comment (#) and step lines leave a set @pending intact until the
			// scenario consumes it; a blank line between scenarios is harmless.
		}
	}
	return out
}

// allTestFuncNames walks the module for `func TestXxx(` declarations, excluding
// this file (whose regex literal would otherwise look like a definition).
func allTestFuncNames(t *testing.T, root string) map[string]bool {
	t.Helper()
	self := "scenario_coverage_test.go"
	names := map[string]bool{}
	err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if d.IsDir() {
			if d.Name() == ".git" {
				return filepath.SkipDir
			}
			return nil
		}
		if !strings.HasSuffix(path, "_test.go") || filepath.Base(path) == self {
			return nil
		}
		b, err := os.ReadFile(path)
		if err != nil {
			return err
		}
		for _, line := range strings.Split(string(b), "\n") {
			if m := testFuncRe.FindStringSubmatch(line); m != nil {
				names[m[1]] = true
			}
		}
		return nil
	})
	if err != nil {
		t.Fatalf("walk module: %v", err)
	}
	return names
}

// moduleRoot walks up from the working directory to the dir containing go.mod.
func moduleRoot(t *testing.T) string {
	t.Helper()
	dir, err := os.Getwd()
	if err != nil {
		t.Fatalf("getwd: %v", err)
	}
	for {
		if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {
			return dir
		}
		parent := filepath.Dir(dir)
		if parent == dir {
			t.Fatal("go.mod not found walking up from cwd")
		}
		dir = parent
	}
}