"""Tests for scripts/autoresearch_start.py — jepa-fx-risk#11 Phase A scaffold. Success criterion: `autoresearch start ` scaffolds a runnable run dir from a ready leaf; refuses non-ready nodes; strips candidate_metric; records provenance. """ import importlib.util import json import sys from pathlib import Path import pytest # Load the module without executing main() _SCRIPT = Path(__file__).parent.parent / "scripts" / "autoresearch_start.py" def _import(): spec = importlib.util.spec_from_file_location("autoresearch_start", _SCRIPT) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) return mod @pytest.fixture() def mod(): return _import() @pytest.fixture() def backlog(tmp_path): data = { "strategic_question": "Test strategic question?", "generated_at": "2026-06-27T00:00:00Z", "nodes": [ { "id": "rq-01", "question": "Does X improve Y?", "case_type": "autoresearch-loop", "data": "obtainable", "method": "adjacent", "falsifiable": "yes", "candidate_metric": " val_vol_r2", # leading space — bypass test "depends_on": [], "status": "autoresearch-ready", "track": "autoresearch", "converged": True, "survived_review": True, }, { "id": "rq-02", "question": "Not ready yet?", "case_type": "empirical-study", "data": "obtainable", "method": "adjacent", "falsifiable": "yes", "candidate_metric": None, "depends_on": [], "status": "needs-metric", "track": "study", "converged": True, "survived_review": True, }, { "id": "rq-03", "question": "A spike.", "case_type": "spike", "data": "have", "method": "yes-named", "falsifiable": "yes", "candidate_metric": None, "depends_on": [], "status": "spike-ready", "track": "spike", "converged": True, "survived_review": True, }, ], } p = tmp_path / "backlog.json" p.write_text(json.dumps(data)) return p @pytest.fixture() def fake_train_py(tmp_path): """Minimal train.py placeholder for scaffold tests.""" src = tmp_path / "train_template.py" src.write_text("# train.py placeholder\n") return src # --------------------------------------------------------------------------- # fail-closed: refuse non-autoresearch-ready nodes # --------------------------------------------------------------------------- class TestRefuseNonReady: def test_refuses_needs_metric(self, mod, backlog, fake_train_py, tmp_path): run_dir = tmp_path / "runs" / "rq-02" with pytest.raises(SystemExit) as exc: mod.scaffold_run(backlog, "rq-02", run_dir, fake_train_py) assert exc.value.code != 0 def test_refuses_spike_ready(self, mod, backlog, fake_train_py, tmp_path): run_dir = tmp_path / "runs" / "rq-03" with pytest.raises(SystemExit) as exc: mod.scaffold_run(backlog, "rq-03", run_dir, fake_train_py) assert exc.value.code != 0 def test_refuses_missing_rq_id(self, mod, backlog, fake_train_py, tmp_path): run_dir = tmp_path / "runs" / "rq-99" with pytest.raises(SystemExit) as exc: mod.scaffold_run(backlog, "rq-99", run_dir, fake_train_py) assert exc.value.code != 0 def test_refuses_existing_run_dir(self, mod, backlog, fake_train_py, tmp_path): run_dir = tmp_path / "runs" / "rq-01" run_dir.mkdir(parents=True) with pytest.raises(SystemExit) as exc: mod.scaffold_run(backlog, "rq-01", run_dir, fake_train_py) assert exc.value.code != 0 # --------------------------------------------------------------------------- # scaffold structure: correct files created # --------------------------------------------------------------------------- class TestScaffoldStructure: @pytest.fixture(autouse=True) def _scaffold(self, mod, backlog, fake_train_py, tmp_path): self.run_dir = tmp_path / "runs" / "rq-01" mod.scaffold_run(backlog, "rq-01", self.run_dir, fake_train_py) def test_run_dir_created(self): assert self.run_dir.is_dir() def test_program_md_created(self): assert (self.run_dir / "program.md").exists() def test_run_json_created(self): assert (self.run_dir / "run.json").exists() def test_train_py_copied(self): assert (self.run_dir / "train.py").exists() assert (self.run_dir / "train.py").read_text() == "# train.py placeholder\n" # --------------------------------------------------------------------------- # program.md content # --------------------------------------------------------------------------- class TestProgramMd: @pytest.fixture(autouse=True) def _scaffold(self, mod, backlog, fake_train_py, tmp_path): self.run_dir = tmp_path / "runs" / "rq-01" mod.scaffold_run(backlog, "rq-01", self.run_dir, fake_train_py) self.content = (self.run_dir / "program.md").read_text() def test_contains_hypothesis(self): assert "Does X improve Y?" in self.content def test_metric_key_stripped(self): # candidate_metric had leading space " val_vol_r2" — must be stripped assert "`val_vol_r2`" in self.content assert "` val_vol_r2`" not in self.content def test_contains_strategic_question(self): assert "Test strategic question?" in self.content def test_contains_council_node(self): assert "rq-01" in self.content # --------------------------------------------------------------------------- # run.json provenance # --------------------------------------------------------------------------- class TestRunJson: @pytest.fixture(autouse=True) def _scaffold(self, mod, backlog, fake_train_py, tmp_path): self.run_dir = tmp_path / "runs" / "rq-01" mod.scaffold_run(backlog, "rq-01", self.run_dir, fake_train_py) self.run = json.loads((self.run_dir / "run.json").read_text()) def test_strategic_question_in_provenance(self): assert self.run["strategic_question"] == "Test strategic question?" def test_council_node_in_provenance(self): assert self.run["council_node"] == "rq-01" def test_metric_stripped_in_provenance(self): assert self.run["metric"] == "val_vol_r2" assert self.run["metric"] == self.run["metric"].strip() def test_generated_at_present(self): assert "generated_at" in self.run def test_max_iters_present(self): assert "max_iters" in self.run # --------------------------------------------------------------------------- # load_backlog helper # --------------------------------------------------------------------------- class TestLoadBacklog: def test_loads_json(self, mod, backlog): data = mod.load_backlog(str(backlog)) assert data["strategic_question"] == "Test strategic question?" assert len(data["nodes"]) == 3 def test_missing_file_raises(self, mod, tmp_path): with pytest.raises((FileNotFoundError, SystemExit)): mod.load_backlog(str(tmp_path / "nonexistent.json"))