jepa-fx-risk/tests/test_autoresearch_start.py

"""Tests for scripts/autoresearch_start.py — jepa-fx-risk#11 Phase A scaffold.

Success criterion: `autoresearch start <backlog.json> <rq-id>` scaffolds a
runnable run dir from a ready leaf; refuses non-ready nodes; strips
candidate_metric; records provenance.
"""

import importlib.util
import json
import sys
from pathlib import Path

import pytest

# Load the module without executing main()
_SCRIPT = Path(__file__).parent.parent / "scripts" / "autoresearch_start.py"


def _import():
    spec = importlib.util.spec_from_file_location("autoresearch_start", _SCRIPT)
    mod = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(mod)
    return mod


@pytest.fixture()
def mod():
    return _import()


@pytest.fixture()
def backlog(tmp_path):
    data = {
        "strategic_question": "Test strategic question?",
        "generated_at": "2026-06-27T00:00:00Z",
        "nodes": [
            {
                "id": "rq-01",
                "question": "Does X improve Y?",
                "case_type": "autoresearch-loop",
                "data": "obtainable",
                "method": "adjacent",
                "falsifiable": "yes",
                "candidate_metric": " val_vol_r2",  # leading space — bypass test
                "depends_on": [],
                "status": "autoresearch-ready",
                "track": "autoresearch",
                "converged": True,
                "survived_review": True,
            },
            {
                "id": "rq-02",
                "question": "Not ready yet?",
                "case_type": "empirical-study",
                "data": "obtainable",
                "method": "adjacent",
                "falsifiable": "yes",
                "candidate_metric": None,
                "depends_on": [],
                "status": "needs-metric",
                "track": "study",
                "converged": True,
                "survived_review": True,
            },
            {
                "id": "rq-03",
                "question": "A spike.",
                "case_type": "spike",
                "data": "have",
                "method": "yes-named",
                "falsifiable": "yes",
                "candidate_metric": None,
                "depends_on": [],
                "status": "spike-ready",
                "track": "spike",
                "converged": True,
                "survived_review": True,
            },
        ],
    }
    p = tmp_path / "backlog.json"
    p.write_text(json.dumps(data))
    return p


@pytest.fixture()
def fake_train_py(tmp_path):
    """Minimal train.py placeholder for scaffold tests."""
    src = tmp_path / "train_template.py"
    src.write_text("# train.py placeholder\n")
    return src


# ---------------------------------------------------------------------------
# fail-closed: refuse non-autoresearch-ready nodes
# ---------------------------------------------------------------------------

class TestRefuseNonReady:
    def test_refuses_needs_metric(self, mod, backlog, fake_train_py, tmp_path):
        run_dir = tmp_path / "runs" / "rq-02"
        with pytest.raises(SystemExit) as exc:
            mod.scaffold_run(backlog, "rq-02", run_dir, fake_train_py)
        assert exc.value.code != 0

    def test_refuses_spike_ready(self, mod, backlog, fake_train_py, tmp_path):
        run_dir = tmp_path / "runs" / "rq-03"
        with pytest.raises(SystemExit) as exc:
            mod.scaffold_run(backlog, "rq-03", run_dir, fake_train_py)
        assert exc.value.code != 0

    def test_refuses_missing_rq_id(self, mod, backlog, fake_train_py, tmp_path):
        run_dir = tmp_path / "runs" / "rq-99"
        with pytest.raises(SystemExit) as exc:
            mod.scaffold_run(backlog, "rq-99", run_dir, fake_train_py)
        assert exc.value.code != 0

    def test_refuses_existing_run_dir(self, mod, backlog, fake_train_py, tmp_path):
        run_dir = tmp_path / "runs" / "rq-01"
        run_dir.mkdir(parents=True)
        with pytest.raises(SystemExit) as exc:
            mod.scaffold_run(backlog, "rq-01", run_dir, fake_train_py)
        assert exc.value.code != 0


# ---------------------------------------------------------------------------
# scaffold structure: correct files created
# ---------------------------------------------------------------------------

class TestScaffoldStructure:
    @pytest.fixture(autouse=True)
    def _scaffold(self, mod, backlog, fake_train_py, tmp_path):
        self.run_dir = tmp_path / "runs" / "rq-01"
        mod.scaffold_run(backlog, "rq-01", self.run_dir, fake_train_py)

    def test_run_dir_created(self):
        assert self.run_dir.is_dir()

    def test_program_md_created(self):
        assert (self.run_dir / "program.md").exists()

    def test_run_json_created(self):
        assert (self.run_dir / "run.json").exists()

    def test_train_py_copied(self):
        assert (self.run_dir / "train.py").exists()
        assert (self.run_dir / "train.py").read_text() == "# train.py placeholder\n"


# ---------------------------------------------------------------------------
# program.md content
# ---------------------------------------------------------------------------

class TestProgramMd:
    @pytest.fixture(autouse=True)
    def _scaffold(self, mod, backlog, fake_train_py, tmp_path):
        self.run_dir = tmp_path / "runs" / "rq-01"
        mod.scaffold_run(backlog, "rq-01", self.run_dir, fake_train_py)
        self.content = (self.run_dir / "program.md").read_text()

    def test_contains_hypothesis(self):
        assert "Does X improve Y?" in self.content

    def test_metric_key_stripped(self):
        # candidate_metric had leading space " val_vol_r2" — must be stripped
        assert "`val_vol_r2`" in self.content
        assert "` val_vol_r2`" not in self.content

    def test_contains_strategic_question(self):
        assert "Test strategic question?" in self.content

    def test_contains_council_node(self):
        assert "rq-01" in self.content


# ---------------------------------------------------------------------------
# run.json provenance
# ---------------------------------------------------------------------------

class TestRunJson:
    @pytest.fixture(autouse=True)
    def _scaffold(self, mod, backlog, fake_train_py, tmp_path):
        self.run_dir = tmp_path / "runs" / "rq-01"
        mod.scaffold_run(backlog, "rq-01", self.run_dir, fake_train_py)
        self.run = json.loads((self.run_dir / "run.json").read_text())

    def test_strategic_question_in_provenance(self):
        assert self.run["strategic_question"] == "Test strategic question?"

    def test_council_node_in_provenance(self):
        assert self.run["council_node"] == "rq-01"

    def test_metric_stripped_in_provenance(self):
        assert self.run["metric"] == "val_vol_r2"
        assert self.run["metric"] == self.run["metric"].strip()

    def test_generated_at_present(self):
        assert "generated_at" in self.run

    def test_max_iters_present(self):
        assert "max_iters" in self.run


# ---------------------------------------------------------------------------
# load_backlog helper
# ---------------------------------------------------------------------------

class TestLoadBacklog:
    def test_loads_json(self, mod, backlog):
        data = mod.load_backlog(str(backlog))
        assert data["strategic_question"] == "Test strategic question?"
        assert len(data["nodes"]) == 3

    def test_missing_file_raises(self, mod, tmp_path):
        with pytest.raises((FileNotFoundError, SystemExit)):
            mod.load_backlog(str(tmp_path / "nonexistent.json"))