feat(data): EUR/USD M1 fetch + daily realized-vol prep (toy slice, #2/#11)

fetch_data.py politely pulls EUR/USD M1 from histdata.com (maintained package
handles the anti-hotlink token; per-year, spaced). prepare_data.py (LOCKED per
Phase-1 contract) parses M1 -> daily series with realized_vol = the val_vol_r2
target (sqrt sum of squared intraday returns). Verified on 2019-2021: 937 days,
March-2020 COVID RV spike 5.1x over 2019 median — real signal, target works.
Data gitignored (DVC/MinIO = #10).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

.gitignore

@@ -28,3 +28,9 @@ go.work.sum
 # Project-specific
 bin/
 *.templ.go
+
+# python venv (autoresearch loop)
+.venv/
+
+# downloaded + processed market data (track via DVC/MinIO, #10 — not git)
+data/

requirements.txt

@@ -0,0 +1,8 @@
+# Python deps for the autoresearch loop (train.py + scripts). Install torch from
+# the cu130 index FIRST (koala Blackwell sm_120, torch 2.12.1+cu130 verified):
+#   pip install torch --index-url https://download.pytorch.org/whl/cu130
+#   pip install -r requirements.txt
+numpy>=2.0
+pandas>=2.2
+pyarrow>=16
+histdata>=1.3        # histdata.com downloader (handles the tk token politely)

scripts/check_gpu.py

@@ -0,0 +1,21 @@
+"""Phase-0 compute gate (brain wiki/jepa-fx/facts/autoresearch-integration-phase1):
+PyTorch cu130 must see the koala Blackwell GPU and compute before any experiment.
+
+    python scripts/check_gpu.py   # exits 0 if the GPU is usable, 1 otherwise
+
+Note: koala shares this 12GB card with the llama-swap LLM stack. The autoresearch
+agent should run on iguana/berget models so koala's GPU stays free for train.py.
+"""
+import sys
+import torch
+
+print("torch", torch.__version__)
+if not torch.cuda.is_available():
+    print("CUDA NOT AVAILABLE — gate BLOCKED")
+    sys.exit(1)
+print("device:", torch.cuda.get_device_name(0))
+print("capability: sm_%d%d" % torch.cuda.get_device_capability(0))
+x = torch.randn(2000, 2000, device="cuda")
+(x @ x).sum().item()
+torch.cuda.synchronize()
+print("GPU matmul OK — Phase-0 compute gate GREEN")

scripts/fetch_data.py

@@ -0,0 +1,31 @@
+"""Fetch EUR/USD M1 bars from histdata.com (free, research use).
+
+Polite: one request per year, spaced; past years query month=None. Uses the
+maintained `histdata` package which handles histdata's anti-hotlink tk token.
+Output: data/raw/DAT_ASCII_EURUSD_M1_<year>.zip
+
+    YEARS=2019,2020,2021 python scripts/fetch_data.py
+"""
+import os
+import time
+
+from histdata import download_hist_data
+from histdata.api import Platform as P, TimeFrame as T
+
+YEARS = [y.strip() for y in os.environ.get("YEARS", "2019,2020,2021").split(",")]
+
+
+def main():
+    os.makedirs("data/raw", exist_ok=True)
+    for yr in YEARS:
+        f = download_hist_data(
+            year=yr, month=None, pair="eurusd",
+            platform=P.GENERIC_ASCII, time_frame=T.ONE_MINUTE,
+            output_directory="data/raw",
+        )
+        print("fetched", yr, "->", f)
+        time.sleep(2)  # be a good citizen
+
+
+if __name__ == "__main__":
+    main()

scripts/prepare_data.py

@@ -0,0 +1,57 @@
+"""LOCKED data pipeline (toy) — agent must NOT edit (brain Phase-1 contract).
+
+Parses histdata EUR/USD M1 zips → daily series with realized volatility (the
+val_vol_r2 target = 1-day realized vol from intraday squared returns).
+Output: data/processed/eurusd_daily.parquet  [date, close, ret, realized_vol].
+"""
+import glob
+import os
+import zipfile
+
+import numpy as np
+import pandas as pd
+
+RAW = "data/raw"
+OUT = "data/processed/eurusd_daily.parquet"
+
+
+def load_m1() -> pd.DataFrame:
+    frames = []
+    for zp in sorted(glob.glob(os.path.join(RAW, "DAT_ASCII_EURUSD_M1_*.zip"))):
+        with zipfile.ZipFile(zp) as z:
+            csv = [n for n in z.namelist() if n.endswith(".csv")][0]
+            with z.open(csv) as f:
+                df = pd.read_csv(
+                    f, sep=";", header=None,
+                    names=["dt", "open", "high", "low", "close", "vol"],
+                )
+        df["ts"] = pd.to_datetime(df["dt"], format="%Y%m%d %H%M%S")
+        frames.append(df[["ts", "close"]])
+    out = pd.concat(frames).sort_values("ts").reset_index(drop=True)
+    return out
+
+
+def main():
+    m1 = load_m1()
+    m1["r"] = np.log(m1["close"]).diff()
+    m1["day"] = m1["ts"].dt.normalize()
+    daily = m1.groupby("day").agg(
+        close=("close", "last"),
+        realized_vol=("r", lambda x: np.sqrt(np.nansum(x.values ** 2))),
+        n_min=("r", "count"),
+    ).reset_index()
+    daily = daily[daily["n_min"] > 60]  # drop thin days (holidays)
+    daily["ret"] = np.log(daily["close"]).diff()
+    daily = daily.dropna().reset_index(drop=True)
+    os.makedirs(os.path.dirname(OUT), exist_ok=True)
+    daily[["day", "close", "ret", "realized_vol"]].rename(columns={"day": "date"}).to_parquet(OUT)
+    print("rows:", len(daily), "| dates:", daily["day"].min().date(), "→", daily["day"].max().date())
+    # sanity: the COVID crash (March 2020) must show a realized-vol spike
+    rv = daily.set_index("day")["realized_vol"]
+    mar20 = rv["2020-03-01":"2020-03-31"].max()
+    typ = rv["2019-01-01":"2019-12-31"].median()
+    print("median 2019 RV: %.5f | max Mar-2020 RV: %.5f | spike x%.1f" % (typ, mar20, mar20 / typ))
+
+
+if __name__ == "__main__":
+    main()