From 485fdaa9f9fb7a870d1896a974d04b2863c09bb6 Mon Sep 17 00:00:00 2001
From: Mathias <mthbqv@gmail.com>
Date: Tue, 23 Jun 2026 16:49:59 +0200
Subject: [PATCH] feat(data): EUR/USD M1 fetch + daily realized-vol prep (toy
 slice, #2/#11)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

fetch_data.py politely pulls EUR/USD M1 from histdata.com (maintained package
handles the anti-hotlink token; per-year, spaced). prepare_data.py (LOCKED per
Phase-1 contract) parses M1 -> daily series with realized_vol = the val_vol_r2
target (sqrt sum of squared intraday returns). Verified on 2019-2021: 937 days,
March-2020 COVID RV spike 5.1x over 2019 median — real signal, target works.
Data gitignored (DVC/MinIO = #10).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 .gitignore              |  3 +++
 requirements.txt        |  8 +++---
 scripts/fetch_data.py   | 31 ++++++++++++++++++++++
 scripts/prepare_data.py | 57 +++++++++++++++++++++++++++++++++++++++++
 4 files changed, 96 insertions(+), 3 deletions(-)
 create mode 100644 scripts/fetch_data.py
 create mode 100644 scripts/prepare_data.py

diff --git a/.gitignore b/.gitignore
index 255061a..ef8c4cc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -31,3 +31,6 @@ bin/
 
 # python venv (autoresearch loop)
 .venv/
+
+# downloaded + processed market data (track via DVC/MinIO, #10 — not git)
+data/
diff --git a/requirements.txt b/requirements.txt
index 6d2b8ef..2a1a6f2 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,8 @@
-# Python deps for the autoresearch loop (train.py + scripts). The Go side
-# (data pipeline, eval harness) is separate. Install torch from the cu130 index:
+# Python deps for the autoresearch loop (train.py + scripts). Install torch from
+# the cu130 index FIRST (koala Blackwell sm_120, torch 2.12.1+cu130 verified):
 #   pip install torch --index-url https://download.pytorch.org/whl/cu130
 #   pip install -r requirements.txt
-# koala = Blackwell sm_120, driver R610; torch 2.12.1+cu130 verified 2026-06-23.
 numpy>=2.0
+pandas>=2.2
+pyarrow>=16
+histdata>=1.3        # histdata.com downloader (handles the tk token politely)
diff --git a/scripts/fetch_data.py b/scripts/fetch_data.py
new file mode 100644
index 0000000..343af2f
--- /dev/null
+++ b/scripts/fetch_data.py
@@ -0,0 +1,31 @@
+"""Fetch EUR/USD M1 bars from histdata.com (free, research use).
+
+Polite: one request per year, spaced; past years query month=None. Uses the
+maintained `histdata` package which handles histdata's anti-hotlink tk token.
+Output: data/raw/DAT_ASCII_EURUSD_M1_<year>.zip
+
+    YEARS=2019,2020,2021 python scripts/fetch_data.py
+"""
+import os
+import time
+
+from histdata import download_hist_data
+from histdata.api import Platform as P, TimeFrame as T
+
+YEARS = [y.strip() for y in os.environ.get("YEARS", "2019,2020,2021").split(",")]
+
+
+def main():
+    os.makedirs("data/raw", exist_ok=True)
+    for yr in YEARS:
+        f = download_hist_data(
+            year=yr, month=None, pair="eurusd",
+            platform=P.GENERIC_ASCII, time_frame=T.ONE_MINUTE,
+            output_directory="data/raw",
+        )
+        print("fetched", yr, "->", f)
+        time.sleep(2)  # be a good citizen
+
+
+if __name__ == "__main__":
+    main()
diff --git a/scripts/prepare_data.py b/scripts/prepare_data.py
new file mode 100644
index 0000000..cc2e717
--- /dev/null
+++ b/scripts/prepare_data.py
@@ -0,0 +1,57 @@
+"""LOCKED data pipeline (toy) — agent must NOT edit (brain Phase-1 contract).
+
+Parses histdata EUR/USD M1 zips → daily series with realized volatility (the
+val_vol_r2 target = 1-day realized vol from intraday squared returns).
+Output: data/processed/eurusd_daily.parquet  [date, close, ret, realized_vol].
+"""
+import glob
+import os
+import zipfile
+
+import numpy as np
+import pandas as pd
+
+RAW = "data/raw"
+OUT = "data/processed/eurusd_daily.parquet"
+
+
+def load_m1() -> pd.DataFrame:
+    frames = []
+    for zp in sorted(glob.glob(os.path.join(RAW, "DAT_ASCII_EURUSD_M1_*.zip"))):
+        with zipfile.ZipFile(zp) as z:
+            csv = [n for n in z.namelist() if n.endswith(".csv")][0]
+            with z.open(csv) as f:
+                df = pd.read_csv(
+                    f, sep=";", header=None,
+                    names=["dt", "open", "high", "low", "close", "vol"],
+                )
+        df["ts"] = pd.to_datetime(df["dt"], format="%Y%m%d %H%M%S")
+        frames.append(df[["ts", "close"]])
+    out = pd.concat(frames).sort_values("ts").reset_index(drop=True)
+    return out
+
+
+def main():
+    m1 = load_m1()
+    m1["r"] = np.log(m1["close"]).diff()
+    m1["day"] = m1["ts"].dt.normalize()
+    daily = m1.groupby("day").agg(
+        close=("close", "last"),
+        realized_vol=("r", lambda x: np.sqrt(np.nansum(x.values ** 2))),
+        n_min=("r", "count"),
+    ).reset_index()
+    daily = daily[daily["n_min"] > 60]  # drop thin days (holidays)
+    daily["ret"] = np.log(daily["close"]).diff()
+    daily = daily.dropna().reset_index(drop=True)
+    os.makedirs(os.path.dirname(OUT), exist_ok=True)
+    daily[["day", "close", "ret", "realized_vol"]].rename(columns={"day": "date"}).to_parquet(OUT)
+    print("rows:", len(daily), "| dates:", daily["day"].min().date(), "→", daily["day"].max().date())
+    # sanity: the COVID crash (March 2020) must show a realized-vol spike
+    rv = daily.set_index("day")["realized_vol"]
+    mar20 = rv["2020-03-01":"2020-03-31"].max()
+    typ = rv["2019-01-01":"2019-12-31"].median()
+    print("median 2019 RV: %.5f | max Mar-2020 RV: %.5f | spike x%.1f" % (typ, mar20, mar20 / typ))
+
+
+if __name__ == "__main__":
+    main()