feat(multipair): G10 multi-pair pipeline + USE_MULTIPAIR knob (Option C)

- prepare_hourly.py: parameterize PAIR env var; OUT_DEFAULT per-pair; load_m1_from_zips(pair=)
- prepare_multipair.py: inner-join 5-pair hourly parquets on datetime → wide parquet
  cols: datetime, {pair}_ret, {pair}_rv × n_pairs; eurusd_rv = target
- fetch_multipair.py: download GBPUSD/USDJPY/USDCHF/AUDUSD M1 2008-2023 from histdata
- train.py: USE_MULTIPAIR knob (JEPA_USE_MULTIPAIR=1); build() reads multipair parquet
  with n_channels = n_pairs × 2; target = eurusd_rv
- Taskfile: data:fetch:multipair, data:prepare:pair, data:prepare:multipair, data:test updated
- 7 new tests in test_multipair.py; 34/35 pass (1 SKIP until multipair parquet built)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-06-26 14:00:57 +02:00
co-authored by Claude Sonnet 4.6
parent b2bc01ba9e
commit bd8962f997
6 changed files with 281 additions and 11 deletions
+72
View File
@@ -0,0 +1,72 @@
"""Merge per-pair hourly parquets into a single wide multipair parquet.
Each pair contributes two features: {pair}_ret and {pair}_rv (realized vol).
The merge is an INNER JOIN on datetime — only hours present in ALL pairs are kept.
The target for train.py remains eurusd_rv.
Output: data/processed/eurusd_multipair.parquet
python scripts/prepare_multipair.py
PROCESSED=data/processed python scripts/prepare_multipair.py
"""
import os
import pandas as pd
PAIRS = ["eurusd", "gbpusd", "usdjpy", "usdchf", "audusd"]
PROCESSED_DEFAULT = "data/processed"
OUT_DEFAULT = "data/processed/eurusd_multipair.parquet"
def merge_pair_parquets(pair_dfs: dict) -> pd.DataFrame:
"""Inner-join hourly DataFrames from multiple pairs on datetime.
Args:
pair_dfs: dict mapping pair name (e.g. "eurusd") to hourly DataFrame
with columns [datetime, close, ret, realized_vol, ...].
Returns:
Wide DataFrame with columns:
datetime, {pair}_ret, {pair}_rv for each pair.
"""
merged = None
for pair, df in pair_dfs.items():
sub = df[["datetime", "ret", "realized_vol"]].copy()
sub = sub.rename(columns={"ret": f"{pair}_ret", "realized_vol": f"{pair}_rv"})
sub = sub.set_index("datetime")
if merged is None:
merged = sub
else:
merged = merged.join(sub, how="inner")
return merged.reset_index()
def build_multipair_parquet(
processed_dir: str = PROCESSED_DEFAULT,
out_path: str = OUT_DEFAULT,
pairs: list = None,
) -> None:
if pairs is None:
pairs = PAIRS
pair_dfs = {}
for pair in pairs:
path = os.path.join(processed_dir, f"{pair}_hourly.parquet")
if not os.path.exists(path):
raise FileNotFoundError(
f"{pair}_hourly.parquet not found at {path} — run prepare_hourly.py for this pair first"
)
df = pd.read_parquet(path)
pair_dfs[pair] = df
merged = merge_pair_parquets(pair_dfs)
merged.to_parquet(out_path, index=False)
n_pairs = len(pairs)
n_ch = n_pairs * 2
print(f"Multipair parquet: {len(merged):,} rows × {n_ch} feature channels ({n_pairs} pairs)")
print(f"Date range: {merged['datetime'].min()}{merged['datetime'].max()}")
print(f"Written: {out_path}")
if __name__ == "__main__":
processed_dir = os.environ.get("PROCESSED", PROCESSED_DEFAULT)
build_multipair_parquet(processed_dir=processed_dir)