generated from mathias/template-go-web
feat(multipair): G10 multi-pair pipeline + USE_MULTIPAIR knob (Option C)
- prepare_hourly.py: parameterize PAIR env var; OUT_DEFAULT per-pair; load_m1_from_zips(pair=)
- prepare_multipair.py: inner-join 5-pair hourly parquets on datetime → wide parquet
cols: datetime, {pair}_ret, {pair}_rv × n_pairs; eurusd_rv = target
- fetch_multipair.py: download GBPUSD/USDJPY/USDCHF/AUDUSD M1 2008-2023 from histdata
- train.py: USE_MULTIPAIR knob (JEPA_USE_MULTIPAIR=1); build() reads multipair parquet
with n_channels = n_pairs × 2; target = eurusd_rv
- Taskfile: data:fetch:multipair, data:prepare:pair, data:prepare:multipair, data:test updated
- 7 new tests in test_multipair.py; 34/35 pass (1 SKIP until multipair parquet built)
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,72 @@
|
||||
"""Merge per-pair hourly parquets into a single wide multipair parquet.
|
||||
|
||||
Each pair contributes two features: {pair}_ret and {pair}_rv (realized vol).
|
||||
The merge is an INNER JOIN on datetime — only hours present in ALL pairs are kept.
|
||||
The target for train.py remains eurusd_rv.
|
||||
|
||||
Output: data/processed/eurusd_multipair.parquet
|
||||
|
||||
python scripts/prepare_multipair.py
|
||||
PROCESSED=data/processed python scripts/prepare_multipair.py
|
||||
"""
|
||||
import os
|
||||
import pandas as pd
|
||||
|
||||
PAIRS = ["eurusd", "gbpusd", "usdjpy", "usdchf", "audusd"]
|
||||
|
||||
PROCESSED_DEFAULT = "data/processed"
|
||||
OUT_DEFAULT = "data/processed/eurusd_multipair.parquet"
|
||||
|
||||
|
||||
def merge_pair_parquets(pair_dfs: dict) -> pd.DataFrame:
|
||||
"""Inner-join hourly DataFrames from multiple pairs on datetime.
|
||||
|
||||
Args:
|
||||
pair_dfs: dict mapping pair name (e.g. "eurusd") to hourly DataFrame
|
||||
with columns [datetime, close, ret, realized_vol, ...].
|
||||
Returns:
|
||||
Wide DataFrame with columns:
|
||||
datetime, {pair}_ret, {pair}_rv for each pair.
|
||||
"""
|
||||
merged = None
|
||||
for pair, df in pair_dfs.items():
|
||||
sub = df[["datetime", "ret", "realized_vol"]].copy()
|
||||
sub = sub.rename(columns={"ret": f"{pair}_ret", "realized_vol": f"{pair}_rv"})
|
||||
sub = sub.set_index("datetime")
|
||||
if merged is None:
|
||||
merged = sub
|
||||
else:
|
||||
merged = merged.join(sub, how="inner")
|
||||
|
||||
return merged.reset_index()
|
||||
|
||||
|
||||
def build_multipair_parquet(
|
||||
processed_dir: str = PROCESSED_DEFAULT,
|
||||
out_path: str = OUT_DEFAULT,
|
||||
pairs: list = None,
|
||||
) -> None:
|
||||
if pairs is None:
|
||||
pairs = PAIRS
|
||||
pair_dfs = {}
|
||||
for pair in pairs:
|
||||
path = os.path.join(processed_dir, f"{pair}_hourly.parquet")
|
||||
if not os.path.exists(path):
|
||||
raise FileNotFoundError(
|
||||
f"{pair}_hourly.parquet not found at {path} — run prepare_hourly.py for this pair first"
|
||||
)
|
||||
df = pd.read_parquet(path)
|
||||
pair_dfs[pair] = df
|
||||
|
||||
merged = merge_pair_parquets(pair_dfs)
|
||||
merged.to_parquet(out_path, index=False)
|
||||
n_pairs = len(pairs)
|
||||
n_ch = n_pairs * 2
|
||||
print(f"Multipair parquet: {len(merged):,} rows × {n_ch} feature channels ({n_pairs} pairs)")
|
||||
print(f"Date range: {merged['datetime'].min()} → {merged['datetime'].max()}")
|
||||
print(f"Written: {out_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
processed_dir = os.environ.get("PROCESSED", PROCESSED_DEFAULT)
|
||||
build_multipair_parquet(processed_dir=processed_dir)
|
||||
Reference in New Issue
Block a user