generated from mathias/template-go-web
mathiasandClaude Sonnet 4.6
bd8962f997
- prepare_hourly.py: parameterize PAIR env var; OUT_DEFAULT per-pair; load_m1_from_zips(pair=)
- prepare_multipair.py: inner-join 5-pair hourly parquets on datetime → wide parquet
cols: datetime, {pair}_ret, {pair}_rv × n_pairs; eurusd_rv = target
- fetch_multipair.py: download GBPUSD/USDJPY/USDCHF/AUDUSD M1 2008-2023 from histdata
- train.py: USE_MULTIPAIR knob (JEPA_USE_MULTIPAIR=1); build() reads multipair parquet
with n_channels = n_pairs × 2; target = eurusd_rv
- Taskfile: data:fetch:multipair, data:prepare:pair, data:prepare:multipair, data:test updated
- 7 new tests in test_multipair.py; 34/35 pass (1 SKIP until multipair parquet built)
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
73 lines
2.4 KiB
Python
73 lines
2.4 KiB
Python
"""Merge per-pair hourly parquets into a single wide multipair parquet.
|
||
|
||
Each pair contributes two features: {pair}_ret and {pair}_rv (realized vol).
|
||
The merge is an INNER JOIN on datetime — only hours present in ALL pairs are kept.
|
||
The target for train.py remains eurusd_rv.
|
||
|
||
Output: data/processed/eurusd_multipair.parquet
|
||
|
||
python scripts/prepare_multipair.py
|
||
PROCESSED=data/processed python scripts/prepare_multipair.py
|
||
"""
|
||
import os
|
||
import pandas as pd
|
||
|
||
PAIRS = ["eurusd", "gbpusd", "usdjpy", "usdchf", "audusd"]
|
||
|
||
PROCESSED_DEFAULT = "data/processed"
|
||
OUT_DEFAULT = "data/processed/eurusd_multipair.parquet"
|
||
|
||
|
||
def merge_pair_parquets(pair_dfs: dict) -> pd.DataFrame:
|
||
"""Inner-join hourly DataFrames from multiple pairs on datetime.
|
||
|
||
Args:
|
||
pair_dfs: dict mapping pair name (e.g. "eurusd") to hourly DataFrame
|
||
with columns [datetime, close, ret, realized_vol, ...].
|
||
Returns:
|
||
Wide DataFrame with columns:
|
||
datetime, {pair}_ret, {pair}_rv for each pair.
|
||
"""
|
||
merged = None
|
||
for pair, df in pair_dfs.items():
|
||
sub = df[["datetime", "ret", "realized_vol"]].copy()
|
||
sub = sub.rename(columns={"ret": f"{pair}_ret", "realized_vol": f"{pair}_rv"})
|
||
sub = sub.set_index("datetime")
|
||
if merged is None:
|
||
merged = sub
|
||
else:
|
||
merged = merged.join(sub, how="inner")
|
||
|
||
return merged.reset_index()
|
||
|
||
|
||
def build_multipair_parquet(
|
||
processed_dir: str = PROCESSED_DEFAULT,
|
||
out_path: str = OUT_DEFAULT,
|
||
pairs: list = None,
|
||
) -> None:
|
||
if pairs is None:
|
||
pairs = PAIRS
|
||
pair_dfs = {}
|
||
for pair in pairs:
|
||
path = os.path.join(processed_dir, f"{pair}_hourly.parquet")
|
||
if not os.path.exists(path):
|
||
raise FileNotFoundError(
|
||
f"{pair}_hourly.parquet not found at {path} — run prepare_hourly.py for this pair first"
|
||
)
|
||
df = pd.read_parquet(path)
|
||
pair_dfs[pair] = df
|
||
|
||
merged = merge_pair_parquets(pair_dfs)
|
||
merged.to_parquet(out_path, index=False)
|
||
n_pairs = len(pairs)
|
||
n_ch = n_pairs * 2
|
||
print(f"Multipair parquet: {len(merged):,} rows × {n_ch} feature channels ({n_pairs} pairs)")
|
||
print(f"Date range: {merged['datetime'].min()} → {merged['datetime'].max()}")
|
||
print(f"Written: {out_path}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
processed_dir = os.environ.get("PROCESSED", PROCESSED_DEFAULT)
|
||
build_multipair_parquet(processed_dir=processed_dir)
|