"""Merge per-pair hourly parquets into a single wide multipair parquet. Each pair contributes two features: {pair}_ret and {pair}_rv (realized vol). The merge is an INNER JOIN on datetime — only hours present in ALL pairs are kept. The target for train.py remains eurusd_rv. Output: data/processed/eurusd_multipair.parquet python scripts/prepare_multipair.py PROCESSED=data/processed python scripts/prepare_multipair.py """ import os import pandas as pd PAIRS = ["eurusd", "gbpusd", "usdjpy", "usdchf", "audusd"] PROCESSED_DEFAULT = "data/processed" OUT_DEFAULT = "data/processed/eurusd_multipair.parquet" def merge_pair_parquets(pair_dfs: dict) -> pd.DataFrame: """Inner-join hourly DataFrames from multiple pairs on datetime. Args: pair_dfs: dict mapping pair name (e.g. "eurusd") to hourly DataFrame with columns [datetime, close, ret, realized_vol, ...]. Returns: Wide DataFrame with columns: datetime, {pair}_ret, {pair}_rv for each pair. """ merged = None for pair, df in pair_dfs.items(): sub = df[["datetime", "ret", "realized_vol"]].copy() sub = sub.rename(columns={"ret": f"{pair}_ret", "realized_vol": f"{pair}_rv"}) sub = sub.set_index("datetime") if merged is None: merged = sub else: merged = merged.join(sub, how="inner") return merged.reset_index() def build_multipair_parquet( processed_dir: str = PROCESSED_DEFAULT, out_path: str = OUT_DEFAULT, pairs: list = None, ) -> None: if pairs is None: pairs = PAIRS pair_dfs = {} for pair in pairs: path = os.path.join(processed_dir, f"{pair}_hourly.parquet") if not os.path.exists(path): raise FileNotFoundError( f"{pair}_hourly.parquet not found at {path} — run prepare_hourly.py for this pair first" ) df = pd.read_parquet(path) pair_dfs[pair] = df merged = merge_pair_parquets(pair_dfs) merged.to_parquet(out_path, index=False) n_pairs = len(pairs) n_ch = n_pairs * 2 print(f"Multipair parquet: {len(merged):,} rows × {n_ch} feature channels ({n_pairs} pairs)") print(f"Date range: {merged['datetime'].min()} → {merged['datetime'].max()}") print(f"Written: {out_path}") if __name__ == "__main__": processed_dir = os.environ.get("PROCESSED", PROCESSED_DEFAULT) build_multipair_parquet(processed_dir=processed_dir)