generated from mathias/template-go-web
feat(multipair): G10 multi-pair pipeline + USE_MULTIPAIR knob (Option C)
- prepare_hourly.py: parameterize PAIR env var; OUT_DEFAULT per-pair; load_m1_from_zips(pair=)
- prepare_multipair.py: inner-join 5-pair hourly parquets on datetime → wide parquet
cols: datetime, {pair}_ret, {pair}_rv × n_pairs; eurusd_rv = target
- fetch_multipair.py: download GBPUSD/USDJPY/USDCHF/AUDUSD M1 2008-2023 from histdata
- train.py: USE_MULTIPAIR knob (JEPA_USE_MULTIPAIR=1); build() reads multipair parquet
with n_channels = n_pairs × 2; target = eurusd_rv
- Taskfile: data:fetch:multipair, data:prepare:pair, data:prepare:multipair, data:test updated
- 7 new tests in test_multipair.py; 34/35 pass (1 SKIP until multipair parquet built)
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,48 @@
|
||||
"""Fetch G10 FX M1 data from histdata.com for all pairs except EURUSD (already fetched).
|
||||
|
||||
Each pair's zips go into data/raw/{pair}/ to avoid collisions.
|
||||
Output: data/raw/gbpusd/DAT_ASCII_GBPUSD_M1_YYYY.zip etc.
|
||||
|
||||
python scripts/fetch_multipair.py
|
||||
PAIRS=gbpusd,usdjpy YEARS=2020,2021 python scripts/fetch_multipair.py
|
||||
"""
|
||||
import os
|
||||
import time
|
||||
|
||||
from histdata import download_hist_data
|
||||
from histdata.api import Platform as P, TimeFrame as T
|
||||
|
||||
PAIRS_DEFAULT = ["gbpusd", "usdjpy", "usdchf", "audusd"]
|
||||
YEARS_DEFAULT = list(range(2008, 2024))
|
||||
|
||||
|
||||
def main():
|
||||
pairs_env = os.environ.get("PAIRS", "")
|
||||
pairs = [p.strip() for p in pairs_env.split(",")] if pairs_env else PAIRS_DEFAULT
|
||||
|
||||
years_env = os.environ.get("YEARS", "")
|
||||
years = [int(y.strip()) for y in years_env.split(",")] if years_env else YEARS_DEFAULT
|
||||
|
||||
for pair in pairs:
|
||||
out_dir = f"data/raw/{pair}"
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
print(f"\n=== {pair.upper()} ===")
|
||||
for yr in years:
|
||||
out_path = os.path.join(out_dir, f"DAT_ASCII_{pair.upper()}_M1_{yr}.zip")
|
||||
if os.path.exists(out_path):
|
||||
print(f" {yr} already present, skip")
|
||||
continue
|
||||
try:
|
||||
f = download_hist_data(
|
||||
year=str(yr), month=None, pair=pair,
|
||||
platform=P.GENERIC_ASCII, time_frame=T.ONE_MINUTE,
|
||||
output_directory=out_dir,
|
||||
)
|
||||
print(f" fetched {yr} → {f}")
|
||||
except Exception as e:
|
||||
print(f" {yr} FAILED: {e}")
|
||||
time.sleep(2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user