generated from mathias/template-go-web
mathiasandClaude Sonnet 4.6
bd8962f997
- prepare_hourly.py: parameterize PAIR env var; OUT_DEFAULT per-pair; load_m1_from_zips(pair=)
- prepare_multipair.py: inner-join 5-pair hourly parquets on datetime → wide parquet
cols: datetime, {pair}_ret, {pair}_rv × n_pairs; eurusd_rv = target
- fetch_multipair.py: download GBPUSD/USDJPY/USDCHF/AUDUSD M1 2008-2023 from histdata
- train.py: USE_MULTIPAIR knob (JEPA_USE_MULTIPAIR=1); build() reads multipair parquet
with n_channels = n_pairs × 2; target = eurusd_rv
- Taskfile: data:fetch:multipair, data:prepare:pair, data:prepare:multipair, data:test updated
- 7 new tests in test_multipair.py; 34/35 pass (1 SKIP until multipair parquet built)
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
49 lines
1.6 KiB
Python
49 lines
1.6 KiB
Python
"""Fetch G10 FX M1 data from histdata.com for all pairs except EURUSD (already fetched).
|
|
|
|
Each pair's zips go into data/raw/{pair}/ to avoid collisions.
|
|
Output: data/raw/gbpusd/DAT_ASCII_GBPUSD_M1_YYYY.zip etc.
|
|
|
|
python scripts/fetch_multipair.py
|
|
PAIRS=gbpusd,usdjpy YEARS=2020,2021 python scripts/fetch_multipair.py
|
|
"""
|
|
import os
|
|
import time
|
|
|
|
from histdata import download_hist_data
|
|
from histdata.api import Platform as P, TimeFrame as T
|
|
|
|
PAIRS_DEFAULT = ["gbpusd", "usdjpy", "usdchf", "audusd"]
|
|
YEARS_DEFAULT = list(range(2008, 2024))
|
|
|
|
|
|
def main():
|
|
pairs_env = os.environ.get("PAIRS", "")
|
|
pairs = [p.strip() for p in pairs_env.split(",")] if pairs_env else PAIRS_DEFAULT
|
|
|
|
years_env = os.environ.get("YEARS", "")
|
|
years = [int(y.strip()) for y in years_env.split(",")] if years_env else YEARS_DEFAULT
|
|
|
|
for pair in pairs:
|
|
out_dir = f"data/raw/{pair}"
|
|
os.makedirs(out_dir, exist_ok=True)
|
|
print(f"\n=== {pair.upper()} ===")
|
|
for yr in years:
|
|
out_path = os.path.join(out_dir, f"DAT_ASCII_{pair.upper()}_M1_{yr}.zip")
|
|
if os.path.exists(out_path):
|
|
print(f" {yr} already present, skip")
|
|
continue
|
|
try:
|
|
f = download_hist_data(
|
|
year=str(yr), month=None, pair=pair,
|
|
platform=P.GENERIC_ASCII, time_frame=T.ONE_MINUTE,
|
|
output_directory=out_dir,
|
|
)
|
|
print(f" fetched {yr} → {f}")
|
|
except Exception as e:
|
|
print(f" {yr} FAILED: {e}")
|
|
time.sleep(2)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|