generated from mathias/template-go-web
feat(data): EUR/USD hourly pipeline + 2008-2023 M1 dataset (#2)
- scripts/prepare_hourly.py: M1→hourly aggregation (realized_vol = sqrt(Σr²), MIN_BARS=30 threshold, no weekend rows, year-based split preserved) - tests/test_prepare_hourly.py: 5 TDD tests, all green - train.py: USE_HOURLY=True, WINDOW=240 (10-day), PATCH_LEN=24 (1-day patches); build() prefers eurusd_hourly.parquet, falls back to daily; EXPORT BLOCK updated - Taskfile.yml: data:fetch:historical, data:prepare:hourly, data:prepare:all, data:test - 98,591 hourly rows (2008-2023) covering GFC, Euro crisis, Brexit, COVID, Fed cycle Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -18,8 +18,9 @@ import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
# --- agent-tunable knobs ---
|
||||
WINDOW = 60
|
||||
PATCH_LEN = 10 # non-overlapping patches (6 tokens per window)
|
||||
USE_HOURLY = True # prefer eurusd_hourly.parquet when available
|
||||
WINDOW = 240 # hourly: 10 trading days; if USE_HOURLY=False reset to 60
|
||||
PATCH_LEN = 24 # hourly: 1-day patches (10 tokens); if USE_HOURLY=False reset to 10
|
||||
D_MODEL = 128
|
||||
DEPTH = 2
|
||||
N_HEADS = 4
|
||||
@@ -120,9 +121,20 @@ class HorizonPredictor(nn.Module):
|
||||
# ── Data ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
def build():
|
||||
"""Year-based split: encoder trains on 2019-2021; probe evaluates on 2022-2023 OOS."""
|
||||
df = pd.read_parquet("data/processed/eurusd_daily.parquet").reset_index(drop=True)
|
||||
df["date"] = pd.to_datetime(df["date"])
|
||||
"""Year-based split: encoder trains on ≤2021; probe evaluates on ≥2022 OOS.
|
||||
|
||||
Uses eurusd_hourly.parquet when USE_HOURLY=True and the file exists;
|
||||
falls back to eurusd_daily.parquet otherwise.
|
||||
"""
|
||||
import os
|
||||
hourly_path = "data/processed/eurusd_hourly.parquet"
|
||||
daily_path = "data/processed/eurusd_daily.parquet"
|
||||
if USE_HOURLY and os.path.exists(hourly_path):
|
||||
df = pd.read_parquet(hourly_path).reset_index(drop=True)
|
||||
df["date"] = pd.to_datetime(df["datetime"])
|
||||
else:
|
||||
df = pd.read_parquet(daily_path).reset_index(drop=True)
|
||||
df["date"] = pd.to_datetime(df["date"])
|
||||
feats = df[["ret", "realized_vol"]].to_numpy(np.float32)
|
||||
target = df["realized_vol"].to_numpy(np.float32)
|
||||
tr_idx = df.index[df["date"].dt.year <= 2021].tolist()
|
||||
@@ -195,8 +207,14 @@ def main():
|
||||
# Set EXPORT_EMBEDDINGS=1 to write embeddings.json for the Go eval harness.
|
||||
import os
|
||||
if os.environ.get("EXPORT_EMBEDDINGS") == "1":
|
||||
df2 = pd.read_parquet("data/processed/eurusd_daily.parquet").reset_index(drop=True)
|
||||
df2["date"] = pd.to_datetime(df2["date"])
|
||||
hourly_path2 = "data/processed/eurusd_hourly.parquet"
|
||||
daily_path2 = "data/processed/eurusd_daily.parquet"
|
||||
if USE_HOURLY and os.path.exists(hourly_path2):
|
||||
df2 = pd.read_parquet(hourly_path2).reset_index(drop=True)
|
||||
df2["date"] = pd.to_datetime(df2["datetime"])
|
||||
else:
|
||||
df2 = pd.read_parquet(daily_path2).reset_index(drop=True)
|
||||
df2["date"] = pd.to_datetime(df2["date"])
|
||||
tr_mask = df2["date"].dt.year <= 2021
|
||||
feats2 = df2[["ret", "realized_vol"]].to_numpy(np.float32)
|
||||
mu2 = feats2[tr_mask].mean(0); sd2 = feats2[tr_mask].std(0) + 1e-8
|
||||
|
||||
Reference in New Issue
Block a user