generated from mathias/template-go-web
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
50635cd80e | ||
|
|
a0d180e37d |
@@ -28,9 +28,3 @@ go.work.sum
|
|||||||
# Project-specific
|
# Project-specific
|
||||||
bin/
|
bin/
|
||||||
*.templ.go
|
*.templ.go
|
||||||
|
|
||||||
# python venv (autoresearch loop)
|
|
||||||
.venv/
|
|
||||||
|
|
||||||
# downloaded + processed market data (track via DVC/MinIO, #10 — not git)
|
|
||||||
data/
|
|
||||||
|
|||||||
@@ -1,8 +0,0 @@
|
|||||||
# Python deps for the autoresearch loop (train.py + scripts). Install torch from
|
|
||||||
# the cu130 index FIRST (koala Blackwell sm_120, torch 2.12.1+cu130 verified):
|
|
||||||
# pip install torch --index-url https://download.pytorch.org/whl/cu130
|
|
||||||
# pip install -r requirements.txt
|
|
||||||
numpy>=2.0
|
|
||||||
pandas>=2.2
|
|
||||||
pyarrow>=16
|
|
||||||
histdata>=1.3 # histdata.com downloader (handles the tk token politely)
|
|
||||||
@@ -1,21 +0,0 @@
|
|||||||
"""Phase-0 compute gate (brain wiki/jepa-fx/facts/autoresearch-integration-phase1):
|
|
||||||
PyTorch cu130 must see the koala Blackwell GPU and compute before any experiment.
|
|
||||||
|
|
||||||
python scripts/check_gpu.py # exits 0 if the GPU is usable, 1 otherwise
|
|
||||||
|
|
||||||
Note: koala shares this 12GB card with the llama-swap LLM stack. The autoresearch
|
|
||||||
agent should run on iguana/berget models so koala's GPU stays free for train.py.
|
|
||||||
"""
|
|
||||||
import sys
|
|
||||||
import torch
|
|
||||||
|
|
||||||
print("torch", torch.__version__)
|
|
||||||
if not torch.cuda.is_available():
|
|
||||||
print("CUDA NOT AVAILABLE — gate BLOCKED")
|
|
||||||
sys.exit(1)
|
|
||||||
print("device:", torch.cuda.get_device_name(0))
|
|
||||||
print("capability: sm_%d%d" % torch.cuda.get_device_capability(0))
|
|
||||||
x = torch.randn(2000, 2000, device="cuda")
|
|
||||||
(x @ x).sum().item()
|
|
||||||
torch.cuda.synchronize()
|
|
||||||
print("GPU matmul OK — Phase-0 compute gate GREEN")
|
|
||||||
@@ -1,31 +0,0 @@
|
|||||||
"""Fetch EUR/USD M1 bars from histdata.com (free, research use).
|
|
||||||
|
|
||||||
Polite: one request per year, spaced; past years query month=None. Uses the
|
|
||||||
maintained `histdata` package which handles histdata's anti-hotlink tk token.
|
|
||||||
Output: data/raw/DAT_ASCII_EURUSD_M1_<year>.zip
|
|
||||||
|
|
||||||
YEARS=2019,2020,2021 python scripts/fetch_data.py
|
|
||||||
"""
|
|
||||||
import os
|
|
||||||
import time
|
|
||||||
|
|
||||||
from histdata import download_hist_data
|
|
||||||
from histdata.api import Platform as P, TimeFrame as T
|
|
||||||
|
|
||||||
YEARS = [y.strip() for y in os.environ.get("YEARS", "2019,2020,2021").split(",")]
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
os.makedirs("data/raw", exist_ok=True)
|
|
||||||
for yr in YEARS:
|
|
||||||
f = download_hist_data(
|
|
||||||
year=yr, month=None, pair="eurusd",
|
|
||||||
platform=P.GENERIC_ASCII, time_frame=T.ONE_MINUTE,
|
|
||||||
output_directory="data/raw",
|
|
||||||
)
|
|
||||||
print("fetched", yr, "->", f)
|
|
||||||
time.sleep(2) # be a good citizen
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@@ -1,57 +0,0 @@
|
|||||||
"""LOCKED data pipeline (toy) — agent must NOT edit (brain Phase-1 contract).
|
|
||||||
|
|
||||||
Parses histdata EUR/USD M1 zips → daily series with realized volatility (the
|
|
||||||
val_vol_r2 target = 1-day realized vol from intraday squared returns).
|
|
||||||
Output: data/processed/eurusd_daily.parquet [date, close, ret, realized_vol].
|
|
||||||
"""
|
|
||||||
import glob
|
|
||||||
import os
|
|
||||||
import zipfile
|
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
import pandas as pd
|
|
||||||
|
|
||||||
RAW = "data/raw"
|
|
||||||
OUT = "data/processed/eurusd_daily.parquet"
|
|
||||||
|
|
||||||
|
|
||||||
def load_m1() -> pd.DataFrame:
|
|
||||||
frames = []
|
|
||||||
for zp in sorted(glob.glob(os.path.join(RAW, "DAT_ASCII_EURUSD_M1_*.zip"))):
|
|
||||||
with zipfile.ZipFile(zp) as z:
|
|
||||||
csv = [n for n in z.namelist() if n.endswith(".csv")][0]
|
|
||||||
with z.open(csv) as f:
|
|
||||||
df = pd.read_csv(
|
|
||||||
f, sep=";", header=None,
|
|
||||||
names=["dt", "open", "high", "low", "close", "vol"],
|
|
||||||
)
|
|
||||||
df["ts"] = pd.to_datetime(df["dt"], format="%Y%m%d %H%M%S")
|
|
||||||
frames.append(df[["ts", "close"]])
|
|
||||||
out = pd.concat(frames).sort_values("ts").reset_index(drop=True)
|
|
||||||
return out
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
m1 = load_m1()
|
|
||||||
m1["r"] = np.log(m1["close"]).diff()
|
|
||||||
m1["day"] = m1["ts"].dt.normalize()
|
|
||||||
daily = m1.groupby("day").agg(
|
|
||||||
close=("close", "last"),
|
|
||||||
realized_vol=("r", lambda x: np.sqrt(np.nansum(x.values ** 2))),
|
|
||||||
n_min=("r", "count"),
|
|
||||||
).reset_index()
|
|
||||||
daily = daily[daily["n_min"] > 60] # drop thin days (holidays)
|
|
||||||
daily["ret"] = np.log(daily["close"]).diff()
|
|
||||||
daily = daily.dropna().reset_index(drop=True)
|
|
||||||
os.makedirs(os.path.dirname(OUT), exist_ok=True)
|
|
||||||
daily[["day", "close", "ret", "realized_vol"]].rename(columns={"day": "date"}).to_parquet(OUT)
|
|
||||||
print("rows:", len(daily), "| dates:", daily["day"].min().date(), "→", daily["day"].max().date())
|
|
||||||
# sanity: the COVID crash (March 2020) must show a realized-vol spike
|
|
||||||
rv = daily.set_index("day")["realized_vol"]
|
|
||||||
mar20 = rv["2020-03-01":"2020-03-31"].max()
|
|
||||||
typ = rv["2019-01-01":"2019-12-31"].median()
|
|
||||||
print("median 2019 RV: %.5f | max Mar-2020 RV: %.5f | spike x%.1f" % (typ, mar20, mar20 / typ))
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
Reference in New Issue
Block a user