From 07111c67c09fb22eeb1f198891793607479b0723 Mon Sep 17 00:00:00 2001 From: fredmaloggia Date: Tue, 25 Nov 2025 21:41:59 +0100 Subject: [PATCH] continuazione refactoring --- AGENT.md | 67 +++++----- Trading Pattern Recon w Hurst v.3.1.6.py | 61 ++++----- config/pattern_knn_config.json | 40 +++++- equity_from_log.py | 46 +++---- shared_utils.py | 13 +- signals_daily_kNN_prod_v.2.py | 156 ++++++++++------------- 6 files changed, 192 insertions(+), 191 deletions(-) diff --git a/AGENT.md b/AGENT.md index 3f624f3..adf2ad8 100644 --- a/AGENT.md +++ b/AGENT.md @@ -1,41 +1,38 @@ -# Agenda tecnica - Trading repo +# Trading - Note di progetto -## Struttura e moduli -- `signals_daily_kNN_prod_v.2.py`: generatore segnali EOD -> ordini t+1 (Equal_Weight/Risk_Parity); kNN pattern long-only, ranking top-N, sizing equal weight / risk parity con cap; fetch prezzi open via API Euronext; scrive audit `trades_audit_log.csv`, snapshot Excel datato e `open_trades/*.csv`, copia su Dropbox locale. -- `Trading Pattern Recon w Hurst v.3.1.6.py`: workflow di ricerca/backtest (Hurst + pattern, forward walk, costruzione portafogli, metriche/grafici). Versioni precedenti in `archivio/` (3.1.1-3.1.5) + vecchio `signals_daily_kNN_prod.py` come riferimento storico. -- `equity_from_log.py`: ricostruisce rendimenti/equity per strategia a partire da `trades_audit_log.csv` usando rendimenti DB; produce CSV debug ed equity + PNG drawdown. -- `check_universe_prices.py`: validazione/fetch prezzi Euronext per universo, con retry; salva Excel datato. -- `shared_utils.py`: helper comuni (config loader, connessione DB via `connection.txt`, z-score, libreria pattern kNN, Hurst). Config JSON in `config/pattern_knn_config.json`. +## Scopo +Sintesi unica per struttura del repo, flussi operativi e punti aperti da rivedere. Questo file unifica le note precedenti (AGENT/REVIEW_NOTES). -## Dipendenze e I/O esterni -- Librerie: pandas, numpy, sqlalchemy+pyodbc, matplotlib, openpyxl (per Excel), urllib/ssl per API HTTP. -- DB SQL Server via stored procedure (`opt_RendimentoGiornaliero1_ALL` di default) e credenziali in `connection.txt`. -- API prezzi Euronext `fin.scorer.app` (open/prevClose) per ordini e checker; rete richiesta. -- File system: numerosi artefatti CSV/XLSX/PNG (trades, segnali, equity, heatmap) e path Dropbox hardcoded `C:\Users\Admin\Dropbox\...`. +## Struttura dei file principali +- signals_daily_kNN_prod_v.2.py: generatore giornaliero di segnali kNN (Top-N, Equal Weight/Risk Parity) con fetch prezzi open Euronext, regole di rischio, aggiornamento open_trades, audit log, export Excel e copia opzionale su Dropbox locale. +- Trading Pattern Recon w Hurst v.3.1.6.py: ricerca/backtest Hurst+kNN (walk-forward long-only, ranking portafogli, metriche e plot); esporta CSV/XLSX/PNG in Output/ e plot/. +- equity_from_log.py: ricostruzione equity da trades_audit_log.csv, fetch rendimenti DB, salvataggio CSV (returns/debug/equity) e PNG (drawdown). +- shared_utils.py: utility comuni (loader config JSON, connessione DB via connection.txt, z_norm, libreria pattern, Hurst map). +- config/pattern_knn_config.json: parametri per pattern, ranking, signals, DB, equity. +- Dati: Input/Universo per Trading System.xlsx, open_trades/*.csv, Output/ file datati, plot/ per grafici (plots/ legacy rimosso). -## Flussi principali -- **Signals daily**: carica universo Excel -> ritorni DB per tutti gli ISIN -> Hurst map (251-252d) -> segnali kNN (WP=60, HA=10, k=25, theta per-ISIN da Hurst/100) -> ranking buy unico -> target Top-N (MAX_OPEN=15) -> per ogni strategia confronto con posizioni aperte (`open_trades/*.csv`), risk-exit SL/TP/Trailing/Time/Weak -> costruisce ordini OPEN/CLOSE t+1 con sizing e fetch prezzi open (cache unica) -> aggiorna audit log e snapshot Excel. -- **Backtest/ricerca**: script Hurst+kNN su intero storico, walk-forward long-only, selezione portafogli dinamica, metriche finali e grafici (equity, heatmap, composizioni) salvati sotto root/`plots_by_topN`. -- **Equity da log**: legge audit, scarica rendimenti DB, normalizza (gestisce percentuali/log-return), calcola rendimento giornaliero medio ponderato per strategia e salva CSV/PNG. -- **Price checker**: controlla universo e tenta 3 fetch sequenziali per ISIN-mercato, loggando esito in Excel datato. +## Dipendenze e integrazioni esterne +- Python: pandas, numpy, sqlalchemy+pyodbc, matplotlib, openpyxl, urllib/ssl. +- DB SQL Server via stored procedure opt_RendimentoGiornaliero1_ALL (default) e credenziali in connection.txt. +- API prezzi Euronext fin.scorer.app per open/prevClose; rete richiesta. +- Path Dropbox hardcoded per copie output. -## Artefatti e dati -- Input: `Universo per Trading System.xlsx`, `connection.txt` (credenziali DB), file signals/grafici storici, `open_trades/*.csv` correnti, `trades_audit_log.csv` cumulativo. -- Output: file datati `*_signals.xlsx`, `daily_returns_by_strategy.csv`, `equity_by_strategy.csv/png`, heatmap/composition/equity PNG, report `trades_report.xlsx`, summary `final_metrics*.xlsx`. +## Flussi operativi +- Signals daily: carica universo, rendimenti DB per tutti gli ISIN, Hurst map 251-252d, segnali kNN (WP=60, HA=10, k=25, theta da Hurst/100), ranking unico Top-N (MAX_OPEN=15), confronto con open_trades, regole SL/TP/Trailing/Time/Weak, fetch prezzi open con cache unica, scrive audit log, open_trades e Excel datati (con copia Dropbox opzionale). +- Backtest/ricerca: run storiche Hurst+kNN, walk-forward, selezione portafogli dinamica, metriche (CAGR/vol/DD/Heal/Hurst) e plot; esporta heatmap/equity/composition in Output/ e plot/. +- Equity da log: legge audit, scarica rendimenti DB, normalizza, calcola rendimento giornaliero per strategia e salva CSV debug/equity + PNG drawdown. +- Price checker: verifica prezzi universo da API Euronext con 3 retry e log in Excel datato. -## Punti critici / fragilita -- Config duplicata/override nei due script principali: valori richiesti via `require_value` subito rimpiazzati da `.get` con default diversi; rischio incoerenze tra run. -- Hardcode percorsi e naming (Dropbox, file universi, stored proc) senza parametri CLI -> difficile rilocare/automatizzare ed errori silenziosi se path mancano. -- Fetch DB e API sequenziali per ISIN (nessun batching/parallelismo), con print blocking e retry lineari: potenziale collo di bottiglia e timeout su universi ampi. -- Gestione errori best-effort: `_db_fetch_returns` ignora asset mancanti e continua, rischiando gap nascosti nelle serie; `_fetch_open_prices_once` usa `_safe_to_float` senza log dettagliato. -- Audit/logging: `append_audit_rows` concatena senza dedupe/concorrenza; nessun controllo su idempotenza o coerenza tra `open_trades` e audit. -- Sizing RP: guarda inv-vol 60d senza controllo di correlazione/covarianza; cap applicato prima della rinormalizzazione puo lasciare cash residuo non tracciato. -- Risk-exit: calcolo PnL/Drawdown dalla serie daily con possibili `NaN`; nessun controllo timezone/duplicati; `WeakDays` usato ma poco gestito. -- Dipendenze esterne: blocchi se `connection.txt` o `config` mancanti; API HTTP senza fallback/caching persistente. -- Qualita codice: duplicazioni (config, utility importate due volte), nomi con spazi, script monolitici senza test; archivio versioni ridondanti. +## Percorsi e artefatti +- Input: Input/Universo per Trading System.xlsx, connection.txt, open_trades/*.csv, trades_audit_log.csv cumulativo. +- Output: Output/*_signals.xlsx, daily_returns_by_strategy.csv, equity_by_strategy.csv/png, trades_report.xlsx, final_metrics*.xlsx, grafici in plot/ (unica cartella immagini; contenuti da plots/ spostati qui). -## Collo di bottiglia e aree di miglioramento -- I/O seriale verso DB e API e il punto piu lento e fragile (rete). Cache e parallelizzazione mancano. -- Persistenza su file Excel/CSV come single source of truth rende difficile la coerenza e l'automazione; non c'e validazione schema/typen. -- Gestione configurazione e percorsi centralizzata ma hardcoded e duplicata; assente modello a pacchetto/moduli riusabili. -- Monitoraggio/error handling minimale: molti `print`, nessun livello di log o alerting per run fallite. +## Punti aperti da rivedere +- Config duplicata/override tra script e JSON; servono fonte unica e parametri CLI. +- Percorsi hardcoded (Dropbox, stored proc, universi) e naming misto Output/output; mancano controlli robusti quando path o file non esistono. +- Fetch DB/API sequenziale per ISIN e calcoli kNN/Hurst single-thread; valutare caching e parallelismo. +- Error handling/logging minimale: molti print, pochi log strutturati; audit senza dedupe/idempotenza e sync limitata con open_trades. +- Sizing risk parity usa solo vol 60d senza correlazioni; cap prima della rinormalizzazione lascia cash non tracciato. +- Risk-exit basate su serie con possibili NaN/duplicati; nessun controllo timezone; WeakDays poco gestito. +- Pulizia artefatti: uniformare output su Output/ e plot/, evitare nuove cartelle legacy; valutare validazione schema per Excel/CSV. +- Test e modularita limitati: script monolitici con duplicazioni (pattern/ranking), nessun pacchetto riusabile e nomi con spazi. diff --git a/Trading Pattern Recon w Hurst v.3.1.6.py b/Trading Pattern Recon w Hurst v.3.1.6.py index 9bd303b..0706049 100644 --- a/Trading Pattern Recon w Hurst v.3.1.6.py +++ b/Trading Pattern Recon w Hurst v.3.1.6.py @@ -50,7 +50,7 @@ SAVE_PNG = globals().get("SAVE_PNG", True) def savefig_safe(path, **kwargs): """ Save a matplotlib figure to disk safely, honoring SAVE_PNG. - Usage: savefig_safe("plots/myfig.png", dpi=150, bbox_inches="tight") + Usage: savefig_safe("plot/myfig.png", dpi=150, bbox_inches="tight") """ if not SAVE_PNG or _plt_sf is None: return @@ -71,21 +71,21 @@ def savefig_safe(path, **kwargs): # ========================================= # PARAMETRI GLOBALI # ========================================= -OUTPUT_DIR = Path("output") -PLOT_DIR = Path("plot") -OUTPUT_DIR.mkdir(parents=True, exist_ok=True) -PLOT_DIR.mkdir(parents=True, exist_ok=True) CONFIG = load_config() DB_CONFIG = require_section(CONFIG, "db") PATTERN_CONFIG = require_section(CONFIG, "pattern") TAGGING_CONFIG = require_section(CONFIG, "tagging") RANKING_CONFIG = require_section(CONFIG, "ranking") -DB_CONFIG = CONFIG.get("db", {}) -PATTERN_CONFIG = CONFIG.get("pattern", {}) -TAGGING_CONFIG = CONFIG.get("tagging", {}) -RANKING_CONFIG = CONFIG.get("ranking", {}) +PATHS_CONFIG = require_section(CONFIG, "paths") +HURST_CONFIG = CONFIG.get("hurst", {}) +RUN_CONFIG = CONFIG.get("run", {}) -UNIVERSO_XLSX = "Input/Universo per Trading System.xlsx" +OUTPUT_DIR = Path(PATHS_CONFIG.get("output_dir", "output")) +PLOT_DIR = Path(PATHS_CONFIG.get("plot_dir", "plot")) +OUTPUT_DIR.mkdir(parents=True, exist_ok=True) +PLOT_DIR.mkdir(parents=True, exist_ok=True) + +UNIVERSO_XLSX = PATHS_CONFIG.get("input_universe", "Input/Universo per Trading System.xlsx") # Export OUTPUT_HURST_XLSX = OUTPUT_DIR / "hurst_by_isin.xlsx" @@ -119,34 +119,18 @@ Z_REV = float(require_value(TAGGING_CONFIG, "z_rev", "tagging")) Z_VOL = float(require_value(TAGGING_CONFIG, "z_vol", "tagging")) STD_COMP_PCT = float(require_value(TAGGING_CONFIG, "std_comp_pct", "tagging")) -DAYS_PER_YEAR = 252 - TOP_N_MAX = int(require_value(RANKING_CONFIG, "top_n_max", "ranking")) # numero massimo di asset ammessi RP_MAX_WEIGHT = require_value(RANKING_CONFIG, "rp_max_weight", "ranking") # 2 x 1/15 ≈ 0.1333 = 13,33% if RP_MAX_WEIGHT is None: RP_MAX_WEIGHT = 2 / max(TOP_N_MAX, 1) else: RP_MAX_WEIGHT = float(RP_MAX_WEIGHT) -STORED_PROC = DB_CONFIG.get("stored_proc", "opt_RendimentoGiornaliero1_ALL") -N_BARS = DB_CONFIG.get("n_bars", 1305) -PTF_CURR = DB_CONFIG.get("ptf_curr", "EUR") +HURST_MIN_LENGTH = int(HURST_CONFIG.get("min_length", 200)) +HURST_WIN_GRID = HURST_CONFIG.get("win_grid") +HURST_MIN_SEGMENTS = int(HURST_CONFIG.get("min_segments", 1)) -# Pattern-matching (iper-parametri) -WP = PATTERN_CONFIG.get("wp", 60) # lunghezza finestra pattern (barre) -HA = PATTERN_CONFIG.get("ha", 10) # orizzonte outcome (barre) -KNN_K = PATTERN_CONFIG.get("knn_k", 25) # numero di vicini -THETA = PATTERN_CONFIG.get("theta", 0.005) # soglia su outcome per generare segnale -EMBARGO = PATTERN_CONFIG.get("embargo", WP + HA) - -# Tagging rule-based (soglie) -Z_REV = TAGGING_CONFIG.get("z_rev", 2.0) -Z_VOL = TAGGING_CONFIG.get("z_vol", 2.0) -STD_COMP_PCT = TAGGING_CONFIG.get("std_comp_pct", 0.15) - -DAYS_PER_YEAR = 252 - -TOP_N_MAX = RANKING_CONFIG.get("top_n_max", 15) # numero massimo di asset ammessi -RP_MAX_WEIGHT = RANKING_CONFIG.get("rp_max_weight", 2 / max(TOP_N_MAX, 1)) # 2 x 1/15 ≈ 0.1333 = 13,33% +DAYS_PER_YEAR = int(RUN_CONFIG.get("days_per_year", 252)) +TOP_N = int(RUN_CONFIG.get("top_n_default", TOP_N_MAX)) # ========================================= # UTILS GENERALI @@ -169,13 +153,15 @@ def format_eta(seconds): return f"{minutes}m {secs:02d}s" # ================= HURST (sui RENDIMENTI) ================= -def hurst_rs_returns(r, win_grid=None, min_seg=1): +def hurst_rs_returns(r, win_grid=None, min_seg=None): r = pd.Series(r).dropna().astype("float64").values n = len(r) - if n < 200: + seg_min = HURST_MIN_SEGMENTS if min_seg is None else int(min_seg) + if n < HURST_MIN_LENGTH: return np.nan if win_grid is None: - base = np.array([16,24,32,48,64,96,128,192,256,384], dtype=int) + base = HURST_WIN_GRID or [16,24,32,48,64,96,128,192,256,384] + base = np.array(base, dtype=int) win_grid = [w for w in base if w <= n//2] if len(win_grid) < 4: max_w = max(16, n//4) @@ -186,7 +172,7 @@ def hurst_rs_returns(r, win_grid=None, min_seg=1): for w in win_grid: if w < 8 or w > n: continue m = n//w - if m < min_seg: continue + if m < seg_min: continue rs_list = [] for i in range(m): seg = r[i*w:(i+1)*w] @@ -211,12 +197,13 @@ def hurst_rs_returns(r, win_grid=None, min_seg=1): def hurst_dfa_returns(r, win_grid=None): r = pd.Series(r).dropna().astype("float64").values n = len(r) - if n < 200: + if n < HURST_MIN_LENGTH: return np.nan r_dm = r - np.mean(r) y = np.cumsum(r_dm) if win_grid is None: - base = np.array([16,24,32,48,64,96,128,192,256], dtype=int) + base = HURST_WIN_GRID or [16,24,32,48,64,96,128,192,256] + base = np.array(base, dtype=int) win_grid = [w for w in base if w <= n//2] if len(win_grid) < 4: max_w = max(16, n//4) diff --git a/config/pattern_knn_config.json b/config/pattern_knn_config.json index 4aa5685..b5c02c7 100644 --- a/config/pattern_knn_config.json +++ b/config/pattern_knn_config.json @@ -1,7 +1,7 @@ { "db": { "stored_proc": "opt_RendimentoGiornaliero1_ALL", - "n_bars": 1305, + "n_bars": 1260, "ptf_curr": "EUR" }, "pattern": { @@ -37,5 +37,43 @@ "Equal_Weight", "Risk_Parity" ] + }, + "paths": { + "base_dir": ".", + "input_universe": "Input/Universo per Trading System.xlsx", + "connection_txt": "connection.txt", + "output_dir": "output", + "plot_dir": "plot", + "open_trades_dir": "open_trades", + "audit_log_csv": "output/trades_audit_log.csv" + }, + "hurst": { + "lookback": null, + "min_length": 200, + "win_grid": [ + 16, + 24, + 32, + 48, + 64, + 96, + 128, + 192, + 256, + 384 + ], + "min_segments": 1 + }, + "prices": { + "base_url": "https://fin.scorer.app/finance/euronext/price", + "max_retry": 3, + "sleep_sec": 0.1, + "timeout": 10 + }, + "run": { + "business_days_only": true, + "seed": 42, + "top_n_default": 15, + "days_per_year": 252 } } diff --git a/equity_from_log.py b/equity_from_log.py index c7dc87e..79f6dd2 100644 --- a/equity_from_log.py +++ b/equity_from_log.py @@ -31,10 +31,18 @@ from shared_utils import ( # PATH & OUTPUT # ============================================================================= BASE_DIR = Path(__file__).resolve().parent -OUTPUT_DIR = BASE_DIR / "output" -PLOT_DIR = BASE_DIR / "plot" -AUDIT_LOG_CSV = OUTPUT_DIR / "trades_audit_log.csv" -CONNECTION_TXT = BASE_DIR / "connection.txt" +CONFIG = None +try: + CONFIG = load_config() + PATHS_CONFIG = require_section(CONFIG, "paths") +except Exception as exc: # pragma: no cover - best effort + print(f"[WARN] Config non disponibile ({exc}); uso i percorsi di default.") + PATHS_CONFIG = {} + +OUTPUT_DIR = BASE_DIR / PATHS_CONFIG.get("output_dir", "output") +PLOT_DIR = BASE_DIR / PATHS_CONFIG.get("plot_dir", "plot") +AUDIT_LOG_CSV = BASE_DIR / PATHS_CONFIG.get("audit_log_csv", OUTPUT_DIR / "trades_audit_log.csv") +CONNECTION_TXT = BASE_DIR / PATHS_CONFIG.get("connection_txt", "connection.txt") OUT_DAILY_CSV = OUTPUT_DIR / "daily_returns_by_strategy.csv" OUT_EQUITY_CSV = OUTPUT_DIR / "equity_by_strategy.csv" @@ -44,16 +52,13 @@ PLOT_DD = PLOT_DIR / "drawdown_by_strategy.png" # Stored procedure SP_NAME_DEFAULT = "opt_RendimentoGiornaliero1_ALL" -SP_N_DEFAULT = 1305 +SP_N_DEFAULT = 1260 PTF_CURR_DEFAULT = "EUR" -# prova a leggere il file di configurazione condiviso; se manca si usano i default -CONFIG = None try: - CONFIG = load_config() - DB_CONFIG = require_section(CONFIG, "db") + DB_CONFIG = require_section(CONFIG, "db") if CONFIG else {} except Exception as exc: # pragma: no cover - best effort - print(f"[WARN] Config non disponibile ({exc}); uso i default interni.") + print(f"[WARN] Config DB non disponibile ({exc}); uso i default interni.") DB_CONFIG = {} else: SP_NAME_DEFAULT = str(DB_CONFIG.get("stored_proc", SP_NAME_DEFAULT)) @@ -61,24 +66,19 @@ else: PTF_CURR_DEFAULT = str(DB_CONFIG.get("ptf_curr", PTF_CURR_DEFAULT)) # Strategie valide (tutto il resto viene ignorato) -# "Aggressiva_Crypto" è stata rimossa perché non deve più essere processata +# "Aggressiva_Crypto" ? stata rimossa perch? non deve pi? essere processata DEFAULT_STRATEGIES = ["Equal_Weight", "Risk_Parity"] VALID_STRATEGIES = DEFAULT_STRATEGIES -try: # opzionale: sezione dedicata nel config per personalizzare la whitelist - EQUITY_CFG = CONFIG.get("equity_log", {}) if CONFIG else {} -except NameError: # pragma: no cover - CONFIG non definito se load_config fallisce - EQUITY_CFG = {} - -if EQUITY_CFG: - raw_whitelist = EQUITY_CFG.get("strategy_whitelist") - if raw_whitelist: - whitelist = [str(x).strip() for x in raw_whitelist if str(x).strip()] - if whitelist: - VALID_STRATEGIES = whitelist -VALID_STRATEGIES = ["Equal_Weight", "Risk_Parity"] +EQUITY_CFG = CONFIG.get("equity_log", {}) if CONFIG else {} +raw_whitelist = EQUITY_CFG.get("strategy_whitelist") if isinstance(EQUITY_CFG, dict) else None +if raw_whitelist: + whitelist = [str(x).strip() for x in raw_whitelist if str(x).strip()] + if whitelist: + VALID_STRATEGIES = whitelist # ============================================================================= + # FETCH RENDIMENTI DAL DB # ============================================================================= def fetch_returns_from_db(isins, start_date, end_date) -> pd.DataFrame: diff --git a/shared_utils.py b/shared_utils.py index 54ef3d3..1d6982a 100644 --- a/shared_utils.py +++ b/shared_utils.py @@ -206,16 +206,23 @@ def hurst_rs(series: pd.Series) -> Optional[float]: return float(h) -def build_hurst_map(returns_long: pd.DataFrame, lookback: int = 252) -> Dict[str, float]: +def build_hurst_map( + returns_long: pd.DataFrame, + lookback: Optional[int] = None, + min_length: int = 100, +) -> Dict[str, float]: if returns_long.empty: return {} ret_wide = returns_long.pivot(index="Date", columns="ISIN", values="Ret").sort_index() hurst_map: Dict[str, float] = {} for isin in ret_wide.columns: series = ret_wide[isin].dropna().astype(float) - if len(series) < max(lookback, 100): + if len(series) < max(1, int(min_length)): continue - h_val = hurst_rs(series.iloc[-lookback:]) + window = len(series) if lookback is None else min(len(series), int(lookback)) + if window <= 0: + continue + h_val = hurst_rs(series.iloc[-window:]) if h_val is None or not np.isfinite(h_val): continue hurst_map[str(isin)] = float(h_val) diff --git a/signals_daily_kNN_prod_v.2.py b/signals_daily_kNN_prod_v.2.py index cdf8a4e..1fbaaf8 100644 --- a/signals_daily_kNN_prod_v.2.py +++ b/signals_daily_kNN_prod_v.2.py @@ -54,32 +54,32 @@ from shared_utils import ( ) # ========================= -# CONFIG -# ========================= -CONFIG = load_config() -DB_CONFIG = require_section(CONFIG, "db") -PATTERN_CONFIG = require_section(CONFIG, "pattern") -TAGGING_CONFIG = require_section(CONFIG, "tagging") -RANKING_CONFIG = require_section(CONFIG, "ranking") -SIGNALS_CONFIG = require_section(CONFIG, "signals") -DB_CONFIG = CONFIG.get("db", {}) -PATTERN_CONFIG = CONFIG.get("pattern", {}) -TAGGING_CONFIG = CONFIG.get("tagging", {}) -RANKING_CONFIG = CONFIG.get("ranking", {}) -SIGNALS_CONFIG = CONFIG.get("signals", {}) - -BASE_DIR = Path(".") -OUTPUT_DIR = BASE_DIR / "output" -# Universe now expected inside Input folder -UNIVERSO_XLSX = BASE_DIR / "Input" / "Universo per Trading System.xlsx" -CONNECTION_TXT = BASE_DIR / "connection.txt" -AUDIT_LOG_CSV = OUTPUT_DIR / "trades_audit_log.csv" -OPEN_TRADES_DIR = BASE_DIR / "open_trades" -DROPBOX_EXPORT_DIR = Path(r"C:\Users\Admin\Dropbox\Condivisa Lavoro\Segnali di trading su ETF") - -def _dated_signals_filename() -> Path: - date_prefix = pd.Timestamp.today().strftime("%Y%m%d") - return OUTPUT_DIR / f"{date_prefix}_signals.xlsx" +# CONFIG +# ========================= +CONFIG = load_config() +DB_CONFIG = require_section(CONFIG, "db") +PATTERN_CONFIG = require_section(CONFIG, "pattern") +TAGGING_CONFIG = require_section(CONFIG, "tagging") +RANKING_CONFIG = require_section(CONFIG, "ranking") +SIGNALS_CONFIG = require_section(CONFIG, "signals") +PATHS_CONFIG = require_section(CONFIG, "paths") +HURST_CONFIG = CONFIG.get("hurst", {}) +PRICES_CONFIG = CONFIG.get("prices", {}) +RUN_CONFIG = CONFIG.get("run", {}) + +BASE_DIR = Path(PATHS_CONFIG.get("base_dir", ".")).resolve() +OUTPUT_DIR = BASE_DIR / PATHS_CONFIG.get("output_dir", "output") +PLOT_DIR = BASE_DIR / PATHS_CONFIG.get("plot_dir", "plot") +# Universe now expected inside Input folder +UNIVERSO_XLSX = BASE_DIR / PATHS_CONFIG.get("input_universe", "Input/Universo per Trading System.xlsx") +CONNECTION_TXT = BASE_DIR / PATHS_CONFIG.get("connection_txt", "connection.txt") +AUDIT_LOG_CSV = BASE_DIR / PATHS_CONFIG.get("audit_log_csv", OUTPUT_DIR / "trades_audit_log.csv") +OPEN_TRADES_DIR = BASE_DIR / PATHS_CONFIG.get("open_trades_dir", "open_trades") +DROPBOX_EXPORT_DIR = Path(r"C:\Users\Admin\Dropbox\Condivisa Lavoro\Segnali di trading su ETF") + +def _dated_signals_filename() -> Path: + date_prefix = pd.Timestamp.today().strftime("%Y%m%d") + return OUTPUT_DIR / f"{date_prefix}_signals.xlsx" # Stored procedure / parametri DB SP_NAME_DEFAULT = str(require_value(DB_CONFIG, "stored_proc", "db")) @@ -115,45 +115,17 @@ else: RP_MAX_WEIGHT = float(RP_MAX_WEIGHT) # Sizing -BASE_CAPITAL_PER_STRATEGY = float(require_value(SIGNALS_CONFIG, "base_capital_per_strategy", "signals")) -MIN_TRADE_NOTIONAL = float(require_value(SIGNALS_CONFIG, "min_trade_notional", "signals")) -RISK_PARITY_LOOKBACK = int(require_value(SIGNALS_CONFIG, "risk_parity_lookback", "signals")) -SP_NAME_DEFAULT = DB_CONFIG.get("stored_proc", "opt_RendimentoGiornaliero1_ALL") -SP_N_DEFAULT = DB_CONFIG.get("n_bars", 1305) -PTF_CURR_DEFAULT = DB_CONFIG.get("ptf_curr", "EUR") - -# Pattern recognition (come backtest) -WP = PATTERN_CONFIG.get("wp", 60) -HA = PATTERN_CONFIG.get("ha", 10) -KNN_K = PATTERN_CONFIG.get("knn_k", 25) -THETA = PATTERN_CONFIG.get("theta", 0.005) # 0,005% in decimali (identico al backtest) -Z_REV = TAGGING_CONFIG.get("z_rev", 2.0) -Z_VOL = TAGGING_CONFIG.get("z_vol", 2.0) -STD_COMP_PCT = TAGGING_CONFIG.get("std_comp_pct", 0.15) - -# Exit rules (identiche al backtest) -SL_BPS = SIGNALS_CONFIG.get("sl_bps", 300.0) -TP_BPS = SIGNALS_CONFIG.get("tp_bps", 800.0) -TRAIL_BPS = SIGNALS_CONFIG.get("trail_bps", 300.0) -TIME_STOP_BARS = SIGNALS_CONFIG.get("time_stop_bars", 20) -THETA_EXIT = SIGNALS_CONFIG.get("theta_exit", 0.0) # soglia debolezza -WEAK_DAYS_EXIT = SIGNALS_CONFIG.get("weak_days_exit") # uscita IMMEDIATA in caso di debolezza (come backtest) - -# Ranking e selezione Top-N per APERTURE -MAX_OPEN = SIGNALS_CONFIG.get("max_open", 15) # cap strumenti aperti oggi (come backtest) - -# Allineamento al backtest v3.1.5 per il cap del Risk Parity -TOP_N_MAX = RANKING_CONFIG.get("top_n_max", MAX_OPEN) -RP_MAX_WEIGHT = RANKING_CONFIG.get("rp_max_weight", 2 / max(TOP_N_MAX, 1)) # ≈ 0.1333 = 13,33% per singolo asset - -# Sizing -BASE_CAPITAL_PER_STRATEGY = SIGNALS_CONFIG.get("base_capital_per_strategy", 100.0) -MIN_TRADE_NOTIONAL = SIGNALS_CONFIG.get("min_trade_notional", 0.01) -RISK_PARITY_LOOKBACK = SIGNALS_CONFIG.get("risk_parity_lookback", 60) - -# Calendario -BUSINESS_DAYS_ONLY = True -SEED = 42 +BASE_CAPITAL_PER_STRATEGY = float(require_value(SIGNALS_CONFIG, "base_capital_per_strategy", "signals")) +MIN_TRADE_NOTIONAL = float(require_value(SIGNALS_CONFIG, "min_trade_notional", "signals")) +RISK_PARITY_LOOKBACK = int(require_value(SIGNALS_CONFIG, "risk_parity_lookback", "signals")) +HURST_LOOKBACK = HURST_CONFIG.get("lookback", None) +HURST_MIN_LENGTH = int(HURST_CONFIG.get("min_length", 200)) +OPEN_PRICE_BASE_URL = str(PRICES_CONFIG.get("base_url", "https://fin.scorer.app/finance/euronext/price")) +OPEN_MAX_RETRY = int(PRICES_CONFIG.get("max_retry", 3)) +OPEN_SLEEP_SEC = float(PRICES_CONFIG.get("sleep_sec", 0.1)) +OPEN_TIMEOUT = float(PRICES_CONFIG.get("timeout", 10)) +BUSINESS_DAYS_ONLY = bool(RUN_CONFIG.get("business_days_only", True)) +SEED = int(RUN_CONFIG.get("seed", 42)) warnings.filterwarnings("ignore") np.random.seed(SEED) @@ -241,34 +213,30 @@ def _db_fetch_returns(conn_str: str, out_all = out_all.dropna(subset=["Date"]).sort_values(["ISIN", "Date"]).reset_index(drop=True) return out_all[["Date", "ISIN", "Ret"]] -# ========================= -# UNIVERSO + OPEN PRICE API (schema checker) -# ========================= -OPEN_MAX_RETRY = 3 -OPEN_SLEEP_SEC = 0.1 -OPEN_TIMEOUT = 10 - -def load_universe(path: Path) -> pd.DataFrame: - df = pd.read_excel(path) - if "ISIN" not in df.columns: - raise KeyError("Nel file Universo manca la colonna 'ISIN'") - df["ISIN"] = df["ISIN"].astype(str).str.strip() +# ========================= +# UNIVERSO + OPEN PRICE API (schema checker) +# ========================= +def load_universe(path: Path) -> pd.DataFrame: + df = pd.read_excel(path) + if "ISIN" not in df.columns: + raise KeyError("Nel file Universo manca la colonna 'ISIN'") + df["ISIN"] = df["ISIN"].astype(str).str.strip() for col in ["Asset Class", "Mercato", "TickerOpen"]: if col not in df.columns: df[col] = "" df[col] = df[col].astype(str).str.strip() return df -def _build_symbol_euronext(row: pd.Series) -> Tuple[str, str]: - isin = str(row.get("ISIN", "")).strip() - venue = str(row.get("Mercato", "")).strip() - tok = str(row.get("TickerOpen", "") or "").strip() - base = "https://fin.scorer.app/finance/euronext/price" - if tok and "-" in tok and tok.split("-")[0].upper() == isin.upper(): - return base, tok - if isin and venue: - return base, f"{isin}-{venue}" - return base, isin +def _build_symbol_euronext(row: pd.Series) -> Tuple[str, str]: + isin = str(row.get("ISIN", "")).strip() + venue = str(row.get("Mercato", "")).strip() + tok = str(row.get("TickerOpen", "") or "").strip() + base = OPEN_PRICE_BASE_URL + if tok and "-" in tok and tok.split("-")[0].upper() == isin.upper(): + return base, tok + if isin and venue: + return base, f"{isin}-{venue}" + return base, isin def get_open_price(isin: str, universe: pd.DataFrame) -> Optional[float]: """ @@ -732,11 +700,15 @@ def main_run(run_date: Optional[dt.date] = None): n_bars=SP_N_DEFAULT, ptf_curr=PTF_CURR_DEFAULT, ) - if returns_long.empty: - raise RuntimeError("Nessun rendimento disponibile dal DB (SP vuota?).") - - # 2b) Hurst map per ISIN (stessa logica concettuale del backtest) - hurst_map = build_hurst_map(returns_long, lookback=252) + if returns_long.empty: + raise RuntimeError("Nessun rendimento disponibile dal DB (SP vuota?).") + + # 2b) Hurst map per ISIN (stessa logica concettuale del backtest) + hurst_map = build_hurst_map( + returns_long, + lookback=HURST_LOOKBACK or SP_N_DEFAULT, + min_length=HURST_MIN_LENGTH, + ) # 3) Segnali EOD su D con THETA = Hurst/100 per ISIN sig_df = generate_signals_today(universe, returns_long, today, hurst_map=hurst_map)