Applicato .env, creato yaml, esportata configurazione, inserita verifica dati scaricati da DB

20240929 Ottimizzatore Versione 2.2 UK.py

@@ -7,15 +7,21 @@ from pypfopt.exceptions import OptimizationError
 import matplotlib.pyplot as plt
 import os
 import sys
+from dotenv import load_dotenv
+import yaml
 
 # Cartelle di input/output/plot
 OUTPUT_DIR = "Output"
 INPUT_DIR = "Input"
 PLOT_DIR = "Plot"
+CONFIG_FILE = "config.yaml"
 
 os.makedirs(OUTPUT_DIR, exist_ok=True)
 os.makedirs(INPUT_DIR, exist_ok=True)
 os.makedirs(PLOT_DIR, exist_ok=True)
+load_dotenv()
+
+GAP_FFILL_LIMIT_DAYS = 5  # forward-fill limit per buchi di calendario
 
 def excel_path(filename: str) -> str:
     """Percorso completo per i file Excel di output."""
@@ -25,13 +31,74 @@ def plot_path(filename: str) -> str:
     """Percorso completo per i file di grafico."""
     return os.path.join(PLOT_DIR, filename)
 
-# Configurazione della connessione al database
-username = 'readonly'
-password = 'e8nqtSa39L4Le3'
-host = '26.69.45.60'
-database = 'FirstSolutionDB'
-port = 1433
+# Configurazione della connessione al database (variabili d'ambiente) e parametri portafoglio
+def load_db_config():
+    req = ["DB_USERNAME", "DB_PASSWORD", "DB_HOST", "DB_NAME"]
+    missing = [k for k in req if not os.getenv(k)]
+    if missing:
+        raise RuntimeError(f"Variabili d'ambiente mancanti per il DB: {', '.join(missing)}")
+    return {
+        "username": os.getenv("DB_USERNAME"),
+        "password": os.getenv("DB_PASSWORD"),
+        "host": os.getenv("DB_HOST"),
+        "port": os.getenv("DB_PORT", "1433"),
+        "database": os.getenv("DB_NAME"),
+    }
+
+DEFAULT_VOL_TARGETS = [
+    {"years": 5, "target_vol": 0.06, "name": "VAR3_GBP"},
+]
+
+DEFAULT_ASSET_CLASS_LIMITS = {
+    'Azionari': 0.75,
+    'Obbligazionari': 0.75,
+    'Metalli Preziosi': 0.20,
+    'Materie Prime': 0.05,
+    'Immobiliare': 0.05,
+    'Criptovalute': 0.05,
+    'Monetari': 0.10
+}
+
+def load_targets_and_limits(config_file: str, profile: str = "uk"):
+    """Legge target di volatilità e limiti asset class dal file di configurazione."""
+    cfg = {}
+    try:
+        with open(config_file, "r", encoding="utf-8") as f:
+            cfg = yaml.safe_load(f) or {}
+    except FileNotFoundError:
+        cfg = {}
+
+    vt_cfg = cfg.get("volatility_targets", {})
+    vt_list = []
+    if isinstance(vt_cfg, dict):
+        vt_list = vt_cfg.get(profile) or vt_cfg.get("default") or []
+    elif isinstance(vt_cfg, list):
+        vt_list = vt_cfg
+    if not vt_list:
+        vt_list = DEFAULT_VOL_TARGETS
+
+    volatility_targets_local = {
+        (int(item["years"]), float(item["target_vol"])): item["name"]
+        for item in vt_list
+        if "years" in item and "target_vol" in item and "name" in item
+    }
+
+    asset_limits_cfg = cfg.get("asset_class_limits") or {}
+    if not asset_limits_cfg:
+        asset_limits_cfg = DEFAULT_ASSET_CLASS_LIMITS
+    asset_class_limits_local = {k: float(v) for k, v in asset_limits_cfg.items()}
+
+    return volatility_targets_local, asset_class_limits_local
+
+db_cfg = load_db_config()
+username = db_cfg["username"]
+password = db_cfg["password"]
+host = db_cfg["host"]
+database = db_cfg["database"]
+port = db_cfg["port"]
 connection_string = f"mssql+pyodbc://{username}:{password}@{host}:{port}/{database}?driver=ODBC+Driver+17+for+SQL+Server"
+CONFIG_PROFILE = os.getenv("CONFIG_PROFILE", "uk")
+volatility_targets, asset_class_limits_cfg = load_targets_and_limits(CONFIG_FILE, CONFIG_PROFILE)
 
 try:
     # Crea l'Engine
@@ -54,6 +121,85 @@ template_df = pd.read_excel(template_path)
 file_path = os.path.join(INPUT_DIR, 'Universo ETF per ottimizzatore UK.xlsx')
 df = pd.read_excel(file_path, usecols=['ISIN', 'Nome', 'Categoria', 'Asset Class', 'PesoMax', 'Codice Titolo'],dtype={'Codice Titolo':str})
 
+# =========================
+# VALIDAZIONE DATI RENDIMENTI
+# =========================
+def _gap_ranges_missing(series: pd.Series, all_dates: pd.DatetimeIndex, max_entries: int = 3):
+    """Restituisce fino a max_entries intervalli di date mancanti (start, end, len)."""
+    missing_idx = series[series.isna()].index
+    if missing_idx.empty:
+        return []
+    positions = [all_dates.get_loc(ts) for ts in missing_idx if ts in all_dates]
+    if not positions:
+        return []
+    ranges = []
+    start_pos = positions[0]
+    end_pos = positions[0]
+    for pos in positions[1:]:
+        if pos == end_pos + 1:
+            end_pos = pos
+        else:
+            ranges.append((all_dates[start_pos], all_dates[end_pos], end_pos - start_pos + 1))
+            start_pos = end_pos = pos
+    ranges.append((all_dates[start_pos], all_dates[end_pos], end_pos - start_pos + 1))
+    return ranges[:max_entries]
+
+
+def preprocess_returns(temp_df: pd.DataFrame, isin: str, all_dates: pd.DatetimeIndex,
+                       gap_ffill_limit: int = GAP_FFILL_LIMIT_DAYS) -> pd.Series:
+    """Pulizia/validazione serie rendimenti (tipi, duplicati, buchi, niente fillna(0))."""
+    required_cols = {"Px_Date", "RendimentoGiornaliero"}
+    missing_cols = required_cols.difference(temp_df.columns)
+    if missing_cols:
+        print(f"[{isin}] Colonne mancanti {missing_cols}, asset ignorato.")
+        return pd.Series(dtype=float)
+
+    df_local = temp_df.copy()
+    df_local["Px_Date"] = pd.to_datetime(df_local["Px_Date"], errors="coerce").dt.normalize()
+    df_local["RendimentoGiornaliero"] = pd.to_numeric(df_local["RendimentoGiornaliero"], errors="coerce")
+
+    before_rows = len(df_local)
+    df_local = df_local.dropna(subset=["Px_Date", "RendimentoGiornaliero"])
+    dropped_na = before_rows - len(df_local)
+
+    dup_count = df_local.duplicated(subset=["Px_Date"]).sum()
+    if dup_count > 0:
+        print(f"[{isin}] Drop duplicati su Px_Date: {dup_count}")
+    df_local = df_local.drop_duplicates(subset=["Px_Date"], keep="last").sort_values("Px_Date")
+
+    if df_local.empty:
+        print(f"[{isin}] Nessuna riga valida dopo cleaning, asset ignorato.")
+        return pd.Series(dtype=float)
+
+    df_local["RendimentoGiornaliero"] = df_local["RendimentoGiornaliero"] / 100.0
+
+    series = df_local.set_index("Px_Date")["RendimentoGiornaliero"].reindex(all_dates)
+    missing_before_fill = int(series.isna().sum())
+    if missing_before_fill > 0:
+        gaps = _gap_ranges_missing(series, all_dates, max_entries=3)
+        if gaps:
+            gaps_str = "; ".join([f"{g[0].date()}->{g[1].date()} ({g[2]} gg)" for g in gaps])
+            print(f"[{isin}] Date mancanti prima del fill (prime): {gaps_str}")
+
+    series_ffill = series.ffill(limit=gap_ffill_limit)
+    first_valid = series_ffill.first_valid_index()
+    last_valid = series_ffill.last_valid_index()
+    if first_valid is None or last_valid is None:
+        print(f"[{isin}] Serie vuota dopo forward-fill, asset ignorato.")
+        return pd.Series(dtype=float)
+
+    series_ffill = series_ffill.loc[first_valid:last_valid]
+    residual_missing = int(series_ffill.isna().sum())
+    if residual_missing > 0:
+        print(f"[{isin}] {residual_missing} valori mancanti non coperti dal forward-fill (limite {gap_ffill_limit}), righe scartate.")
+        series_ffill = series_ffill.dropna()
+
+    coverage_days = series_ffill.shape[0]
+    if dropped_na > 0 or missing_before_fill > 0:
+        print(f"[{isin}] Righe totali: {before_rows}, drop NA: {dropped_na}, copertura finale: {coverage_days} giorni.")
+
+    return series_ffill
+
 # Intervallo di date degli ultimi 5 anni, escludendo sabati e domeniche
 end_date = pd.Timestamp.now().normalize() - pd.Timedelta(days=1)
 start_date = end_date - pd.DateOffset(years=5)
@@ -72,29 +218,23 @@ for isin in df['ISIN'].unique():
         if temp_df.empty:
             print(f"Nessun dato recuperato per {isin}, skipping...")
             continue
-        temp_df['Px_Date'] = pd.to_datetime(temp_df['Px_Date'], format='%Y-%m-%d').dt.normalize()
-        temp_df.set_index('Px_Date', inplace=True)
-        temp_df['RendimentoGiornaliero'] = temp_df['RendimentoGiornaliero'] / 100
-        final_df[isin] = temp_df['RendimentoGiornaliero'].reindex(all_dates)
+        clean_series = preprocess_returns(temp_df, isin, all_dates, gap_ffill_limit=GAP_FFILL_LIMIT_DAYS)
+        if clean_series.empty:
+            print(f"Nessun dato valido per {isin} dopo la validazione, asset ignorato.")
+            continue
+        final_df[isin] = clean_series.reindex(all_dates)
         isin_from_db.add(isin)
-        print(f"Dati recuperati per {isin}: {final_df[isin].count()} righe di dati non-null prelevate.")
+        non_null = int(final_df[isin].count())
+        missing_left = int(final_df[isin].isna().sum())
+        print(f"Dati recuperati per {isin}: {non_null} righe valide, mancanti residui: {missing_left}.")
     except SQLAlchemyError as e:
         print(f"Errore durante l'esecuzione della stored procedure per {isin}:", e)
 
-final_df.fillna(0, inplace=True)
-
-# Configurazione degli obiettivi di volatilità
-volatility_targets = {
-    (5, 0.06): 'VAR3_GBP',
-    #(1, 0.12): 'VAR6_1Y',
-    #(3, 0.12): 'VAR6_3Y',
-    (5, 0.12): 'VAR6_GBP',
-    (5, 0.18): 'VAR9_GBP'
-}
-
-# Definizione del numero di giorni lavorativi per anno
-days_per_year = 252
-riskfree_rate = 0.02
+final_df = final_df.loc[:, final_df.notna().any()]  # elimina asset senza dati utili
+if final_df.shape[1] == 0:
+    print("Nessun ISIN valido dopo la validazione dei rendimenti, uscita.")
+    sys.exit(1)
+final_df = final_df.dropna(how='all')
 
 # Ottimizzazione per ciascun target di volatilità e salvataggio dei risultati
 optimized_weights = pd.DataFrame()
@@ -111,21 +251,14 @@ for (years, target_vol), name in volatility_targets.items():
     
     # Aggiunta dei vincoli per le categorie e le asset class
     categories_limits = df.groupby('Categoria')['PesoMax'].max().to_dict()
-    asset_class_limits = {
-        'Azionari': 0.75,
-        'Obbligazionari': 0.75,
-        'Metalli Preziosi': 0.20,
-        'Materie Prime': 0.05,
-        'Immobiliare': 0.05
-    }
-    
+
     for category, max_weight in categories_limits.items():
         isin_list = df[df['Categoria'] == category]['ISIN'].tolist()
         category_idx = [period_df.columns.get_loc(isin) for isin in isin_list if isin in period_df.columns]
         ef.add_constraint(lambda w: sum(w[i] for i in category_idx) <= max_weight)
         ef.add_constraint(lambda w: sum(w[i] for i in category_idx) >= 0)
-    
-    for asset_class, max_weight in asset_class_limits.items():
+
+    for asset_class, max_weight in asset_class_limits_cfg.items():
         isin_list = df[df['Asset Class'] == asset_class]['ISIN'].tolist()
         asset_class_idx = [period_df.columns.get_loc(isin) for isin in isin_list if isin in period_df.columns]
         ef.add_constraint(lambda w: sum(w[i] for i in asset_class_idx) <= max_weight)
@@ -162,7 +295,7 @@ for (years, target_vol), name in volatility_targets.items():
         print(f"File {output_file_path} saved successfully.")
         
         # Grafico a torta per ciascun portafoglio ottimizzato
-        asset_allocations = {asset: 0 for asset in asset_class_limits}
+        asset_allocations = {asset: 0 for asset in asset_class_limits_cfg}
         for isin, weight in weights.items():
             asset_class = df.loc[df['ISIN'] == isin, 'Asset Class'].values[0]
             asset_allocations[asset_class] += weight

20251007 Ottimizzatore Versione 2.5.2.py

@@ -13,6 +13,8 @@ import os
 import numpy as np
 import pandas as pd
 import matplotlib.pyplot as plt
+from dotenv import load_dotenv
+import yaml
 
 from sqlalchemy import create_engine, text
 from sqlalchemy.exc import SQLAlchemyError
@@ -25,10 +27,14 @@ from pypfopt.exceptions import OptimizationError
 OUTPUT_DIR = "Output"
 PLOT_DIR = "Plot"
 INPUT_DIR = "Input"
+CONFIG_FILE = "config.yaml"
 
 os.makedirs(OUTPUT_DIR, exist_ok=True)
 os.makedirs(PLOT_DIR, exist_ok=True)
 os.makedirs(INPUT_DIR, exist_ok=True)
+load_dotenv()
+
+GAP_FFILL_LIMIT_DAYS = 5  # forward-fill limit per buchi di calendario
 
 def excel_path(filename: str) -> str:
     """Costruisce il percorso completo per un file Excel nella cartella di output."""
@@ -245,27 +251,28 @@ def portfolio_path_metrics(period_df: pd.DataFrame,
         "Hmin_100m_5Y": hmin_5y_months
     }
 
-# --- Lettura parametri dal file connection.txt ---
-params = {}
-with open("connection.txt", "r") as f:
-    for line in f:
-        line = line.strip()
-        if line and not line.startswith("#"):
-            key, value = line.split("=", 1)
-            params[key.strip()] = value.strip()
+def load_db_config():
+    """Recupera i parametri di connessione dal set di variabili d'ambiente."""
+    required_keys = ["DB_USERNAME", "DB_PASSWORD", "DB_HOST", "DB_NAME"]
+    missing = [k for k in required_keys if not os.getenv(k)]
+    if missing:
+        raise RuntimeError(f"Variabili d'ambiente mancanti per il DB: {', '.join(missing)}")
 
-username = params.get("username")
-password = params.get("password")
-host = params.get("host")
-port = params.get("port", "1433")
-database = params.get("database")
+    return {
+        "username": os.getenv("DB_USERNAME"),
+        "password": os.getenv("DB_PASSWORD"),
+        "host": os.getenv("DB_HOST"),
+        "port": os.getenv("DB_PORT", "1433"),
+        "database": os.getenv("DB_NAME"),
+    }
 
+db_cfg = load_db_config()
 connection_string = (
-    f"mssql+pyodbc://{username}:{password}@{host}:{port}/{database}"
+    f"mssql+pyodbc://{db_cfg['username']}:{db_cfg['password']}@{db_cfg['host']}:{db_cfg['port']}/{db_cfg['database']}"
     "?driver=ODBC+Driver+17+for+SQL+Server"
 )
 
-print("Connection string letta correttamente")
+print("Connection string configurata da variabili d'ambiente.")
 
 # =========================
 # CONNESSIONE AL DB
@@ -292,6 +299,85 @@ df = pd.read_excel(
     dtype={'Codice Titolo': str}
 )
 
+# =========================
+# VALIDAZIONE DATI RENDIMENTI
+# =========================
+def _gap_ranges_missing(series: pd.Series, all_dates: pd.DatetimeIndex, max_entries: int = 3):
+    """Restituisce fino a max_entries intervalli di date mancanti (start, end, len)."""
+    missing_idx = series[series.isna()].index
+    if missing_idx.empty:
+        return []
+    positions = [all_dates.get_loc(ts) for ts in missing_idx if ts in all_dates]
+    if not positions:
+        return []
+    ranges = []
+    start_pos = positions[0]
+    end_pos = positions[0]
+    for pos in positions[1:]:
+        if pos == end_pos + 1:
+            end_pos = pos
+        else:
+            ranges.append((all_dates[start_pos], all_dates[end_pos], end_pos - start_pos + 1))
+            start_pos = end_pos = pos
+    ranges.append((all_dates[start_pos], all_dates[end_pos], end_pos - start_pos + 1))
+    return ranges[:max_entries]
+
+
+def preprocess_returns(temp_df: pd.DataFrame, isin: str, all_dates: pd.DatetimeIndex,
+                       gap_ffill_limit: int = GAP_FFILL_LIMIT_DAYS) -> pd.Series:
+    """Pulizia/validazione serie rendimenti (tipi, duplicati, buchi, niente fillna(0))."""
+    required_cols = {"Px_Date", "RendimentoGiornaliero"}
+    missing_cols = required_cols.difference(temp_df.columns)
+    if missing_cols:
+        print(f"[{isin}] Colonne mancanti {missing_cols}, asset ignorato.")
+        return pd.Series(dtype=float)
+
+    df_local = temp_df.copy()
+    df_local["Px_Date"] = pd.to_datetime(df_local["Px_Date"], errors="coerce").dt.normalize()
+    df_local["RendimentoGiornaliero"] = pd.to_numeric(df_local["RendimentoGiornaliero"], errors="coerce")
+
+    before_rows = len(df_local)
+    df_local = df_local.dropna(subset=["Px_Date", "RendimentoGiornaliero"])
+    dropped_na = before_rows - len(df_local)
+
+    dup_count = df_local.duplicated(subset=["Px_Date"]).sum()
+    if dup_count > 0:
+        print(f"[{isin}] Drop duplicati su Px_Date: {dup_count}")
+    df_local = df_local.drop_duplicates(subset=["Px_Date"], keep="last").sort_values("Px_Date")
+
+    if df_local.empty:
+        print(f"[{isin}] Nessuna riga valida dopo cleaning, asset ignorato.")
+        return pd.Series(dtype=float)
+
+    df_local["RendimentoGiornaliero"] = df_local["RendimentoGiornaliero"] / 100.0
+
+    series = df_local.set_index("Px_Date")["RendimentoGiornaliero"].reindex(all_dates)
+    missing_before_fill = int(series.isna().sum())
+    if missing_before_fill > 0:
+        gaps = _gap_ranges_missing(series, all_dates, max_entries=3)
+        if gaps:
+            gaps_str = "; ".join([f"{g[0].date()}->{g[1].date()} ({g[2]} gg)" for g in gaps])
+            print(f"[{isin}] Date mancanti prima del fill (prime): {gaps_str}")
+
+    series_ffill = series.ffill(limit=gap_ffill_limit)
+    first_valid = series_ffill.first_valid_index()
+    last_valid = series_ffill.last_valid_index()
+    if first_valid is None or last_valid is None:
+        print(f"[{isin}] Serie vuota dopo forward-fill, asset ignorato.")
+        return pd.Series(dtype=float)
+
+    series_ffill = series_ffill.loc[first_valid:last_valid]
+    residual_missing = int(series_ffill.isna().sum())
+    if residual_missing > 0:
+        print(f"[{isin}] {residual_missing} valori mancanti non coperti dal forward-fill (limite {gap_ffill_limit}), righe scartate.")
+        series_ffill = series_ffill.dropna()
+
+    coverage_days = series_ffill.shape[0]
+    if dropped_na > 0 or missing_before_fill > 0:
+        print(f"[{isin}] Righe totali: {before_rows}, drop NA: {dropped_na}, copertura finale: {coverage_days} giorni.")
+
+    return series_ffill
+
 # =========================
 # SERIE STORICHE RENDIMENTI
 # =========================
@@ -310,35 +396,78 @@ for isin in df['ISIN'].unique():
         if temp_df.empty:
             print(f"Nessun dato recuperato per {isin}, skipping...")
             continue
-        temp_df['Px_Date'] = pd.to_datetime(temp_df['Px_Date'], format='%Y-%m-%d', errors='coerce').dt.normalize()
-        temp_df = temp_df.dropna(subset=['Px_Date'])
-        temp_df.set_index('Px_Date', inplace=True)
-        temp_df['RendimentoGiornaliero'] = temp_df['RendimentoGiornaliero'] / 100
-        final_df[isin] = temp_df['RendimentoGiornaliero'].reindex(all_dates)
+        clean_series = preprocess_returns(temp_df, isin, all_dates, gap_ffill_limit=GAP_FFILL_LIMIT_DAYS)
+        if clean_series.empty:
+            print(f"Nessun dato valido per {isin} dopo la validazione, asset ignorato.")
+            continue
+        final_df[isin] = clean_series.reindex(all_dates)
         isin_from_db.add(isin)
-        print(f"Dati recuperati per {isin}: {final_df[isin].count()} righe di dati non-null prelevate.")
+        non_null = int(final_df[isin].count())
+        missing_left = int(final_df[isin].isna().sum())
+        print(f"Dati recuperati per {isin}: {non_null} righe valide, mancanti residui: {missing_left}.")
     except SQLAlchemyError as e:
         print(f"Errore durante l'esecuzione della stored procedure per {isin}:", e)
 
-final_df.fillna(0, inplace=True)
+final_df = final_df.loc[:, final_df.notna().any()]  # elimina asset senza dati utili
+if final_df.shape[1] == 0:
+    print("Nessun ISIN valido dopo la validazione dei rendimenti, uscita.")
+    sys.exit(1)
+final_df = final_df.dropna(how='all')
 
 # -------- H_min sempre su 5 anni (21 gg = 1 mese) --------
 five_year_df = final_df.loc[end_date - pd.DateOffset(years=5): end_date]
 
 # =========================
-# CONFIGURAZIONE OBIETTIVI
+# CONFIGURAZIONE OBIETTIVI (da config.yaml con fallback)
 # =========================
-volatility_targets = {
-    # (1, 0.06): 'VAR3_1Y',
-    # (3, 0.06): 'VAR3_3Y',
-    (5, 0.06): 'VAR3_5Y',
-    (1, 0.12): 'VAR6_1Y',
-    (3, 0.12): 'VAR6_3Y',
-    (5, 0.12): 'VAR6_5Y',
-    # (1, 0.18): 'VAR9_1Y',
-    # (3, 0.18): 'VAR9_3Y',
-    (5, 0.18): 'VAR9_5Y'
+DEFAULT_VOL_TARGETS = [
+    {"years": 5, "target_vol": 0.06, "name": "VAR3_5Y"},
+    {"years": 1, "target_vol": 0.12, "name": "VAR6_1Y"},
+    {"years": 3, "target_vol": 0.12, "name": "VAR6_3Y"},
+    {"years": 5, "target_vol": 0.12, "name": "VAR6_5Y"},
+    {"years": 5, "target_vol": 0.18, "name": "VAR9_5Y"},
+]
+
+DEFAULT_ASSET_CLASS_LIMITS = {
+    'Azionari': 0.75, 'Obbligazionari': 0.75,
+    'Metalli Preziosi': 0.20, 'Materie Prime': 0.05,
+    'Immobiliare': 0.05, 'Criptovalute': 0.05, 'Monetari': 0.10
 }
+
+def load_targets_and_limits(config_file: str, profile: str = "default"):
+    """Legge target di volatilità e limiti asset class dal file di configurazione."""
+    cfg = {}
+    try:
+        with open(config_file, "r", encoding="utf-8") as f:
+            cfg = yaml.safe_load(f) or {}
+    except FileNotFoundError:
+        cfg = {}
+
+    vt_cfg = cfg.get("volatility_targets", {})
+    vt_list = []
+    if isinstance(vt_cfg, dict):
+        vt_list = vt_cfg.get(profile) or vt_cfg.get("default") or []
+    elif isinstance(vt_cfg, list):
+        vt_list = vt_cfg
+    if not vt_list:
+        vt_list = DEFAULT_VOL_TARGETS
+
+    volatility_targets_local = {
+        (int(item["years"]), float(item["target_vol"])): item["name"]
+        for item in vt_list
+        if "years" in item and "target_vol" in item and "name" in item
+    }
+
+    asset_limits_cfg = cfg.get("asset_class_limits") or {}
+    if not asset_limits_cfg:
+        asset_limits_cfg = DEFAULT_ASSET_CLASS_LIMITS
+    asset_class_limits_local = {k: float(v) for k, v in asset_limits_cfg.items()}
+
+    return volatility_targets_local, asset_class_limits_local
+
+CONFIG_PROFILE = os.getenv("CONFIG_PROFILE", "default")
+volatility_targets, asset_class_limits_cfg = load_targets_and_limits(CONFIG_FILE, CONFIG_PROFILE)
+
 days_per_year = 252
 riskfree_rate = 0.02
 
@@ -432,12 +561,7 @@ for (years, target_vol), name in volatility_targets.items():
             ef.add_constraint(lambda w, idxs=idxs, maxw=maxw: sum(w[i] for i in idxs) <= maxw)
 
     # Vincoli per Asset Class
-    asset_class_limits = {
-        'Azionari': 0.75, 'Obbligazionari': 0.75,
-        'Metalli Preziosi': 0.20, 'Materie Prime': 0.05,
-        'Immobiliare': 0.05, 'Criptovalute': 0.05, 'Monetari': 0.1
-    }
-    for ac, maxw in asset_class_limits.items():
+    for ac, maxw in asset_class_limits_cfg.items():
         isin_list = df[df['Asset Class'] == ac]['ISIN'].tolist()
         idxs = [period_df.columns.get_loc(isin) for isin in isin_list if isin in period_df.columns]
         if idxs:
@@ -478,13 +602,16 @@ for (years, target_vol), name in volatility_targets.items():
                 results_rows.append(row)
 
         results_full_df = pd.DataFrame(results_rows, columns=template_cols)
-        output_df = pd.concat([template_df.iloc[0:0], results_full_df], ignore_index=True)
+        if results_full_df.empty:
+            output_df = template_df.iloc[0:0].copy()
+        else:
+            output_df = results_full_df.reindex(columns=template_cols)
         output_file_path = excel_path(f'PTFOPT{name}.xlsx')
         output_df.to_excel(output_file_path, index=False)
         print(f"File {output_file_path} saved successfully.")
 
         # --- Pie chart asset allocation (se ci sono pesi > 0) ---
-        asset_allocations = {asset: 0 for asset in asset_class_limits}
+        asset_allocations = {asset: 0 for asset in asset_class_limits_cfg}
         for isin, weight in weights.items():
             r_sel = df.loc[df['ISIN'] == isin]
             if r_sel.empty:

20251119 Ottimizzatore Lite v.1.0.py

@@ -11,6 +11,8 @@ import os
 import numpy as np
 import pandas as pd
 import matplotlib.pyplot as plt
+from dotenv import load_dotenv
+import yaml
 
 from sqlalchemy import create_engine, text
 from sqlalchemy.exc import SQLAlchemyError
@@ -23,10 +25,14 @@ from pypfopt.exceptions import OptimizationError
 OUTPUT_DIR = "Output"
 INPUT_DIR = "Input"
 PLOT_DIR = "Plot"
+CONFIG_FILE = "config.yaml"
 
 os.makedirs(OUTPUT_DIR, exist_ok=True)
 os.makedirs(INPUT_DIR, exist_ok=True)
 os.makedirs(PLOT_DIR, exist_ok=True)
+load_dotenv()
+
+GAP_FFILL_LIMIT_DAYS = 5  # forward-fill limit per buchi di calendario
 
 def excel_path(filename: str) -> str:
     """Costruisce il percorso completo per un file Excel nella cartella di output."""
@@ -187,27 +193,28 @@ def h_min_100(returns: pd.Series, month_len: int = 21):
 
     return np.nan, np.nan
 
-# --- Lettura parametri dal file connection.txt ---
-params = {}
-with open("connection.txt", "r") as f:
-    for line in f:
-        line = line.strip()
-        if line and not line.startswith("#"):
-            key, value = line.split("=", 1)
-            params[key.strip()] = value.strip()
+def load_db_config():
+    """Recupera i parametri di connessione dal set di variabili d'ambiente."""
+    required_keys = ["DB_USERNAME", "DB_PASSWORD", "DB_HOST", "DB_NAME"]
+    missing = [k for k in required_keys if not os.getenv(k)]
+    if missing:
+        raise RuntimeError(f"Variabili d'ambiente mancanti per il DB: {', '.join(missing)}")
 
-username = params.get("username")
-password = params.get("password")
-host = params.get("host")
-port = params.get("port", "1433")
-database = params.get("database")
+    return {
+        "username": os.getenv("DB_USERNAME"),
+        "password": os.getenv("DB_PASSWORD"),
+        "host": os.getenv("DB_HOST"),
+        "port": os.getenv("DB_PORT", "1433"),
+        "database": os.getenv("DB_NAME"),
+    }
 
+db_cfg = load_db_config()
 connection_string = (
-    f"mssql+pyodbc://{username}:{password}@{host}:{port}/{database}"
+    f"mssql+pyodbc://{db_cfg['username']}:{db_cfg['password']}@{db_cfg['host']}:{db_cfg['port']}/{db_cfg['database']}"
     "?driver=ODBC+Driver+17+for+SQL+Server"
 )
 
-print("Connection string letta correttamente")
+print("Connection string configurata da variabili d'ambiente.")
 
 # =========================
 # CONNESSIONE AL DB
@@ -234,6 +241,85 @@ df = pd.read_excel(
     dtype={'Codice Titolo': str}
 )
 
+# =========================
+# VALIDAZIONE DATI RENDIMENTI
+# =========================
+def _gap_ranges_missing(series: pd.Series, all_dates: pd.DatetimeIndex, max_entries: int = 3):
+    """Restituisce fino a max_entries intervalli di date mancanti (start, end, len)."""
+    missing_idx = series[series.isna()].index
+    if missing_idx.empty:
+        return []
+    positions = [all_dates.get_loc(ts) for ts in missing_idx if ts in all_dates]
+    if not positions:
+        return []
+    ranges = []
+    start_pos = positions[0]
+    end_pos = positions[0]
+    for pos in positions[1:]:
+        if pos == end_pos + 1:
+            end_pos = pos
+        else:
+            ranges.append((all_dates[start_pos], all_dates[end_pos], end_pos - start_pos + 1))
+            start_pos = end_pos = pos
+    ranges.append((all_dates[start_pos], all_dates[end_pos], end_pos - start_pos + 1))
+    return ranges[:max_entries]
+
+
+def preprocess_returns(temp_df: pd.DataFrame, isin: str, all_dates: pd.DatetimeIndex,
+                       gap_ffill_limit: int = GAP_FFILL_LIMIT_DAYS) -> pd.Series:
+    """Pulizia/validazione serie rendimenti (tipi, duplicati, buchi, niente fillna(0))."""
+    required_cols = {"Px_Date", "RendimentoGiornaliero"}
+    missing_cols = required_cols.difference(temp_df.columns)
+    if missing_cols:
+        print(f"[{isin}] Colonne mancanti {missing_cols}, asset ignorato.")
+        return pd.Series(dtype=float)
+
+    df_local = temp_df.copy()
+    df_local["Px_Date"] = pd.to_datetime(df_local["Px_Date"], errors="coerce").dt.normalize()
+    df_local["RendimentoGiornaliero"] = pd.to_numeric(df_local["RendimentoGiornaliero"], errors="coerce")
+
+    before_rows = len(df_local)
+    df_local = df_local.dropna(subset=["Px_Date", "RendimentoGiornaliero"])
+    dropped_na = before_rows - len(df_local)
+
+    dup_count = df_local.duplicated(subset=["Px_Date"]).sum()
+    if dup_count > 0:
+        print(f"[{isin}] Drop duplicati su Px_Date: {dup_count}")
+    df_local = df_local.drop_duplicates(subset=["Px_Date"], keep="last").sort_values("Px_Date")
+
+    if df_local.empty:
+        print(f"[{isin}] Nessuna riga valida dopo cleaning, asset ignorato.")
+        return pd.Series(dtype=float)
+
+    df_local["RendimentoGiornaliero"] = df_local["RendimentoGiornaliero"] / 100.0
+
+    series = df_local.set_index("Px_Date")["RendimentoGiornaliero"].reindex(all_dates)
+    missing_before_fill = int(series.isna().sum())
+    if missing_before_fill > 0:
+        gaps = _gap_ranges_missing(series, all_dates, max_entries=3)
+        if gaps:
+            gaps_str = "; ".join([f"{g[0].date()}->{g[1].date()} ({g[2]} gg)" for g in gaps])
+            print(f"[{isin}] Date mancanti prima del fill (prime): {gaps_str}")
+
+    series_ffill = series.ffill(limit=gap_ffill_limit)
+    first_valid = series_ffill.first_valid_index()
+    last_valid = series_ffill.last_valid_index()
+    if first_valid is None or last_valid is None:
+        print(f"[{isin}] Serie vuota dopo forward-fill, asset ignorato.")
+        return pd.Series(dtype=float)
+
+    series_ffill = series_ffill.loc[first_valid:last_valid]
+    residual_missing = int(series_ffill.isna().sum())
+    if residual_missing > 0:
+        print(f"[{isin}] {residual_missing} valori mancanti non coperti dal forward-fill (limite {gap_ffill_limit}), righe scartate.")
+        series_ffill = series_ffill.dropna()
+
+    coverage_days = series_ffill.shape[0]
+    if dropped_na > 0 or missing_before_fill > 0:
+        print(f"[{isin}] Righe totali: {before_rows}, drop NA: {dropped_na}, copertura finale: {coverage_days} giorni.")
+
+    return series_ffill
+
 # =========================
 # SERIE STORICHE RENDIMENTI
 # =========================
@@ -252,35 +338,77 @@ for isin in df['ISIN'].unique():
         if temp_df.empty:
             print(f"Nessun dato recuperato per {isin}, skipping...")
             continue
-        temp_df['Px_Date'] = pd.to_datetime(temp_df['Px_Date'], format='%Y-%m-%d', errors='coerce').dt.normalize()
-        temp_df = temp_df.dropna(subset=['Px_Date'])
-        temp_df.set_index('Px_Date', inplace=True)
-        temp_df['RendimentoGiornaliero'] = temp_df['RendimentoGiornaliero'] / 100
-        final_df[isin] = temp_df['RendimentoGiornaliero'].reindex(all_dates)
+        clean_series = preprocess_returns(temp_df, isin, all_dates, gap_ffill_limit=GAP_FFILL_LIMIT_DAYS)
+        if clean_series.empty:
+            print(f"Nessun dato valido per {isin} dopo la validazione, asset ignorato.")
+            continue
+        final_df[isin] = clean_series.reindex(all_dates)
         isin_from_db.add(isin)
-        print(f"Dati recuperati per {isin}: {final_df[isin].count()} righe di dati non-null prelevate.")
+        non_null = int(final_df[isin].count())
+        missing_left = int(final_df[isin].isna().sum())
+        print(f"Dati recuperati per {isin}: {non_null} righe valide, mancanti residui: {missing_left}.")
     except SQLAlchemyError as e:
         print(f"Errore durante l'esecuzione della stored procedure per {isin}:", e)
 
-final_df.fillna(0, inplace=True)
+final_df = final_df.loc[:, final_df.notna().any()]  # elimina asset senza dati utili
+if final_df.shape[1] == 0:
+    print("Nessun ISIN valido dopo la validazione dei rendimenti, uscita.")
+    sys.exit(1)
+final_df = final_df.dropna(how='all')
 
 # -------- H_min sempre su 5 anni (21 gg = 1 mese) --------
 five_year_df = final_df.loc[end_date - pd.DateOffset(years=5): end_date]
 
 # =========================
-# CONFIGURAZIONE OBIETTIVI
+# CONFIGURAZIONE OBIETTIVI (da config.yaml con fallback)
 # =========================
-volatility_targets = {
-    # (1, 0.06): 'VAR3_1Y',
-    # (3, 0.06): 'VAR3_3Y',
-    (5, 0.06): 'VAR3_5Y',
-    (1, 0.12): 'VAR6_1Y',
-    (3, 0.12): 'VAR6_3Y',
-    (5, 0.12): 'VAR6_5Y',
-    # (1, 0.18): 'VAR9_1Y',
-    # (3, 0.18): 'VAR9_3Y',
-    (5, 0.18): 'VAR9_5Y'
+DEFAULT_VOL_TARGETS = [
+    {"years": 5, "target_vol": 0.06, "name": "VAR3_5Y"},
+    {"years": 1, "target_vol": 0.12, "name": "VAR6_1Y"},
+    {"years": 3, "target_vol": 0.12, "name": "VAR6_3Y"},
+    {"years": 5, "target_vol": 0.12, "name": "VAR6_5Y"},
+    {"years": 5, "target_vol": 0.18, "name": "VAR9_5Y"},
+]
+
+DEFAULT_ASSET_CLASS_LIMITS = {
+    'Azionari': 0.75, 'Obbligazionari': 0.75,
+    'Metalli Preziosi': 0.20, 'Materie Prime': 0.05,
+    'Immobiliare': 0.05, 'Criptovalute': 0.05, 'Monetari': 0.10
 }
+
+def load_targets_and_limits(config_file: str, profile: str = "default"):
+    """Legge target di volatilità e limiti asset class dal file di configurazione."""
+    cfg = {}
+    try:
+        with open(config_file, "r", encoding="utf-8") as f:
+            cfg = yaml.safe_load(f) or {}
+    except FileNotFoundError:
+        cfg = {}
+
+    vt_cfg = cfg.get("volatility_targets", {})
+    vt_list = []
+    if isinstance(vt_cfg, dict):
+        vt_list = vt_cfg.get(profile) or vt_cfg.get("default") or []
+    elif isinstance(vt_cfg, list):
+        vt_list = vt_cfg
+    if not vt_list:
+        vt_list = DEFAULT_VOL_TARGETS
+
+    volatility_targets_local = {
+        (int(item["years"]), float(item["target_vol"])): item["name"]
+        for item in vt_list
+        if "years" in item and "target_vol" in item and "name" in item
+    }
+
+    asset_limits_cfg = cfg.get("asset_class_limits") or {}
+    if not asset_limits_cfg:
+        asset_limits_cfg = DEFAULT_ASSET_CLASS_LIMITS
+    asset_class_limits_local = {k: float(v) for k, v in asset_limits_cfg.items()}
+
+    return volatility_targets_local, asset_class_limits_local
+
+CONFIG_PROFILE = os.getenv("CONFIG_PROFILE", "default")
+volatility_targets, asset_class_limits_cfg = load_targets_and_limits(CONFIG_FILE, CONFIG_PROFILE)
 days_per_year = 252
 riskfree_rate = 0.02
 
@@ -374,16 +502,11 @@ for (years, target_vol), name in volatility_targets.items():
             ef.add_constraint(lambda w, idxs=idxs, maxw=maxw: sum(w[i] for i in idxs) <= maxw)
 
     # Vincoli per Asset Class
-    asset_class_limits = {
-        'Azionari': 0.75, 'Obbligazionari': 0.75,
-        'Metalli Preziosi': 0.20, 'Materie Prime': 0.05,
-        'Immobiliare': 0.05, 'Criptovalute': 0.05, 'Monetari': 0.1
-    }
-    for ac, maxw in asset_class_limits.items():
+    for ac, maxw in asset_class_limits_cfg.items():
         isin_list = df[df['Asset Class'] == ac]['ISIN'].tolist()
         idxs = [period_df.columns.get_loc(isin) for isin in isin_list if isin in period_df.columns]
         if idxs:
-            ef.add_constraint(lambda w, idxs=idxs, maxw=maxw: sum(w[i] for i in idxs) <= maxw)
+            ef.add_constraint(lambda w, idxs=idxs, maxw=maxw: w[idxs].sum() <= maxw)
 
     # ---------- Risoluzione ----------
     try:
@@ -420,13 +543,16 @@ for (years, target_vol), name in volatility_targets.items():
                 results_rows.append(row)
 
         results_full_df = pd.DataFrame(results_rows, columns=template_cols)
-        output_df = pd.concat([template_df.iloc[0:0], results_full_df], ignore_index=True)
+        if results_full_df.empty:
+            output_df = template_df.iloc[0:0].copy()
+        else:
+            output_df = results_full_df.reindex(columns=template_cols)
         output_file_path = excel_path(f'PTFOPT{name}.xlsx')
         output_df.to_excel(output_file_path, index=False)
         print(f"File {output_file_path} saved successfully.")
 
         # --- Pie chart asset allocation: salva in Output senza mostrare ---
-        asset_allocations = {asset: 0 for asset in ['Azionari', 'Obbligazionari', 'Metalli Preziosi', 'Materie Prime', 'Immobiliare', 'Criptovalute', 'Monetari']}
+        asset_allocations = {asset: 0 for asset in asset_class_limits_cfg}
         for isin, weight in weights.items():
             r_sel = df.loc[df['ISIN'] == isin]
             if r_sel.empty:

README.md

@@ -0,0 +1,31 @@
+# Ottimizzatore - Setup rapido
+
+## Prerequisiti
+- Python 3.10+ con pip
+- Driver ODBC 17 per SQL Server installato
+
+## Configurazione credenziali
+1. Copia `.env.example` in `.env`.
+2. Imposta le variabili:
+   - `DB_USERNAME`
+   - `DB_PASSWORD`
+   - `DB_HOST`
+   - `DB_PORT` (default 1433)
+   - `DB_NAME`
+3. Mantieni `.env` fuori dal controllo versione (gia' ignorato).
+
+## Installazione dipendenze
+```
+pip install -r requirements.txt
+```
+
+## Configurazione portafogli (config.yaml)
+- Parametri variabili (target di volatilità, naming portafogli, limiti per Asset Class) stanno in `config.yaml`.
+- Profili:
+  - `default`: usato da tutti gli script principali (v2.6, 2.5.2, Lite).
+  - `uk`: usato di default da `20240929 Ottimizzatore Versione 2.2 UK.py` (oppure imposta `CONFIG_PROFILE=uk`).
+- Per modificare target/limiti, edita `config.yaml` senza toccare il codice. Se il file manca o è incompleto, i default interni vengono usati.
+
+## Esecuzione
+Esegui lo script desiderato (es. `python "Sviluppo/20251022 Ottimizzatore Versione 2.6.py"`).
+Assicurati che `.env` sia presente o le variabili siano esportate nel processo.

Sviluppo/20251022 Ottimizzatore Versione 2.6 - R^2.py

@@ -13,6 +13,8 @@ import os
 import numpy as np
 import pandas as pd
 import matplotlib.pyplot as plt
+from dotenv import load_dotenv
+import yaml
 
 from sqlalchemy import create_engine, text
 from sqlalchemy.exc import SQLAlchemyError
@@ -25,10 +27,12 @@ from pypfopt.exceptions import OptimizationError
 OUTPUT_DIR = "Output"
 INPUT_DIR = "Input"
 PLOT_DIR = "Plot"
+CONFIG_FILE = "config.yaml"
 
 os.makedirs(OUTPUT_DIR, exist_ok=True)
 os.makedirs(INPUT_DIR, exist_ok=True)
 os.makedirs(PLOT_DIR, exist_ok=True)
+load_dotenv()
 
 def excel_path(filename: str) -> str:
     """Percorso completo per i file Excel di output."""
@@ -43,23 +47,60 @@ optimized_weights_phase2 = pd.DataFrame()
 summary_data_phase2 = []
 
 # =========================
-# CONFIGURAZIONE OBIETTIVI
+# CONFIGURAZIONE OBIETTIVI (da config.yaml con fallback)
 # =========================
-volatility_targets = {
-    # (1, 0.06): 'VAR3_1Y',
-    # (3, 0.06): 'VAR3_3Y',
-    (5, 0.06): 'VAR3_5Y',
-    (1, 0.12): 'VAR6_1Y',
-    (3, 0.12): 'VAR6_3Y',
-    (5, 0.12): 'VAR6_5Y',
-    # (1, 0.18): 'VAR9_1Y',
-    # (3, 0.18): 'VAR9_3Y',
-    (5, 0.18): 'VAR9_5Y'
+DEFAULT_VOL_TARGETS = [
+    {"years": 5, "target_vol": 0.06, "name": "VAR3_5Y"},
+    {"years": 1, "target_vol": 0.12, "name": "VAR6_1Y"},
+    {"years": 3, "target_vol": 0.12, "name": "VAR6_3Y"},
+    {"years": 5, "target_vol": 0.12, "name": "VAR6_5Y"},
+    {"years": 5, "target_vol": 0.18, "name": "VAR9_5Y"},
+]
+
+DEFAULT_ASSET_CLASS_LIMITS = {
+    'Azionari': 0.75, 'Obbligazionari': 0.75,
+    'Metalli Preziosi': 0.20, 'Materie Prime': 0.05,
+    'Immobiliare': 0.05, 'Criptovalute': 0.05, 'Monetari': 0.10
 }
+
+def load_targets_and_limits(config_file: str, profile: str = "default"):
+    """Legge target di volatilità e limiti asset class dal file di configurazione."""
+    cfg = {}
+    try:
+        with open(config_file, "r", encoding="utf-8") as f:
+            cfg = yaml.safe_load(f) or {}
+    except FileNotFoundError:
+        cfg = {}
+
+    vt_cfg = cfg.get("volatility_targets", {})
+    vt_list = []
+    if isinstance(vt_cfg, dict):
+        vt_list = vt_cfg.get(profile) or vt_cfg.get("default") or []
+    elif isinstance(vt_cfg, list):
+        vt_list = vt_cfg
+    if not vt_list:
+        vt_list = DEFAULT_VOL_TARGETS
+
+    volatility_targets_local = {
+        (int(item["years"]), float(item["target_vol"])): item["name"]
+        for item in vt_list
+        if "years" in item and "target_vol" in item and "name" in item
+    }
+
+    asset_limits_cfg = cfg.get("asset_class_limits") or {}
+    if not asset_limits_cfg:
+        asset_limits_cfg = DEFAULT_ASSET_CLASS_LIMITS
+    asset_class_limits_local = {k: float(v) for k, v in asset_limits_cfg.items()}
+
+    return volatility_targets_local, asset_class_limits_local
+
+CONFIG_PROFILE = os.getenv("CONFIG_PROFILE", "default")
+volatility_targets, asset_class_limits_cfg = load_targets_and_limits(CONFIG_FILE, CONFIG_PROFILE)
 days_per_year = 252
 riskfree_rate = 0.02
 
 mu_ph2_floor = 0.9
+GAP_FFILL_LIMIT_DAYS = 5  # forward-fill limit per buchi di calendario
 
 # ---------------------------------
 # Utility per R^2 sull’equity line
@@ -315,6 +356,85 @@ df = pd.read_excel(
     dtype={'Codice Titolo': str}
 )
 
+# =========================
+# VALIDAZIONE DATI RENDIMENTI
+# =========================
+def _gap_ranges_missing(series: pd.Series, all_dates: pd.DatetimeIndex, max_entries: int = 3):
+    """Restituisce fino a max_entries intervalli di date mancanti (start, end, len)."""
+    missing_idx = series[series.isna()].index
+    if missing_idx.empty:
+        return []
+    positions = [all_dates.get_loc(ts) for ts in missing_idx if ts in all_dates]
+    if not positions:
+        return []
+    ranges = []
+    start_pos = positions[0]
+    end_pos = positions[0]
+    for pos in positions[1:]:
+        if pos == end_pos + 1:
+            end_pos = pos
+        else:
+            ranges.append((all_dates[start_pos], all_dates[end_pos], end_pos - start_pos + 1))
+            start_pos = end_pos = pos
+    ranges.append((all_dates[start_pos], all_dates[end_pos], end_pos - start_pos + 1))
+    return ranges[:max_entries]
+
+
+def preprocess_returns(temp_df: pd.DataFrame, isin: str, all_dates: pd.DatetimeIndex,
+                       gap_ffill_limit: int = GAP_FFILL_LIMIT_DAYS) -> pd.Series:
+    """Pulizia/validazione serie rendimenti (tipi, duplicati, buchi, niente fillna(0))."""
+    required_cols = {"Px_Date", "RendimentoGiornaliero"}
+    missing_cols = required_cols.difference(temp_df.columns)
+    if missing_cols:
+        print(f"[{isin}] Colonne mancanti {missing_cols}, asset ignorato.")
+        return pd.Series(dtype=float)
+
+    df_local = temp_df.copy()
+    df_local["Px_Date"] = pd.to_datetime(df_local["Px_Date"], errors="coerce").dt.normalize()
+    df_local["RendimentoGiornaliero"] = pd.to_numeric(df_local["RendimentoGiornaliero"], errors="coerce")
+
+    before_rows = len(df_local)
+    df_local = df_local.dropna(subset=["Px_Date", "RendimentoGiornaliero"])
+    dropped_na = before_rows - len(df_local)
+
+    dup_count = df_local.duplicated(subset=["Px_Date"]).sum()
+    if dup_count > 0:
+        print(f"[{isin}] Drop duplicati su Px_Date: {dup_count}")
+    df_local = df_local.drop_duplicates(subset=["Px_Date"], keep="last").sort_values("Px_Date")
+
+    if df_local.empty:
+        print(f"[{isin}] Nessuna riga valida dopo cleaning, asset ignorato.")
+        return pd.Series(dtype=float)
+
+    df_local["RendimentoGiornaliero"] = df_local["RendimentoGiornaliero"] / 100.0
+
+    series = df_local.set_index("Px_Date")["RendimentoGiornaliero"].reindex(all_dates)
+    missing_before_fill = int(series.isna().sum())
+    if missing_before_fill > 0:
+        gaps = _gap_ranges_missing(series, all_dates, max_entries=3)
+        if gaps:
+            gaps_str = "; ".join([f"{g[0].date()}->{g[1].date()} ({g[2]} gg)" for g in gaps])
+            print(f"[{isin}] Date mancanti prima del fill (prime): {gaps_str}")
+
+    series_ffill = series.ffill(limit=gap_ffill_limit)
+    first_valid = series_ffill.first_valid_index()
+    last_valid = series_ffill.last_valid_index()
+    if first_valid is None or last_valid is None:
+        print(f"[{isin}] Serie vuota dopo forward-fill, asset ignorato.")
+        return pd.Series(dtype=float)
+
+    series_ffill = series_ffill.loc[first_valid:last_valid]
+    residual_missing = int(series_ffill.isna().sum())
+    if residual_missing > 0:
+        print(f"[{isin}] {residual_missing} valori mancanti non coperti dal forward-fill (limite {gap_ffill_limit}), righe scartate.")
+        series_ffill = series_ffill.dropna()
+
+    coverage_days = series_ffill.shape[0]
+    if dropped_na > 0 or missing_before_fill > 0:
+        print(f"[{isin}] Righe totali: {before_rows}, drop NA: {dropped_na}, copertura finale: {coverage_days} giorni.")
+
+    return series_ffill
+
 # =========================
 # SERIE STORICHE RENDIMENTI
 # =========================
@@ -333,17 +453,23 @@ for isin in df['ISIN'].unique():
         if temp_df.empty:
             print(f"Nessun dato recuperato per {isin}, skipping...")
             continue
-        temp_df['Px_Date'] = pd.to_datetime(temp_df['Px_Date'], format='%Y-%m-%d', errors='coerce').dt.normalize()
-        temp_df = temp_df.dropna(subset=['Px_Date'])
-        temp_df.set_index('Px_Date', inplace=True)
-        temp_df['RendimentoGiornaliero'] = temp_df['RendimentoGiornaliero'] / 100
-        final_df[isin] = temp_df['RendimentoGiornaliero'].reindex(all_dates)
+        clean_series = preprocess_returns(temp_df, isin, all_dates, gap_ffill_limit=GAP_FFILL_LIMIT_DAYS)
+        if clean_series.empty:
+            print(f"Nessun dato valido per {isin} dopo la validazione, asset ignorato.")
+            continue
+        final_df[isin] = clean_series.reindex(all_dates)
         isin_from_db.add(isin)
-        print(f"Dati recuperati per {isin}: {final_df[isin].count()} righe di dati non-null prelevate.")
+        non_null = int(final_df[isin].count())
+        missing_left = int(final_df[isin].isna().sum())
+        print(f"Dati recuperati per {isin}: {non_null} righe valide, mancanti residui: {missing_left}.")
     except SQLAlchemyError as e:
         print(f"Errore durante l'esecuzione della stored procedure per {isin}:", e)
 
-final_df.fillna(0, inplace=True)
+final_df = final_df.loc[:, final_df.notna().any()]  # elimina asset senza dati utili
+if final_df.shape[1] == 0:
+    print("Nessun ISIN valido dopo la validazione dei rendimenti, uscita.")
+    sys.exit(1)
+final_df = final_df.dropna(how='all')
 
 # -------- H_min sempre su 5 anni (21 gg = 1 mese) --------
 five_year_df = final_df.loc[end_date - pd.DateOffset(years=5): end_date]
@@ -438,12 +564,7 @@ for (years, target_vol), name in volatility_targets.items():
             ef.add_constraint(lambda w, idxs=idxs, maxw=maxw: sum(w[i] for i in idxs) <= maxw)
 
     # Vincoli per Asset Class
-    asset_class_limits = {
-        'Azionari': 0.75, 'Obbligazionari': 0.75,
-        'Metalli Preziosi': 0.20, 'Materie Prime': 0.05,
-        'Immobiliare': 0.05, 'Criptovalute': 0.05, 'Monetari': 0.1
-    }
-    for ac, maxw in asset_class_limits.items():
+    for ac, maxw in asset_class_limits_cfg.items():
         isin_list = df[df['Asset Class'] == ac]['ISIN'].tolist()
         idxs = [period_df.columns.get_loc(isin) for isin in isin_list if isin in period_df.columns]
         if idxs:
@@ -484,13 +605,16 @@ for (years, target_vol), name in volatility_targets.items():
                 results_rows.append(row)
 
         results_full_df = pd.DataFrame(results_rows, columns=template_cols)
-        output_df = pd.concat([template_df.iloc[0:0], results_full_df], ignore_index=True)
+        if results_full_df.empty:
+            output_df = template_df.iloc[0:0].copy()
+        else:
+            output_df = results_full_df.reindex(columns=template_cols)
         output_file_path = excel_path(f'PTFOPT{name}.xlsx')
         output_df.to_excel(output_file_path, index=False)
         print(f"File {output_file_path} saved successfully.")
 
         # --- Pie chart asset allocation (se ci sono pesi > 0) ---
-        asset_allocations = {asset: 0 for asset in asset_class_limits}
+        asset_allocations = {asset: 0 for asset in asset_class_limits_cfg}
         for isin, weight in weights.items():
             r_sel = df.loc[df['ISIN'] == isin]
             if r_sel.empty:
@@ -707,12 +831,7 @@ if cp is not None:
             if idxs:
                 ef_h.add_constraint(lambda w, idxs=idxs, maxw=maxw: cp.sum(w[idxs]) <= maxw)
 
-        asset_class_limits = {
-            'Azionari': 0.75, 'Obbligazionari': 0.75,
-            'Metalli Preziosi': 0.20, 'Materie Prime': 0.05,
-            'Immobiliare': 0.05, 'Criptovalute': 0.05, 'Monetari': 0.1
-        }
-        for ac, maxw in asset_class_limits.items():
+        for ac, maxw in asset_class_limits_cfg.items():
             isin_list = df[df['Asset Class'] == ac]['ISIN'].tolist()
             idxs = [period_df_p.columns.get_loc(isin) for isin in isin_list if isin in period_df_p.columns]
             if idxs:
@@ -873,9 +992,12 @@ else:
             row['peso']    = float(weight * 99)  # allineato a Fase 1
             results_rows.append(row)
 
-        # Prepara il foglio con l’intestazione del template + righe risultato
+        # Prepara il foglio con l'intestazione del template + righe risultato
         results_full_df = pd.DataFrame(results_rows, columns=template_cols)
-        output_df = pd.concat([template_df.iloc[0:0], results_full_df], ignore_index=True)
+        if results_full_df.empty:
+            output_df = template_df.iloc[0:0].copy()
+        else:
+            output_df = results_full_df.reindex(columns=template_cols)
 
         # NOME FILE: identico al naming di Fase 1
         output_file_path = excel_path(f'PTFOPT{name}_PH2.xlsx')

Sviluppo/20251022 Ottimizzatore Versione 2.6.py

@@ -13,6 +13,8 @@ import os
 import numpy as np
 import pandas as pd
 import matplotlib.pyplot as plt
+from dotenv import load_dotenv
+import yaml
 
 from sqlalchemy import create_engine, text
 from sqlalchemy.exc import SQLAlchemyError
@@ -25,10 +27,12 @@ from pypfopt.exceptions import OptimizationError
 OUTPUT_DIR = "Output"
 INPUT_DIR = "Input"
 PLOT_DIR = "Plot"
+CONFIG_FILE = "config.yaml"
 
 os.makedirs(OUTPUT_DIR, exist_ok=True)
 os.makedirs(INPUT_DIR, exist_ok=True)
 os.makedirs(PLOT_DIR, exist_ok=True)
+load_dotenv()
 
 def excel_path(filename: str) -> str:
     """Percorso completo per i file Excel di output."""
@@ -43,23 +47,61 @@ optimized_weights_phase1_heal = pd.DataFrame()
 summary_data_phase1_heal = []
 
 # =========================
-# CONFIGURAZIONE OBIETTIVI
+# CONFIGURAZIONE OBIETTIVI (da config.yaml con fallback)
 # =========================
-volatility_targets = {
-    # (1, 0.06): 'VAR3_1Y',
-    # (3, 0.06): 'VAR3_3Y',
-    (5, 0.06): 'VAR3_5Y',
-    (1, 0.12): 'VAR6_1Y',
-    (3, 0.12): 'VAR6_3Y',
-    (5, 0.12): 'VAR6_5Y',
-    # (1, 0.18): 'VAR9_1Y',
-    # (3, 0.18): 'VAR9_3Y',
-    (5, 0.18): 'VAR9_5Y'
+DEFAULT_VOL_TARGETS = [
+    {"years": 5, "target_vol": 0.06, "name": "VAR3_5Y"},
+    {"years": 1, "target_vol": 0.12, "name": "VAR6_1Y"},
+    {"years": 3, "target_vol": 0.12, "name": "VAR6_3Y"},
+    {"years": 5, "target_vol": 0.12, "name": "VAR6_5Y"},
+    {"years": 5, "target_vol": 0.18, "name": "VAR9_5Y"},
+]
+
+DEFAULT_ASSET_CLASS_LIMITS = {
+    'Azionari': 0.75, 'Obbligazionari': 0.75,
+    'Metalli Preziosi': 0.20, 'Materie Prime': 0.05,
+    'Immobiliare': 0.05, 'Criptovalute': 0.05, 'Monetari': 0.10
 }
+
+def load_targets_and_limits(config_file: str, profile: str = "default"):
+    """Legge target di volatilità e limiti asset class dal file di configurazione."""
+    cfg = {}
+    try:
+        with open(config_file, "r", encoding="utf-8") as f:
+            cfg = yaml.safe_load(f) or {}
+    except FileNotFoundError:
+        cfg = {}
+
+    vt_cfg = cfg.get("volatility_targets", {})
+    vt_list = []
+    if isinstance(vt_cfg, dict):
+        vt_list = vt_cfg.get(profile) or vt_cfg.get("default") or []
+    elif isinstance(vt_cfg, list):
+        vt_list = vt_cfg
+    if not vt_list:
+        vt_list = DEFAULT_VOL_TARGETS
+
+    volatility_targets_local = {
+        (int(item["years"]), float(item["target_vol"])): item["name"]
+        for item in vt_list
+        if "years" in item and "target_vol" in item and "name" in item
+    }
+
+    asset_limits_cfg = cfg.get("asset_class_limits") or {}
+    if not asset_limits_cfg:
+        asset_limits_cfg = DEFAULT_ASSET_CLASS_LIMITS
+    asset_class_limits_local = {k: float(v) for k, v in asset_limits_cfg.items()}
+
+    return volatility_targets_local, asset_class_limits_local
+
+CONFIG_PROFILE = os.getenv("CONFIG_PROFILE", "default")
+volatility_targets, asset_class_limits_cfg = load_targets_and_limits(CONFIG_FILE, CONFIG_PROFILE)
+
 days_per_year = 252
 riskfree_rate = 0.02
 
 mu_heal_floor = 0.85
+GAP_FFILL_LIMIT_DAYS = 5  # forward-fill limit per buchi di calendario
 
 # ---------------------------------
 # Utility per R^2 sull’equity line
@@ -268,27 +310,28 @@ def portfolio_path_metrics(period_df: pd.DataFrame,
         "Hmin_100m_5Y": hmin_5y_months
     }
 
-# --- Lettura parametri dal file connection.txt ---
-params = {}
-with open("connection.txt", "r") as f:
-    for line in f:
-        line = line.strip()
-        if line and not line.startswith("#"):
-            key, value = line.split("=", 1)
-            params[key.strip()] = value.strip()
+def load_db_config():
+    """Recupera i parametri di connessione dal set di variabili d'ambiente."""
+    required_keys = ["DB_USERNAME", "DB_PASSWORD", "DB_HOST", "DB_NAME"]
+    missing = [k for k in required_keys if not os.getenv(k)]
+    if missing:
+        raise RuntimeError(f"Variabili d'ambiente mancanti per il DB: {', '.join(missing)}")
 
-username = params.get("username")
-password = params.get("password")
-host = params.get("host")
-port = params.get("port", "1433")
-database = params.get("database")
+    return {
+        "username": os.getenv("DB_USERNAME"),
+        "password": os.getenv("DB_PASSWORD"),
+        "host": os.getenv("DB_HOST"),
+        "port": os.getenv("DB_PORT", "1433"),
+        "database": os.getenv("DB_NAME"),
+    }
 
+db_cfg = load_db_config()
 connection_string = (
-    f"mssql+pyodbc://{username}:{password}@{host}:{port}/{database}"
+    f"mssql+pyodbc://{db_cfg['username']}:{db_cfg['password']}@{db_cfg['host']}:{db_cfg['port']}/{db_cfg['database']}"
     "?driver=ODBC+Driver+17+for+SQL+Server"
 )
 
-print("Connection string letta correttamente")
+print("Connection string configurata da variabili d'ambiente.")
 
 # =========================
 # CONNESSIONE AL DB
@@ -315,6 +358,85 @@ df = pd.read_excel(
     dtype={'Codice Titolo': str}
 )
 
+# =========================
+# VALIDAZIONE DATI RENDIMENTI
+# =========================
+def _gap_ranges_missing(series: pd.Series, all_dates: pd.DatetimeIndex, max_entries: int = 3):
+    """Restituisce fino a max_entries intervalli di date mancanti (start, end, len)."""
+    missing_idx = series[series.isna()].index
+    if missing_idx.empty:
+        return []
+    positions = [all_dates.get_loc(ts) for ts in missing_idx if ts in all_dates]
+    if not positions:
+        return []
+    ranges = []
+    start_pos = positions[0]
+    end_pos = positions[0]
+    for pos in positions[1:]:
+        if pos == end_pos + 1:
+            end_pos = pos
+        else:
+            ranges.append((all_dates[start_pos], all_dates[end_pos], end_pos - start_pos + 1))
+            start_pos = end_pos = pos
+    ranges.append((all_dates[start_pos], all_dates[end_pos], end_pos - start_pos + 1))
+    return ranges[:max_entries]
+
+
+def preprocess_returns(temp_df: pd.DataFrame, isin: str, all_dates: pd.DatetimeIndex,
+                       gap_ffill_limit: int = GAP_FFILL_LIMIT_DAYS) -> pd.Series:
+    """Pulizia/validazione serie rendimenti (tipi, duplicati, buchi, niente fillna(0))."""
+    required_cols = {"Px_Date", "RendimentoGiornaliero"}
+    missing_cols = required_cols.difference(temp_df.columns)
+    if missing_cols:
+        print(f"[{isin}] Colonne mancanti {missing_cols}, asset ignorato.")
+        return pd.Series(dtype=float)
+
+    df_local = temp_df.copy()
+    df_local["Px_Date"] = pd.to_datetime(df_local["Px_Date"], errors="coerce").dt.normalize()
+    df_local["RendimentoGiornaliero"] = pd.to_numeric(df_local["RendimentoGiornaliero"], errors="coerce")
+
+    before_rows = len(df_local)
+    df_local = df_local.dropna(subset=["Px_Date", "RendimentoGiornaliero"])
+    dropped_na = before_rows - len(df_local)
+
+    dup_count = df_local.duplicated(subset=["Px_Date"]).sum()
+    if dup_count > 0:
+        print(f"[{isin}] Drop duplicati su Px_Date: {dup_count}")
+    df_local = df_local.drop_duplicates(subset=["Px_Date"], keep="last").sort_values("Px_Date")
+
+    if df_local.empty:
+        print(f"[{isin}] Nessuna riga valida dopo cleaning, asset ignorato.")
+        return pd.Series(dtype=float)
+
+    df_local["RendimentoGiornaliero"] = df_local["RendimentoGiornaliero"] / 100.0
+
+    series = df_local.set_index("Px_Date")["RendimentoGiornaliero"].reindex(all_dates)
+    missing_before_fill = int(series.isna().sum())
+    if missing_before_fill > 0:
+        gaps = _gap_ranges_missing(series, all_dates, max_entries=3)
+        if gaps:
+            gaps_str = "; ".join([f"{g[0].date()}->{g[1].date()} ({g[2]} gg)" for g in gaps])
+            print(f"[{isin}] Date mancanti prima del fill (prime): {gaps_str}")
+
+    series_ffill = series.ffill(limit=gap_ffill_limit)
+    first_valid = series_ffill.first_valid_index()
+    last_valid = series_ffill.last_valid_index()
+    if first_valid is None or last_valid is None:
+        print(f"[{isin}] Serie vuota dopo forward-fill, asset ignorato.")
+        return pd.Series(dtype=float)
+
+    series_ffill = series_ffill.loc[first_valid:last_valid]
+    residual_missing = int(series_ffill.isna().sum())
+    if residual_missing > 0:
+        print(f"[{isin}] {residual_missing} valori mancanti non coperti dal forward-fill (limite {gap_ffill_limit}), righe scartate.")
+        series_ffill = series_ffill.dropna()
+
+    coverage_days = series_ffill.shape[0]
+    if dropped_na > 0 or missing_before_fill > 0:
+        print(f"[{isin}] Righe totali: {before_rows}, drop NA: {dropped_na}, copertura finale: {coverage_days} giorni.")
+
+    return series_ffill
+
 # =========================
 # SERIE STORICHE RENDIMENTI
 # =========================
@@ -333,17 +455,23 @@ for isin in df['ISIN'].unique():
         if temp_df.empty:
             print(f"Nessun dato recuperato per {isin}, skipping...")
             continue
-        temp_df['Px_Date'] = pd.to_datetime(temp_df['Px_Date'], format='%Y-%m-%d', errors='coerce').dt.normalize()
-        temp_df = temp_df.dropna(subset=['Px_Date'])
-        temp_df.set_index('Px_Date', inplace=True)
-        temp_df['RendimentoGiornaliero'] = temp_df['RendimentoGiornaliero'] / 100
-        final_df[isin] = temp_df['RendimentoGiornaliero'].reindex(all_dates)
+        clean_series = preprocess_returns(temp_df, isin, all_dates, gap_ffill_limit=GAP_FFILL_LIMIT_DAYS)
+        if clean_series.empty:
+            print(f"Nessun dato valido per {isin} dopo la validazione, asset ignorato.")
+            continue
+        final_df[isin] = clean_series.reindex(all_dates)
         isin_from_db.add(isin)
-        print(f"Dati recuperati per {isin}: {final_df[isin].count()} righe di dati non-null prelevate.")
+        non_null = int(final_df[isin].count())
+        missing_left = int(final_df[isin].isna().sum())
+        print(f"Dati recuperati per {isin}: {non_null} righe valide, mancanti residui: {missing_left}.")
     except SQLAlchemyError as e:
         print(f"Errore durante l'esecuzione della stored procedure per {isin}:", e)
 
-final_df.fillna(0, inplace=True)
+final_df = final_df.loc[:, final_df.notna().any()]  # elimina asset senza dati utili
+if final_df.shape[1] == 0:
+    print("Nessun ISIN valido dopo la validazione dei rendimenti, uscita.")
+    sys.exit(1)
+final_df = final_df.dropna(how='all')
 
 # -------- H_min sempre su 5 anni (21 gg = 1 mese) --------
 five_year_df = final_df.loc[end_date - pd.DateOffset(years=5): end_date]
@@ -438,12 +566,7 @@ for (years, target_vol), name in volatility_targets.items():
             ef.add_constraint(lambda w, idxs=idxs, maxw=maxw: sum(w[i] for i in idxs) <= maxw)
 
     # Vincoli per Asset Class
-    asset_class_limits = {
-        'Azionari': 0.75, 'Obbligazionari': 0.75,
-        'Metalli Preziosi': 0.20, 'Materie Prime': 0.05,
-        'Immobiliare': 0.05, 'Criptovalute': 0.05, 'Monetari': 0.1
-    }
-    for ac, maxw in asset_class_limits.items():
+    for ac, maxw in asset_class_limits_cfg.items():
         isin_list = df[df['Asset Class'] == ac]['ISIN'].tolist()
         idxs = [period_df.columns.get_loc(isin) for isin in isin_list if isin in period_df.columns]
         if idxs:
@@ -484,13 +607,16 @@ for (years, target_vol), name in volatility_targets.items():
                 results_rows.append(row)
 
         results_full_df = pd.DataFrame(results_rows, columns=template_cols)
-        output_df = pd.concat([template_df.iloc[0:0], results_full_df], ignore_index=True)
+        if results_full_df.empty:
+            output_df = template_df.iloc[0:0].copy()
+        else:
+            output_df = results_full_df.reindex(columns=template_cols)
         output_file_path = excel_path(f'PTFOPT{name}.xlsx')
         output_df.to_excel(output_file_path, index=False)
         print(f"File {output_file_path} saved successfully.")
 
         # --- Pie chart asset allocation (se ci sono pesi > 0) ---
-        asset_allocations = {asset: 0 for asset in asset_class_limits}
+        asset_allocations = {asset: 0 for asset in asset_class_limits_cfg}
         for isin, weight in weights.items():
             r_sel = df.loc[df['ISIN'] == isin]
             if r_sel.empty:
@@ -707,12 +833,7 @@ if cp is not None:
             if idxs:
                 ef_h.add_constraint(lambda w, idxs=idxs, maxw=maxw: cp.sum(w[idxs]) <= maxw)
 
-        asset_class_limits = {
-            'Azionari': 0.75, 'Obbligazionari': 0.75,
-            'Metalli Preziosi': 0.20, 'Materie Prime': 0.05,
-            'Immobiliare': 0.05, 'Criptovalute': 0.05, 'Monetari': 0.1
-        }
-        for ac, maxw in asset_class_limits.items():
+        for ac, maxw in asset_class_limits_cfg.items():
             isin_list = df[df['Asset Class'] == ac]['ISIN'].tolist()
             idxs = [period_df_p.columns.get_loc(isin) for isin in isin_list if isin in period_df_p.columns]
             if idxs:

TRACCIA_IMPROVEMENTS.md

@@ -28,12 +28,18 @@
 - CPU: covarianze e Solvers EfficientFrontier per ogni combinazione durata/volatilita'.
 - IO file: generazione multipla di Excel e plot per portafoglio.
 
-## Direzioni di miglioramento (linee guida, non implementate)
-- Sicurezza: rimuovere/securizzare `connection.txt`, usare variabili ambiente o secret store; separare credenziali dal repo.
-- Configurazione: estrarre target/vincoli in file di config (yaml/json); centralizzare costanti.
-- Gestione dipendenze: aggiungere requirements/lockfile e script setup.
-- Architettura: modularizzare (data fetch, metriche, optimizer, export), riuso tra versioni, eliminare duplicati.
-- Dati: validazione input, gestione missing diversa da fillna(0), log warning sui buchi/ISIN mancanti.
-- Performance: fetch batch/caching, eventuale parallelizzazione o memoization di covarianze/metriche.
-- Testing/qualita': introdurre test per metriche e vincoli, logging strutturato, controlli su overwrite output.
-- Observability: report riepilogo a schermo/logs strutturati piu' dei soli print.
+## Avanzamento miglioramenti
+- [x] Modularizzazione base e riuso funzioni (fetch dati, metriche, optimizer, export) tra 2.6/2.5.x/Lite.
+- [x] Configurazione esterna: target, vincoli e naming in `config.yaml`; costanti centralizzate.
+- [x] Sicurezza credenziali DB: rimosso uso diretto di `connection.txt`, lettura da `.env`/variabili ambiente nel codice MSSQL.
+- [ ] Validazione dati su input/rendimenti (in corso).
+- [ ] Performance/caching/parallel: da attivare dopo la validazione.
+- [ ] Testing/logging/observability strutturata: da aggiungere dopo stabilizzazione schema dati.
+
+## Piano dettaglio: Validazione dati (punto 4)
+- Tipi e schema: forzare date in datetime, rendimenti numerici, errore se colonne richieste mancano; sorting per data.
+- Duplicati: drop duplicati per (ISIN, data) con log warning del conteggio scartato.
+- Periodi mancanti: rilevare buchi di calendario per ISIN, log warning con range; opzionale reindex business days + forward-fill limitato.
+- Missing values: vietato `fillna(0)` sui rendimenti; drop NA iniziali/finali, forward-fill limitato (es. 5 giorni) per buchi interni, altrimenti drop e log.
+- ISIN mancanti: se la stored proc non ritorna dati o la serie e' vuota, log warning e saltare l'asset dall'ottimizzazione.
+- Report: riepilogo per profilo/ISIN con righe scartate, buchi individuati e fill applicati; stampa su stdout e file in `Output/` per audit.