Qionk/a-share-quant
0
1"""2涨跌方向预测 - 二分类模块3========================4独立于回归预测管线，使用 GARCH(1,1) 波动率作为核心特征，5XGBoost + ElasticNet LogisticRegression 二分类器，6严格扩展窗口时间序列验证，智能参数推荐。7 8防泄漏保证:91. 仅使用扩展窗口 (expanding window) 划分，禁止随机 shuffle / k-fold102. GARCH(1,1) 条件波动率仅在各折训练集上拟合，验证集用前向预测113. 所有量价衍生特征（成交量变化率、相对成交量、量价配合度、放量上涨、缩量下跌）保留124. create_clf_features 对时刻 i 仅使用 [i-look_back, i-1] 的信息135. 目标变量 目标涨跌 = 涨跌标签.shift(-1)，预测的是下一日涨跌14"""15 16import time17import warnings18import numpy as np19import pandas as pd20from dataclasses import dataclass, field21from typing import List, Tuple, Dict, Optional, Callable22 23from sklearn.linear_model import LogisticRegression24from sklearn.preprocessing import StandardScaler25from sklearn.metrics import roc_auc_score26from scipy.stats import pearsonr27 28from .features import compute_technical_indicators, time_series_split29from .preprocessing import preprocess_data30from .volatility import fit_garch31 32warnings.filterwarnings("ignore")33 34# ── 分类特征列（排除目标涨跌、目标收益率，保留所有量价衍生特征） ──35 36CLF_FEATURE_COLS = [37 # 价格变化38 "pct_change", "日收益率",39 # 多周期动量40 "ret_2d", "ret_3d", "ret_5d", "ret_10d",41 # 均线偏离度42 "close_ma5_bias", "close_ma10_bias", "close_ma20_bias", "ma5_ma10_cross",43 # 波动率44 "volatility_5d", "volatility_10d", "volatility_20d",45 # 技术指标46 "rsi", "rsi_6", "rsi_14", "dif", "macd",47 # K线形态48 "upper_shadow", "lower_shadow", "body_ratio", "high_low_range",49 # 量价配合50 "成交量变化率", "相对成交量", "量价配合度",51 "放量上涨", "缩量下跌",52 # 成交量多周期53 "vol_chg_3d", "vol_chg_5d", "vol_ratio_5d",54 # ── 科技股专用 ──55 # 换手率系列56 "turnover", "turnover_ma5", "turnover_ma10",57 "turnover_bias", "turnover_accel",58 "cum_turnover_5d", "cum_turnover_10d",59 # 跳空缺口60 "gap", "gap_abs", "gap_up", "gap_down",61 # 价格位置62 "price_pos_5d", "price_pos_10d", "price_pos_20d",63 "drawdown_5d", "drawdown_10d", "drawdown_20d",64 # 连涨连跌65 "streak_up", "streak_down",66 # ATR67 "atr_5", "atr_14", "atr_ratio",68 # 涨停板效应69 "near_limit_up", "near_limit_down",70 "limit_up_count_10d", "limit_down_count_10d",71 # 相对强弱（需指数数据，可选）72 "excess_ret", "excess_ret_5d", "excess_ret_10d", "excess_ret_20d",73]74 75 76# ── 数据容器 ──77 78@dataclass79class ClfResult:80 """单分类器训练结果（所有字段对应 OOS 样本）"""81 model_name: str82 model_object: object = None83 # OOS 预测与标签（按折拼接，时间顺序）84 oos_probabilities: np.ndarray = field(default_factory=lambda: np.array([]))85 oos_predictions: np.ndarray = field(default_factory=lambda: np.array([]))86 oos_actuals: np.ndarray = field(default_factory=lambda: np.array([]))87 oos_returns: np.ndarray = field(default_factory=lambda: np.array([]))88 oos_future_ret: np.ndarray = field(default_factory=lambda: np.array([]))89 oos_next_day_ret: np.ndarray = field(default_factory=lambda: np.array([]))90 oos_dates: np.ndarray = field(default_factory=lambda: np.array([]))91 # CV 指标92 fold_metrics: dict = field(default_factory=dict)93 overall_metrics: dict = field(default_factory=dict)94 # 特征重要性（XGBoost 多折平均）95 feature_importance: dict = field(default_factory=dict)96 training_time: float = 0.097 feature_names: list = field(default_factory=list)98 # 最新交易日实时预测（目标涨跌为 NaN 的最后一行）99 latest_proba: float = np.nan100 latest_date: object = None101 102 103# ── 特征工程 ──104 105def create_clf_features(106 df: pd.DataFrame,107 feature_cols: List[str],108 look_back: int,109) -> Tuple[np.ndarray, np.ndarray, List[str], pd.DatetimeIndex,110 np.ndarray, pd.DatetimeIndex]:111 """112 构建分类滞后特征（严格防泄漏）。113 114 对时刻 i，仅使用 [i-look_back, i-1] 的数据构造特征，115 目标 y[i] = df.iloc[i]['目标涨跌'] 预测的是 i→i+1 的涨跌方向。116 117 最后一天（目标涨跌为 NaN）的特征也会构建，但 y 中对应位置为 NaN，118 单独返回为 X_latest / latest_date，供实时预测使用。119 120 返回: (X, y, feature_names, dates, X_latest, latest_date)121 X: (n_labeled, n_features * look_back) 有标签样本特征122 y: (n_labeled,) 0/1 标签（仅非 NaN 行）123 feature_names: "col_lagN" 格式的特征名124 dates: 有标签样本的对齐日期125 X_latest: (1, n_features * look_back) 最新日特征（可能为空）126 latest_date: 最新日日期（可能为 None）127 """128 if "目标涨跌" not in df.columns:129 raise ValueError("df 中缺少 '目标涨跌' 列，请先调用 preprocess_data()")130 131 missing = [c for c in feature_cols if c not in df.columns]132 if missing:133 raise ValueError(f"df 中缺少以下特征列: {missing}")134 135 # 只保留需要的列 + 目标136 cols = feature_cols + ["目标涨跌"]137 df_sub = df[cols].copy()138 # 仅删除特征列有 NaN 的行，保留目标涨跌为 NaN 的最后一天139 df_sub = df_sub.dropna(subset=feature_cols)140 141 if len(df_sub) <= look_back:142 raise ValueError(f"有效数据量 {len(df_sub)} <= look_back {look_back}，无法构造特征")143 144 n = len(df_sub)145 X_list = []146 for i in range(look_back, n):147 # 仅使用 i-look_back 到 i-1 的数据，不含 i（防泄漏要求 4）148 row = df_sub[feature_cols].iloc[i - look_back:i].values.flatten()149 X_list.append(row)150 151 X_all = np.array(X_list, dtype=np.float64)152 y_all = df_sub["目标涨跌"].values[look_back:]153 dates_all = df_sub.index[look_back:]154 155 # 分离有标签和无标签样本156 labeled_mask = ~np.isnan(y_all)157 X = X_all[labeled_mask]158 y = y_all[labeled_mask].astype(int)159 dates = dates_all[labeled_mask]160 161 # 最新一天（目标涨跌为 NaN）162 X_latest = np.array([], dtype=np.float64)163 latest_date = None164 unlabeled_mask = ~labeled_mask165 if unlabeled_mask.any():166 X_latest = X_all[unlabeled_mask][-1:]167 latest_date = dates_all[unlabeled_mask][-1]168 169 # 构建特征名170 feature_names = []171 for col in feature_cols:172 for lag in range(look_back, 0, -1):173 feature_names.append(f"{col}_lag{lag}")174 175 return X, y, feature_names, dates, X_latest, latest_date176 177 178# ── 模型构建 ──179 180def build_xgb_classifier(params: dict):181 """XGBoost 二分类器"""182 import xgboost as xgb183 return xgb.XGBClassifier(184 n_estimators=params.get("n_estimators", 100),185 max_depth=params.get("max_depth", 6),186 learning_rate=params.get("learning_rate", 0.1),187 subsample=params.get("subsample", 0.8),188 colsample_bytree=params.get("colsample_bytree", 0.8),189 min_child_weight=params.get("min_child_weight", 1),190 reg_alpha=params.get("reg_alpha", 0.0),191 reg_lambda=params.get("reg_lambda", 1.0),192 objective="binary:logistic",193 eval_metric="logloss",194 random_state=42,195 n_jobs=-1,196 verbosity=0,197 )198 199 200def build_elasticnet(params: dict):201 """ElasticNet LogisticRegression"""202 return LogisticRegression(203 C=params.get("C", 1.0),204 l1_ratio=params.get("l1_ratio", 0.15),205 penalty="elasticnet",206 solver="saga",207 max_iter=params.get("max_iter", 5000),208 tol=params.get("tol", 1e-3),209 random_state=42,210 n_jobs=-1,211 )212 213 214# ── GARCH 波动率特征（防泄漏） ──215 216def _expanding_hist_vol(returns: np.ndarray, min_window: int = 20) -> np.ndarray:217 """扩展窗口历史波动率（GARCH 失败时的回退方案）。218 每一点仅使用该点之前的数据。"""219 vol = np.zeros(len(returns))220 for i in range(len(returns)):221 w = max(i + 1, min_window)222 vol[i] = np.std(returns[max(0, i - w + 1):i + 1])223 return vol224 225 226def _extract_garch_vol_from_model(garch_result) -> np.ndarray:227 """从已拟合的 GARCH 模型提取条件波动率序列。228 conditional_volatility[t] 是 GARCH 模型对时刻 t 的预测波动率，229 基于 t-1 之前的信息——由 arch 库保证。"""230 cond_vol = np.asarray(garch_result.conditional_volatility, dtype=np.float64)231 # 处理 NaN（GARCH 初始化阶段可能产生）232 mask = np.isnan(cond_vol)233 if mask.any():234 first_valid = cond_vol[~mask][0] if (~mask).any() else 0.01235 cond_vol[mask] = first_valid236 return cond_vol237 238 239def _forecast_garch_vol(garch_result, horizon: int) -> np.ndarray:240 """对未来 horizon 期前向预测 GARCH 波动率（仅使用训练集信息）。"""241 try:242 forecast = garch_result.forecast(horizon=horizon)243 var_forecast = forecast.variance.iloc[-1].values244 vol = np.sqrt(np.maximum(var_forecast, 0))245 vol = np.nan_to_num(vol, nan=0.01)246 return vol247 except Exception:248 return np.full(horizon, 0.01)249 250 251# ── 评估指标 ──252 253def calculate_classification_metrics(254 y_true: np.ndarray,255 y_pred_proba: np.ndarray,256 returns: np.ndarray,257 predictions: np.ndarray,258 future_ret: np.ndarray = None,259 forecast_days: int = 1,260 next_day_ret: np.ndarray = None,261) -> dict:262 """263 计算分类及策略指标。264 265 策略逻辑: 预测涨(prob>=threshold)则T日收盘买入、T+1日收盘卖出，266 预测跌则空仓(收益=0)。回测收益 = next_day_ret * predictions（下一日收益）。267 若 next_day_ret 为 None，回退为 returns。268 269 返回:270 auc, ic, ic_pvalue,271 cum_return, ann_return, ann_volatility, sharpe, max_dd, win_rate,272 profit_loss_ratio, accuracy, precision, recall, confusion273 """274 metrics = {}275 276 # AUC277 try:278 unique = np.unique(y_true)279 if len(unique) >= 2:280 metrics["auc"] = float(roc_auc_score(y_true, y_pred_proba))281 else:282 metrics["auc"] = np.nan283 except ValueError:284 metrics["auc"] = np.nan285 286 # IC: 预测概率与下一日实际收益的相关性287 pnl = next_day_ret if next_day_ret is not None else returns288 mask = ~(np.isnan(y_pred_proba) | np.isnan(pnl))289 if mask.sum() >= 10:290 ic, ic_pv = pearsonr(y_pred_proba[mask], pnl[mask])291 metrics["ic"] = float(ic)292 metrics["ic_pvalue"] = float(ic_pv)293 else:294 metrics["ic"] = np.nan295 metrics["ic_pvalue"] = np.nan296 297 # 策略收益: 预测涨做多(future_ret)，预测跌空仓(收益=0)298 n = len(predictions)299 # 过滤 future_ret 为 NaN 的行（持有期超过数据末尾时无法验证）300 pnl_slice = pnl[:n]301 valid = ~np.isnan(pnl_slice)302 n_valid = valid.sum()303 strategy_returns = pnl_slice[valid] * predictions[valid]304 if n_valid == 0:305 strategy_returns = np.array([0.0])306 cum_series = np.cumprod(1 + strategy_returns)307 cum_return = cum_series[-1] - 1308 metrics["cum_return"] = float(cum_return)309 310 # 年化收益（252 交易日，按有效样本数）311 if cum_return > -1 and n_valid > 0:312 metrics["ann_return"] = float((1 + cum_return) ** (252 / n_valid) - 1)313 else:314 metrics["ann_return"] = -1.0315 316 # 年化波动率317 if np.std(strategy_returns) > 0 and n_valid > 0:318 metrics["ann_volatility"] = float(np.std(strategy_returns) * np.sqrt(252))319 else:320 metrics["ann_volatility"] = 0.0321 322 # Sharpe（无风险利率 = 0）323 if np.std(strategy_returns) > 0 and n_valid > 0:324 metrics["sharpe"] = float(np.sqrt(252) * np.mean(strategy_returns) / np.std(strategy_returns))325 else:326 metrics["sharpe"] = 0.0327 328 # 最大回撤329 running_max = np.maximum.accumulate(cum_series)330 drawdowns = (cum_series - running_max) / running_max331 metrics["max_dd"] = float(drawdowns.min())332 333 # 胜率（做多时的正确率，按N日持有期收益方向，仅有效行）334 long_mask = predictions[valid] == 1335 if long_mask.sum() > 0:336 metrics["win_rate"] = float((pnl_slice[valid][long_mask] > 0).mean())337 else:338 metrics["win_rate"] = 0.0339 340 # 盈亏比: 平均正收益 / 平均负收益的绝对值341 pos_ret = strategy_returns[strategy_returns > 0]342 neg_ret = strategy_returns[strategy_returns < 0]343 if len(neg_ret) > 0 and np.abs(neg_ret.mean()) > 1e-12:344 metrics["profit_loss_ratio"] = float(pos_ret.mean() / np.abs(neg_ret.mean())) if len(pos_ret) > 0 else np.inf345 else:346 metrics["profit_loss_ratio"] = np.inf if len(pos_ret) > 0 else 0.0347 348 # 混淆矩阵349 tp = int(((predictions == 1) & (y_true[:n] == 1)).sum())350 tn = int(((predictions == 0) & (y_true[:n] == 0)).sum())351 fp = int(((predictions == 1) & (y_true[:n] == 0)).sum())352 fn = int(((predictions == 0) & (y_true[:n] == 1)).sum())353 total = tp + tn + fp + fn354 metrics["confusion"] = {"tp": tp, "tn": tn, "fp": fp, "fn": fn}355 metrics["accuracy"] = (tp + tn) / total if total > 0 else 0.0356 metrics["precision"] = tp / (tp + fp) if (tp + fp) > 0 else 0.0357 metrics["recall"] = tp / (tp + fn) if (tp + fn) > 0 else 0.0358 359 return metrics360 361 362# ── 扩展窗口训练引擎 ──363 364def run_expanding_window(365 X: np.ndarray,366 y: np.ndarray,367 returns: np.ndarray,368 dates: np.ndarray,369 feature_names: List[str],370 models: List[str],371 params: Dict[str, dict],372 n_splits: int = 5,373 min_train_size: int = 100,374 progress_cb: Optional[Callable] = None,375 future_ret: np.ndarray = None,376 forecast_days: int = 1,377 next_day_ret: np.ndarray = None,378 X_latest: np.ndarray = None,379 latest_date: object = None,380 all_returns: np.ndarray = None,381) -> Dict[str, ClfResult]:382 """383 严格扩展窗口时间序列训练 + 验证（禁止随机 shuffle / k-fold）。384 385 每折:386 - 训练集 = [:train_end]（含所有历史数据）387 - 验证集 = [train_end:train_end+fold_size]（紧接训练集之后）388 - GARCH(1,1) 仅在训练集上拟合 → training cond_vol（已由 arch 库保证无前视）389 - 验证集 GARCH vol = 前向预测（仅使用训练集参数）390 391 X, y, returns, dates: 已对齐的完整数据（由 create_clf_features 产出）392 models: ["XGBoost", "ElasticNet"] 子集393 params: {"XGBoost": {...}, "ElasticNet": {...}}394 future_ret: N日持有期收益（用于回测P&L），None则回退为returns395 forecast_days: 持有天数396 397 返回: {model_name: ClfResult}398 """399 results = {}400 401 for model_name in models:402 t0 = time.time()403 404 result = ClfResult(model_name=model_name)405 result.feature_names = feature_names + ["garch_vol"]406 407 # 仅使用扩展窗口划分（要求 2+3）408 splits = time_series_split(len(X), n_splits=n_splits, min_train_size=min_train_size)409 if len(splits) == 0:410 split_point = int(len(X) * 0.8)411 splits = [(np.arange(split_point), np.arange(split_point, len(X)))]412 413 all_oos_probs = []414 all_oos_preds = []415 all_oos_actuals = []416 all_oos_returns = []417 all_oos_dates = []418 all_oos_future_ret = []419 all_oos_next_day_ret = []420 fold_metrics_list = []421 fold_importances = []422 fold_count = len(splits)423 last_fold_scaler = None # ElasticNet 最后一折的 scaler424 last_fold_garch_model = None # 最后一折的 GARCH 模型425 last_fold_train_returns = None426 427 for fold_i, (train_idx, val_idx) in enumerate(splits):428 if progress_cb:429 pct = fold_i / fold_count430 msg = f"[{model_name}] 折 {fold_i + 1}/{fold_count}"431 progress_cb(pct, msg)432 433 X_train = X[train_idx]434 y_train = y[train_idx]435 X_val = X[val_idx]436 y_val = y[val_idx]437 val_dates = dates[val_idx]438 train_returns = returns[train_idx]439 val_returns = returns[val_idx]440 val_future_ret = future_ret[val_idx] if future_ret is not None else val_returns441 val_next_day_ret = next_day_ret[val_idx] if next_day_ret is not None else val_returns442 443 # ── A. 仅在训练集上拟合 GARCH（要求 4: 防未来信息泄漏） ──444 garch_model = fit_garch(train_returns, p=1, q=1, dist='t')445 446 if garch_model is not None:447 garch_vol_train = _extract_garch_vol_from_model(garch_model)448 garch_vol_val = _forecast_garch_vol(garch_model, len(val_idx))449 else:450 # GARCH 不收敛，回退为扩展历史波动率（仍仅用训练集）451 garch_vol_train = _expanding_hist_vol(train_returns, min_window=20)452 garch_vol_val = np.full(len(val_idx), np.std(train_returns[-60:]) if len(train_returns) >= 60 else np.std(train_returns))453 454 # 长度对齐（GARCH 可能少返回首元素）455 if len(garch_vol_train) < len(X_train):456 pad_len = len(X_train) - len(garch_vol_train)457 garch_vol_train = np.concatenate([np.full(pad_len, garch_vol_train[0]), garch_vol_train])458 elif len(garch_vol_train) > len(X_train):459 garch_vol_train = garch_vol_train[-len(X_train):]460 461 garch_vol_train = np.nan_to_num(garch_vol_train, nan=0.01)462 garch_vol_val = np.nan_to_num(garch_vol_val, nan=0.01)463 464 X_train_aug = np.column_stack([X_train, garch_vol_train.reshape(-1, 1)])465 X_val_aug = np.column_stack([X_val, garch_vol_val.reshape(-1, 1)])466 467 # ── B. 训练 + 预测 ──468 if model_name == "XGBoost":469 clf = build_xgb_classifier(params.get(model_name, {}))470 clf.fit(X_train_aug, y_train)471 proba = clf.predict_proba(X_val_aug)[:, 1]472 # 收集每折特征重要性（后续求平均）473 fold_importances.append(dict(zip(474 feature_names + ["garch_vol"],475 clf.feature_importances_,476 )))477 elif model_name == "ElasticNet":478 scaler = StandardScaler()479 X_train_scaled = scaler.fit_transform(X_train_aug)480 X_val_scaled = scaler.transform(X_val_aug)481 clf = build_elasticnet(params.get(model_name, {}))482 clf.fit(X_train_scaled, y_train)483 proba = clf.predict_proba(X_val_scaled)[:, 1]484 else:485 raise ValueError(f"不支持的模型: {model_name}")486 487 preds = (proba >= 0.5).astype(int)488 489 all_oos_probs.append(proba)490 all_oos_preds.append(preds)491 all_oos_actuals.append(y_val)492 all_oos_returns.append(val_returns)493 all_oos_future_ret.append(val_future_ret)494 all_oos_next_day_ret.append(val_next_day_ret)495 all_oos_dates.append(val_dates)496 497 # 折级指标（策略P&L用 next_day_ret）498 fold_metrics = calculate_classification_metrics(499 y_val, proba, val_returns, preds,500 future_ret=val_future_ret, forecast_days=forecast_days,501 next_day_ret=val_next_day_ret,502 )503 fold_metrics_list.append(fold_metrics)504 505 result.model_object = clf506 if model_name == "ElasticNet":507 last_fold_scaler = scaler508 last_fold_garch_model = garch_model509 last_fold_train_returns = train_returns510 511 # ── 聚合 OOS（按时序拼接，非末 N 个） ──512 result.oos_probabilities = np.concatenate(all_oos_probs)513 result.oos_predictions = np.concatenate(all_oos_preds).astype(int)514 result.oos_actuals = np.concatenate(all_oos_actuals).astype(int)515 result.oos_returns = np.concatenate(all_oos_returns)516 result.oos_future_ret = np.concatenate(all_oos_future_ret)517 result.oos_next_day_ret = np.concatenate(all_oos_next_day_ret)518 result.oos_dates = np.concatenate(all_oos_dates)519 520 # ── 最新交易日实时预测 ──521 if X_latest is not None and len(X_latest) > 0 and result.model_object is not None:522 try:523 # 用最后一折的训练集收益率构造 GARCH vol524 _ret_for_garch = all_returns if all_returns is not None else last_fold_train_returns525 if _ret_for_garch is not None:526 gm = fit_garch(_ret_for_garch, p=1, q=1, dist='t')527 if gm is not None:528 garch_vol_latest = _forecast_garch_vol(gm, 1)529 else:530 vol_std = np.std(_ret_for_garch[-60:]) if len(_ret_for_garch) >= 60 else np.std(_ret_for_garch)531 garch_vol_latest = np.array([vol_std])532 else:533 garch_vol_latest = np.array([0.01])534 garch_vol_latest = np.nan_to_num(garch_vol_latest, nan=0.01)535 536 X_lat_aug = np.column_stack([X_latest, garch_vol_latest.reshape(-1, 1)])537 538 if model_name == "ElasticNet" and last_fold_scaler is not None:539 X_lat_aug = last_fold_scaler.transform(X_lat_aug)540 541 latest_proba = float(result.model_object.predict_proba(X_lat_aug)[0, 1])542 result.latest_proba = latest_proba543 result.latest_date = latest_date544 except Exception:545 pass546 547 # 特征重要性：多折平均（XGBoost）548 if fold_importances:549 avg_imp = {}550 for key in fold_importances[0]:551 avg_imp[key] = float(np.mean([fi.get(key, 0) for fi in fold_importances]))552 result.feature_importance = avg_imp553 554 # 折级指标平均555 if fold_metrics_list:556 avg_metrics = {}557 for key in fold_metrics_list[0]:558 vals = [m[key] for m in fold_metrics_list559 if not (isinstance(m[key], float) and np.isnan(m[key]))]560 if key == "confusion":561 avg_metrics[key] = {562 k: int(np.mean([m["confusion"][k] for m in fold_metrics_list]))563 for k in fold_metrics_list[0]["confusion"]564 }565 elif vals:566 avg_metrics[key] = float(np.mean(vals))567 else:568 avg_metrics[key] = np.nan569 result.fold_metrics = {570 "fold_scores": fold_metrics_list,571 "cv_avg": avg_metrics,572 }573 574 # 全量 OOS 指标（策略P&L用 next_day_ret）575 result.overall_metrics = calculate_classification_metrics(576 result.oos_actuals, result.oos_probabilities,577 result.oos_returns, result.oos_predictions,578 future_ret=result.oos_future_ret, forecast_days=forecast_days,579 next_day_ret=result.oos_next_day_ret,580 )581 582 result.training_time = time.time() - t0583 results[model_name] = result584 585 if progress_cb:586 progress_cb(1.0, f"[{model_name}] 完成 ({result.training_time:.1f}s)")587 588 return results589 590 591# ── 智能参数推荐 ──592 593def get_recommended_params(n_samples: int, n_features: int = 30) -> dict:594 """根据有效样本量返回推荐参数（针对科技股优化）。595 596 科技股特点：波动大、换手高、趋势短 → 短回溯 + 浅树 + 强正则。597 598 7 档:599 tiny < 200600 small 200-500601 medium 500-800602 med_large 800-1200603 large 1200-2000604 xlarge 2000-3000605 xxlarge > 3000606 """607 if n_samples < 200:608 return {609 "mode": "tiny",610 "xgb": {611 "n_estimators": 60, "max_depth": 2, "learning_rate": 0.08,612 "subsample": 0.75, "colsample_bytree": 0.75,613 "min_child_weight": 8, "reg_alpha": 0.5, "reg_lambda": 3.0,614 },615 "elasticnet": {616 "C": 0.5, "l1_ratio": 0.15, "max_iter": 5000, "tol": 1e-3,617 },618 "look_back": 5,619 "n_splits": 3,620 }621 elif n_samples < 500:622 return {623 "mode": "small",624 "xgb": {625 "n_estimators": 80, "max_depth": 2, "learning_rate": 0.06,626 "subsample": 0.75, "colsample_bytree": 0.75,627 "min_child_weight": 8, "reg_alpha": 0.4, "reg_lambda": 3.0,628 },629 "elasticnet": {630 "C": 0.3, "l1_ratio": 0.12, "max_iter": 5000, "tol": 1e-3,631 },632 "look_back": 6,633 "n_splits": 4,634 }635 elif n_samples < 800:636 return {637 "mode": "medium",638 "xgb": {639 "n_estimators": 100, "max_depth": 2, "learning_rate": 0.05,640 "subsample": 0.75, "colsample_bytree": 0.75,641 "min_child_weight": 8, "reg_alpha": 0.4, "reg_lambda": 3.0,642 },643 "elasticnet": {644 "C": 0.2, "l1_ratio": 0.10, "max_iter": 5000, "tol": 1e-4,645 },646 "look_back": 8,647 "n_splits": 5,648 }649 elif n_samples < 1200:650 return {651 "mode": "med_large",652 "xgb": {653 "n_estimators": 120, "max_depth": 2, "learning_rate": 0.05,654 "subsample": 0.75, "colsample_bytree": 0.75,655 "min_child_weight": 8, "reg_alpha": 0.4, "reg_lambda": 3.0,656 },657 "elasticnet": {658 "C": 0.15, "l1_ratio": 0.10, "max_iter": 5000, "tol": 1e-4,659 },660 "look_back": 8,661 "n_splits": 6,662 }663 elif n_samples < 2000:664 return {665 "mode": "large",666 "xgb": {667 "n_estimators": 150, "max_depth": 3, "learning_rate": 0.04,668 "subsample": 0.75, "colsample_bytree": 0.70,669 "min_child_weight": 10, "reg_alpha": 0.4, "reg_lambda": 3.0,670 },671 "elasticnet": {672 "C": 0.10, "l1_ratio": 0.08, "max_iter": 8000, "tol": 1e-4,673 },674 "look_back": 8,675 "n_splits": 7,676 }677 elif n_samples < 3000:678 return {679 "mode": "xlarge",680 "xgb": {681 "n_estimators": 180, "max_depth": 3, "learning_rate": 0.03,682 "subsample": 0.70, "colsample_bytree": 0.65,683 "min_child_weight": 12, "reg_alpha": 0.5, "reg_lambda": 3.5,684 },685 "elasticnet": {686 "C": 0.08, "l1_ratio": 0.06, "max_iter": 8000, "tol": 1e-4,687 },688 "look_back": 10,689 "n_splits": 8,690 }691 else:692 return {693 "mode": "xxlarge",694 "xgb": {695 "n_estimators": 200, "max_depth": 3, "learning_rate": 0.02,696 "subsample": 0.70, "colsample_bytree": 0.60,697 "min_child_weight": 12, "reg_alpha": 0.5, "reg_lambda": 4.0,698 },699 "elasticnet": {700 "C": 0.05, "l1_ratio": 0.05, "max_iter": 10000, "tol": 1e-4,701 },702 "look_back": 10,703 "n_splits": 8,704 }705 706 707def check_params_deviation(current: Dict[str, dict], recommended: dict) -> List[str]:708 """对比当前参数与推荐参数，返回警告列表。"""709 warnings_list = []710 711 xgb_cur = current.get("XGBoost", {})712 xgb_rec = recommended.get("xgb", {})713 for param, rec_val in xgb_rec.items():714 cur_val = xgb_cur.get(param)715 if cur_val is not None and cur_val != rec_val:716 warnings_list.append(717 f"XGBoost.{param}: 当前={cur_val}, 推荐={rec_val}"718 )719 720 en_cur = current.get("ElasticNet", {})721 en_rec = recommended.get("elasticnet", {})722 for param, rec_val in en_rec.items():723 cur_val = en_cur.get(param)724 if cur_val is not None and cur_val != rec_val:725 warnings_list.append(726 f"ElasticNet.{param}: 当前={cur_val}, 推荐={rec_val}"727 )728 729 return warnings_list730 731 732# ── 动态融合权重 ──733 734def _compute_ensemble_weights(result_a: 'ClfResult', result_b: 'ClfResult') -> dict:735 """基于 AUC(40%) + Sharpe(30%) + 胜率(30%) 综合评分计算动态权重"""736 737 def _score(r):738 m = r.overall_metrics739 auc = m.get("auc", 0.5)740 sharpe = m.get("sharpe", 0.0)741 win_rate = m.get("win_rate", 0.5)742 if np.isnan(auc) or auc < 0.5:743 auc = 0.5744 if np.isnan(sharpe) or sharpe < 0:745 sharpe = 0.0746 if np.isnan(win_rate) or win_rate < 0:747 win_rate = 0.0748 return 0.4 * auc + 0.3 * sharpe + 0.3 * win_rate749 750 score_a = _score(result_a)751 score_b = _score(result_b)752 total = score_a + score_b753 754 if total <= 0:755 w_a, w_b = 0.5, 0.5756 else:757 w_a = score_a / total758 w_b = score_b / total759 760 return {result_a.model_name: w_a, result_b.model_name: w_b}761 762 763def _load_index_returns(stock_code: str, start_date, end_date) -> Optional[pd.Series]:764 """765 根据股票代码自动选择对应板块指数，返回日收益率 Series。766 科技股(创业板300xxx/科创板688xxx) → 创业板指(399006)767 主板 → 沪深300(000300)768 加载失败返回 None（不影响训练）769 """770 if not stock_code:771 return None772 try:773 import akshare as ak774 if stock_code.startswith("3"):775 index_code = "399006"776 elif stock_code.startswith("688"):777 index_code = "399006"778 else:779 index_code = "000300"780 781 sd = pd.Timestamp(start_date).strftime("%Y%m%d")782 ed = pd.Timestamp(end_date).strftime("%Y%m%d")783 idx_df = ak.index_zh_a_hist(symbol=index_code, period="daily",784 start_date=sd, end_date=ed)785 if idx_df is None or idx_df.empty:786 return None787 idx_df["日期"] = pd.to_datetime(idx_df["日期"])788 idx_df = idx_df.set_index("日期").sort_index()789 idx_ret = idx_df["涨跌幅"] / 100.0790 idx_ret.index.name = None791 return idx_ret792 except Exception:793 return None794 795 796# ── 特征预筛选 ──797 798def _screen_features(799 df_ind: pd.DataFrame,800 available_features: List[str],801 look_back: int,802 top_n: int = 50,803) -> Tuple[List[str], List[dict]]:804 """805 快速 XGBoost 预训练，按 base 特征聚合重要性，保留 top_n 个。806 返回: (selected_features, eliminated_records)807 """808 import xgboost as xgb809 810 X, y, feature_names, _, _, _ = create_clf_features(df_ind, available_features, look_back)811 812 if len(X) < 50:813 return available_features, []814 815 split_idx = int(len(X) * 0.8)816 X_train, X_val = X[:split_idx], X[split_idx:]817 y_train, y_val = y[:split_idx], y[split_idx:]818 819 clf = xgb.XGBClassifier(820 n_estimators=100, max_depth=4, learning_rate=0.05,821 subsample=0.8, colsample_bytree=0.8,822 objective="binary:logistic", eval_metric="logloss",823 random_state=42, n_jobs=-1, verbosity=0,824 early_stopping_rounds=10,825 )826 clf.fit(X_train, y_train, eval_set=[(X_val, y_val)], verbose=False)827 828 importances = clf.feature_importances_829 830 base_importance = {}831 for i, feat_name in enumerate(feature_names):832 base = feat_name.rsplit("_lag", 1)[0]833 base_importance[base] = base_importance.get(base, 0.0) + importances[i]834 835 sorted_feats = sorted(base_importance.items(), key=lambda x: x[1], reverse=True)836 837 selected = [f for f, _ in sorted_feats[:top_n]]838 eliminated = []839 for rank, (feat, imp) in enumerate(sorted_feats[top_n:], start=top_n + 1):840 eliminated.append({"feature": feat, "importance": float(imp), "rank": rank})841 842 return selected, eliminated843 844 845def _log_eliminated_features(stock_code: str, records: List[dict], kept_count: int):846 """将淘汰特征写入 MySQL 日志表"""847 if not records or not stock_code:848 return849 try:850 from .stock_data_store import _get_conn851 conn = _get_conn()852 if not conn:853 return854 cur = conn.cursor()855 cur.execute("""856 CREATE TABLE IF NOT EXISTS clf_feature_elimination_log (857 id INT AUTO_INCREMENT PRIMARY KEY,858 stock_code VARCHAR(20) NOT NULL,859 feature_name VARCHAR(100) NOT NULL,860 importance DOUBLE,861 rank_in_total INT,862 total_features INT,863 kept_features INT,864 eliminated_at DATETIME NOT NULL,865 INDEX idx_fe_stock (stock_code),866 INDEX idx_fe_feature (feature_name)867 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4868 """)869 from datetime import datetime870 now = datetime.now()871 total = kept_count + len(records)872 rows = [(stock_code, r["feature"], r["importance"], r["rank"], total, kept_count, now)873 for r in records]874 cur.executemany(875 "INSERT INTO clf_feature_elimination_log "876 "(stock_code, feature_name, importance, rank_in_total, total_features, kept_features, eliminated_at) "877 "VALUES (%s,%s,%s,%s,%s,%s,%s)",878 rows)879 cur.close()880 conn.close()881 except Exception:882 pass883 884 885# ── 完整管线 ──886 887def run_classifier_pipeline(888 df: pd.DataFrame,889 selected_models: List[str],890 params: Dict[str, dict],891 look_back: int = 20,892 n_splits: int = 5,893 progress_cb: Optional[Callable] = None,894 forecast_days: int = 1,895 threshold: float = 0.5,896 stock_code: str = None,897 feature_screen: bool = False,898 top_n_features: int = 50,899):900 """901 分类器完整管线。902 903 1. preprocess_data → 日收益率 / future_ret / 目标涨跌 / 量价衍生特征904 2. compute_technical_indicators → MA / MACD / RSI / 布林带 / OBV 等905 3. create_clf_features → 滞后特征展平（严格防泄漏）906 4. run_expanding_window → 扩展窗口训练+验证 + GARCH 注入907 5. 概率融合 → final_proba = (proba_a + proba_b) / 2, signal = final_proba >= threshold908 909 df: 原始 OHLCV DataFrame（index 为日期）910 selected_models: ["XGBoost", "ElasticNet"] 子集911 params: 每个模型的超参数字典912 look_back: 特征回溯天数913 n_splits: 扩展窗口折数914 progress_cb: callable(pct, msg) 进度回调915 forecast_days: 持有天数（N日持有期收益）916 threshold: 融合概率阈值917 918 返回: (results: Dict[str, ClfResult], ensemble_result: dict | None)919 """920 if progress_cb:921 progress_cb(0.0, "数据预处理中...")922 923 # 0. 尝试加载板块指数日收益率（用于相对强弱特征）924 index_returns = _load_index_returns(stock_code, df.index.min(), df.index.max())925 926 # 1. 预处理（添加 日收益率 / future_ret / 目标涨跌 / 成交量变化率 / 相对成交量 /927 # 量价配合度 / 放量上涨 / 缩量下跌）928 df_proc = preprocess_data(df, forecast_days=forecast_days, index_returns=index_returns)929 930 # 2. 技术指标（添加 MA / MACD / RSI / 布林带 / OBV / vol_ma / vwap 等，931 # 全部仅用历史数据，无前视）932 df_ind = compute_technical_indicators(df_proc)933 934 # 3. 筛选可用特征（仅保留 df 中实际存在的列）935 available_features = [c for c in CLF_FEATURE_COLS if c in df_ind.columns]936 if len(available_features) < 3:937 raise ValueError(f"可用特征不足 ({len(available_features)})，请检查数据完整性")938 939 # 3.5 特征预筛选：XGBoost 快速训练 → 保留 top_n base 特征940 _screening_info = None941 if feature_screen and len(available_features) > top_n_features:942 if progress_cb:943 progress_cb(0.05, f"特征预筛选: {len(available_features)} 个特征中选取 top {top_n_features}...")944 _orig_count = len(available_features)945 available_features, _eliminated = _screen_features(946 df_ind, available_features, look_back, top_n=top_n_features)947 _log_eliminated_features(stock_code, _eliminated, len(available_features))948 _screening_info = {949 "original": _orig_count,950 "kept": len(available_features),951 "eliminated": _eliminated,952 "kept_features": available_features[:],953 }954 if progress_cb:955 progress_cb(0.08, f"特征筛选完成: {_orig_count} → {len(available_features)}")956 957 if progress_cb:958 progress_cb(0.1, "构造滞后特征...")959 960 # 4. 构造特征和目标（X: 纯滞后特征, y: 目标涨跌）961 X, y, feature_names, aligned_dates, X_latest, latest_date = create_clf_features(df_ind, available_features, look_back)962 963 # 5. 用 create_clf_features 返回的对齐日期做标签索引（防 dropna 偏移）964 returns = df_ind.loc[aligned_dates, "日收益率"].values965 future_ret = df_ind.loc[aligned_dates, "future_ret"].values966 next_day_ret = df_ind.loc[aligned_dates, "next_day_ret"].values967 dates = np.array(aligned_dates)968 969 # 全量收益率序列（用于 GARCH 拟合最新日预测）970 all_returns = df_ind["日收益率"].dropna().values971 972 if progress_cb:973 progress_cb(0.15, f"有效样本: {len(X)} 行 × {len(feature_names)} 特征")974 975 # 6. 扩展窗口训练 + 验证976 results = run_expanding_window(977 X=X, y=y, returns=returns, dates=dates,978 feature_names=feature_names,979 models=selected_models,980 params=params,981 n_splits=n_splits,982 progress_cb=progress_cb,983 future_ret=future_ret,984 forecast_days=forecast_days,985 next_day_ret=next_day_ret,986 X_latest=X_latest,987 latest_date=latest_date,988 all_returns=all_returns,989 )990 991 # ── 7. 概率融合（如果两个模型都选中） ──992 ensemble_result = None993 if len(selected_models) == 2:994 model_a, model_b = selected_models[0], selected_models[1]995 if model_a in results and model_b in results:996 proba_a = results[model_a].oos_probabilities997 proba_b = results[model_b].oos_probabilities998 n_common = min(len(proba_a), len(proba_b))999 1000 # 动态权重：基于 AUC(40%) + Sharpe(30%) + 胜率(30%) 综合评分1001 weights = _compute_ensemble_weights(results[model_a], results[model_b])1002 w_a, w_b = weights[model_a], weights[model_b]1003 1004 fused_proba = w_a * proba_a[:n_common] + w_b * proba_b[:n_common]1005 fused_signal = (fused_proba >= threshold).astype(int)1006 1007 y_common = results[model_a].oos_actuals[:n_common]1008 ret_common = results[model_a].oos_returns[:n_common]1009 fut_common = results[model_a].oos_future_ret[:n_common]1010 ndr_common = results[model_a].oos_next_day_ret[:n_common]1011 dates_common = results[model_a].oos_dates[:n_common]1012 1013 ensemble_metrics = calculate_classification_metrics(1014 y_common, fused_proba,1015 ret_common, fused_signal,1016 future_ret=fut_common,1017 forecast_days=forecast_days,1018 next_day_ret=ndr_common,1019 )1020 1021 ensemble_result = {1022 "fused_proba": fused_proba,1023 "fused_signal": fused_signal,1024 "metrics": ensemble_metrics,1025 "weights": {model_a: float(w_a), model_b: float(w_b)},1026 "oos_dates": dates_common,1027 "oos_returns": ret_common,1028 "oos_future_ret": fut_common,1029 "oos_next_day_ret": ndr_common,1030 "oos_actuals": y_common,1031 "threshold": threshold,1032 "forecast_days": forecast_days,1033 }1034 1035 # 融合最新日预测1036 lat_a = results[model_a].latest_proba1037 lat_b = results[model_b].latest_proba1038 if not np.isnan(lat_a) and not np.isnan(lat_b):1039 ensemble_result["latest_proba"] = float(w_a * lat_a + w_b * lat_b)1040 ensemble_result["latest_date"] = results[model_a].latest_date1041 ensemble_result["latest_signal"] = int(ensemble_result["latest_proba"] >= threshold)1042 1043 if progress_cb:1044 progress_cb(1.0, "涨跌预测完成!")1045 1046 if ensemble_result and _screening_info:1047 ensemble_result["screening_info"] = _screening_info1048 1049 return results, ensemble_result, _screening_info1050 1051 1052# ── 自动调参：随机搜索 + 早停 ──1053 1054_TUNE_SEARCH_SPACE = {1055 "look_back": [5, 10, 15, 20, 30, 40],1056 "forecast_days": [1, 2, 3, 5],1057 "n_splits": [4, 5, 6, 7, 8],1058 "xgb_learning_rate": [0.01, 0.03, 0.05, 0.08, 0.1],1059 "xgb_n_estimators": [80, 100, 150, 200, 300],1060 "xgb_max_depth": [3, 4, 5, 6],1061 "xgb_subsample": [0.6, 0.7, 0.8, 0.9],1062 "xgb_colsample_bytree": [0.5, 0.6, 0.7, 0.8],1063 "xgb_min_child_weight": [1, 3, 5, 8, 12],1064 "xgb_reg_alpha": [0, 0.1, 0.3, 0.5, 1.0],1065 "xgb_reg_lambda": [1.0, 2.0, 3.0, 4.0],1066 "en_C": [0.05, 0.1, 0.3, 0.5, 1.0],1067 "en_l1_ratio": [0.05, 0.1, 0.15, 0.3, 0.5],1068}1069 1070 1071def auto_tune_classifier(1072 df: pd.DataFrame,1073 stock_code: str = None,1074 target_auc: float = 0.53,1075 max_trials: int = 30,1076 selected_models: List[str] = None,1077 trial_cb: Optional[Callable] = None,1078) -> dict:1079 """1080 随机搜索参数空间，找到 ensemble AUC >= target_auc 的参数组合。1081 1082 trial_cb: callable(trial_idx, max_trials, trial_result_dict) 每轮回调1083 返回: {"best_params": {...}, "best_auc": float, "trials": [...], "found": bool}1084 """1085 import random as _rand1086 1087 if selected_models is None:1088 selected_models = ["XGBoost", "ElasticNet"]1089 1090 trials = []1091 best_auc = 0.01092 best_params = None1093 tried = set()1094 1095 for trial_idx in range(max_trials):1096 # 随机采样参数1097 sample = {k: _rand.choice(v) for k, v in _TUNE_SEARCH_SPACE.items()}1098 sample_key = tuple(sorted(sample.items()))1099 if sample_key in tried:1100 continue1101 tried.add(sample_key)1102 1103 params = {1104 "XGBoost": {1105 "learning_rate": sample["xgb_learning_rate"],1106 "n_estimators": sample["xgb_n_estimators"],1107 "max_depth": sample["xgb_max_depth"],1108 "subsample": sample["xgb_subsample"],1109 "colsample_bytree": sample["xgb_colsample_bytree"],1110 "min_child_weight": sample["xgb_min_child_weight"],1111 "reg_alpha": sample["xgb_reg_alpha"],1112 "reg_lambda": sample["xgb_reg_lambda"],1113 },1114 "ElasticNet": {1115 "C": sample["en_C"],1116 "l1_ratio": sample["en_l1_ratio"],1117 "max_iter": 5000,1118 "tol": 1e-3,1119 },1120 }1121 1122 t0 = time.time()1123 try:1124 results, ensemble_result, _ = run_classifier_pipeline(1125 df=df,1126 selected_models=selected_models,1127 params=params,1128 look_back=sample["look_back"],1129 n_splits=sample["n_splits"],1130 forecast_days=sample["forecast_days"],1131 threshold=0.5,1132 stock_code=stock_code,1133 )1134 except Exception:1135 continue1136 elapsed = round(time.time() - t0, 1)1137 1138 # 提取 AUC1139 auc = np.nan1140 if ensemble_result and ensemble_result.get("metrics"):1141 auc = ensemble_result["metrics"].get("auc", np.nan)1142 elif results:1143 first_model = list(results.keys())[0]1144 auc = results[first_model].overall_metrics.get("auc", np.nan)1145 1146 if np.isnan(auc):1147 continue1148 1149 trial_result = {1150 "trial": trial_idx + 1,1151 "look_back": sample["look_back"],1152 "forecast_days": sample["forecast_days"],1153 "n_splits": sample["n_splits"],1154 "lr": sample["xgb_learning_rate"],1155 "depth": sample["xgb_max_depth"],1156 "n_est": sample["xgb_n_estimators"],1157 "auc": round(auc, 4),1158 "elapsed": elapsed,1159 "params": params,1160 "sample": sample,1161 }1162 trials.append(trial_result)1163 1164 if auc > best_auc:1165 best_auc = auc1166 best_params = sample1167 1168 if trial_cb:1169 trial_cb(trial_idx + 1, max_trials, trial_result)1170 1171 if auc >= target_auc:1172 break1173 1174 return {1175 "best_params": best_params,1176 "best_auc": round(best_auc, 4),1177 "trials": trials,1178 "found": best_auc >= target_auc,1179 }1180 1181 1182# ── 贝叶斯优化调参 (Optuna TPE) ──1183 1184def auto_tune_optuna(1185 df: pd.DataFrame,1186 stock_code: str = None,1187 target_auc: float = 0.53,1188 max_trials: int = 20,1189 selected_models: List[str] = None,1190 trial_cb: Optional[Callable] = None,1191) -> dict:1192 """1193 使用 Optuna TPE 贝叶斯优化搜索最佳参数，收敛更快。1194 1195 trial_cb: callable(trial_idx, max_trials, trial_result_dict) 每轮回调1196 返回: {"best_params": {...}, "best_auc": float, "trials": [...], "found": bool}1197 """1198 import optuna1199 optuna.logging.set_verbosity(optuna.logging.WARNING)1200