Daveabc12/Backtesting-App
1
1# New version ---
2import streamlit as st
3import pandas as pd
4import os
5import numpy as np
6from datetime import date, datetime
7import plotly.graph_objects as go
8import itertools
9import json
10# --- CORRECTED IMPORTS: Moved MACD ---
11from ta.volatility import BollingerBands
12from ta.momentum import RSIIndicator # Removed MACD from here
13from ta.trend import ADXIndicator, MACD
14from ta.volume import MFIIndicator # Added MFI here
15# --- [NEW] ADDED ATR FOR INTELLIGENT EXIT ---
16from ta.volatility import AverageTrueRange
17# --- [END NEW] ---
18from multiprocessing import Pool, cpu_count
19from functools import partial
20from dateutil.relativedelta import relativedelta
21from datetime import timedelta
22
23# --- 0. Settings Management Functions ---
24CONFIG_FILE = "config.json"
25VETO_CONFIG_FILE = "veto_config.json"
26TOP_SETUPS_FILE = "top_setups.json"
27USER_SETUPS_FILE = "user_advisor_setups.json"
28MARKOV_SETUP_FILE = "best_markov.json"
29
30def save_settings(params_to_save):
31 with open(CONFIG_FILE, 'w', encoding='utf-8') as f:
32 json.dump(params_to_save, f, indent=4)
33 st.sidebar.success("Settings saved as default!")
34
35# --- Replacement for load_settings function ---
36def load_settings():
37 default_structure = {
38 "large_ma_period": 70,
39 "bband_period": 32,
40 "bband_std_dev": 1.4,
41 "confidence_threshold": 95,
42
43 "long_entry_threshold_pct": 0.01,
44 "long_exit_ma_threshold_pct": 0.0,
45 "long_trailing_stop_loss_pct": 0.2,
46 "long_delay_days": 0,
47
48 "short_entry_threshold_pct": 0.0,
49 "short_exit_ma_threshold_pct": 0.0,
50 "short_trailing_stop_loss_pct": 0.2,
51 "short_delay_days": 0,
52
53 "use_rsi": True, "rsi_w": 1.5,
54 "rsi_logic": "Level",
55 "primary_driver": "Bollinger Bands",
56
57 "exit_logic_type": "Intelligent (ADX/MACD/ATR)",
58 "exit_confidence_threshold": 40,
59 "smart_exit_atr_period": 14,
60 "smart_exit_atr_multiplier": 3.0,
61 "intelligent_tsl_pct": 0.2,
62 "norm_lookback_years": 1,
63 'use_rolling_benchmark': False,
64 "benchmark_rank": 99,
65 "use_ma_floor_filter": True,
66 "catcher_stop_pct": 0.03,
67 "use_vol": False, "vol_w": 0.5,
68 "use_trend": False, "trend_w": 2.0,
69 "use_volume": False, "volume_w": 0.5,
70
71 "use_adx_filter": False, "adx_threshold": 10.0,
72 "adx_period": 14,
73
74 "use_macd": False, "macd_w": 2.0,
75 "use_ma_slope": False, "ma_slope_w": 0.5,
76 "use_markov": False, "markov_w": 1.0,
77
78 "max_trading_days": 60,
79 "max_long_duration": 60,
80 "max_short_duration": 3
81 }
82
83 if os.path.exists(CONFIG_FILE):
84 try:
85 with open(CONFIG_FILE, 'r', encoding='utf-8') as f:
86 loaded = json.load(f)
87 for key in default_structure:
88 if key in loaded:
89 default_structure[key] = loaded[key]
90 except (json.JSONDecodeError, Exception) as e:
91 print(f"Error loading config.json: {e}. Using default settings.")
92 return default_structure
93
94# --- UPDATED: Save a list of veto setups ---
95def save_veto_setup(veto_setups_list): # Takes a list now
96 with open(VETO_CONFIG_FILE, 'w', encoding='utf-8') as f:
97 # Save the list directly
98 json.dump(veto_setups_list, f, indent=4)
99 st.sidebar.success(f"Saved {len(veto_setups_list)} Veto filter(s) as default!")
100
101# --- UPDATED: Load a list of veto setups, robustly ---
102def load_veto_setup():
103 veto_list = [] # Default to empty list
104 if os.path.exists(VETO_CONFIG_FILE):
105 try:
106 with open(VETO_CONFIG_FILE, 'r', encoding='utf-8') as f:
107 loaded_data = json.load(f)
108 # Ensure it's loaded as a list
109 if isinstance(loaded_data, list):
110 veto_list = loaded_data
111 elif isinstance(loaded_data, dict): # Handle old single dict format
112 veto_list = [loaded_data]
113 except (json.JSONDecodeError, Exception) as e:
114 print(f"Error loading veto config: {e}. Using empty list.")
115 # Keep veto_list as empty
116 pass
117 return veto_list
118
119# --- UPDATED: Includes new factors, num_setups=8, removed infinity filter ---
120def save_top_setups(results_df, side, num_setups=8): # Default is now 8
121 df = results_df.copy()
122
123 if df.empty:
124 st.sidebar.warning(f"No valid setups found for {side.title()} to save.")
125 return
126
127 deduplication_cols = [
128 'Conf. Threshold', 'Avg Profit/Trade', 'Ticker G/B Ratio', 'Trade G/B Ratio',
129 'Winning Tickers', 'Losing Tickers', 'Avg Entry Conf.',
130 'Good Score', 'Bad Score', 'Norm. Score %', 'Total Trades'
131 ]
132
133 factor_cols = ['RSI', 'Volatility', 'TREND', 'Volume', 'MACD', 'MA Slope']
134 existing_factor_cols = [col for col in factor_cols if col in df.columns]
135 if existing_factor_cols:
136 df['FactorsOn'] = df[existing_factor_cols].apply(lambda row: (row == 'On').sum(), axis=1)
137 else:
138 df['FactorsOn'] = 0
139
140 # --- CONSISTENCY FIX: Sort by the Weighted Score ("Norm. Score %") ---
141 # If Norm. Score % exists, use it. Otherwise fall back to Strategy Score or Trade G/B Ratio.
142 sort_col = 'Trade G/B Ratio' # Default fallback
143 if 'Norm. Score %' in df.columns:
144 sort_col = 'Norm. Score %'
145 elif 'Strategy Score' in df.columns:
146 sort_col = 'Strategy Score'
147
148 if sort_col in df.columns:
149 df[sort_col] = df[sort_col].fillna(-np.inf)
150 # Sort by Score (Descending), then by Complexity (Ascending - simpler is better)
151 df = df.sort_values(
152 by=[sort_col, 'FactorsOn'],
153 ascending=[False, True]
154 )
155 else:
156 st.sidebar.error(f"Could not sort top setups: '{sort_col}' column missing.")
157 return
158
159 existing_cols_for_dedup = [col for col in deduplication_cols if col in df.columns]
160 deduplicated_df = df.drop_duplicates(subset=existing_cols_for_dedup, keep='first')
161
162 top_setups = deduplicated_df.head(num_setups).to_dict('records')
163
164 if os.path.exists(TOP_SETUPS_FILE):
165 try:
166 with open(TOP_SETUPS_FILE, 'r', encoding='utf-8') as f:
167 all_top_setups = json.load(f)
168 except json.JSONDecodeError:
169 all_top_setups = {}
170 else:
171 all_top_setups = {}
172
173 all_top_setups[side] = top_setups
174
175 with open(TOP_SETUPS_FILE, 'w', encoding='utf-8') as f:
176 json.dump(all_top_setups, f, indent=4)
177
178 st.sidebar.success(f"Top {len(top_setups)} unique {side.title()} setups saved! (Sorted by {sort_col})")
179
180def load_top_setups():
181 if os.path.exists(TOP_SETUPS_FILE):
182 try:
183 with open(TOP_SETUPS_FILE, 'r', encoding='utf-8') as f:
184 return json.load(f)
185 except json.JSONDecodeError:
186 return None # Return None if file is corrupt
187 return None
188
189@st.cache_data # Use caching for efficiency
190def load_proverbs():
191 # Default to an empty list
192 proverbs_list = []
193 if os.path.exists("proverbs.json"):
194 try:
195 with open("proverbs.json", 'r', encoding='utf-8') as f:
196 loaded_data = json.load(f)
197 # Ensure it's a list
198 if isinstance(loaded_data, list):
199 proverbs_list = loaded_data
200 except (json.JSONDecodeError, Exception) as e:
201 print(f"Error loading proverbs.json: {e}. Using fallback.")
202 # Keep proverbs_list empty or provide a default
203 # proverbs_list = ["Error loading proverbs."]
204 # Add a fallback if the list is empty after trying to load
205 if not proverbs_list:
206 proverbs_list = ["Have a great trading day!"] # Fallback message
207 return proverbs_list
208
209def load_user_setups():
210 """Loads user-defined advisor setups from a JSON file."""
211
212 # 1. Update Template to include Notes
213 default_row_template = {
214 "Run": False,
215 "Notes": "", # <--- NEW FIELD FOR ADVICE
216 "RSI": "Off", "Volatility": "Off", "TREND": "Off", "Volume": "Off",
217 "MACD": "Off", "MA Slope": "Off", "Markov": "Off",
218 "ADX Filter": "Off",
219 "Conf. Threshold": 50,
220 "Large MA Period": 50, "Bollinger Band Period": 20, "Bollinger Band Std Dev": 2.0,
221 "Catcher Offset (%)": 3.0,
222 "Long Entry Threshold (%)": 0.0, "Long Exit Threshold (%)": 0.0, "Long Stop Loss (%)": 8.0, "Long Delay (Days)": 0,
223 "Short Entry Threshold (%)": 0.0, "Short Exit Threshold (%)": 0.0, "Short Stop Loss (%)": 8.0, "Short Delay (Days)": 0,
224 "Z_Avg_Profit": 0.0, "Z_Num_Trades": 0, "Z_WL_Ratio": 0.0
225 }
226
227 # Default Examples
228 default_setup = [default_row_template.copy()]
229
230 # Pad with defaults
231 while len(default_setup) < 20:
232 default_setup.append(default_row_template.copy())
233
234 if os.path.exists(USER_SETUPS_FILE):
235 try:
236 with open(USER_SETUPS_FILE, 'r', encoding='utf-8') as f:
237 user_setups = json.load(f)
238 if isinstance(user_setups, list):
239 processed_setups = []
240 for setup in user_setups:
241 full_setup = default_row_template.copy()
242 full_setup.update(setup) # Merge loaded data
243 processed_setups.append(full_setup)
244
245 while len(processed_setups) < 20:
246 processed_setups.append(default_row_template.copy())
247 return processed_setups[:20]
248 else:
249 return default_setup
250 except Exception as e:
251 print(f"Error loading {USER_SETUPS_FILE}: {e}. Using defaults.")
252 return default_setup
253 return default_setup
254
255# --- UPDATED: Callback to Save Setup (Added Notes) ---
256def add_setup_to_user_list():
257 try:
258 stats = st.session_state.get('last_run_stats', {})
259 if not stats:
260 st.toast("⚠️ No stats found. Run analysis first.", icon="⚠️")
261 return
262
263 def get_weight_or_off(toggle_key, weight_key):
264 if st.session_state.get(toggle_key, False):
265 return round(st.session_state.get(weight_key, 1.0), 2)
266 return "Off"
267
268 adx_val = "Off"
269 if st.session_state.get("use_adx_filter", False):
270 val = st.session_state.get("adx_threshold", 25.0)
271 adx_val = max(20.0, min(30.0, val))
272
273 new_setup = {
274 "Run": True,
275 "Notes": "Auto-Saved Setup", # <--- Default note for auto-saves
276 "RSI": get_weight_or_off('use_rsi', 'rsi_w'),
277 "Volatility": get_weight_or_off('use_vol', 'vol_w'),
278 "TREND": get_weight_or_off('use_trend', 'trend_w'),
279 "Volume": get_weight_or_off('use_volume', 'volume_w'),
280 "MACD": get_weight_or_off('use_macd', 'macd_w'),
281 "MA Slope": get_weight_or_off('use_ma_slope', 'ma_slope_w'),
282 "Markov": get_weight_or_off('use_markov', 'markov_w'),
283 "ADX Filter": adx_val,
284
285 "Conf. Threshold": st.session_state.confidence_slider,
286 "Large MA Period": st.session_state.ma_period,
287 "Bollinger Band Period": st.session_state.bb_period,
288 "Bollinger Band Std Dev": st.session_state.bb_std,
289
290 "Long Entry Threshold (%)": st.session_state.long_entry,
291 "Long Exit Threshold (%)": st.session_state.long_exit,
292 "Long Stop Loss (%)": st.session_state.long_sl,
293 "Long Delay (Days)": st.session_state.long_delay,
294
295 "Short Entry Threshold (%)": st.session_state.short_entry,
296 "Short Exit Threshold (%)": st.session_state.short_exit,
297 "Short Stop Loss (%)": st.session_state.short_sl,
298 "Short Delay (Days)": st.session_state.short_delay,
299
300 "Z_Avg_Profit": stats.get("Z_Avg_Profit", 0.0) * 100.0,
301 "Z_Num_Trades": stats.get("Z_Num_Trades", 0),
302 "Z_WL_Ratio": stats.get("Z_WL_Ratio", 0.0)
303 }
304
305 current_setups = st.session_state.get("user_setups_data", [])
306 non_empty_setups = [s for s in current_setups if not is_row_blank(s)]
307 default_row_template = {k:v for k,v in load_user_setups()[0].items()}
308 non_empty_setups.append(new_setup)
309 while len(non_empty_setups) < 20: non_empty_setups.append(default_row_template.copy())
310
311 final_setups = non_empty_setups[:20]
312 save_user_setups(final_setups)
313
314 # Refresh State
315 processed_setups = []
316 for s in final_setups:
317 processed_setups.append(s.copy()) # Simple copy is enough now
318 st.session_state["user_setups_data"] = processed_setups
319
320 st.toast("✅ Setup saved!", icon="✅")
321 st.session_state.run_user_advisor_setup = True
322 st.rerun()
323
324 except Exception as e:
325 st.error(f"Could not save setup: {e}")
326
327def save_user_setups(setups_list):
328 """Saves user-defined advisor setups to a JSON file."""
329 try:
330 # Ensure we only save 20
331 with open(USER_SETUPS_FILE, 'w', encoding='utf-8') as f:
332 json.dump(setups_list[:20], f, indent=4) # <-- CHANGED TO 20
333 st.success("User-defined setups saved!")
334 except Exception as e:
335 st.error(f"Error saving user setups: {e}")
336
337def save_markov_setup(setup_dict):
338 """Saves the best Markov setup to a JSON file."""
339 try:
340 with open(MARKOV_SETUP_FILE, 'w', encoding='utf-8') as f:
341 json.dump(setup_dict, f, indent=4)
342 st.sidebar.success("Best Markov setup saved as default!")
343 except Exception as e:
344 st.sidebar.error(f"Error saving Markov setup: {e}")
345
346def load_markov_setup():
347 """Loads the best Markov setup from a JSON file."""
348 if os.path.exists(MARKOV_SETUP_FILE):
349 try:
350 with open(MARKOV_SETUP_FILE, 'r', encoding='utf-8') as f:
351 return json.load(f)
352 except Exception as e:
353 print(f"Error loading {MARKOV_SETUP_FILE}: {e}")
354 return None
355 return None
356
357# --- UPDATED: Helper to accept filename (Replaces the old load_completed_setups) ---
358def load_completed_setups(filename):
359 """Reads the existing CSV and returns a set of completed configurations."""
360 completed_configs = set()
361 if os.path.exists(filename):
362 try:
363 # We only need to read the config columns: Factors and Weights
364 df_completed = pd.read_csv(filename, usecols=['RSI', 'Volatility', 'TREND', 'Volume', 'MACD', 'MA Slope', 'Markov',
365 'RSI W', 'Volatility W', 'TREND W', 'Volume W', 'MACD W', 'MA Slope W', 'Markov W'],
366 dtype={'RSI': str, 'Volatility': str, 'TREND': str, 'Volume': str, 'MACD': str, 'MA Slope': str, 'Markov': str})
367
368 for index, row in df_completed.iterrows():
369 toggles = tuple(row[['RSI', 'Volatility', 'TREND', 'Volume', 'MACD', 'MA Slope', 'Markov']].values)
370 weights = tuple(row[['RSI W', 'Volatility W', 'TREND W', 'Volume W', 'MACD W', 'MA Slope W', 'Markov W']].values)
371 completed_configs.add((toggles, weights))
372 except Exception as e:
373 print(f"Warning: Error loading CSV for checkpointing: {e}. Starting from scratch.")
374 return completed_configs
375
376# --- 1. Data Loading and Cleaning Functions ---
377@st.cache_data(ttl=300) # Added TTL to auto-refresh cache every 5 mins
378def load_all_data(folder_path):
379 if not os.path.exists(folder_path):
380 st.error(f"Folder '{folder_path}' not found.")
381 return None, None
382
383 all_files = [f for f in os.listdir(folder_path) if f.endswith('.csv')]
384 all_files.sort() # Ensure we load in chronological order (e.g. 2023, then 2024)
385
386 if not all_files:
387 st.error("No CSV files found in the 'csv_data' folder.")
388 return None, None
389
390 df_list = []
391 error_messages = []
392
393 for file_name in all_files:
394 file_path = os.path.join(folder_path, file_name)
395 try:
396 # 1. Read CSV without parsing dates yet (load as strings to be safe)
397 df = pd.read_csv(file_path, header=0, index_col=0, encoding='utf-8')
398
399 # 2. FORCE ISO PARSING (YYYY-MM-DD)
400 # We disable 'dayfirst' because ISO puts Year first.
401 df.index = pd.to_datetime(df.index, format='%Y-%m-%d', errors='coerce')
402
403 # 3. Drop rows with invalid dates
404 df = df[df.index.notna()]
405
406 if df.empty:
407 error_messages.append(f"Warning: Skipped {file_name}, no valid dates found.")
408 continue
409
410 df_list.append(df)
411 except Exception as e:
412 error_messages.append(f"Could not read {file_name}. Error: {e}")
413
414 if not df_list:
415 return None, "No data could be loaded successfully."
416
417 # 4. CONSOLIDATE
418 try:
419 master_df = pd.concat(df_list)
420
421 # 5. DEDUPLICATE (Keep the newest version of any overlapping date)
422 # This prevents "Double Dots" if 2024-2025.csv and 2025-NOW.csv overlap
423 if master_df.index.has_duplicates:
424 master_df = master_df[~master_df.index.duplicated(keep='last')]
425
426 # 6. FORCE SORT (The Final "Zig-Zag" Killer)
427 # Ensures Jan 1st always comes before Jan 2nd
428 master_df.sort_index(inplace=True)
429
430 # 7. NUMERIC CONVERSION (From your original code)
431 # Ensures all price columns are numbers, not strings
432 for col in master_df.columns:
433 master_df[col] = pd.to_numeric(master_df[col], errors='coerce')
434
435 for msg in error_messages: st.warning(msg)
436
437 return master_df, f"Successfully combined data from {len(df_list)} files."
438
439 except Exception as e:
440 return None, f"Critical Error merging data: {e}"
441
442def clean_data_and_report_outliers(df):
443 """
444 Cleans data using a 'Rolling Median' filter (User Defined: 1 Month Window).
445 Adapts to long-term trends while catching sudden 'Pence vs Pound' glitches.
446 """
447 outlier_report = []
448 # Identify price columns (exclude _Volume, _High, _Low)
449 price_columns = [col for col in df.columns if not any(x in str(col) for x in ['_Volume', '_High', '_Low'])]
450
451 for ticker in price_columns:
452 if ticker in df.columns:
453 # Ensure numeric
454 series = pd.to_numeric(df[ticker], errors='coerce')
455
456 # --- ROLLING MEDIAN FILTER (1 Month / 20 Days) ---
457 # center=True looks at 10 days before and 10 days after (if available) to find the 'true' level.
458 # min_periods=1 ensures it works even at the start of the file.
459 rolling_median = series.rolling(window=20, center=True, min_periods=1).median()
460
461 # 1. Catch the 'Pence vs Pounds' Crash (e.g., 154 -> 1.56)
462 # Logic: If price is < 20% of the recent median (an 80% drop).
463 low_threshold = rolling_median * 0.20
464
465 # 2. Catch massive data spikes (e.g., 1.56 -> 154 if logic reversed)
466 # Logic: If price is > 5x the recent median (500% spike).
467 high_threshold = rolling_median * 5.0
468
469 bad_data_mask = (series < low_threshold) | (series > high_threshold)
470 bad_days = series[bad_data_mask].index
471
472 if not bad_days.empty:
473 df.loc[bad_days, ticker] = np.nan
474 outlier_report.append({'Ticker': ticker, 'Type': 'Rolling Filter', 'Count': len(bad_days)})
475
476 return df, outlier_report
477
478# --- 2. Custom Backtesting Engine ---
479
480# --- [UPDATED] 9-Factor Version (with MFI & SuperTrend) ---
481def calculate_confidence_score(df, primary_driver,
482 use_rsi, use_volatility, use_trend, use_volume, use_macd, use_ma_slope, use_markov,
483 use_mfi, use_supertrend,
484 rsi_w, vol_w, trend_w, vol_w_val, macd_w, ma_slope_w, markov_w,
485 mfi_w, supertrend_w,
486 bband_params,
487 best_markov_setup=None):
488
489 long_score = pd.Series(0.0, index=df.index)
490 short_score = pd.Series(0.0, index=df.index)
491
492 # --- Pre-calculate Markov State if needed ---
493 if use_markov and best_markov_setup and 'RunUp_State' not in df.columns:
494 run_up_period = best_markov_setup.get('Run-Up Period', 10)
495 df['RunUp_Return'] = df['Close'].pct_change(periods=run_up_period)
496 df['RunUp_State'] = df['RunUp_Return'].apply(lambda x: 'Up' if x > 0 else 'Down')
497
498 # --- BBand Factor ---
499 if primary_driver != 'Bollinger Bands' and 'bband_lower' in df.columns:
500 bb_weight = 1.0
501 long_entry_pct = bband_params.get('long_entry_threshold_pct', 0.0)
502 short_entry_pct = bband_params.get('short_entry_threshold_pct', 0.0)
503 long_bb_trigger_price = df['bband_lower'] * (1 - long_entry_pct)
504 short_bb_trigger_price = df['bband_upper'] * (1 + short_entry_pct)
505 long_score_range = (df['large_ma'] - long_bb_trigger_price).replace(0, np.nan)
506 short_score_range = (short_bb_trigger_price - df['large_ma']).replace(0, np.nan)
507 long_score += ((df['large_ma'] - df['Close']) / long_score_range).clip(0, 1).fillna(0) * bb_weight
508 short_score += ((df['Close'] - df['large_ma']) / short_score_range).clip(0, 1).fillna(0) * bb_weight
509
510 # Factor 1: RSI
511 if primary_driver != 'RSI Crossover' and use_rsi and 'RSI' in df.columns:
512 long_score += ((30 - df['RSI']) / 30).clip(0, 1).fillna(0) * rsi_w
513 short_score += ((df['RSI'] - 70) / 30).clip(0, 1).fillna(0) * rsi_w
514
515 # Factor 2: Volatility
516 if use_volatility and 'Volatility_p' in df.columns:
517 vol_signal = (df['Volatility_p'] > 0.025).astype(float) * vol_w
518 long_score += vol_signal; short_score += vol_signal
519
520 # Factor 3: Trend
521 if use_trend and 'SMA_200' in df.columns and 'Close' in df.columns:
522 valid_sma = (df['SMA_200'] != 0) & df['SMA_200'].notna()
523 pct_dist = pd.Series(0.0, index=df.index)
524 pct_dist.loc[valid_sma] = df.loc[valid_sma].apply(lambda row: (row['Close'] - row['SMA_200']) / row['SMA_200'] if row['SMA_200'] != 0 else 0, axis=1)
525 long_score += (pct_dist / 0.10).clip(0, 1).fillna(0) * trend_w
526 short_score += (-pct_dist / 0.10).clip(0, 1).fillna(0) * trend_w
527
528 # Factor 4: Volume
529 if use_volume and 'Volume_Ratio' in df.columns:
530 vol_spike_signal = ((df['Volume_Ratio'] - 1.75) / 2.25).clip(0, 1).fillna(0) * vol_w_val
531 long_score += vol_spike_signal; short_score += vol_spike_signal
532
533 # Factor 5: MACD
534 if primary_driver != 'MACD Crossover' and use_macd and 'MACD_line' in df.columns:
535 macd_cross_long = (df['MACD_line'].shift(1) < df['MACD_signal'].shift(1)) & (df['MACD_line'] >= df['MACD_signal'])
536 macd_cross_short = (df['MACD_line'].shift(1) > df['MACD_signal'].shift(1)) & (df['MACD_line'] <= df['MACD_signal'])
537 long_score += macd_cross_long.astype(float) * macd_w * 0.6
538 short_score += macd_cross_short.astype(float) * macd_w * 0.6
539 long_score += (df['MACD_hist'] > 0).astype(float) * macd_w * 0.4
540 short_score += (df['MACD_hist'] < 0).astype(float) * macd_w * 0.4
541
542 # Factor 6: MA Slope
543 if primary_driver != 'MA Slope' and use_ma_slope and 'ma_slope' in df.columns:
544 long_score += (df['ma_slope'] > 0).astype(float) * ma_slope_w
545 short_score += (df['ma_slope'] < 0).astype(float) * ma_slope_w
546
547 # Factor 7: Markov State
548 if primary_driver != 'Markov State' and use_markov and best_markov_setup and 'RunUp_State' in df.columns:
549 strategy = best_markov_setup.get('Strategy')
550 if strategy == 'Down -> Up': long_score += (df['RunUp_State'] == 'Down').astype(float) * markov_w
551 elif strategy == 'Up -> Up': long_score += (df['RunUp_State'] == 'Up').astype(float) * markov_w
552 elif strategy == 'Up -> Down': short_score += (df['RunUp_State'] == 'Up').astype(float) * markov_w
553 elif strategy == 'Down -> Down': short_score += (df['RunUp_State'] == 'Down').astype(float) * markov_w
554
555 # Factor 8: MFI (Money Flow Index) [IMPROVED]
556 if use_mfi and 'MFI' in df.columns:
557 # Binary Logic: If MFI < 20 (Oversold), give full weight.
558 # This fixes the "tiny score" issue.
559 long_score += (df['MFI'] < 25).astype(float) * mfi_w
560 short_score += (df['MFI'] > 75).astype(float) * mfi_w
561
562 # Factor 9: SuperTrend [IMPROVED]
563 if use_supertrend and 'SuperTrend' in df.columns:
564 # Long Score: Price > SuperTrend (Trend is Bullish)
565 long_score += (df['Close'] > df['SuperTrend']).astype(float) * supertrend_w
566 # Short Score: Price < SuperTrend (Trend is Bearish)
567 short_score += (df['Close'] < df['SuperTrend']).astype(float) * supertrend_w
568
569 return long_score.fillna(0), short_score.fillna(0)
570
571# --- FULL REPLACEMENT FOR run_backtest FUNCTION ---
572def run_backtest(data, params,
573 use_rsi, use_volatility, use_trend, use_volume, use_macd, use_ma_slope, use_markov, use_mfi, use_supertrend, # Added MFI/ST
574 rsi_w, vol_w, trend_w, vol_w_val, macd_w, ma_slope_w, markov_w, mfi_w, supertrend_w, # Added MFI/ST weights
575 use_adx_filter, adx_threshold, rsi_logic,
576 adx_period=14,
577 veto_setups_list=None,
578 primary_driver='Bollinger Bands',
579 markov_setup=None,
580 exit_logic_type='Standard (Price-Based)',
581 exit_confidence_threshold=50,
582 smart_trailing_stop_pct=0.05,
583 long_score_95_percentile=None,
584 short_score_95_percentile=None,
585 benchmark_rank=0.95,
586 smart_exit_atr_period=14,
587 smart_exit_atr_multiplier=3.0,
588 intelligent_tsl_pct=1.0,
589 analysis_start_date=None,
590 analysis_end_date=None,
591 benchmark_lookback_years=None,
592 use_rolling_benchmark=False):
593
594 df = data.copy()
595 required_cols = ['Close']
596 if 'High' not in df.columns: df['High'] = df['Close']
597 if 'Low' not in df.columns: df['Low'] = df['Close']
598 required_cols.extend(['High', 'Low'])
599 if 'Volume' in df.columns: required_cols.append('Volume')
600
601 df['Close'] = pd.to_numeric(df['Close'], errors='coerce').replace(0, np.nan)
602 df.dropna(subset=['Close'], inplace=True)
603
604 min_ma_period = params.get('large_ma_period', 50)
605
606 min_lookback_needed = min_ma_period
607 if (primary_driver == 'Markov State' or use_markov) and markov_setup is not None:
608 min_lookback_needed = max(min_ma_period, markov_setup.get('Run-Up Period', 10))
609
610 if len(df) < min_lookback_needed or len(df) < params.get('bband_period', 20) or len(df) < 30:
611 return 0, 0, 0.0, 0.0, None, ([], [], [], []), [], (0, 0, 0, 0), ([], []), (None, None, None, None), (0, 0, 0, 0, 0, 0)
612
613 # --- Indicator Calculations ---
614 df['large_ma'] = df['Close'].rolling(window=min_ma_period).mean().ffill()
615 df['ma_slope'] = df['large_ma'].diff(periods=3).ffill()
616
617 bband_period = params.get('bband_period', 20)
618 if len(df) >= bband_period:
619 try:
620 indicator_bb = BollingerBands(close=df['Close'], window=bband_period, window_dev=params['bband_std_dev'])
621 df['bband_lower'] = indicator_bb.bollinger_lband()
622 df['bband_upper'] = indicator_bb.bollinger_hband()
623 except Exception: df['bband_lower'], df['bband_upper'] = np.nan, np.nan
624 else: df['bband_lower'], df['bband_upper'] = np.nan, np.nan
625
626 if len(df) >= 14:
627 try:
628 indicator_rsi = RSIIndicator(close=df['Close'], window=14)
629 df['RSI'] = indicator_rsi.rsi()
630 except Exception: df['RSI'] = np.nan
631
632 df['Volatility_p'] = df['Close'].pct_change().rolling(window=14).std()
633
634 try:
635 indicator_adx = ADXIndicator(high=df['High'], low=df['Low'], close=df['Close'], window=adx_period, fillna=True)
636 df['ADX'] = indicator_adx.adx().ffill()
637 except Exception: df['ADX'] = np.nan
638
639 try:
640 atr_window = int(smart_exit_atr_period)
641 indicator_atr = AverageTrueRange(high=df['High'], low=df['Low'], close=df['Close'], window=atr_window, fillna=True)
642 df['ATR'] = indicator_atr.average_true_range()
643 except Exception: df['ATR'] = np.nan
644 else:
645 df['RSI'], df['Volatility_p'], df['ADX'], df['ATR'] = np.nan, np.nan, np.nan, np.nan
646
647# --- [NEW] MFI & SUPERTREND CALCULATIONS (Fixed) ---
648 if len(df) >= 14:
649 # 1. MFI Calculation
650 try:
651 mfi_ind = MFIIndicator(high=df['High'], low=df['Low'], close=df['Close'], volume=df['Volume'], window=14, fillna=True)
652 df['MFI'] = mfi_ind.money_flow_index()
653 except Exception: df['MFI'] = 50.0
654
655 # 2. Proper Recursive SuperTrend Calculation
656 try:
657 st_period = 10
658 st_multiplier = 3.0
659
660 # Calculate ATR
661 high_low = df['High'] - df['Low']
662 high_close = np.abs(df['High'] - df['Close'].shift())
663 low_close = np.abs(df['Low'] - df['Close'].shift())
664 ranges = pd.concat([high_low, high_close, low_close], axis=1)
665 true_range = np.max(ranges, axis=1)
666 atr = true_range.rolling(st_period).mean().fillna(0)
667
668 # Basic Bands
669 hl2 = (df['High'] + df['Low']) / 2
670 basic_upper = hl2 + (st_multiplier * atr)
671 basic_lower = hl2 - (st_multiplier * atr)
672
673 # Initialize Final Bands
674 final_upper = basic_upper.copy()
675 final_lower = basic_lower.copy()
676 trend = np.zeros(len(df), dtype=int)
677 supertrend = np.zeros(len(df))
678
679 # Recursive Loop (Accurate Logic)
680 close = df['Close'].values
681 bu = basic_upper.values
682 bl = basic_lower.values
683 fu = final_upper.values
684 fl = final_lower.values
685
686 # 1 = Uptrend, -1 = Downtrend
687 curr_trend = 1
688
689 for i in range(1, len(df)):
690 # Calculate Final Upper Band
691 if bu[i] < fu[i-1] or close[i-1] > fu[i-1]:
692 fu[i] = bu[i]
693 else:
694 fu[i] = fu[i-1]
695
696 # Calculate Final Lower Band
697 if bl[i] > fl[i-1] or close[i-1] < fl[i-1]:
698 fl[i] = bl[i]
699 else:
700 fl[i] = fl[i-1]
701
702 # Determine Trend
703 if curr_trend == 1 and close[i] < fl[i]:
704 curr_trend = -1
705 elif curr_trend == -1 and close[i] > fu[i]:
706 curr_trend = 1
707
708 trend[i] = curr_trend
709 supertrend[i] = fl[i] if curr_trend == 1 else fu[i]
710
711 df['SuperTrend'] = supertrend
712
713 except Exception as e:
714 # print(f"ST Error: {e}") # Debug if needed
715 df['SuperTrend'] = np.nan
716 else:
717 df['MFI'], df['SuperTrend'] = 50.0, np.nan
718
719 df['SMA_200'] = df['Close'].rolling(window=200, min_periods=1).mean()
720
721 if 'Volume' in df.columns:
722 df['Volume'] = pd.to_numeric(df['Volume'], errors='coerce').fillna(0)
723 df['Volume_MA50'] = df['Volume'].rolling(window=50, min_periods=1).mean()
724 df['Volume_Ratio'] = df.apply(lambda row: row['Volume'] / row['Volume_MA50'] if row['Volume_MA50'] > 0 else 0, axis=1)
725 df['Volume_Ratio'] = df['Volume_Ratio'].replace([np.inf, -np.inf], 0).fillna(0)
726 else: df['Volume_Ratio'] = 0.0
727
728 if len(df) >= 26:
729 indicator_macd = MACD(close=df['Close'], window_slow=26, window_fast=12, window_sign=9, fillna=True)
730 df['MACD_line'] = indicator_macd.macd().ffill()
731 df['MACD_signal'] = indicator_macd.macd_signal().ffill()
732 df['MACD_hist'] = indicator_macd.macd_diff().ffill()
733 else: df['MACD_line'], df['MACD_signal'], df['MACD_hist'] = np.nan, np.nan, np.nan
734
735 if (primary_driver == 'Markov State' or use_markov) and markov_setup is not None:
736 run_up_period = markov_setup.get('Run-Up Period', 10)
737 df['RunUp_Return'] = df['Close'].pct_change(periods=run_up_period)
738 df['RunUp_State'] = df['RunUp_Return'].apply(lambda x: 'Up' if x > 0 else 'Down')
739
740 bband_params_for_score = {
741 'long_entry_threshold_pct': params.get('long_entry_threshold_pct', 0.0),
742 'short_entry_threshold_pct': params.get('short_entry_threshold_pct', 0.0)
743 }
744
745 raw_long_score, raw_short_score = calculate_confidence_score(df,
746 primary_driver,
747 use_rsi, use_volatility, use_trend, use_volume, use_macd, use_ma_slope, use_markov,
748 use_mfi, use_supertrend, # <--- NEW
749 rsi_w, vol_w, trend_w, vol_w_val, macd_w, ma_slope_w, markov_w,
750 mfi_w, supertrend_w, # <--- NEW
751 bband_params_for_score,
752 best_markov_setup=markov_setup
753 )
754
755 # 1. GLOBAL (STATIC) MODE - "The Crystal Ball"
756 # Used for "Elite Filtering" of open trades.
757 if not use_rolling_benchmark:
758 if long_score_95_percentile is None:
759 long_scores_gt_zero = raw_long_score[raw_long_score > 0]
760 val = long_scores_gt_zero.quantile(benchmark_rank) if not long_scores_gt_zero.empty else 1.0
761 long_95 = pd.Series(val, index=df.index)
762 else:
763 long_95 = pd.Series(long_score_95_percentile, index=df.index)
764
765 if short_score_95_percentile is None:
766 short_scores_gt_zero = raw_short_score[raw_short_score > 0]
767 val = short_scores_gt_zero.quantile(benchmark_rank) if not short_scores_gt_zero.empty else 1.0
768 short_95 = pd.Series(val, index=df.index)
769 else:
770 short_95 = pd.Series(short_score_95_percentile, index=df.index)
771
772 # 2. ADAPTIVE (ROLLING) MODE - "Real World"
773 # Used for realistic historical simulation.
774 else:
775 if long_score_95_percentile is None:
776 # Default to 1 year if lookback is missing
777 years = benchmark_lookback_years if (benchmark_lookback_years is not None and benchmark_lookback_years > 0) else 1
778 window_days = int(years * 252)
779 # Calculate Rolling Percentile
780 long_95 = raw_long_score.rolling(window=window_days, min_periods=50).quantile(benchmark_rank).fillna(method='bfill')
781 else:
782 long_95 = pd.Series(long_score_95_percentile, index=df.index)
783
784 if short_score_95_percentile is None:
785 years = benchmark_lookback_years if (benchmark_lookback_years is not None and benchmark_lookback_years > 0) else 1
786 window_days = int(years * 252)
787 short_95 = raw_short_score.rolling(window=window_days, min_periods=50).quantile(benchmark_rank).fillna(method='bfill')
788 else:
789 short_95 = pd.Series(short_score_95_percentile, index=df.index)
790
791 # Safety: Ensure we don't divide by zero
792 long_95 = long_95.replace(0, 1.0)
793 short_95 = short_95.replace(0, 1.0)
794
795 # Final Calculation (Removing 'if' check to avoid Series Ambiguity Error)
796 df['long_confidence_score'] = (raw_long_score / long_95 * 100).clip(0, 100).fillna(0.0)
797 df['short_confidence_score'] = (raw_short_score / short_95 * 100).clip(0, 100).fillna(0.0)
798
799 apply_veto = bool(veto_setups_list)
800 if apply_veto:
801 df['any_long_veto_trigger'] = False; df['any_short_veto_trigger'] = False
802 for veto_setup in veto_setups_list:
803 veto_threshold = veto_setup.get('Conf. Threshold', 0)
804 veto_rsi_on = veto_setup.get('RSI') == 'On'; veto_vol_on = veto_setup.get('Volatility') == 'On'
805 veto_trend_on = veto_setup.get('TREND') == 'On'; veto_volume_on = veto_setup.get('Volume') == 'On'
806 veto_macd_on = veto_setup.get('MACD', 'Off') == 'On'; veto_ma_slope_on = veto_setup.get('MA Slope', 'Off') == 'On'
807 veto_markov_on = False
808
809 veto_long_raw, veto_short_raw = calculate_confidence_score(df,
810 'Bollinger Bands',
811 veto_rsi_on, veto_vol_on, veto_trend_on, veto_volume_on, veto_macd_on, veto_ma_slope_on, veto_markov_on,
812 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
813 bband_params_for_score,
814 best_markov_setup=markov_setup
815 )
816 veto_long_norm = (veto_long_raw / 3.0) * 100
817
818 current_long_veto_trigger = (veto_long_norm >= veto_threshold); current_short_veto_trigger = (veto_long_norm >= veto_threshold)
819 df['any_long_veto_trigger'] |= current_long_veto_trigger; df['any_short_veto_trigger'] |= current_long_veto_trigger
820
821 # --- Entry Trigger Logic ---
822 bb_long_signal = df['Close'] < (df['bband_lower'] * (1 - params['long_entry_threshold_pct']))
823 bb_short_signal = df['Close'] > (df['bband_upper'] * (1 + params['short_entry_threshold_pct']))
824
825 if rsi_logic == "Crossover":
826 rsi_long_signal = (df['RSI'].shift(1) < 30) & (df['RSI'] >= 30) & df['RSI'].notna()
827 rsi_short_signal = (df['RSI'].shift(1) > 70) & (df['RSI'] <= 70) & df['RSI'].notna()
828 else: # Level
829 rsi_long_signal = (df['RSI'] <= 30) & df['RSI'].notna()
830 rsi_short_signal = (df['RSI'] >= 70) & df['RSI'].notna()
831
832 macd_long_signal = (df['MACD_line'].shift(1) < df['MACD_signal'].shift(1)) & (df['MACD_line'] >= df['MACD_signal'])
833 macd_short_signal = (df['MACD_line'].shift(1) > df['MACD_signal'].shift(1)) & (df['MACD_line'] <= df['MACD_signal'])
834 ma_slope_long_signal = (df['ma_slope'].shift(1) <= 0) & (df['ma_slope'] > 0)
835 ma_slope_short_signal = (df['ma_slope'].shift(1) >= 0) & (df['ma_slope'] < 0)
836
837 if primary_driver == 'RSI Crossover':
838 base_long_trigger = rsi_long_signal; base_short_trigger = rsi_short_signal
839 elif primary_driver == 'MACD Crossover':
840 base_long_trigger = macd_long_signal; base_short_trigger = macd_short_signal
841 elif primary_driver == 'MA Slope':
842 base_long_trigger = ma_slope_long_signal; base_short_trigger = ma_slope_short_signal
843 elif primary_driver == 'Markov State' and markov_setup is not None:
844 strategy = markov_setup.get('Strategy')
845 if strategy == 'Down -> Up':
846 base_long_trigger = (df['RunUp_State'] == 'Down'); base_short_trigger = pd.Series(False, index=df.index)
847 elif strategy == 'Up -> Up':
848 base_long_trigger = (df['RunUp_State'] == 'Up'); base_short_trigger = pd.Series(False, index=df.index)
849 elif strategy == 'Up -> Down':
850 base_long_trigger = pd.Series(False, index=df.index); base_short_trigger = (df['RunUp_State'] == 'Up')
851 elif strategy == 'Down -> Down':
852 base_long_trigger = pd.Series(False, index=df.index); base_short_trigger = (df['RunUp_State'] == 'Down')
853 else:
854 base_long_trigger = pd.Series(False, index=df.index); base_short_trigger = pd.Series(False, index=df.index)
855 else:
856 base_long_trigger = bb_long_signal; base_short_trigger = bb_short_signal
857
858 if use_adx_filter and 'ADX' in df.columns and df['ADX'].notna().any():
859 adx_allows_entry = (df['ADX'] < adx_threshold).fillna(False)
860 else: adx_allows_entry = True
861
862 ma_is_valid = pd.notna(df['large_ma'])
863
864 # [SURGICAL PATCH START] --------------------------------------------------
865 # Check if ALL confidence indicators are disabled
866 # CORRECTED: Added use_mfi and use_supertrend to this list so the score isn't ignored when they are On.
867 all_indicators_off = not any([
868 use_rsi, use_volatility, use_trend, use_volume, use_macd, use_ma_slope, use_markov,
869 use_mfi, use_supertrend
870 ])
871
872 # If indicators are OFF, we bypass the confidence check (allow raw signal)
873 # Otherwise, we enforce the confidence threshold as usual
874 long_entry_trigger = base_long_trigger & adx_allows_entry & ((df['long_confidence_score'] >= params['confidence_threshold']) | all_indicators_off) & ma_is_valid
875 short_entry_trigger = base_short_trigger & adx_allows_entry & ((df['short_confidence_score'] >= params['confidence_threshold']) | all_indicators_off) & ma_is_valid
876 # [SURGICAL PATCH END] ----------------------------------------------------
877
878 # [FIX] Filter entries by BOTH Start and End date
879 if analysis_start_date is not None:
880 start_mask = df.index >= pd.Timestamp(analysis_start_date)
881 long_entry_trigger &= start_mask
882 short_entry_trigger &= start_mask
883
884 if analysis_end_date is not None:
885 end_mask = df.index <= pd.Timestamp(analysis_end_date)
886 long_entry_trigger &= end_mask
887 short_entry_trigger &= end_mask
888
889 if apply_veto:
890 long_entry_trigger &= ~df['any_long_veto_trigger']
891 short_entry_trigger &= ~df['any_short_veto_trigger']
892
893 potential_long_price_exit = (df['Close'] >= (df['large_ma'] * (1 + params['long_exit_ma_threshold_pct']))) | (df['Close'] >= df['bband_upper'])
894 potential_short_price_exit = (df['Close'] <= (df['large_ma'] * (1 - params['short_exit_ma_threshold_pct']))) | (df['Close'] <= df['bband_lower'])
895
896 long_entry_prices = df['Close'].where(long_entry_trigger).ffill()
897 short_entry_prices = df['Close'].where(short_entry_trigger).ffill()
898
899 if exit_logic_type == 'Intelligent (ADX/MACD/ATR)':
900 adx_rising = (df['ADX'] > df['ADX'].shift(1)).fillna(False)
901 adx_strong = (df['ADX'] > adx_threshold)
902 macd_bullish = (df['MACD_line'] > df['MACD_signal'])
903 stay_in_long_trade = (adx_rising | adx_strong) & macd_bullish
904 base_long_exit_trigger = potential_long_price_exit & (~stay_in_long_trade)
905
906 macd_bearish = (df['MACD_line'] < df['MACD_signal'])
907 stay_in_short_trade = (adx_rising | adx_strong) & macd_bearish
908 base_short_exit_trigger = potential_short_price_exit & (~stay_in_short_trade)
909 else:
910 long_is_in_profit = (df['Close'] > long_entry_prices).fillna(False)
911 short_is_in_profit = (df['Close'] < short_entry_prices).fillna(False)
912 base_long_exit_trigger = potential_long_price_exit & long_is_in_profit
913 base_short_exit_trigger = potential_short_price_exit & short_is_in_profit
914
915 df['long_signal'] = np.nan; df.loc[long_entry_trigger, 'long_signal'] = 1; df.loc[base_long_exit_trigger, 'long_signal'] = 0
916 df['short_signal'] = np.nan; df.loc[short_entry_trigger, 'short_signal'] = -1; df.loc[base_short_exit_trigger, 'short_signal'] = 0
917
918 if primary_driver == 'Markov State' and markov_setup is not None:
919 limit_long = markov_setup.get('Future Period', 5); limit_short = markov_setup.get('Future Period', 5)
920 else:
921 limit_long = params.get('max_long_duration', 120); limit_short = params.get('max_short_duration', 120)
922
923 temp_long_pos = df['long_signal'].ffill().fillna(0)
924 temp_short_pos = df['short_signal'].ffill().fillna(0)
925
926 long_groups = (~(temp_long_pos == 1)).cumsum(); df['days_in_long_trade'] = df.groupby(long_groups).cumcount(); df.loc[temp_long_pos == 0, 'days_in_long_trade'] = 0
927 short_groups = (~(temp_short_pos == -1)).cumsum(); df['days_in_short_trade'] = df.groupby(short_groups).cumcount(); df.loc[temp_short_pos == 0, 'days_in_short_trade'] = 0
928
929 long_time_exit_trigger = (df['days_in_long_trade'] > limit_long) & (temp_long_pos == 1)
930 short_time_exit_trigger = (df['days_in_short_trade'] > limit_short) & (temp_short_pos == -1)
931
932 df.loc[long_time_exit_trigger, 'long_signal'] = 0; df.loc[short_time_exit_trigger, 'short_signal'] = 0
933
934 df['long_entry_price_static'] = df['Close'].where(df['long_signal'].shift(1) == 0).ffill().bfill()
935 df['short_entry_price_static'] = df['Close'].where(df['short_signal'].shift(1) == 0).ffill().bfill()
936
937 # [SURGICAL UPDATE START] --- Catcher Offset Logic ---
938 # We retrieve the offset percentage from params (default to 0.0 if missing)
939 catcher_offset = params.get('catcher_stop_pct', 0.0)
940
941 # Calculate the Adjusted Floor/Ceiling
942 # Positive Offset = Higher Floor (Profit for Long, Profit for Short)
943 # Negative Offset = Lower Floor (Loss allowance for Long, Loss allowance for Short)
944 long_breakeven_floor = df['long_entry_price_static'] * (1 + catcher_offset)
945 short_breakeven_floor = df['short_entry_price_static'] * (1 - catcher_offset)
946 # [SURGICAL UPDATE END] -----------------------------
947
948 use_ma_floor_filter = params.get('use_ma_floor_filter', False)
949
950 standard_tsl_pct_long = params.get('long_trailing_stop_loss_pct', 0)
951 standard_tsl_pct_short = params.get('short_trailing_stop_loss_pct', 0)
952
953 intelligent_tsl_pct_long = intelligent_tsl_pct if exit_logic_type == 'Intelligent (ADX/MACD/ATR)' else 0.0
954 intelligent_tsl_pct_short = intelligent_tsl_pct if exit_logic_type == 'Intelligent (ADX/MACD/ATR)' else 0.0
955
956 long_tsl_exit = pd.Series(False, index=df.index); short_tsl_exit = pd.Series(False, index=df.index)
957
958 if primary_driver != 'Markov State':
959 # --- LONG TSL ---
960 has_hit_target = potential_long_price_exit
961 # Determine which TSL percentage to use (Standard vs Intelligent)
962 tsl_to_use_long = np.where(has_hit_target, intelligent_tsl_pct_long, standard_tsl_pct_long) if use_ma_floor_filter else (intelligent_tsl_pct_long if exit_logic_type == 'Intelligent (ADX/MACD/ATR)' else standard_tsl_pct_long)
963 if not isinstance(tsl_to_use_long, pd.Series): tsl_to_use_long = pd.Series(tsl_to_use_long, index=df.index)
964
965 # Only run calculation if TSL is active (> 0)
966 if (tsl_to_use_long > 0).any():
967 in_long_trade = (df['long_signal'].ffill().fillna(0) == 1)
968 long_high_water_mark = df['High'].where(in_long_trade).groupby((~in_long_trade).cumsum()).cummax()
969 tsl_from_hwm = long_high_water_mark * (1 - tsl_to_use_long)
970
971 # [FIX] Apply Catcher floor ONLY if target has been hit (Persistent during trade)
972 if use_ma_floor_filter:
973 trade_groups = (~in_long_trade).cumsum()
974 # [FIX] Cast to float before cummax to avoid Object dtype error
975 target_hit_persistent = has_hit_target.where(in_long_trade).astype(float).groupby(trade_groups).cummax().fillna(0).astype(bool)
976 long_tsl_price = np.where(target_hit_persistent, np.maximum(tsl_from_hwm, long_breakeven_floor), tsl_from_hwm)
977 else:
978 long_tsl_price = tsl_from_hwm
979
980 long_tsl_exit = in_long_trade & (df['Close'] < long_tsl_price)
981 df.loc[long_tsl_exit, 'long_signal'] = 0
982
983 # --- SHORT TSL ---
984 has_hit_target = potential_short_price_exit
985 # Determine which TSL percentage to use (Standard vs Intelligent)
986 tsl_to_use_short = np.where(has_hit_target, intelligent_tsl_pct_short, standard_tsl_pct_short) if use_ma_floor_filter else (intelligent_tsl_pct_short if exit_logic_type == 'Intelligent (ADX/MACD/ATR)' else standard_tsl_pct_short)
987 if not isinstance(tsl_to_use_short, pd.Series): tsl_to_use_short = pd.Series(tsl_to_use_short, index=df.index)
988
989 # Only run calculation if TSL is active (> 0)
990 if (tsl_to_use_short > 0).any():
991 in_short_trade = (df['short_signal'].ffill().fillna(0) == -1)
992 short_low_water_mark = df['Low'].where(in_short_trade).groupby((~in_short_trade).cumsum()).cummin()
993 tsl_from_lwm = short_low_water_mark * (1 + tsl_to_use_short)
994
995 # [FIX] Apply Catcher floor ONLY if target has been hit (Persistent during trade)
996 if use_ma_floor_filter:
997 trade_groups = (~in_short_trade).cumsum()
998 # [FIX] Cast to float before cummax to avoid Object dtype error
999 target_hit_persistent = has_hit_target.where(in_short_trade).astype(float).groupby(trade_groups).cummax().fillna(0).astype(bool)
1000 short_tsl_price = np.where(target_hit_persistent, np.minimum(tsl_from_lwm, short_breakeven_floor), tsl_from_lwm)
1001 else:
1002 short_tsl_price = tsl_from_lwm
1003
1004 short_tsl_exit = in_short_trade & (df['Close'] > short_tsl_price)
1005 df.loc[short_tsl_exit, 'short_signal'] = 0
1006
1007 df['long_position'] = df['long_signal'].ffill().fillna(0)
1008 df['short_position'] = df['short_signal'].ffill().fillna(0)
1009
1010 if params['long_delay_days'] > 0: df['long_position'] = df['long_position'].shift(params['long_delay_days']).fillna(0)
1011 if params['short_delay_days'] > 0: df['short_position'] = df['short_position'].shift(params['short_delay_days']).fillna(0)
1012
1013 df['daily_return'] = df['Close'].pct_change()
1014 df['long_strategy_return'] = df['long_position'].shift(1) * df['daily_return']
1015 df['short_strategy_return'] = df['short_position'].shift(1) * df['daily_return']
1016 final_long_pnl = (1 + df['long_strategy_return'].fillna(0)).prod(skipna=True) - 1
1017 final_short_pnl = (1 + df['short_strategy_return'].fillna(0)).prod(skipna=True) - 1
1018
1019 long_entries = df[(df['long_position'] == 1) & (df['long_position'].shift(1) == 0)]
1020 long_exits = df[(df['long_position'] == 0) & (df['long_position'].shift(1) == 1)]
1021 short_entries = df[(df['short_position'] == -1) & (df['short_position'].shift(1) == 0)]
1022 short_exits = df[(df['short_position'] == 0) & (df['short_position'].shift(1) == -1)]
1023
1024 if not df.empty:
1025 end_date = df.index.max();
1026 else: end_date = pd.NaT;
1027
1028 # --- CHANGED: COLLECT ALL HISTORY (Removed "30 day" filter) ---
1029 all_historical_trades = []
1030
1031 long_trade_profits, long_durations, first_long_entry_date, last_long_exit_date = [], [], None, None
1032 short_trade_profits, short_durations, first_short_entry_date, last_short_exit_date = [], [], None, None
1033 long_profit_take_count, long_tsl_count, long_time_exit_count = 0, 0, 0
1034 short_profit_take_count, short_tsl_count, short_time_exit_count = 0, 0, 0
1035 df_indices = pd.Series(range(len(df)), index=df.index)
1036
1037 for idx, row in long_entries.iterrows():
1038 if first_long_entry_date is None: first_long_entry_date = idx
1039 future_exits = long_exits[long_exits.index > idx]
1040 if not future_exits.empty:
1041 exit_row = future_exits.iloc[0]; last_long_exit_date = exit_row.name
1042 exit_date = exit_row.name
1043 is_tsl = long_tsl_exit.loc[exit_row.name]; is_time = long_time_exit_trigger.loc[exit_row.name]
1044
1045 if is_tsl: long_tsl_count += 1
1046 elif is_time: long_time_exit_count += 1
1047 else: long_profit_take_count += 1
1048
1049 profit = (exit_row['Close'] / row['Close']) - 1 if pd.notna(exit_row['Close']) and pd.notna(row['Close']) and row['Close'] != 0 else np.nan
1050 long_trade_profits.append(profit)
1051
1052 # APPEND ALL TRADES (No Date Filter)
1053 all_historical_trades.append({'Side': 'Long', 'Date Open': idx, 'Date Closed': exit_date, 'Start Confidence': row.get('long_confidence_score', np.nan), 'Final % P/L': profit, 'Status': 'Closed', 'Exit Reason': 'TSL' if is_tsl else ('Time' if is_time else 'Profit')})
1054
1055 try: long_durations.append(df_indices.loc[exit_row.name] - df_indices.loc[idx])
1056 except KeyError: long_durations.append(np.nan)
1057 avg_long_profit_per_trade = np.nanmean(long_trade_profits) if long_trade_profits else 0.0
1058
1059 for idx, row in short_entries.iterrows():
1060 if first_short_entry_date is None: first_short_entry_date = idx
1061 future_exits = short_exits[short_exits.index > idx]
1062 if not future_exits.empty:
1063 exit_row = future_exits.iloc[0]; last_short_exit_date = exit_row.name
1064 exit_date = exit_row.name
1065 is_tsl = short_tsl_exit.loc[exit_row.name]; is_time = short_time_exit_trigger.loc[exit_row.name]
1066
1067 if is_tsl: short_tsl_count += 1
1068 elif is_time: short_time_exit_count += 1
1069 else: short_profit_take_count += 1
1070
1071 profit = ((exit_row['Close'] / row['Close']) - 1) * -1 if pd.notna(exit_row['Close']) and pd.notna(row['Close']) and row['Close'] != 0 else np.nan
1072 short_trade_profits.append(profit)
1073
1074 # APPEND ALL TRADES (No Date Filter)
1075 all_historical_trades.append({'Side': 'Short', 'Date Open': idx, 'Date Closed': exit_date, 'Start Confidence': row.get('short_confidence_score', np.nan), 'Final % P/L': profit, 'Status': 'Closed', 'Exit Reason': 'TSL' if is_tsl else ('Time' if is_time else 'Profit')})
1076
1077 try: short_durations.append(df_indices.loc[exit_row.name] - df_indices.loc[idx])
1078 except KeyError: short_durations.append(np.nan)
1079 avg_short_profit_per_trade = np.nanmean(short_trade_profits) if short_trade_profits else 0.0
1080
1081 long_wins = sum(1 for p in long_trade_profits if pd.notna(p) and p > 0); long_losses = sum(1 for p in long_trade_profits if pd.notna(p) and p < 0)
1082 short_wins = sum(1 for p in short_trade_profits if pd.notna(p) and p > 0); short_losses = sum(1 for p in short_trade_profits if pd.notna(p) and p < 0)
1083
1084 long_trades_log = [{'date': idx, 'price': row['Close'], 'confidence': row.get('long_confidence_score', np.nan)} for idx, row in long_entries.iterrows()]
1085 short_trades_log = [{'date': idx, 'price': row['Close'], 'confidence': row.get('short_confidence_score', np.nan)} for idx, row in short_entries.iterrows()]
1086
1087 # --- ROBUST OPEN TRADE CAPTURE ---
1088 open_trades = []
1089 if not df.empty and 'Close' in df.columns:
1090 last_close = df['Close'].iloc[-1]
1091
1092 # 1. Check Long
1093 if df['long_position'].iloc[-1] == 1:
1094 if not long_entries.empty:
1095 last_entry_time = long_entries.index[-1]
1096 last_entry = long_entries.loc[last_entry_time]
1097 entry_price = last_entry['Close']
1098 entry_conf = last_entry.get('long_confidence_score', np.nan)
1099 else:
1100 last_entry_time = pd.NaT
1101 entry_price = df['long_entry_price_static'].iloc[-1]
1102 entry_conf = np.nan
1103
1104 if pd.notna(last_close) and pd.notna(entry_price) and entry_price != 0:
1105 pnl = (last_close / entry_price) - 1
1106 open_trades.append({
1107 'Side': 'Long',
1108 'Date Open': last_entry_time,
1109 'Date Closed': pd.NaT,
1110 'Start Confidence': entry_conf,
1111 'Final % P/L': pnl,
1112 'Status': 'Open',
1113 'Exit Reason': 'N/A (Open)'
1114 })
1115
1116 # 2. Check Short
1117 if df['short_position'].iloc[-1] == -1:
1118 if not short_entries.empty:
1119 last_entry_time = short_entries.index[-1]
1120 last_entry = short_entries.loc[last_entry_time]
1121 entry_price = last_entry['Close']
1122 entry_conf = last_entry.get('short_confidence_score', np.nan)
1123 else:
1124 last_entry_time = pd.NaT
1125 entry_price = df['short_entry_price_static'].iloc[-1]
1126 entry_conf = np.nan
1127
1128 if pd.notna(last_close) and pd.notna(entry_price) and entry_price != 0:
1129 pnl = ((last_close / entry_price) - 1) * -1
1130 open_trades.append({
1131 'Side': 'Short',
1132 'Date Open': last_entry_time,
1133 'Date Closed': pd.NaT,
1134 'Start Confidence': entry_conf,
1135 'Final % P/L': pnl,
1136 'Status': 'Open',
1137 'Exit Reason': 'N/A (Open)'
1138 })
1139 # ---------------------------------
1140
1141 # Combine: [All Historical Closed] + [Current Open]
1142 open_trades.extend(all_historical_trades)
1143 df.sort_index(inplace=True)
1144
1145 trade_dates = (first_long_entry_date, last_long_exit_date, first_short_entry_date, last_short_exit_date)
1146 long_durations = [d for d in long_durations if pd.notna(d)]; short_durations = [d for d in short_durations if pd.notna(d)]
1147 avg_long_profit = float(avg_long_profit_per_trade) if pd.notna(avg_long_profit_per_trade) else 0.0
1148 avg_short_profit = float(avg_short_profit_per_trade) if pd.notna(avg_short_profit_per_trade) else 0.0
1149 final_long_pnl_float = float(final_long_pnl) if pd.notna(final_long_pnl) else 0.0
1150 final_short_pnl_float = float(final_short_pnl) if pd.notna(final_short_pnl) else 0.0
1151 final_trade_logs = (long_trades_log, long_exits.index, short_trades_log, short_exits.index)
1152 exit_breakdown = (long_profit_take_count, long_tsl_count, long_time_exit_count, short_profit_take_count, short_tsl_count, short_time_exit_count)
1153
1154 return final_long_pnl_float, final_short_pnl_float, avg_long_profit, avg_short_profit, df, final_trade_logs, open_trades, (long_wins, long_losses, short_wins, short_losses), (long_durations, short_durations), trade_dates, exit_breakdown
1155
1156# --- 3. Charting and Display Functions ---
1157def generate_long_plot(df, trades, ticker):
1158 fig = go.Figure()
1159 # Add Price and MA Lines
1160 fig.add_trace(go.Scatter(x=df.index, y=df['Close'], mode='lines', name='Close Price', line=dict(color='blue')))
1161 if 'large_ma' in df.columns:
1162 fig.add_trace(go.Scatter(x=df.index, y=df['large_ma'], mode='lines', name='Large MA', line=dict(color='orange', dash='dash')))
1163 # Add Bollinger Bands
1164 if 'bband_upper' in df.columns and 'bband_lower' in df.columns:
1165 fig.add_trace(go.Scatter(x=df.index, y=df['bband_upper'], mode='lines', name='Upper Band', line=dict(color='gray', width=0.5)))
1166 fig.add_trace(go.Scatter(x=df.index, y=df['bband_lower'], mode='lines', name='Lower Band', line=dict(color='gray', width=0.5), fill='tonexty', fillcolor='rgba(211,211,211,0.2)'))
1167
1168 # Unpack Trades Tuple: (Long Entries Log, Long Exits Index, Short Entries Log, Short Exits Index)
1169 long_entries_log, long_exits, _, _ = trades
1170
1171 # 1. Plot Long Entries
1172 if long_entries_log:
1173 dates = [t['date'] for t in long_entries_log]
1174 prices = [t['price'] for t in long_entries_log]
1175 scores = [f"Confidence: {t['confidence']:.0f}%" for t in long_entries_log]
1176
1177 # Filter out entries that fall outside the dataframe's date range (just in case)
1178 valid_points = [(d, p, s) for d, p, s in zip(dates, prices, scores) if d in df.index]
1179 if valid_points:
1180 v_dates, v_prices, v_scores = zip(*valid_points)
1181 fig.add_trace(go.Scatter(x=v_dates, y=v_prices, mode='markers', name='Long Entry',
1182 marker=dict(color='green', symbol='triangle-up', size=12),
1183 text=v_scores, hoverinfo='text'))
1184
1185 # 2. Plot Long Exits
1186 # Ensure long_exits is not empty and contains valid dates found in df
1187 if not long_exits.empty:
1188 valid_exits = [date for date in long_exits if date in df.index]
1189 if valid_exits:
1190 exit_prices = df.loc[valid_exits, 'Close']
1191 fig.add_trace(go.Scatter(x=exit_prices.index, y=exit_prices, mode='markers', name='Long Exit',
1192 marker=dict(color='darkgreen', symbol='x', size=8)))
1193
1194 fig.update_layout(title=f'Long Trades for {ticker}', xaxis_title='Date', yaxis_title='Price', legend_title="Indicator")
1195 return fig
1196
1197def generate_short_plot(df, trades, ticker):
1198 fig = go.Figure()
1199 # Add Price and MA Lines
1200 fig.add_trace(go.Scatter(x=df.index, y=df['Close'], mode='lines', name='Close Price', line=dict(color='blue')))