Figure31/pattern-finder
0
1"""2Feature extraction module for price pattern analysis.3Extracts statistical and technical features from price data for4dimensionality reduction and pattern family identification.5"""6 7import numpy as np8import pandas as pd9from typing import Dict, List, Tuple, Optional, Union10from sklearn.decomposition import PCA11from sklearn.preprocessing import StandardScaler12 13class PriceFeatureExtractor:14 """Extracts features from OHLC price data for pattern analysis."""15 16 def __init__(self, window_size: int = 30, n_components: int = 2):17 """18 Initialize the feature extractor.19 20 Args:21 window_size: Size of the sliding window for feature extraction22 n_components: Number of dimensions to reduce to23 """24 self.window_size = window_size25 self.method = 'pca' # Only PCA is supported26 self.n_components = n_components27 self.scaler = StandardScaler()28 29 def extract_features(self, ohlc_df: pd.DataFrame) -> pd.DataFrame:30 """31 Extract features from OHLC price data.32 33 Args:34 ohlc_df: DataFrame with 'open', 'high', 'low', 'close' columns35 36 Returns:37 DataFrame of extracted features38 """39 # Create a copy to avoid modifying the original40 price_data = ohlc_df[['open', 'high', 'low', 'close']].copy()41 42 # Calculate returns and price relationships43 price_data['returns'] = np.log(price_data['close'] / price_data['close'].shift(1))44 price_data['high_low_range'] = price_data['high'] - price_data['low']45 price_data['body_size'] = abs(price_data['close'] - price_data['open'])46 price_data['upper_shadow'] = price_data.apply(47 lambda x: x['high'] - max(x['open'], x['close']), axis=148 )49 price_data['lower_shadow'] = price_data.apply(50 lambda x: min(x['open'], x['close']) - x['low'], axis=151 )52 53 # Drop NaN values from returns calculation54 price_data = price_data.dropna()55 56 # Calculate statistical features57 features = {}58 59 try:60 # Price action features - with added error checking61 features['trend'] = (price_data['close'].iloc[-1] / price_data['close'].iloc[0]) - 1 if len(price_data) > 0 else 062 features['volatility'] = price_data['returns'].std() * np.sqrt(len(price_data)) if len(price_data) > 0 else 063 64 # Calculate max drawdown65 if len(price_data) > 0:66 rolling_max = price_data['close'].cummax()67 drawdown = (price_data['close'] / rolling_max - 1)68 features['max_drawdown'] = drawdown.min()69 else:70 features['max_drawdown'] = 071 72 # Momentum indicators - with safe shift operations73 if len(price_data) > 1:74 price_data['momentum_1'] = price_data['close'] / price_data['close'].shift(1) - 175 else:76 price_data['momentum_1'] = pd.Series([0] * len(price_data))77 78 if len(price_data) > 3:79 price_data['momentum_3'] = price_data['close'] / price_data['close'].shift(3) - 180 else:81 price_data['momentum_3'] = pd.Series([0] * len(price_data))82 83 if len(price_data) > 5:84 price_data['momentum_5'] = price_data['close'] / price_data['close'].shift(5) - 185 else:86 price_data['momentum_5'] = pd.Series([0] * len(price_data))87 88 # Price velocity and acceleration89 price_data['velocity'] = price_data['close'].diff().fillna(0)90 price_data['acceleration'] = price_data['velocity'].diff().fillna(0)91 92 # Candlestick features - with division by zero protection93 hlr_mean = price_data['high_low_range'].mean()94 if hlr_mean > 0:95 features['avg_body_ratio'] = price_data['body_size'].mean() / hlr_mean96 features['upper_shadow_ratio'] = price_data['upper_shadow'].mean() / hlr_mean97 features['lower_shadow_ratio'] = price_data['lower_shadow'].mean() / hlr_mean98 else:99 features['avg_body_ratio'] = 0100 features['upper_shadow_ratio'] = 0101 features['lower_shadow_ratio'] = 0102 103 # Statistical features for returns - safely handle empty dataframes104 for col in ['returns', 'high_low_range', 'body_size', 'momentum_1', 'velocity', 'acceleration']:105 features[f'{col}_mean'] = price_data[col].mean() if not price_data[col].empty else 0106 features[f'{col}_std'] = price_data[col].std() if not price_data[col].empty else 0107 # For performance, only use mean and std - skip skewness and kurtosis108 # These are the most important features and less likely to cause numerical errors109 110 # Pattern complexity - count direction changes (safely)111 for col in ['close', 'velocity']:112 try:113 sign_changes = ((np.sign(price_data[col].diff().fillna(0)) != 114 np.sign(price_data[col].diff().shift(1).fillna(0))) & 115 (np.sign(price_data[col].diff().fillna(0)) != 0)).sum()116 features[f'{col}_sign_changes'] = sign_changes / len(price_data) if len(price_data) > 0 else 0117 except:118 features[f'{col}_sign_changes'] = 0119 120 # Autocorrelation features (safely)121 for lag in [1, 2, 3, 5]:122 try:123 if len(price_data) > lag + 1:124 features[f'autocorr_{lag}'] = price_data['returns'].autocorr(lag)125 else:126 features[f'autocorr_{lag}'] = 0127 except:128 features[f'autocorr_{lag}'] = 0129 130 except Exception as e:131 print(f"Error calculating features: {str(e)}")132 # Provide basic fallback features if calculation fails133 features = {134 'trend': 0, 135 'volatility': 0,136 'basic_mean': np.mean(price_data['close']) if len(price_data) > 0 else 0,137 'basic_std': np.std(price_data['close']) if len(price_data) > 0 else 0138 }139 140 # Return as a DataFrame with one row141 return pd.DataFrame([features])142 143 def extract_features_from_windows(self, 144 ohlc_df: pd.DataFrame, 145 window_size: Optional[int] = None, 146 stride: int = 1) -> List[pd.DataFrame]:147 """148 Extract features from sliding windows in price data.149 150 Args:151 ohlc_df: DataFrame with 'open', 'high', 'low', 'close' columns152 window_size: Size of sliding window (default: self.window_size)153 stride: Steps between windows154 155 Returns:156 List of feature DataFrames for each window157 """158 if window_size is None:159 window_size = self.window_size160 161 windows = []162 for i in range(0, len(ohlc_df) - window_size + 1, stride):163 window = ohlc_df.iloc[i:i+window_size]164 windows.append(window)165 166 all_features = []167 for window in windows:168 window_features = self.extract_features(window)169 all_features.append(window_features)170 171 return all_features172 173 def reduce_dimensions(self, features_df: pd.DataFrame) -> Dict:174 """175 Apply dimensionality reduction to extracted features.176 177 Args:178 features_df: DataFrame of features to reduce179 180 Returns:181 Dictionary with reduced data and metadata182 """183 # Scale features184 scaled_features = self.scaler.fit_transform(features_df)185 186 # Apply PCA dimensionality reduction method187 print(f"Using PCA dimensionality reduction")188 189 # PCA reduction190 reducer = PCA(n_components=self.n_components)191 reduced_data = reducer.fit_transform(scaled_features)192 explained_variance = reducer.explained_variance_ratio_.sum()193 194 # Calculate feature importance195 components = reducer.components_196 feature_importance = {197 features_df.columns[i]: np.abs(components[:, i]).sum() 198 for i in range(features_df.shape[1])199 }200 201 # Create result DataFrame202 result_df = pd.DataFrame(203 reduced_data, 204 columns=[f'component_{i+1}' for i in range(self.n_components)]205 )206 207 return {208 'reduced_data': result_df,209 'explained_variance': explained_variance,210 'feature_importance': feature_importance211 }212 213 def find_similar_patterns(self, 214 source_ohlc: pd.DataFrame, 215 historical_ohlc: pd.DataFrame, 216 window_size: Optional[int] = None,217 stride: int = 1,218 top_n: int = 10) -> Tuple[List[Tuple[float, int]], Dict]:219 """220 Find patterns similar to source pattern using PCA feature extraction.221 222 Args:223 source_ohlc: Source pattern OHLC data224 historical_ohlc: Historical OHLC data to search225 window_size: Size of the pattern window226 stride: Steps between windows227 top_n: Number of most similar patterns to return228 229 Returns:230 Tuple of (distances, visualization_data)231 - distances: List of (distance, index) tuples sorted by similarity232 - visualization_data: Dict with data for visualizing feature space233 """234 print(f"Starting feature-based pattern search...")235 print(f"Source pattern size: {len(source_ohlc)} candles")236 print(f"Historical data size: {len(historical_ohlc)} candles")237 if window_size is None:238 window_size = self.window_size239 240 # Extract features from source pattern241 source_features = self.extract_features(source_ohlc)242 243 # Extract features from all historical windows244 windows = []245 window_indices = []246 247 for i in range(0, len(historical_ohlc) - window_size + 1, stride):248 window = historical_ohlc.iloc[i:i+window_size]249 windows.append(window)250 window_indices.append(i)251 252 # Extract features from all windows253 print(f"Extracting features from {len(windows)} windows...")254 all_features = []255 for i, window in enumerate(windows):256 if i % 1000 == 0 and i > 0:257 print(f"Processed {i}/{len(windows)} windows...")258 window_features = self.extract_features(window)259 all_features.append(window_features)260 261 if not all_features:262 print("No features extracted, returning empty results")263 return [], {}264 265 print(f"Combining features from {len(all_features)} windows...")266 # Combine all features including source267 combined_features = pd.concat(all_features, ignore_index=True)268 all_combined = pd.concat([source_features, combined_features], ignore_index=True)269 270 print(f"Applying PCA dimensionality reduction to {all_combined.shape[0]} samples with {all_combined.shape[1]} features...")271 # Apply dimensionality reduction272 reduction_result = self.reduce_dimensions(all_combined)273 274 # Get reduced coordinates275 reduced_data = reduction_result['reduced_data']276 source_reduced = reduced_data.iloc[0].values277 historical_reduced = reduced_data.iloc[1:].values278 279 print(f"Calculating distances between source pattern and {len(historical_reduced)} windows...")280 # Calculate Euclidean distances in reduced space - using vectorized operations for speed281 try:282 # Fast vectorized calculation283 diffs = historical_reduced - source_reduced284 distances = [(np.sqrt((diff**2).sum()), idx) for diff, idx in zip(diffs, window_indices)]285 print(f"Distance calculation complete: found {len(distances)} matches")286 except Exception as e:287 print(f"Error during vectorized distance calculation: {str(e)}")288 # Fall back to loop implementation289 print("Falling back to loop implementation...")290 distances = []291 for i, point in enumerate(historical_reduced):292 if i % 1000 == 0 and i > 0:293 print(f"Processed {i}/{len(historical_reduced)} points...")294 dist = np.sqrt(((source_reduced - point) ** 2).sum())295 window_idx = window_indices[i]296 distances.append((dist, window_idx))297 298 print(f"Sorting {len(distances)} distances...")299 # Sort by distance (ascending)300 sorted_distances = sorted(distances, key=lambda x: x[0])301 302 # Return top N matches and visualization data303 vis_data = {304 'reduced_data': reduction_result['reduced_data'],305 'explained_variance': reduction_result['explained_variance'],306 'feature_importance': reduction_result['feature_importance'],307 'window_indices': [0] + window_indices, # Add source index308 'source_index': 0309 }310 311 return sorted_distances[:top_n], vis_data