glaria/app-project-huggingface
0
1import pandas as pd2import numpy as np3import math4import base645import scipy.special as scsp6from sklearn.tree import _tree7from matplotlib import pyplot as plt8 9 10significance_treshold = 0.0511 12 13def z2p(z):14 """From z-score return p-value."""15 return 2*(1- (0.5 * (1 + scsp.erf(abs(z) / math.sqrt(2)))))16 17def zscore(p1, p2, n1, n2): # p1, p2 proportions18 """ Obtain zscore of 2 proportions sample """19 p = (p1*float(n1) + p2*float(n2))/(float(n1) + float(n2))20 numerator = p1 - p221 denominator = math.sqrt(p*(1-p)*((1/n1)+(1/n2)))22 return numerator/denominator23 24def kadane_algorithm(input_list):25 max_current = max_global = input_list[0]26 start = end = 027 for i in range(1, len(input_list)):28 if input_list[i] > max_current + input_list[i]:29 max_current = input_list[i]30 start = i31 else:32 max_current += input_list[i]33 if max_current > max_global:34 max_global = max_current35 end = i36 return max_global, start, end37 38def kadane_algorithm_mod(input_list):39 curr_sum = max_total = input_list[0]40 start = end = 041 for i in range(1, len(input_list)):42 curr_sum += input_list[i]43 if curr_sum > max_total:44 max_total = curr_sum45 end = i46 if curr_sum < 0:47 curr_sum = 048 curr_sum = 049 for i in range(end, -1, -1): #second iteration for identifying the start of the interval of max sum50 curr_sum += input_list[i]51 if curr_sum == max_total:52 start = i53 break54 return max_total, start, end55 56def get_negative_array(values):57 transformed_list = []58 for value in values:59 if value > 0.5:60 transformed_list.append(-1)61 elif value < -0.5:62 transformed_list.append(1)63 else:64 transformed_list.append(value)65 return transformed_list66 67 68####***Functions used during dataload***####69def infer_datatypes_and_metatypes(dataset: pd.DataFrame) -> pd.DataFrame:70 info_data = {'COLUMN': [], 'DATATYPE': [], 'METATYPE': []}71 72 for column in dataset.columns:73 dtype = dataset[column].dtype74 75 if column == 'CUSTOMERNUMBER':76 info_data['COLUMN'].append(column)77 info_data['DATATYPE'].append('NUMERIC')78 info_data['METATYPE'].append('PK')79 elif column == 'TGCG':80 info_data['COLUMN'].append(column)81 info_data['DATATYPE'].append('STRING')82 info_data['METATYPE'].append('TGCG')83 else:84 info_data['COLUMN'].append(column)85 86 if dtype == 'bool':87 info_data['DATATYPE'].append('BOOL')88 info_data['METATYPE'].append('KPI')89 elif dtype == 'object':90 info_data['DATATYPE'].append('STRING')91 info_data['METATYPE'].append('SF')92 elif dataset[column].nunique() < 10:93 info_data['DATATYPE'].append('NUM_ST')94 info_data['METATYPE'].append('SF')95 else:96 info_data['DATATYPE'].append('NUMERIC')97 info_data['METATYPE'].append('SF')98 99 inferred_info_dataset = pd.DataFrame(info_data)100 return inferred_info_dataset101 102def validate_datatypes_and_metatypes(dataset: pd.DataFrame, info_dataset: pd.DataFrame) -> bool:103 datatype_values = ['BOOL', 'STRING', 'NUM_ST', 'NUMERIC']104 metatype_values = ['TGCG', 'PK', 'KPI', 'SF']105 106 for index, row in info_dataset.iterrows():107 column = row['COLUMN']108 datatype = row['DATATYPE']109 metatype = row['METATYPE']110 111 if datatype not in datatype_values or metatype not in metatype_values:112 return False113 114 if metatype == 'TGCG' and not dataset[column].apply(lambda x: x.lower() in ['target', 'control'] if pd.notnull(x) else True).all():115 return False116 117 if datatype == 'BOOL' and not dataset[column].apply(lambda x: isinstance(x, bool) if pd.notnull(x) else True).all():118 return False119 120 if datatype == 'STRING' and not dataset[column].apply(lambda x: isinstance(x, str) if pd.notnull(x) else True).all():121 return False122 123 if datatype == 'NUM_ST' and not (dataset[column].nunique() < 10 and dataset[column].apply(lambda x: isinstance(x, (int, float)) if pd.notnull(x) else True).all()):124 return False125 126 if datatype == 'NUMERIC' and not dataset[column].apply(lambda x: isinstance(x, (int, float)) if pd.notnull(x) else True).all():127 return False128 129 if metatype == 'KPI' and not dataset[column].apply(lambda x: isinstance(x, (int, float)) if pd.notnull(x) else True).all():130 return False131 132 return True133###*** ***###134 135def format_float(value):136 if isinstance(value, float):137 return "{:.2f}".format(value)138 return value139 140 141def calculate_metrics2(subset, kpi, tgcg_column):142 """Calculates metrics for a specific KPI."""143 metrics = []144 tg_acceptors = subset.loc[subset[tgcg_column] == 'target', kpi].sum()145 tg_total = len(subset.loc[subset[tgcg_column] == 'target'])146 tg_acceptance = round((tg_acceptors / tg_total)*100,2) if tg_total != 0 else 0147 148 cg_acceptors = subset.loc[subset[tgcg_column] == 'control', kpi].sum()149 cg_total = len(subset.loc[subset[tgcg_column] == 'control'])150 cg_acceptance = round((cg_acceptors / cg_total) * 100, 2) if cg_total != 0 else 0151 152 uplift = tg_acceptance - cg_acceptance153 p_value = z2p(zscore(float(tg_acceptors)/float(tg_total), float(cg_acceptors)/float(cg_total),float(tg_total), float(cg_total))) if tg_total != 0 and cg_total != 0 else None154 155 metrics.append([kpi, "{:.2f}".format(tg_acceptors), "{:.2f}".format(tg_acceptance), "{:.2f}".format(cg_acceptors), "{:.2f}".format(cg_acceptance), "{:.2f}".format(uplift), p_value])156 result_df = pd.DataFrame(metrics, columns=["KPI", "TG Acceptors", "TG Acceptance (%)", "CG Acceptors", "CG Acceptance (%)", "Uplift (%)", "P-value"])157 result_df['P-value'] = pd.to_numeric(result_df['P-value'], errors='coerce')158 return result_df159 160 161def highlight_pvalue(row):162 """Highlights rows with P-value <= significance_treshold."""163 if float(row["P-value"]) <= significance_treshold and float(row["Uplift (%)"]) >= 0:164 return ["background-color: #CCFFCC"] * len(row)165 elif float(row["P-value"]) <= significance_treshold and float(row["Uplift (%)"]) < 0:166 return ["background-color: #FFEAEA"] * len(row)167 else:168 return [""] * len(row)169 170def calculate_metrics(df, kpi_columns, tgcg_column):171 """Calculates metrics for a list of KPIs."""172 metrics = []173 for kpi in kpi_columns:174 tg = df[df[tgcg_column] == 'target']175 cg = df[df[tgcg_column] == 'control']176 177 tg_acceptors = tg[kpi].sum()178 tg_total = len(tg)179 tg_acceptance = round((tg_acceptors / tg_total)*100,2) if tg_total != 0 else 0180 181 cg_acceptors = cg[kpi].sum()182 cg_total = len(cg)183 cg_acceptance = round((cg_acceptors / cg_total) * 100, 2) if cg_total != 0 else 0184 185 uplift = tg_acceptance - cg_acceptance186 p_value = z2p(zscore(tg_acceptors/tg_total, cg_acceptors/cg_total, tg_total, cg_total)) if tg_total != 0 and cg_total != 0 else None187 188 metrics.append([kpi, "{:.2f}".format(tg_acceptors), "{:.2f}".format(tg_acceptance), "{:.2f}".format(cg_acceptors), "{:.2f}".format(cg_acceptance), "{:.2f}".format(uplift), p_value])189 190 result_df = pd.DataFrame(metrics, columns=["KPI", "TG Acceptors", "TG Acceptance (%)", "CG Acceptors", "CG Acceptance (%)", "Uplift (%)", "P-value"])191 result_df['P-value'] = pd.to_numeric(result_df['P-value'], errors='coerce')192 193 return result_df194 195def filter_and_display(df, pvalue_threshold, seg_column, unique_value):196 """Filters and displays the results."""197 df = df[df['P-value'] <= pvalue_threshold]198 if not df.empty:199 print(f"Segment: {seg_column} = {unique_value}")200 print(df.to_string(index=False)) # Display the dataframe without the index201 202def download_csv_link(df, filename, message="Click here to download this table"):203 csv = df.to_csv(index=False)204 b64 = base64.b64encode(csv.encode()).decode()205 href = f'<a href="data:file/csv;base64,{b64}" download="{filename}">{message}</a>'206 return href207 208###*** Functions exclusive of the Advanced Analytics page ***###209def oversample(df, group_cols):210 # biggest group in terms of size211 max_size = df[group_cols].value_counts().max()212 # empty df that will be filled in213 df_oversampled = pd.DataFrame()214 # group columns per group_cols and oversample each of the groups215 for group, group_df in df.groupby(group_cols):216 oversampled_group = group_df.sample(max_size, replace=True)217 df_oversampled = pd.concat([df_oversampled, oversampled_group], axis=0)218 219 # return oversampled dataframe220 return df_oversampled221 222def get_rules(tree, feature_names, class_names, class_of_interest):223 """ Extract the rules of a decisition tree algorithm """224 tree_ = tree.tree_225 feature_name = [226 feature_names[i] if i != _tree.TREE_UNDEFINED else "undefined!"227 for i in tree_.feature228 ]229 230 paths = []231 path = []232 233 def recurse(node, path, paths):234 235 if tree_.feature[node] != _tree.TREE_UNDEFINED:236 name = feature_name[node]237 threshold = tree_.threshold[node]238 p1, p2 = list(path), list(path)239 240 # Check if the feature is a result of one-hot encoding241 if '==' in name:242 feature, value = name.split('==')243 p1 += [f"({feature} <> {value})"]244 p2 += [f"({feature} = {value})"]245 else:246 p1 += [f"({name} <= {np.round(threshold, 2)})"]247 p2 += [f"({name} > {np.round(threshold, 2)})"]248 249 recurse(tree_.children_left[node], p1, paths)250 recurse(tree_.children_right[node], p2, paths)251 else:252 path += [(tree_.value[node], tree_.n_node_samples[node])]253 paths += [path]254 255 recurse(0, path, paths)256 257 # sort by samples count258 paths = sorted(paths, key=lambda x: x[-1][1], reverse=True)259 # generate rules260 rules = []261 for path in paths:262 rule = ""263 264 for p in path[:-1]:265 if rule != "":266 rule += " and \n"267 rule += str(p)268 269 if class_names[np.argmax(path[-1][0][0])] == class_of_interest:270 rule += f"\n\n**(samples: {path[-1][1]})**"271 rules.append(rule)272 273 return rules274 275def qini_curve(y_true, uplift_score):276 # Sorting data by the uplift score277 data = pd.DataFrame({'y_true': y_true, 'uplift_score': uplift_score}).sort_values('uplift_score', ascending=False)278 data.reset_index(drop=True, inplace=True)279 280 data['target_cumsum'] = data.y_true.cumsum()281 data['all_cumnum'] = range(1, len(data) + 1)282 283 # Calculating the cumulative uplift as proportion284 data['uplift_cum'] = data['target_cumsum'] / data['all_cumnum'] - data.iloc[0]['target_cumsum'] / len(data)285 data['proportion_targeted'] = data['all_cumnum'] / len(data) # new line to calculate proportion targeted286 287 # Calculating the baseline (random model)288 random_model = data['target_cumsum'].iloc[-1] / len(data) * data['proportion_targeted']289 290 # Creating a figure and an axis291 fig, ax = plt.subplots()292 293 # Drawing the Qini curve with proportion targeted on x-axis294 ax.plot(data['proportion_targeted'], data['uplift_cum'], label='Model')295 296 # Drawing the baseline with proportion targeted297 ax.plot(data['proportion_targeted'], random_model, label='Random')298 299 # Labels and legend300 ax.set_xlabel('Proportion targeted')301 ax.set_ylabel('Cumulative Uplift')302 ax.legend()303 304 # Calculating the Qini area305 qini_area = (data['uplift_cum'] - random_model).sum() / len(data)306 307 return fig, ax, qini_area308 309 310 311