Trust-logic-Test/temporary-donordriver-batch
0
1import subprocess2import pandas as pd3import matplotlib.pyplot as plt4from matplotlib.ticker import FuncFormatter5import gradio as gr6import tempfile7import logging8from PIL import Image9import os10import io11import numpy as np12from itertools import zip_longest13from io import BytesIO14import re15 16logging.basicConfig(level=logging.INFO)17logger = logging.getLogger(__name__)18 19max_outputs = 1020outputs = []21 22variables_dict = {23 "N1": {24 "Variable Name": "Heartfelt opening",25 "Description": "If I see others suffering, my heart goes out to them and I feel compelled to help.",26 "Match": "my heart goes out to them",27 },28 "N2": {29 "Variable Name": "Seeking appreciation/admiration",30 "Description": "I put a lot of effort and energy into good causes (e.g. Schools, charities, fundraisers). A little appreciation goes a long way.",31 "Match": "little appreciation goes a long way",32 },33 "N3": {34 "Variable Name": "Belonging",35 "Description": "Supporting a cause makes me feel like I belong to a community of like-minded people.",36 "Match": "belong to a community",37 },38 "N4": {39 "Variable Name": "Fulfilling my duty",40 "Description": "As a responsible adult, I feel it is simply my duty to help.",41 "Match": "my duty to help",42 },43 "N5": {44 "Variable Name": "Social caretaking",45 "Description": "I feel a strong sense of social responsibility to care for our society.",46 "Match": "care for our society",47 },48 "N6": {49 "Variable Name": "Selfless dedication",50 "Description": "I dedicate a big part of my life to helping others without expecting anything in return.",51 "Match": "helping others without expecting",52 },53 "N7": {54 "Variable Name": "Sharing",55 "Description": "Through my support for charitable causes, I also share my values with others.",56 "Match": "share my values",57 },58 "N8": {59 "Variable Name": "Giving back",60 "Description": "For me, it's simply about giving back.",61 "Match": "giving back",62 },63 "N9": {64 "Variable Name": "Leave a legacy",65 "Description": "I want to leave something lasting with what I do that can be connected to me.",66 "Match": "leave something lasting",67 },68 "N10": {69 "Variable Name": "Fortune insurance",70 "Description": "I think 'what goes around comes around' if I do good, good things will happen to me.",71 "Match": "good things will happen",72 },73 "N11": {74 "Variable Name": "Nurturing",75 "Description": "I love to take care of others and nurture them.",76 "Match": "nurture them",77 },78 "N12": {79 "Variable Name": "Delegating responsibility",80 "Description": "I prefer to simply provide money and delegate the responsible choice to those who know what they’re doing.",81 "Match": "delegate the responsible",82 },83 "N13": {84 "Variable Name": "Absolution",85 "Description": "I sometimes feel that I have it too good and should also give something up.",86 "Match": "give something up",87 },88 "N14": {89 "Variable Name": "Thrill",90 "Description": "Helping others can be a real thrill; either doing it directly, or the feeling of being involved in what is being done.",91 "Match": "a real thrill",92 },93 "N15": {94 "Variable Name": "Sense of own achievement",95 "Description": "Being able to give, I also get a better sense of what I have achieved.",96 "Match": "what I have achieved",97 },98 "N16": {99 "Variable Name": "Break out/Pleasure",100 "Description": "If doing good can also be fun and inspiring, then it's even more reason and pleasure to give/do good.",101 "Match": "If doing good can also be fun and inspiring",102 },103 "N17": {104 "Variable Name": "Learning",105 "Description": "To me it's important to read up and learn about the organizations and projects that I support.",106 "Match": "read up and learn about",107 },108 "N18": {109 "Variable Name": "Broadening horizons",110 "Description": "In supporting charities, I enjoy that I can also broaden my horizons at the same time.",111 "Match": "broaden my horizons",112 },113 "N19": {114 "Variable Name": "Organizing and achieving",115 "Description": "I love bringing people together to get things done, whether that's organizing functions, collecting money or items and taking them to those that need it.",116 "Match": "organizing functions",117 },118 "N20": {119 "Variable Name": "Supporting progress",120 "Description": "I want to see fundamental change and will support the causes that best work towards that goal.",121 "Match": "support the causes",122 },123 "N21": {124 "Variable Name": "Driving progress",125 "Description": "Real change can only be achieved through structural development on my fronts. I get actively involved with my skills, effort and money to help drive long term progress.",126 "Match": "drive long term progress",127 },128 "N22": {129 "Variable Name": "Shaping an ideal future",130 "Description": "I have a clear vision of how the future should be and how to get there.",131 "Match": "how the future should be",132 },133 "N23": {134 "Variable Name": "Investment",135 "Description": "I see charitable giving as an investment that should have a return in continuous improvement. I research and thoroughly check which charity I invest in.",136 "Match": "investment that should have a return",137 },138 "N24": {139 "Variable Name": "Competing",140 "Description": "A little competition is exciting. Whether that's a fun run or raising more money than others. If it's for a good cause, all the better.",141 "Match": "competition is exciting",142 },143 "N25": {144 "Variable Name": "Winning",145 "Description": "Wars are won by winning battles. I want to know we win battles continuously with this charity.",146 "Match": "win battles",147 },148 "N26": {149 "Variable Name": "Personal reward",150 "Description": "Just to see the smiles and happiness on the faces of those I help is incredible rewarding.",151 "Match": "see the smiles and happiness",152 },153 "N27": {154 "Variable Name": "Creating & growing",155 "Description": "It's rewarding to get my hands dirty and create and grow something over time.",156 "Match": "create and grow",157 },158}159 160 161def plot_model_results(results_df, title, variable_mapping_df):162 """163 Plot model results with specific orders and colors for Needs models.164 Args:165 results_df (DataFrame): DataFrame containing needs variable names and their coefficients.166 title (str): Title of the plot.167 variable_mapping_df (DataFrame): DataFrame containing variable name mappings.168 Returns:169 Image: Image object containing the plot.170 """171 172 # Sort results_df by Coefficient in descending order173 results_df = results_df.sort_values(by="Coefficient", ascending=False)174 175 # Define a single color for all bars176 bar_color = "#FBC10C"177 178 # Create the figure and axis179 fig, ax = plt.subplots(figsize=(10, 8))180 181 # Set the x-axis labels with "%" using FuncFormatter182 formatter = FuncFormatter(lambda x, _: f"{x:.0f}%")183 ax.xaxis.set_major_formatter(formatter)184 185 # Create bars in the bar chart186 for i, (coefficient, need) in enumerate(187 zip(results_df["Coefficient_percent"], results_df["Needs"])188 ):189 # Map need to original variable name using variable_mapping_df190 original_variable_name = variable_mapping_df.loc[191 variable_mapping_df["Variable"] == need, "Variable Name"192 ].values[0]193 194 ax.barh(195 original_variable_name,196 coefficient,197 color=bar_color,198 edgecolor="white",199 height=0.8,200 zorder=2,201 )202 ax.text(203 coefficient + 0.5,204 i,205 f"{coefficient:.2f}%",206 va="center",207 color="#8c8b8c",208 zorder=3,209 )210 211 plt.title(title, fontsize=14)212 213 # Remove plot borders214 ax.spines[["top", "right"]].set_color("none")215 216 # Adjust the linewidth of the left spine217 ax.spines["left"].set_linewidth(0.5)218 ax.spines["left"].set_zorder(4)219 220 # Change the colour of y-axis text221 ax.tick_params(axis="y", colors="#8c8b8c", length=0)222 223 # Tighten the layout224 plt.tight_layout()225 226 # Save the figure to a bytes buffer and then to an image227 img_data = io.BytesIO()228 plt.savefig(img_data, format="png", facecolor=fig.get_facecolor(), edgecolor="none")229 img_data.seek(0)230 img = Image.open(img_data)231 plt.close(fig)232 233 return img234 235 236def plot_mean_values(237 df, title, variable_mapping_df, top_num=None, color_code="#1e3799"238):239 """240 Plot mean values for each column from N1 to N27, calculating mean values within the function.241 Optionally display only the top 'top_num' variables if 'top_num' is not None, use specified bar color,242 and replace column names using a mapping DataFrame.243 Args:244 df (DataFrame): DataFrame containing multiple rows with columns from N1 to N27.245 title (str): Display title of the chart.246 variable_mapping_df (DataFrame): DataFrame containing mappings of current column names to original names.247 top_num (int, optional): Number of top mean values to display, if None, display all.248 color_code (str): Hex code for the color of the bars in the bar chart.249 Returns:250 Image: Image object containing the plot.251 """252 253 # Calculate mean values for each column in the DataFrame254 mean_values = df.mean()255 256 # Sort the Series in descending order257 mean_values_sorted = mean_values.sort_values(ascending=False)258 259 # If top_num is specified, slice the Series to include only the top 'top_num' entries260 if top_num is not None:261 mean_values_sorted = mean_values_sorted.head(top_num)262 263 # Create the figure and axis264 fig, ax = plt.subplots(figsize=(10, 8))265 266 # Set the x-axis labels with "%" using FuncFormatter267 formatter = FuncFormatter(lambda x, _: f"{x:.0f}")268 ax.xaxis.set_major_formatter(formatter)269 270 # Set x-axis limits dynamically based on the data271 ax.set_xlim(272 0, mean_values_sorted.max() * 1.15273 ) # Extend a bit more than the max value for visibility274 275 # Create bars in the bar chart276 for i, (column, value) in enumerate(mean_values_sorted.items()):277 # Map the current column name to the original variable name using variable_mapping_df278 original_variable_name = variable_mapping_df.loc[279 variable_mapping_df["Variable"] == column, "Variable Name"280 ].values[0]281 282 ax.barh(283 original_variable_name,284 value,285 color=color_code, # Use the specified color for the bars286 edgecolor="white",287 height=0.6,288 zorder=2,289 )290 ax.text(291 value + 0.5,292 i,293 f"{value:.2f}",294 va="center",295 color="#8c8b8c",296 zorder=3,297 )298 299 plt.title(title, fontsize=14)300 301 # Remove plot borders302 ax.spines[["top", "right"]].set_color("none")303 304 # Adjust the linewidth of the left spine305 ax.spines["left"].set_linewidth(0.5)306 ax.spines["left"].set_zorder(4)307 308 # Change the colour of y-axis text309 ax.tick_params(axis="y", colors="#8c8b8c", length=0)310 311 # Tighten the layout312 plt.tight_layout()313 314 # Save the figure to a bytes buffer and then to an image315 img_data = BytesIO()316 plt.savefig(img_data, format="png", facecolor=fig.get_facecolor(), edgecolor="none")317 img_data.seek(0)318 img = Image.open(img_data)319 plt.close(fig)320 321 return img322 323 324 325def plot_combined_results(needs_df, hygiene_df, fulfillment_df, variable_mapping_df, plot_title):326 import matplotlib.pyplot as plt327 import numpy as np328 from io import BytesIO329 from PIL import Image330 331 # Map variable names332 name_map = variable_mapping_df.set_index("Variable")["Variable Name"].to_dict()333 needs_df = needs_df[needs_df["Needs"].isin(name_map.keys())]334 needs_df["Variable Name"] = needs_df["Needs"].apply(lambda x: name_map[x])335 336 # Get hygiene and fulfilment means337 hygiene_scores = hygiene_df.mean()338 fulfillment_scores = fulfillment_df.mean()339 340 # Remove fulfilment values with .5 scores (like 2.5, 3.5 etc.)341 fulfillment_scores = fulfillment_scores[~(fulfillment_scores % 1 == 0.5)]342 343 # Create the combined DataFrame344 combined_df = pd.DataFrame({345 "Variable Name": needs_df["Variable Name"],346 "Driver": needs_df["Coefficient_percent"],347 "Hygiene": [hygiene_scores.get(n, 0) for n in needs_df["Needs"]],348 "Fulfilment": [fulfillment_scores.get(n, 0) for n in needs_df["Needs"]],349 }).set_index("Variable Name")350 351 # Drop rows where Fulfilment is NaN (due to filtering)352 combined_df = combined_df.dropna(subset=["Fulfilment"])353 354 # Take top 10 by Driver importance355 combined_df = combined_df.sort_values(by="Driver", ascending=False).head(10)356 357 fig, ax1 = plt.subplots(figsize=(10, 8))358 ax2 = ax1.twiny()359 360 # Compact spacing361 width = 0.35362 spacing = 0.5363 indices = np.arange(len(combined_df)) * (2 * width + spacing)364 365 # Invert Y-axis for proper bar layout366 ax1.invert_yaxis()367 368 # Plot Driver scores (orange-red)369 bars_driver = ax2.barh(370 indices,371 combined_df["Driver"],372 height=width,373 color="#FF4D00", # New driver color374 label="Driver",375 )376 377 # Plot Fulfilment scores (soft purple)378 bars_fulfilment = ax1.barh(379 indices + width + 0.05,380 combined_df["Fulfilment"],381 height=width,382 color="#CAC5E0", # New fulfilment color383 label="Fulfilment",384 )385 386 # Set Y-ticks387 ax1.set(yticks=indices + width / 2 + 0.025, yticklabels=combined_df.index)388 389 # Set X-axis labels and limits390 ax1.set_xlabel("Fulfilment Scores (1-5)")391 ax2.set_xlabel("Driver Scores (%)")392 ax1.set_xlim(1, 5)393 ax2.set_xlim(0, 7)394 ax1.xaxis.set_major_locator(plt.MultipleLocator(1)) # Integer ticks only395 ax2.xaxis.set_major_locator(plt.MultipleLocator(1))396 397 # Custom legend398 patches = [399 plt.Rectangle((0, 0), 1, 1, color="#FF4D00"),400 plt.Rectangle((0, 0), 1, 1, color="#CAC5E0"),401 ]402 ax1.legend(403 patches,404 ["Driver", "Fulfilment"],405 loc="upper right",406 bbox_to_anchor=(1.50, 1),407 fontsize=12,408 markerscale=1.5,409 labelspacing=1.2,410 handlelength=2,411 borderpad=1.5,412 )413 414 # Title and layout415 plt.title(plot_title)416 plt.tight_layout()417 plt.subplots_adjust(right=0.75)418 419 # Save to image buffer420 img_data = BytesIO()421 plt.savefig(img_data, format="png", facecolor=fig.get_facecolor(), edgecolor="none", dpi=300)422 img_data.seek(0)423 img = Image.open(img_data)424 plt.close(fig)425 426 return img427 428 429 430 431 432def call_r_script(input_file, text_output_path, csv_output_path_needs):433 """434 Call the R script for Donor Driver analysis.435 Args:436 input_file (str): Path to the input Excel file.437 text_output_path (str): Path to the output text file.438 csv_output_path_needs (str): Path to the output CSV file for Needs.439 """440 441 command = [442 "Rscript",443 "process_data.R",444 input_file,445 text_output_path,446 csv_output_path_needs,447 ]448 449 try:450 subprocess.run(command, check=True)451 except subprocess.CalledProcessError as e:452 logging.error("R script failed with error: %s", e)453 raise RuntimeError(454 "Error executing R script. Please check the input file format."455 )456 457 458def analyze_excel_single(file_path):459 """460 Analyzes a single Excel file containing data and generates Needs Regression Plot, Hygiene Average Plot, Fulfillment Average Plot, and Combined Plot.461 Args:462 file_path (str): Path to the Excel file.463 Returns:464 Image: Image of the Needs regression plot.465 Image: Image of the Hygiene average plot.466 Image: Image of the Fulfillment average plot.467 Image: Image of the Combined plot.468 str: Summary of the analysis.469 """470 471 # Create a temporary directory472 temp_dir = tempfile.mkdtemp()473 474 # Manually construct file paths475 text_output_path = os.path.join(temp_dir, "output.txt")476 csv_output_path_needs = text_output_path.replace(".txt", "_needs.csv")477 478 # Load the variable mapping sheet479 variable_mapping_df = pd.read_excel(file_path, sheet_name="Variable")480 481 # Step 3: Execute factor analysis and linear regression from R482 call_r_script(file_path, text_output_path, csv_output_path_needs)483 484 # Read the output text file485 with open(text_output_path, "r") as file:486 output_text = file.read()487 488 # Get n_samples from output text489 n_samples = output_text.split("data shape: ")[1]490 n_samples = n_samples.split(" rows,")[0]491 492 # Get coefficient percentages493 results_df_needs = pd.read_csv(csv_output_path_needs)494 results_df_needs["Coefficient_percent"] = results_df_needs["Coefficient"] * 100495 496 # Get dataset name497 dataset_name = file_path.split("/")[-1]498 499 # Generate the Needs regression plot500 img_needs = plot_model_results(501 results_df_needs,502 f"Donor Drivers: {dataset_name}\n(What drives your donors needs?)\nn={n_samples}",503 variable_mapping_df,504 )505 506 # Load the Hygiene sheet507 hygiene_df = pd.read_excel(file_path, sheet_name="Needs and Hygiene")508 509 # Calculate Hygiene average score and plot chart510 img_hygiene = plot_mean_values(511 hygiene_df,512 f"Hygiene: {dataset_name}",513 variable_mapping_df,514 top_num=14,515 color_code="#020101",516 )517 518 # Load the Fulfillment sheet519 fulfillment_df = pd.read_excel(file_path, sheet_name="Fulfilment")520 521 # Calculate Fulfillment average score and plot chart522 img_fulfillment = plot_mean_values(523 fulfillment_df,524 f"Fulfillment: {dataset_name}",525 variable_mapping_df,526 top_num=7,527 color_code="#A8A6A4",528 )529 530 # Generate the combined plot for all three metrics531 img_combined = plot_combined_results(532 results_df_needs,533 hygiene_df,534 fulfillment_df,535 variable_mapping_df,536 f"Combined Normalized Results for Donor Drivers and Fulfilment: {dataset_name}\n",537 )538 539 # Arrange needs driver result according to Needs540 results_df_needs = results_df_needs.sort_values(by="Needs", ascending=True)541 542 # Calculate mean values for each hygiene column543 hygiene_mean_values = hygiene_df.mean()544 545 # Create a DataFrame with two columns: "Column Name" and "Mean Value"546 hygiene_mean_df = pd.DataFrame(547 {"Column Name": hygiene_df.columns, "Mean Value": hygiene_mean_values}548 )549 550 # Calculate mean values for each fulfillment column551 fulfillment_mean_values = fulfillment_df.mean()552 553 # Create a DataFrame with two columns: "Column Name" and "Mean Value"554 fulfillment_mean_df = pd.DataFrame(555 {"Column Name": fulfillment_df.columns, "Mean Value": fulfillment_mean_values}556 )557 558 # Merge results_df_needs with hygiene_mean_df on "Needs" and "Column Name"559 merged_df = pd.merge(560 results_df_needs,561 hygiene_mean_df,562 how="inner",563 left_on="Needs",564 right_on="Column Name",565 )566 567 # Merge the above merged DataFrame with fulfillment_mean_values on "Needs" and "Column Name"568 merged_df_2 = pd.merge(569 merged_df, fulfillment_mean_df, how="inner", on="Column Name"570 )571 572 # Merge the above merged DataFrame with variable_mapping_df on "Column Name" and "Variable"573 final_merged_df = pd.merge(574 merged_df_2,575 variable_mapping_df,576 how="inner",577 left_on="Column Name",578 right_on="Variable",579 )580 581 # Create dataframe for combined results582 combined_data = {583 "DonorDriver": final_merged_df["Variable Name"],584 "Driver for giving (%)": final_merged_df["Coefficient_percent"].round(2),585 "Hygiene needs (1-5)": final_merged_df["Mean Value_x"].round(2),586 "Fulfilment (1-5)": final_merged_df["Mean Value_y"].round(2),587 }588 df_combined = pd.DataFrame(combined_data)589 590 # After processing, ensure to delete the temporary files and directory591 os.remove(text_output_path)592 for file_name in os.listdir(temp_dir):593 file_single = os.path.join(temp_dir, file_name)594 if os.path.isfile(file_single):595 os.remove(file_single)596 os.rmdir(temp_dir)597 598 return (599 img_needs,600 img_hygiene,601 img_fulfillment,602 img_combined,603 df_combined,604 output_text,605 )606 607 608def batch_file_processing(file_paths):609 """610 Analyzes all Excel files in a list of file paths and generates the Needs, Hygiene, Fulfillment and Combined plots per dataset.611 Args:612 file_paths (List[str]): List of paths to the Excel files.613 Returns:614 Image: Image of the Needs plot.615 Image: Image of the Hygiene plot.616 Image: Image of the Fulfillment plot.617 Image: Image of the Combined plot.618 str: Summary of the analysis.619 """620 621 # Process each file622 img_needs_list = []623 img_hygiene_list = []624 img_fulfillment_list = []625 img_combined_list = []626 df_combined_list = []627 output_text_list = []628 629 for file_path in file_paths:630 (631 img_needs,632 img_hygiene,633 img_fulfillment,634 img_combined,635 df_combined,636 output_text,637 ) = analyze_excel_single(file_path)638 img_needs_list.append(img_needs)639 img_hygiene_list.append(img_hygiene)640 img_fulfillment_list.append(img_fulfillment)641 img_combined_list.append(img_combined)642 df_combined_list.append(df_combined)643 output_text_list.append(output_text)644 645 return (646 img_needs_list,647 img_hygiene_list,648 img_fulfillment_list,649 img_combined_list,650 df_combined_list,651 output_text_list,652 )653 654def variable_outputs(file_inputs):655 # Call batch file processing and get analysis results656 (657 img_needs_list,658 img_hygiene_list,659 img_fulfillment_list,660 img_combined_list,661 df_combined_list,662 output_text_list,663 ) = batch_file_processing(file_inputs)664 665 # Number of datasets uploaded666 k = len(file_inputs)667 668 # Container for visible plots669 plots_visible = []670 671 # Use zip_longest to iterate over the lists672 for row, (673 img_needs,674 img_hygiene,675 img_fulfillment,676 img_combined,677 df_combined,678 output_text,679 ) in enumerate(680 zip_longest(681 img_needs_list,682 img_hygiene_list,683 img_fulfillment_list,684 img_combined_list,685 df_combined_list,686 output_text_list,687 )688 ):689 # Protect against row >= len(file_inputs)690 if row >= len(file_inputs):691 break692 693 # Get dataset name694 dataset_name = file_inputs[row].split("/")[-1]695 696 # Fallback if the image is None697 if img_combined is None:698 img_combined = None # Will render as an empty image placeholder699 700 # Fallback if the DataFrame is missing or empty701 if df_combined is None or df_combined.empty:702 df_combined = pd.DataFrame(columns=["No data available"])703 704 # Fallback if text is missing705 706 df_combined = df_combined.drop(columns=["Hygiene needs (1-5)"])707 708 plots = [709 gr.Image(710 value=img_combined,711 type="pil",712 label=" ",713 visible=True,714 ),715 gr.Markdown(716 "<span style='font-size:20px; font-weight:bold;'>2) DonorDrivers® Ranking Graph Table </span> <br></br> This table shows you the 27 DonorDriver® with the actual percentage computed vs their current level of fulfilment (from a scale of 0-5 with 5 as highest).",717 visible=True,718 ),719 #gr.Markdown("This table shows you the 27 DonorDriver® with the actual percentage computed vs their current level of fulfilment (from a scale of 0-5 with 5 as highest)"),720 gr.Dataframe(721 value=df_combined,722 headers=list(df_combined.columns),723 interactive=False,724 label=f"{dataset_name}",725 visible=True,726 height=800,727 ),728 ]729 730 plots_visible.extend(plots)731 732 # Placeholders for unused outputs733 plots_invisible = [734 gr.Image(label=" ", visible=False),735 gr.Markdown(value=" ", visible=False),736 737 gr.Dataframe(label=" ", visible=False),738 ]739 740 # Return combined list of visible + invisible placeholders741 return plots_visible + plots_invisible * (max_outputs - k)742 743def reset_outputs():744 # Reset outputs745 outputs = []746 747 # First set visible748 combined_plot = gr.Image(value=None, label=" ", visible=True)749 combined_text = gr.Markdown(value=" ", visible=True)750 combined_df = gr.Dataframe(value=None, label=" ", visible=True)751 outputs.extend([combined_plot, combined_text, combined_df])752 753 # Remaining sets invisible754 for i in range(1, max_outputs):755 combined_plot = gr.Image(value=None, label=" ", visible=False)756 combined_text = gr.Markdown(value=" ", visible=False)757 combined_df = gr.Dataframe(value=None, label=" ", visible=False)758 outputs.extend([combined_plot, combined_text, combined_df])759 760 return outputs761 762 763 764def data_processing(file_path):765 """766 Processes a single CSV file and generates required outputs.767 Args:768 file_path (str): Path to the CSV file.769 Returns:770 str: Path to the processed Excel file.771 """772 try:773 logger.info("Processing CSV file: %s", file_path)774 775 processed_file_path = None776 777 dataset_df = pd.read_csv(file_path)778 779 # List of columns to be removed780 columns_to_remove = [781 "Respondent ID",782 "Collector ID",783 "Start Date",784 "End Date",785 "IP Address",786 "Email Address",787 "First Name",788 "Last Name",789 "Custom Data 1",790 ]791 792 # Remove the specified columns793 dataset_df = dataset_df.drop(columns=columns_to_remove, errors="ignore")794 795 # Columns to search for796 search_texts = [797 "What are your reasons for giving to charity",798 "Here are the same statements again",799 ]800 801 # Convert column names to lowercase802 dataset_df.columns = [col.lower() for col in dataset_df.columns]803 804 # Convert search texts to lowercase805 search_texts = [text.lower() for text in search_texts]806 807 # Find the indices of columns that contain the search texts808 indices = {}809 for search_text in search_texts:810 for i, col_name in enumerate(dataset_df.columns):811 if search_text in col_name:812 indices[search_text] = i813 814 if len(indices) == len(search_texts):815 index_reasons = indices[search_texts[0]]816 index_statements = indices[search_texts[1]]817 818 # Create df_hygiene: columns between index_reasons and index_statements819 df_hygiene = dataset_df.iloc[:, index_reasons:index_statements]820 821 # Create df_fulfilment: columns from index_statements onwards822 df_fulfilment = dataset_df.iloc[:, index_statements:]823 824 # Remove the first row and set the second row as column names825 df_hygiene.columns = df_hygiene.iloc[0]826 df_hygiene = df_hygiene[1:]827 828 df_fulfilment.columns = df_fulfilment.iloc[0]829 df_fulfilment = df_fulfilment[1:]830 831 # Function to normalize text832 def normalize_text(text):833 text = text.lower().strip()834 text = re.sub(835 r"\s+", " ", text836 ) # Replace multiple spaces with single space837 text = re.sub(r"[^\w\s]", "", text) # Remove punctuation838 return text839 840 # Create a dictionary of normalized matches841 match_keywords = {842 key: normalize_text(val["Match"]) for key, val in variables_dict.items()843 }844 845 # Function to filter columns based on match values846 def filter_columns_by_match(df):847 # Normalize column names848 normalized_columns = {normalize_text(col): col for col in df.columns}849 850 # Create a set of matched columns851 matched_columns = set()852 853 for key, match in match_keywords.items():854 for col_normalized in normalized_columns:855 if match in col_normalized:856 matched_columns.add(normalized_columns[col_normalized])857 858 # Convert matched_columns to a list before using it to filter859 matched_columns_list = list(matched_columns)860 861 # Filter the dataframe to retain only matched columns862 filtered_df = df[matched_columns_list]863 return filtered_df864 865 # Filter both dataframes866 df_hygiene_filtered = filter_columns_by_match(df_hygiene)867 df_fulfilment_filtered = filter_columns_by_match(df_fulfilment)868 869 # Remove rows that contain all NaN values870 df_hygiene_filtered = df_hygiene_filtered.dropna(how="all")871 df_fulfilment_filtered = df_fulfilment_filtered.dropna(how="all")872 873 # For any value in all columns that contain " - " (rating),874 # split and only take the first part (digits)875 def split_value(val):876 if isinstance(val, str) and " = " in val:877 return val.split(" = ")[0]878 return val879 880 # Apply the function to all elements of the DataFrame881 df_hygiene_filtered = df_hygiene_filtered.applymap(split_value)882 df_fulfilment_filtered = df_fulfilment_filtered.applymap(split_value)883 884 def convert_to_int(df):885 # Convert columns to numeric values, preserving NaN values886 return df.apply(pd.to_numeric, errors="coerce").astype(pd.Int64Dtype())887 888 # Convert the columns to integers, preserving NaN values889 df_hygiene_filtered = convert_to_int(df_hygiene_filtered)890 df_fulfilment_filtered = convert_to_int(df_fulfilment_filtered)891 892 # Create a reverse mapping from "Match" values to keys893 match_to_key = {894 normalize_text(val["Match"]): key for key, val in variables_dict.items()895 }896 897 # Function to rename columns based on match898 def rename_columns(df):899 # Normalize column names900 normalized_columns = {normalize_text(col): col for col in df.columns}901 902 # Create a mapping from normalized column names to variable_dict keys903 rename_mapping = {}904 for col_normalized, col in normalized_columns.items():905 for match_normalized, key in match_to_key.items():906 if match_normalized in col_normalized:907 rename_mapping[col] = key908 break909 910 # Rename columns in the DataFrame911 df = df.rename(columns=rename_mapping)912 return df913 914 # Rename columns in both filtered dataframes915 df_hygiene_filtered = rename_columns(df_hygiene_filtered)916 df_fulfilment_filtered = rename_columns(df_fulfilment_filtered)917 918 # Define the desired column order919 desired_order = [f"N{i}" for i in range(1, 28)]920 921 # Function to reorder columns922 def reorder_columns(df, desired_order):923 # Reorder columns based on desired_order924 df = df.reindex(925 columns=[col for col in desired_order if col in df.columns]926 )927 return df928 929 # Reorder columns in both dataframes930 df_hygiene_filtered = reorder_columns(df_hygiene_filtered, desired_order)931 df_fulfilment_filtered = reorder_columns(932 df_fulfilment_filtered, desired_order933 )934 935 # Create the DataFrame936 df_variables = pd.DataFrame(937 {938 "Variable": list(variables_dict.keys()),939 "Variable Name": [940 info["Variable Name"] for info in variables_dict.values()941 ],942 "Description": [943 info["Description"] for info in variables_dict.values()944 ],945 }946 )947 948 # Create a temporary directory949 temp_dir = tempfile.mkdtemp()950 logger.info("Created temporary directory for processed file: %s", temp_dir)951 952 # Get dataset name953 dataset_name = file_path.split("/")[-1]954 dataset_name = dataset_name.split(".")[0]955 956 # Save processed df as an Excel file in the temporary directory957 processed_file_path = os.path.join(temp_dir, f"{dataset_name}.xlsx")958 959 # Save to Excel file with multiple sheets960 with pd.ExcelWriter(processed_file_path, engine="openpyxl") as writer:961 df_hygiene_filtered.to_excel(962 writer, sheet_name="Needs and Hygiene", index=False963 )964 df_fulfilment_filtered.to_excel(965 writer, sheet_name="Fulfilment", index=False966 )967 df_variables.to_excel(writer, sheet_name="Variable", index=False)968 else:969 logger.info("Could not find all search texts in the columns.")970 971 return processed_file_path972 except Exception as e:973 logger.error("Error processing CSV file: %s", e)974 raise975 976def process_examples(file_name):977 # Mapping user-friendly names to actual file paths978 mapping = {979 "OD Org": "Overseas Aid.xlsx",980 "Local Poverty Organisation": "Child Aid.xlsx",981 }982 actual_file_name = mapping[file_name[0]] # Retrieve the actual file name983 file_path = f"example_files/{actual_file_name}" # Build the full file path984 985 # Simulate processing the file and return outputs986 outputs = variable_outputs([file_path])987 return outputs988 989 990def process_datasets(file_inputs):991 """992 Processes uploaded datasets and calls appropriate functions based on file type.993 Args:994 file_inputs (List[UploadFile]): List of uploaded files.995 Returns:996 List[gr.Blocks]: List of Gradio output components.997 """998 outputs_list = []999 1000 for file_input in file_inputs:1001 file_path = file_input.name1002 file_extension = os.path.splitext(file_path)[-1].lower()1003 1004 if file_extension == ".xlsx":1005 outputs_list.append(file_path)1006 1007 elif file_extension == ".csv":1008 try:1009 processed_file_path = data_processing(file_path)1010 outputs_list.append(processed_file_path)1011 except Exception as e:1012 logger.error("Error processing file %s: %s", file_path, e)1013 1014 outputs = variable_outputs(outputs_list)1015 1016 return outputs1017 1018js_func = """1019function refresh() {1020 const url = new URL(window.location);1021 if (url.searchParams.get('__theme') !== 'light') {1022 url.searchParams.set('__theme', 'light');1023 window.location.href = url.href;1024 }1025}1026"""1027with gr.Blocks(js=js_func) as demo:1028 with gr.Column():1029 with gr.Row():1030 1031 gr.Markdown(1032 "<span style='font-size:20px; font-weight:bold;'>Choose dataset to analyse ‘Overseas Development Organisation’ or ‘Local Poverty Organisation’ to see the instant analysis of their DonorDrivers®. As a default, the analysis displays the ‘Overseas Development Organisation’ results. Toggle to ‘Local Poverty Organisation’ to see the difference between their DonorDrivers®.</span>",1033 visible=True,)1034 with gr.Row():1035 dataset = gr.Dataset(1036 components=[gr.Textbox(visible=False)],1037 label="",1038 type="values",1039 samples=[1040 ["OD Org"],1041 ["Local Poverty Organisation"],1042 ],1043 )1044 with gr.Row():1045 gr.Markdown(1046 "<span style='font-size:20px; font-weight:bold;'>1) DonorDrivers® Ranking Graph View</span>",1047 visible=True,1048 )1049 with gr.Row():1050 1051 gr.Markdown("""1052 This graph shows you the ranking of the 27 DonorDrivers®1053 highlighting the top motivations for the respondents to give to an1054 organisation vs the current fulfilment level. The baseline impact for1055 each driver is 3.7% (100% divided across the 27 DonorDrivers®).1056 Any percentage above this average indicates higher significance. To1057 make precision impact, we recommend focusing on fulfilling the1058 top six DonorDrivers®.1059 """)1060 1061 1062 1063 1064 1065 1066 with gr.Column():1067 # set default output widgets1068 outputs = reset_outputs()1069 1070 1071 dataset.click(fn=process_examples, inputs=dataset, outputs=outputs)1072 default_sample = ["OD Org"]1073 demo.load(fn=lambda: process_examples(default_sample), inputs=None, outputs=outputs)1074 1075 1076 1077 1078demo.launch(server_name="0.0.0.0")