serJD/speckleAggregateBranches

1import specklepy2from specklepy.api.client import SpeckleClient3from specklepy.api.credentials import get_default_account, get_local_accounts4from specklepy.transports.server import ServerTransport5from specklepy.api import operations6from specklepy.objects.geometry import Polyline, Point, Mesh7import json8import pandas as pd9import numpy as n10from specklepy.api.wrapper import StreamWrapper11import requests12from datetime import datetime13import copy14 15def get_dataframe(objects_raw, return_original_df=False):16    """17    Creates a pandas DataFrame from a list of raw Speckle objects.18    Args:19        objects_raw (list): List of raw Speckle objects.20        return_original_df (bool, optional): If True, the function also returns the original DataFrame before any conversion to numeric. Defaults to False.21    Returns:22        pd.DataFrame or tuple: If return_original_df is False, returns a DataFrame where all numeric columns have been converted to their respective types, 23                               and non-numeric columns are left unchanged. 24                               If return_original_df is True, returns a tuple where the first item is the converted DataFrame, 25                               and the second item is the original DataFrame before conversion.26    This function iterates over the raw Speckle objects, creating a dictionary for each object that excludes the '@Geometry' attribute. 27    These dictionaries are then used to create a pandas DataFrame. 28    The function attempts to convert each column to a numeric type if possible, and leaves it unchanged if not. 29    Non-convertible values in numeric columns are replaced with their original values.30    """31    # dataFrame32    df_data = []33    # Iterate over speckle objects34    for obj_raw in objects_raw:35        obj = obj_raw.__dict__36        df_obj = {k: v for k, v in obj.items() if k != '@Geometry'}37        df_data.append(df_obj)38 39    # Create DataFrame and GeoDataFrame40    df = pd.DataFrame(df_data)41    # Convert columns to float or int if possible, preserving non-convertible values <-42    df_copy = df.copy()43    for col in df.columns:44        df[col] = pd.to_numeric(df[col], errors='coerce')45        df[col].fillna(df_copy[col], inplace=True)46 47    if return_original_df:48        return df, df_copy49    else:50        return df51 52def aggregate_data_optimized(df_a, df_b, uuid_col_name, ref_col_name, exclude_columns):53    # Ensure the uuid_col_name is included for the merging process54    columns_to_use = [col for col in df_a.columns if col not in exclude_columns or col == uuid_col_name]55    56    df_a_filtered = df_a[columns_to_use]57    58    # Perform the merge without adding suffixes, as we intend to overwrite existing columns in df_b59    df_merged = pd.merge(df_b, df_a_filtered, how='left', left_on=ref_col_name, right_on=uuid_col_name, suffixes=(None, '_y'))60 61    # Initialize a dictionary for logging62    log_dict = {63        'info': [],64        'warning': [],65        'summary': {}66    }67 68    # Logging matched and unmatched counts69    matched_count = df_merged[ref_col_name].notnull().sum()70    unmatched_count = df_b.shape[0] - matched_count71 72    log_dict['summary'] = {73        'matched_count': matched_count,74        'unmatched_count': unmatched_count,75        'total_rows_processed': df_b.shape[0]76    }77    log_dict['info'].append("Data aggregation completed successfully.")78 79    # Explicitly overwrite columns in df_b with those from df_a, based on the merge80    for col in columns_to_use:81        if col not in exclude_columns and col != uuid_col_name and f'{col}_y' in df_merged:82            df_merged[col] = df_merged.pop(f'{col}_y')83    84    # Drop any remaining '_y' columns that were not explicitly handled85    df_merged = df_merged.loc[:, ~df_merged.columns.str.endswith('_y')]86 87    # Additionally, if the uuid_col_name is not part of the original df_b columns and is only used for matching, it should be removed88    if uuid_col_name not in df_b.columns:89        df_merged.drop(columns=[uuid_col_name], inplace=True, errors='ignore')90 91    return df_merged, log_dict92 93 94 95def updateStreamAnalysisFast(client, new_data, stream_id, branch_name, geometryGroupPath=None, match_by_id="", return_original=False, comm_message=""):96    if geometryGroupPath is None:97        geometryGroupPath = ["@Speckle", "Geometry"]98 99    branch = client.branch.get(stream_id, branch_name, 2)100    latest_commit = branch.commits.items[0]101    commit = client.commit.get(stream_id, latest_commit.id)102    transport = ServerTransport(client=client, stream_id=stream_id)103    res = operations.receive(commit.referencedObject, transport)104    objects_raw = res[geometryGroupPath[0]][geometryGroupPath[1]]105 106    # Pre-create a mapping from IDs to objects for faster lookup107    id_to_object_map = {obj[match_by_id]: obj for obj in objects_raw} if match_by_id else {i: obj for i, obj in enumerate(objects_raw)}108 109    # Pre-process DataFrame if match_by_id is provided110    if match_by_id:111        new_data.set_index(match_by_id, inplace=True)112 113    # Update objects in a more efficient way using .items()114    for local_id, updates in new_data.iterrows():115        target_object = id_to_object_map.get(str(local_id))116        if target_object:117            for col_name, value in updates.items():118                target_object[col_name] = value119 120    # Send updated objects back to Speckle121    new_objects_raw_speckle_id = operations.send(base=res, transports=[transport])122    commit_id = client.commit.create(stream_id=stream_id, branch_name=branch_name, object_id=new_objects_raw_speckle_id, message=comm_message + "#+SourceCommit: "+latest_commit.id)123    print("commit created")124    if return_original:125        return objects_raw  # as back-up126 127    return commit_id128 129 130 131        132def getSpeckleStream(stream_id,133                     branch_name,134                     client,135                     commit_id=""136                     ):137    """138    Retrieves data from a specific branch of a speckle stream.139    Args:140        stream_id (str): The ID of the speckle stream.141        branch_name (str): The name of the branch within the speckle stream.142        client (specklepy.api.client.Client, optional): A speckle client. Defaults to a global `client`.143        commit_id (str): id of a commit, if nothing is specified, the latest commit will be fetched144    Returns:145        dict: The speckle stream data received from the specified branch.146    This function retrieves the last commit from a specific branch of a speckle stream.147    It uses the provided speckle client to get the branch and commit information, and then 148    retrieves the speckle stream data associated with the last commit.149    It prints out the branch details and the creation dates of the last three commits for debugging purposes.150    """151 152    print("updated A")153 154    # set stream and branch155    try:156        branch = client.branch.get(stream_id, branch_name, 1)157        print(branch)158    except:159        branch = client.branch.get(stream_id, branch_name, 1)160        print(branch)161 162    print("branch info:", branch)163    #[print(ite.createdAt) for ite in branch.commits.items]164 165    if commit_id == "":166        latest_commit = branch.commits.items[0]167        choosen_commit_id = latest_commit.id168        commit = client.commit.get(stream_id, choosen_commit_id)169        print("latest commit ", branch.commits.items[0].createdAt, " was choosen")170    elif type(commit_id) == type("s"): # string, commit uuid171        choosen_commit_id = commit_id172        commit = client.commit.get(stream_id, choosen_commit_id)173        print("provided commit ", choosen_commit_id, " was choosen")174    elif type(commit_id) == type(1): #int 175        latest_commit = branch.commits.items[commit_id]176        choosen_commit_id = latest_commit.id177        commit = client.commit.get(stream_id, choosen_commit_id)178 179 180    print(commit)181    print(commit.referencedObject)182    # get transport183    transport = ServerTransport(client=client, stream_id=stream_id)184    #speckle stream185    res = operations.receive(commit.referencedObject, transport)186 187    return res