serJD/speckleCloneBranch_noConditions
0
1 2import json3import re4from specklepy.transports.server import ServerTransport5from specklepy.api import operations6import time7 8def get_database_properties(database_pages):9 # Iterate through the results (each page corresponds to a row in the database)10 propList = []11 for page in database_pages:12 # Print the name and type of each property in this page13 for prop_name, prop_data in page['properties'].items():14 prop_type = prop_data['type']15 propList.append(prop_name)16 break17 return propList18 19# query full database20def fetch_all_database_pages(client, database_id):21 """22 Fetches all pages from a specified Notion database.23 24 :param client: Initialized Notion client.25 :param database_id: The ID of the Notion database to query.26 :return: A list containing all pages from the database.27 """28 start_cursor = None29 all_pages = []30 31 while True:32 response = client.databases.query(33 **{34 "database_id": database_id,35 "start_cursor": start_cursor36 }37 )38 39 all_pages.extend(response['results'])40 41 # Check if there's more data to fetch42 if response['has_more']:43 start_cursor = response['next_cursor']44 else:45 break46 47 return all_pages48 49 50def get_property_value(page, property_name):51 """52 Extracts the value from a specific property in a Notion page based on its type.53 :param page: The Notion page data as retrieved from the API.54 :param property_name: The name of the property whose value is to be fetched.55 :return: The value or values contained in the specified property, depending on type.56 """57 # Check if the property exists in the page58 if property_name not in page['properties']:59 return None # or raise an error if you prefer60 61 property_data = page['properties'][property_name]62 prop_type = property_data['type']63 64 # Handle 'title' and 'rich_text' types65 if prop_type in ['title', 'rich_text']:66 return ''.join(text_block['text']['content'] for text_block in property_data[prop_type])67 68 # Handle 'number' type69 elif prop_type == 'number':70 return property_data[prop_type]71 72 # Handle 'select' type73 elif prop_type == 'select':74 return property_data[prop_type]['name'] if property_data[prop_type] else None75 76 # Handle 'multi_select' type77 elif prop_type == 'multi_select':78 return [option['name'] for option in property_data[prop_type]]79 80 # Handle 'date' type81 elif prop_type == 'date':82 if property_data[prop_type]['end']:83 return (property_data[prop_type]['start'], property_data[prop_type]['end'])84 else:85 return property_data[prop_type]['start']86 87 # Handle 'relation' type88 elif prop_type == 'relation':89 return [relation['id'] for relation in property_data[prop_type]]90 91 # Handle 'people' type92 elif prop_type == 'people':93 return [person['name'] for person in property_data[prop_type] if 'name' in person]94 95 # Add more handlers as needed for other property types96 97 else:98 # Return None or raise an error for unsupported property types99 return None100 101 102def parse_invalid_json(json_string):103 if json_string == None or json_string == "none":104 return None105 # Replace fancy quotes and single quotes with standard double quotes106 json_string = re.sub(r"[‘’“”]", '"', json_string)107 json_string = json_string.replace("'", '"')108 109 # Add quotes around any unquoted keys110 json_string = re.sub(r'(?<!")(\b\w+\b)(?!"):', r'"\1":', json_string)111 112 # Handle unquoted numeric values or booleans if necessary113 # This part can be customized based on specific requirements114 115 try:116 # Try parsing the corrected string117 return json.loads(json_string)118 except json.JSONDecodeError as e:119 # Handle parsing error (or re-raise the exception)120 print("JSON parsing error:", e)121 print(json_string)122 return None123 124def notionTable2JSON(databaseFUll_pages, kpi_database_pages):125 attributeMetaData = {}126 availableAttributes = {}127 cnt = 0128 for page in databaseFUll_pages:129 attributeData = {130 "name": get_property_value(page, "name"),131 "nameShort": get_property_value(page, "nameShort"),132 "nameLong": get_property_value(page, "nameLong"),133 "description": get_property_value(page, "description"),134 "indicator": get_property_value(page, "indicator"),135 "unit": get_property_value(page, "unit"),136 "unitShort": get_property_value(page, "unitShort"),137 "spatialUnit": get_property_value(page, "spatialUnit"),138 "method": get_property_value(page, "method"),139 "type": get_property_value(page, "type"),140 "colorMapping": parse_invalid_json(get_property_value(page, "colorMapping")),141 "parameter": parse_invalid_json(get_property_value(page, "parameter")),142 "level_1": get_property_value(page, "level_1"),143 "level_2": get_property_value(page, "level_2"),144 "level_3": get_property_value(page, "level_3"),145 146 "dataSet": "ActivityNodes",147 "dataSource": "",148 149 "KPI": [],150 "visualisation": []151 }152 curAttrName = get_property_value(page, "name")153 print(curAttrName)154 lev1 = get_property_value(page, "level_1")155 lev2 = get_property_value(page, "level_2")156 lev3= get_property_value(page, "level_3")157 158 if lev1 != None and lev1 != "NA":159 if lev1 not in availableAttributes:160 availableAttributes[lev1] = {"sub-levels": {}, "values": []}161 162 if lev2 != None and lev2 != "NA":163 if lev2 not in availableAttributes[lev1]["sub-levels"]:164 availableAttributes[lev1]["sub-levels"][lev2] = {"sub-levels": {}, "values": []}165 166 if lev3 != None and lev3 != "NA":167 if lev3 not in availableAttributes[lev1]["sub-levels"][lev2]["sub-levels"]:168 availableAttributes[lev1]["sub-levels"][lev2]["sub-levels"][lev3] = {"values": []}169 availableAttributes[lev1]["sub-levels"][lev2]["sub-levels"][lev3]["values"].append(curAttrName)170 else:171 availableAttributes[lev1]["sub-levels"][lev2]["values"].append(curAttrName)172 else:173 availableAttributes[lev1]["values"].append(curAttrName)174 175 # iterated through list of KPI ref. Ids176 kpiIDs = get_property_value(page, "KPI")177 for kpiID in kpiIDs:178 179 curKPI = get_page_by_id(kpi_database_pages, kpiID)180 KPI_template ={181 "name":get_property_value(curKPI, "name"),182 "type":get_property_value(curKPI, "type"),183 "unit":get_property_value(curKPI, "unit"),184 "color":parse_invalid_json(get_property_value(curKPI, "color")),185 "nameShort": get_property_value(curKPI, "nameShort"),186 "quality":get_property_value(curKPI, "quality"),187 "args": parse_invalid_json(get_property_value(curKPI, "args")),188 "description":get_property_value(curKPI, "description"),189 "interpretrationHigh":get_property_value(curKPI, "interpretrationHigh"),190 "interpretationLow":get_property_value(curKPI, "interpretationLow"),191 }192 # add KPI data to attributeData193 attributeData["KPI"].append(KPI_template)194 195 # add to main dictioanry196 attributeMetaData[get_property_value(page, "name")] = attributeData197 198 print("processed pages:", cnt)199 return attributeMetaData, availableAttributes200 201 202def get_page_by_id(notion_db_pages, page_id):203 for pg in notion_db_pages:204 if pg["id"] == page_id:205 return pg206 207def getSpeckleStream(stream_id,208 branch_name,209 client,210 commit_id=""211 ):212 """213 Retrieves data from a specific branch of a speckle stream.214 215 Args:216 stream_id (str): The ID of the speckle stream.217 branch_name (str): The name of the branch within the speckle stream.218 client (specklepy.api.client.Client, optional): A speckle client. Defaults to a global `client`.219 commit_id (str): id of a commit, if nothing is specified, the latest commit will be fetched220 221 Returns:222 dict: The speckle stream data received from the specified branch.223 224 This function retrieves the last commit from a specific branch of a speckle stream.225 It uses the provided speckle client to get the branch and commit information, and then 226 retrieves the speckle stream data associated with the last commit.227 It prints out the branch details and the creation dates of the last three commits for debugging purposes.228 """229 230 print("updated A")231 232 # set stream and branch233 try:234 branch = client.branch.get(stream_id, branch_name, 3)235 print(branch)236 except:237 branch = client.branch.get(stream_id, branch_name, 1)238 print(branch)239 240 print("last three commits:")241 [print(ite.createdAt) for ite in branch.commits.items]242 243 if commit_id == "":244 latest_commit = branch.commits.items[0]245 choosen_commit_id = latest_commit.id246 commit = client.commit.get(stream_id, choosen_commit_id)247 print("latest commit ", branch.commits.items[0].createdAt, " was choosen")248 elif type(commit_id) == type("s"): # string, commit uuid249 choosen_commit_id = commit_id250 commit = client.commit.get(stream_id, choosen_commit_id)251 print("provided commit ", choosen_commit_id, " was choosen")252 elif type(commit_id) == type(1): #int 253 latest_commit = branch.commits.items[commit_id]254 choosen_commit_id = latest_commit.id255 commit = client.commit.get(stream_id, choosen_commit_id)256 257 258 print(commit)259 print(commit.referencedObject)260 # get transport261 transport = ServerTransport(client=client, stream_id=stream_id)262 #speckle stream263 res = operations.receive(commit.referencedObject, transport)264 265 return res