app.py

import json
import re
import sys
import time
import copy
from notion_client import Client
import copy 

from specklepy.api.client import SpeckleClient
from specklepy.api.credentials import get_default_account, get_local_accounts
from specklepy.transports.server import ServerTransport
from specklepy.api import operations
from specklepy.objects.geometry import Polyline, Point
from specklepy.objects import Base
import os
from functools import wraps

import gradio as gr
import requests
from huggingface_hub import webhook_endpoint, WebhookPayload
from fastapi import Request
import datetime

import gradio as gr


from utils import *

current_directory = os.path.dirname(os.path.abspath(__file__))
# Path to the config.json file
config_file_path = os.path.join(current_directory, "config.json")
with open(config_file_path, 'r') as f:
        config = json.load(f)

# notion login
speckle_token = os.environ.get("SPECKLE_TOKEN")
notion_token = os.environ.get("NOTION_TOKEN")

notion_token = notion_token
notion = Client(auth=notion_token)

# speckle 
CLIENT = SpeckleClient(host="https://speckle.xyz/")
CLIENT.authenticate_with_token(token=speckle_token)


def filter_activityNodeAnalysis(objDict):
    flag = True    
    tot_sqm = 0
    for k, v in objDict.items():
        if k.startswith("lu+"):
            tot_sqm += float(v)
    if tot_sqm <= 0:
        flag = False 

    # COND B: remove non connected 
    if "isConnected" in objDict.keys():
        if not objDict["isConnected"]:
            flag = False
    else:
        flag = False

    return flag



# load config to variables 
def mainFunc(STREAM_ID, SOURCE_BRANCH, TARGET_BRANCH, UUID_COL, ATTR_METADATA, KPI_METADATA, DEFAULT_ATTRIBUTES, DATASET_NAME, UPDATE_SRC):
    print("SOURCE_BRANCH",SOURCE_BRANCH)
    print("TARGET_BRANCH",TARGET_BRANCH)
    if type(TARGET_BRANCH) == type([]):
        TARGET_BRANCH = SOURCE_BRANCH.replace(TARGET_BRANCH[0], TARGET_BRANCH[1])
    print("TARGET_BRANCH Final",TARGET_BRANCH)
    
    # get data from notion
    databaseFUll_pages  = fetch_all_database_pages(notion, ATTR_METADATA)
    time.sleep(1)
    kpi_database_pages = fetch_all_database_pages(notion, KPI_METADATA)
    time.sleep(1)   

    # generate JSON files
    attributeMetaData, availableAttributes = notionTable2JSON(databaseFUll_pages, kpi_database_pages)

    # extract attribute/colum/feature names from notion table 
    attributesOfInterest = DEFAULT_ATTRIBUTES
    attributesOfInterestDetails = {}
    for page in databaseFUll_pages:
        pv = get_property_value(page, "name")
        isUsed = get_property_value(page, "isUsed")
        # can be used for filtering
        attributesOfInterestDetails[pv] = {"isUsed":isUsed}
        attributesOfInterest.append(pv)
        if UUID_COL not in attributesOfInterest:
            attributesOfInterest.append(UUID_COL)


    # get speckle data 
    # get stream
    print("getting source branch, from branch: ",SOURCE_BRANCH)
    stream, inputCommitID = getSpeckleStream(STREAM_ID,
                            SOURCE_BRANCH,
                            CLIENT,
                            commit_id = "")
    time.sleep(2)
    # navigate to list with speckle objects of interest
    try:
      stream_data = stream["@Data"]["@{0}"]
    except:
       print("something went wrong, try again with non-capital d")
    try:
     stream_data = stream["@data"]["@{0}"]
    except:
        print("check on speckle.com how to access the data")
    

    # ======== assamble new stream data ============
    streamData_new = []
    log = {"removedDatapoints":0,"removedID":[], "avg_attrRemoved":0, "removedAttr":[]}
    for i, obj in enumerate(stream_data):
        objDict = obj.__dict__
        # REMOVE DATA POINTS ==============================
        # COND A: no landuses at all.
        pass_flag = True

        if "activity_node_analysis" in SOURCE_BRANCH:
            pass_flag = filter_activityNodeAnalysis(objDict)
        elif "isovist_analysis" in SOURCE_BRANCH:
            # to filter conditions defined yet
            pass
            
    
        if pass_flag == False :
            log["removedDatapoints"] +=1
            log["removedID"].append(objDict[UUID_COL])
            continue

        # REMOVE ATTRIBUTES ===============================
        datanew = Base()
        for k, v in objDict.items():
            if k in attributesOfInterest:
                AoIDetailsFlag = True 
                try:
                    # default to include property 
                    isUsedFlag = attributesOfInterestDetails[k].get("isUsed", True)  
                    if isUsedFlag == False or isUsedFlag == "False" :  
                        AoIDetailsFlag = False
                except KeyError as e:  # Catching specific exceptions for clarity
                    print(f"AoIDetails test failed for key {k}: {e}")
                
                if AoIDetailsFlag:
                    datanew[k] = v
            else:
                log["avg_attrRemoved"] +=1
                log["removedAttr"].append(k)
        
        datanew["dataset"] = DATASET_NAME
        streamData_new.append(datanew)

    log["avg_attrRemoved"] = log["avg_attrRemoved"]/len(stream_data)
    log["removedAttr"] = list(set(log["removedAttr"]))

    stream_new = copy.deepcopy(stream)
    stream_new["@Data"]["@{0}"] = streamData_new
    
    #add additional data to stream
    stream_new["logs"] = json.dumps(log)
    stream_new["attributeMetaData"] = json.dumps(attributeMetaData)
    stream_new["availableAttributes"] = json.dumps(availableAttributes)
    stream_new["datasetMetadata"] = DATASET_NAME


    # set stream and branch
    # Get transport
    transport = ServerTransport(client=CLIENT, stream_id=STREAM_ID)
    
    # Send the data object to the speckle stream
    object_id = operations.send(stream_new, [transport])
    time.sleep(1)
    
    # Create a new commit with the new object
    print("----> attempting to make commit to: ",TARGET_BRANCH )
    commit_id = CLIENT.commit.create(
        STREAM_ID,
        object_id= object_id,
        message = "auto commit from HF; Triggered by: " + UPDATE_SRC + ", removed datapoints: " + str(log["removedDatapoints"]) + "; avg. removed attributes: " + str(int(log["avg_attrRemoved"])) + "#+SourceCommit: " + inputCommitID,
        branch_name=TARGET_BRANCH,
    )

    print(commit_id)

    return "https://speckle.xyz/streams/" + STREAM_ID + "/commits/" + commit_id


@webhook_endpoint
async def update_streams(request: Request):
    # Initialize flag
    should_continue = False
    finalBranchName = None

    # Read the request body as JSON
    payload = await request.json()

    print("============= payload =============")
    print(payload)
    print("============= config ==============")
    print(config)
    print("===================================")

    payload = payload["payload"]

    # webhook calls can come from different sources 
    if payload.get('source') == 'notionTrigger':
        action = payload.get('action')
        streamName = payload.get('streamName')
        branchName = payload.get('branchName')
        update_source = "notionTrigger"

        should_continue = True

    else:
        # Check if the payload structure matches the expected format
        # Assuming payload["event"]["event_name"] gives you the event type
        update_source = "SpeckleWebhook"
        event_name = payload["event"]["event_name"]
        streamName = payload.get("stream", {}).get("name")
        
        # Extract branchName for commit_update events from the "old" commit data
        if event_name == "commit_update":
            branchName = payload.get("event", {}).get("data", {}).get("old", {}).get("branchName")
           
        else:
            branchName = payload.get("event", {}).get("data", {}).get("commit", {}).get("branchName")
        
        
        # List of valid event types
        valid_event_types = ["commit_create", "commit_delete", "commit_update"]
        
        if event_name in valid_event_types:
            if streamName in config:
                if branchName in config[streamName]:
                    stream_config = config[streamName][branchName]
                    should_continue = True
                else:
                    print(f"Branch name {branchName} not found in config for stream {streamName}.")
            else:
                print(f"Stream name {streamName} not found in config.")
        else:
            print(f"Event type {event_name} is not one of the specified types.")

    print("payload branchname", branchName)
    
    # If the flag is True, continue running the main part of the code
    if should_continue:
        config_entry = config[streamName][branchName]
        STREAM_NAME = config_entry["STREAM_NAME"]
        STREAM_ID = config_entry["STREAM_ID"]
        SOURCE_BRANCH = config_entry["SOURCE_BRANCH"]
        TARGET_BRANCH = config_entry["TARGET_BRANCH"]
        UUID_COL = config_entry["UUID_COL"]
        ATTR_METADATA = config_entry["ATTR_METADATA"]
        KPI_METADATA = config_entry["KPI_METADATA"]
        DEFAULT_ATTRIBUTES = config_entry["DEFAULT_ATTRIBUTES"]
        DATASET_NAME = config_entry.get("DATASET_NAME")
        
        # main
        commit_url =mainFunc(STREAM_ID, SOURCE_BRANCH, TARGET_BRANCH, UUID_COL, ATTR_METADATA, KPI_METADATA, DEFAULT_ATTRIBUTES, DATASET_NAME, update_source)
    return commit_url