app.py

import streamlit as st
import streamlit.components.v1 as components
from PIL import Image
import torch
from torchvision.transforms import functional as F
from PIL import Image, ImageEnhance
from htlm_webpage import display_bpmn_xml
import gc
import psutil
import copy


from OCR import text_prediction, filter_text, mapping_text, rescale
from train import prepare_model
from utils import draw_annotations, create_loader, class_dict, arrow_dict, object_dict
from toXML import calculate_pool_bounds, add_diagram_elements
from pathlib import Path
from toXML import create_bpmn_object, create_flow_element
import xml.etree.ElementTree as ET
import numpy as np
from display import draw_stream
from eval import full_prediction
from streamlit_image_comparison import image_comparison
from xml.dom import minidom
from streamlit_cropper import st_cropper
from streamlit_drawable_canvas import st_canvas
from streamlit_image_select import image_select
from utils import find_closest_object
from train import get_faster_rcnn_model, get_arrow_model
import gdown

def get_memory_usage():
    process = psutil.Process()
    mem_info = process.memory_info()
    return mem_info.rss / (1024 ** 2)  # Return memory usage in MB

def clear_memory():
    st.session_state.clear()
    gc.collect()

# Function to read XML content from a file
def read_xml_file(filepath):
    """ Read XML content from a file """
    with open(filepath, 'r', encoding='utf-8') as file:
        return file.read()

# Function to modify bounding box positions based on the given sizes
def modif_box_pos(pred, size):
    modified_pred = copy.deepcopy(pred)  # Make a deep copy of the prediction
    for i, (x1, y1, x2, y2) in enumerate(modified_pred['boxes']):
        center = [(x1 + x2) / 2, (y1 + y2) / 2]
        label = class_dict[modified_pred['labels'][i]]
        if label in size:
            modified_pred['boxes'][i] = [center[0] - size[label][0] / 2, center[1] - size[label][1] / 2, center[0] + size[label][0] / 2, center[1] + size[label][1] / 2]
    return modified_pred['boxes']

# Function to create a BPMN XML file from prediction results
def create_XML(full_pred, text_mapping, scale):
    namespaces = {
        'bpmn': 'http://www.omg.org/spec/BPMN/20100524/MODEL',
        'bpmndi': 'http://www.omg.org/spec/BPMN/20100524/DI',
        'di': 'http://www.omg.org/spec/DD/20100524/DI',
        'dc': 'http://www.omg.org/spec/DD/20100524/DC',
        'xsi': 'http://www.w3.org/2001/XMLSchema-instance'
    }
    
    size_elements = {
        'start': (43.2, 43.2),
        'task': (120, 96),
        'message': (43.2, 43.2),
        'messageEvent': (43.2, 43.2),
        'end': (43.2, 43.2),
        'exclusiveGateway': (60, 60),
        'event': (43.2, 43.2),
        'parallelGateway': (60, 60),
        'dataObject': (48, 72),
        'dataStore': (72, 72),
        'subProcess': (144, 108),
        'eventBasedGateway': (60, 60),
        'timerEvent': (48, 48),
    }


    definitions = ET.Element('bpmn:definitions', {
        'xmlns:xsi': namespaces['xsi'],
        'xmlns:bpmn': namespaces['bpmn'],
        'xmlns:bpmndi': namespaces['bpmndi'],
        'xmlns:di': namespaces['di'],
        'xmlns:dc': namespaces['dc'],
        'targetNamespace': "http://example.bpmn.com",
        'id': "simpleExample"
    })

    #modify the boxes positions
    old_boxes = copy.deepcopy(full_pred)
    full_pred['boxes'] = modif_box_pos(full_pred, size_elements)

    # Create BPMN collaboration element
    collaboration = ET.SubElement(definitions, 'bpmn:collaboration', id='collaboration_1')

    # Create BPMN process elements
    process = []
    for idx in range(len(full_pred['pool_dict'].items())):
        process_id = f'process_{idx+1}'
        process.append(ET.SubElement(definitions, 'bpmn:process', id=process_id, isExecutable='false', name=text_mapping[full_pred['BPMN_id'][list(full_pred['pool_dict'].keys())[idx]]]))

    bpmndi = ET.SubElement(definitions, 'bpmndi:BPMNDiagram', id='BPMNDiagram_1')
    bpmnplane = ET.SubElement(bpmndi, 'bpmndi:BPMNPlane', id='BPMNPlane_1', bpmnElement='collaboration_1')

    full_pred['boxes'] = rescale(scale, full_pred['boxes'])

    # Add diagram elements for each pool
    for idx, (pool_index, keep_elements) in enumerate(full_pred['pool_dict'].items()):
        pool_id = f'participant_{idx+1}'
        pool = ET.SubElement(collaboration, 'bpmn:participant', id=pool_id, processRef=f'process_{idx+1}', name=text_mapping[full_pred['BPMN_id'][list(full_pred['pool_dict'].keys())[idx]]])
        
        # Calculate the bounding box for the pool
        if len(keep_elements) == 0:
            min_x, min_y, max_x, max_y = full_pred['boxes'][pool_index]
            pool_width = max_x - min_x
            pool_height = max_y - min_y
        else:
            min_x, min_y, max_x, max_y = calculate_pool_bounds(full_pred, keep_elements, size_elements)
            pool_width = max_x - min_x + 100  # Adding padding
            pool_height = max_y - min_y + 100  # Adding padding
        
        add_diagram_elements(bpmnplane, pool_id, min_x - 50, min_y - 50, pool_width, pool_height)

    # Create BPMN elements for each pool
    for idx, (pool_index, keep_elements) in enumerate(full_pred['pool_dict'].items()):
        create_bpmn_object(process[idx], bpmnplane, text_mapping, definitions, size_elements, full_pred, keep_elements)

    # Create message flow elements
    message_flows = [i for i, label in enumerate(full_pred['labels']) if class_dict[label] == 'messageFlow']
    for idx in message_flows:
        create_flow_element(bpmnplane, text_mapping, idx, size_elements, full_pred, collaboration, message=True)

    # Create sequence flow elements
    for idx, (pool_index, keep_elements) in enumerate(full_pred['pool_dict'].items()):
        for i in keep_elements:
            if full_pred['labels'][i] == list(class_dict.values()).index('sequenceFlow'):
                create_flow_element(bpmnplane, text_mapping, i, size_elements, full_pred, process[idx], message=False)
    
    # Generate pretty XML string
    tree = ET.ElementTree(definitions)
    rough_string = ET.tostring(definitions, 'utf-8')
    reparsed = minidom.parseString(rough_string)
    pretty_xml_as_string = reparsed.toprettyxml(indent="  ")

    full_pred['boxes'] = rescale(1/scale, full_pred['boxes'])
    full_pred['boxes'] = old_boxes

    return pretty_xml_as_string


# Function to load the models only once and use session state to keep track of it
def load_models():
    with st.spinner('Loading model...'):     
        model_object = get_faster_rcnn_model(len(object_dict))
        model_arrow = get_arrow_model(len(arrow_dict),2)

        url_arrow = 'https://drive.google.com/uc?id=1xwfvo7BgDWz-1jAiJC1DCF0Wp8YlFNWt'
        url_object = 'https://drive.google.com/uc?id=1GiM8xOXG6M6r8J9HTOeMJz9NKu7iumZi'

        # Define paths to save models
        output_arrow = 'model_arrow.pth'
        output_object = 'model_object.pth'

        # Download models using gdown
        if not Path(output_arrow).exists():
            # Download models using gdown
            gdown.download(url_arrow, output_arrow, quiet=False)
        else:
            print('Model arrow downloaded from local')
        if not Path(output_object).exists():
            gdown.download(url_object, output_object, quiet=False)
        else:
            print('Model object downloaded from local')

        # Load models
        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
        model_arrow.load_state_dict(torch.load(output_arrow, map_location=device))
        model_object.load_state_dict(torch.load(output_object, map_location=device))
        st.session_state.model_loaded = True
        st.session_state.model_arrow = model_arrow
        st.session_state.model_object = model_object

# Function to prepare the image for processing
def prepare_image(image, pad=True, new_size=(1333, 1333)):
    original_size = image.size
    # Calculate scale to fit the new size while maintaining aspect ratio
    scale = min(new_size[0] / original_size[0], new_size[1] / original_size[1])
    new_scaled_size = (int(original_size[0] * scale), int(original_size[1] * scale))
    # Resize image to new scaled size
    image = F.resize(image, (new_scaled_size[1], new_scaled_size[0]))

    if pad:
        enhancer = ImageEnhance.Brightness(image)
        image = enhancer.enhance(1.5)  # Adjust the brightness if necessary
        # Pad the resized image to make it exactly the desired size
        padding = [0, 0, new_size[0] - new_scaled_size[0], new_size[1] - new_scaled_size[1]]
        image = F.pad(image, padding, fill=200, padding_mode='edge')

    return new_scaled_size, image

# Function to display various options for image annotation
def display_options(image, score_threshold):
    col1, col2, col3, col4, col5 = st.columns(5)
    with col1:
        write_class = st.toggle("Write Class", value=True)
        draw_keypoints = st.toggle("Draw Keypoints", value=True)
        draw_boxes = st.toggle("Draw Boxes", value=True)
    with col2:
        draw_text = st.toggle("Draw Text", value=False)
        write_text = st.toggle("Write Text", value=False)
        draw_links = st.toggle("Draw Links", value=False)
    with col3:
        write_score = st.toggle("Write Score", value=True)
        write_idx = st.toggle("Write Index", value=False)
    with col4:
        # Define options for the dropdown menu
        dropdown_options = [list(class_dict.values())[i] for i in range(len(class_dict))]
        dropdown_options[0] = 'all'
        selected_option = st.selectbox("Show class", dropdown_options)

    # Draw the annotated image with selected options
    annotated_image = draw_stream(
        np.array(image), prediction=st.session_state.prediction, text_predictions=st.session_state.text_pred,
        draw_keypoints=draw_keypoints, draw_boxes=draw_boxes, draw_links=draw_links, draw_twins=False, draw_grouped_text=draw_text,
        write_class=write_class, write_text=write_text, keypoints_correction=True, write_idx=write_idx, only_print=selected_option,
        score_threshold=score_threshold, write_score=write_score, resize=True, return_image=True, axis=True
    )

    # Display the original and annotated images side by side
    image_comparison(
        img1=annotated_image,
        img2=image,
        label1="Annotated Image",
        label2="Original Image",
        starting_position=99,
        width=1000,
    )

# Function to perform inference on the uploaded image using the loaded models
def perform_inference(model_object, model_arrow, image, score_threshold):
    _, uploaded_image = prepare_image(image, pad=False)
              
    img_tensor = F.to_tensor(prepare_image(image.convert('RGB'))[1])

    # Display original image
    if 'image_placeholder' not in st.session_state:
        image_placeholder = st.empty()  # Create an empty placeholder
    image_placeholder.image(uploaded_image, caption='Original Image', width=1000)

    # Prediction
    _, st.session_state.prediction = full_prediction(model_object, model_arrow, img_tensor, score_threshold=score_threshold, iou_threshold=0.5)

    # Perform OCR on the uploaded image
    ocr_results = text_prediction(uploaded_image)

    # Filter and map OCR results to prediction results
    st.session_state.text_pred = filter_text(ocr_results, threshold=0.5)
    st.session_state.text_mapping = mapping_text(st.session_state.prediction, st.session_state.text_pred, print_sentences=False, percentage_thresh=0.5)
                
    # Remove the original image display
    image_placeholder.empty()

    # Force garbage collection
    gc.collect()

@st.cache_data
def get_image(uploaded_file):
    return Image.open(uploaded_file).convert('RGB')


def main():
    st.set_page_config(layout="wide")

    # Add your company logo banner
    st.image("./images/banner.png", use_column_width=True)

    # Sidebar content
    st.sidebar.header("This BPMN AI model recognition is proposed by: \n ELCA in collaboration with EPFL.")
    st.sidebar.subheader("Instructions:")
    st.sidebar.text("1. Upload you image")
    st.sidebar.text("2. Crop the image \n  (try to put the BPMN diagram \n   in the center of the image)")
    st.sidebar.text("3. Set the score threshold \n   for prediction (default is 0.5)")
    st.sidebar.text("4. Set the scale for the XML file \n  (default is 1.0)")
    st.sidebar.text("5. Click on 'Launch Prediction'")
    st.sidebar.text("6. You can now see the annotation \n   and the BPMN XML result")
    st.sidebar.text("7. You can modify and download \n   the result in right format")

    st.sidebar.subheader("If there is an error, try to:")
    st.sidebar.text("1. Change the score threshold")
    st.sidebar.text("2. Re-crop the image by placing\n   the BPMN diagram in the center\n   of the image")
    st.sidebar.text("3. Re-Launch the prediction")

    st.sidebar.subheader("You can close this sidebar")

    # Set the title of the app
    st.title("BPMN model recognition demo")
    
     # Display current memory usage
    memory_usage = get_memory_usage()
    print(f"Current memory usage: {memory_usage:.2f} MB")

    # Initialize the session state for storing pool bounding boxes
    if 'pool_bboxes' not in st.session_state:
        st.session_state.pool_bboxes = []

    # Load the models using the defined function
    if 'model_object' not in st.session_state or 'model_arrow' not in st.session_state:
        clear_memory()
        load_models()

    model_arrow = st.session_state.model_arrow
    model_object = st.session_state.model_object

    #Create the layout for the app
    col1, col2 = st.columns(2)
    with col1:
        with st.expander("Use example images"):
            img_selected = image_select("If you have no image and just want to test the demo, click on one of these images", ["./images/None.jpg", "./images/example1.jpg", "./images/example2.jpg", "./images/example3.jpg"], 
                                    captions=["None", "Example 1", "Example 2", "Example 3"], index=0, use_container_width=False, return_value="original")
        
        if img_selected== './images/None.jpg':
            print('No example image selected')
            #delete the prediction 
            if 'prediction' in st.session_state:
                del st.session_state['prediction']
            img_selected = None

        # Create a file uploader for the user to upload an image
        if img_selected is not None:
            uploaded_file = img_selected
        else:
            uploaded_file = st.file_uploader("Choose an image from my computer...", type=["jpg", "jpeg", "png"])

    # Display the uploaded image if the user has uploaded an image
    if uploaded_file is not None:
        with st.spinner('Waiting for image display...'): 
            original_image = get_image(uploaded_file)
            col1, col2 = st.columns(2)

            # Create a cropper to allow the user to crop the image and display the cropped image
            with col1:           
                cropped_image = st_cropper(original_image, realtime_update=True, box_color='#0000FF', should_resize_image=True, default_coords=(30, original_image.size[0]-30, 30, original_image.size[1]-30))
            with col2:
                st.image(cropped_image, caption="Cropped Image", use_column_width=False, width=500)
                
        # Display the options for the user to set the score threshold and scale
        if cropped_image is not None:
            col1, col2, col3 = st.columns(3)
            with col1:
                score_threshold = st.slider("Set score threshold for prediction", min_value=0.0, max_value=1.0, value=0.5, step=0.05)
            with col2:
                st.session_state.scale = st.slider("Set scale for XML file", min_value=0.1, max_value=2.0, value=1.0, step=0.1)

            # Launch the prediction when the user clicks the button    
            if st.button("Launch Prediction"):
                st.session_state.crop_image = cropped_image
                with st.spinner('Processing...'):
                    perform_inference(model_object, model_arrow, st.session_state.crop_image, score_threshold)
                    #st.session_state.prediction = modif_box_pos(st.session_state.prediction, object_dict)                    
                    st.balloons()
    else:
        #delete the prediction 
        if 'prediction' in st.session_state:
            del st.session_state['prediction']


    # If the prediction has been made and the user has uploaded an image, display the options for the user to annotate the image
    if 'prediction' in st.session_state and uploaded_file is not None:

        with st.spinner('Waiting for result display...'): 
            display_options(st.session_state.crop_image, score_threshold)

        #if st.session_state.prediction_up==True:
        with st.spinner('Waiting for BPMN modeler...'):
            st.session_state.bpmn_xml = create_XML(st.session_state.prediction.copy(), st.session_state.text_mapping, st.session_state.scale)
            display_bpmn_xml(st.session_state.bpmn_xml)

        # Force garbage collection after display
        gc.collect()

if __name__ == "__main__":
    print('Starting the app...')
    main()