Initial commit

.dockerignore

@@ -0,0 +1,16 @@
+*.csv
+*.pdf
+*.url
+*.jpg
+*.png
+*.ipynb
+examples/*
+processing/*
+output/*
+tools/__pycache__/*
+old_code/*
+tesseract/*
+poppler/*
+build/*
+dist/*
+build_deps/*

.github/workflows/check_file_size.yml

@@ -0,0 +1,16 @@
+name: Check file size
+on:               # or directly `on: [push]` to run the action on every push on any branch
+  pull_request:
+    branches: [main]
+
+  # to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  sync-to-hub:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check large files
+        uses: ActionsDesk/[email protected]
+        with:
+          filesizelimit: 10485760 # this is 10MB so we can sync to HF Spaces

.github/workflows/sync_to_hf.yml

@@ -0,0 +1,20 @@
+name: Sync to Hugging Face hub
+on:
+  push:
+    branches: [main]
+
+  # to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  sync-to-hub:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Push to hub
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        run: git push https://seanpedrickcase:[email protected]/spaces/seanpedrickcase/document_rag_preparation main

.gitignore

@@ -0,0 +1,16 @@
+*.csv
+*.pdf
+*.url
+*.jpg
+*.png
+*.ipynb
+examples/*
+processing/*
+output/*
+tools/__pycache__/*
+old_code/*
+tesseract/*
+poppler/*
+build/*
+dist/*
+build_deps/*

Dockerfile

@@ -0,0 +1,58 @@
+FROM public.ecr.aws/docker/library/python:3.11.9-slim-bookworm
+
+# Install system dependencies. Need to specify -y for poppler to get it to install
+RUN apt-get update \
+    && apt-get install -y \
+        tesseract-ocr -y \
+        poppler-utils -y \
+		libgl1-mesa-glx -y \
+		libglib2.0-0 -y \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /src
+
+COPY requirements.txt .
+
+RUN pip install --no-cache-dir -r requirements.txt
+
+RUN pip install --no-cache-dir gradio==4.31.5
+
+# Set up a new user named "user" with user ID 1000
+RUN useradd -m -u 1000 user
+
+# Change ownership of /home/user directory
+#RUN chown -R user:user /home/user
+
+# Make output folder
+RUN mkdir -p /home/user/app/output && chown -R user:user /home/user/app/output
+RUN mkdir -p /home/user/app/tld && chown -R user:user /home/user/app/tld
+
+# Switch to the "user" user
+USER user
+
+# Set environmental variables
+ENV HOME=/home/user \
+	PATH=/home/user/.local/bin:$PATH \
+    PYTHONPATH=$HOME/app \
+	PYTHONUNBUFFERED=1 \
+	GRADIO_ALLOW_FLAGGING=never \
+	GRADIO_NUM_PORTS=1 \
+	GRADIO_SERVER_NAME=0.0.0.0 \
+	GRADIO_SERVER_PORT=7860 \
+	GRADIO_THEME=huggingface \
+	TLDEXTRACT_CACHE=$HOME/app/tld/.tld_set_snapshot \
+	#GRADIO_TEMP_DIR=$HOME/tmp \
+	#GRADIO_ROOT_PATH=/address-match \
+	# gunicorn keep alive timeout limit extended for GUI-based work - https://github.com/tiangolo/uvicorn-gunicorn-fastapi-docker?tab=readme-ov-file#timeout
+	KEEP_ALIVE=60 \
+	SYSTEM=spaces
+ 
+# Set the working directory to the user's home directory
+WORKDIR $HOME/app
+
+# Copy the current directory contents into the container at $HOME/app setting the owner to the user
+COPY --chown=user . $HOME/app
+#COPY . $HOME/app
+
+CMD ["python", "app.py"]

README.md

@@ -0,0 +1,14 @@
+---
+title: Document RAG preparation
+emoji: 📖
+colorFrom: yellow
+colorTo: purple
+sdk: docker
+app_file: app.py
+pinned: true
+license: apache-2.0
+---
+
+# Document RAG preparation
+
+Extract text from documents and convert into tabular format using the Unstructured package. The outputs can then be used downstream for e.g. RAG/other processes that require tabular data. Currently supports the following file types: .pdf, .docx, .odt, .pptx, .html, text files (.txt, .md., .rst), image files (.png, .jpg, .heic), email exports (.msg, .eml), tabular files (.csv, .xlsx),  code files (.py, .js etc.). Outputs csvs and files in a 'Document' format commonly used as input to vector databases e.g. ChromaDB, or Langchain embedding datastore integrations. See [here](https://docs.unstructured.io/open-source/core-functionality/overview) for more details about what is going on under the hood.

app.py

@@ -0,0 +1,199 @@
+import os
+
+# By default TLDExtract will try to pull files from the internet. I have instead downloaded this file locally to avoid the requirement for an internet connection.
+os.environ['TLDEXTRACT_CACHE'] = 'tld/.tld_set_snapshot'
+
+from tools.helper_functions import ensure_output_folder_exists, add_folder_to_path, custom_regex_load
+from tools.unstructured_funcs import partition_file, clean_elements, export_elements_as_table_to_file, filter_elements_and_metadata, chunk_all_elements, minimum_chunk_length, start_new_chunk_after_end_of_this_element_length, hard_max_character_length_chunks, multipage_sections, overlap_all
+#from tools.aws_functions import load_data_from_aws
+from tools.clean_funcs import pre_clean, full_entity_list, chosen_redact_entities
+import gradio as gr
+import pandas as pd
+import numpy as np
+from typing import Type, List
+from unstructured.documents.elements import Element
+
+# Creating an alias for pandas DataFrame using Type
+PandasDataFrame = Type[pd.DataFrame]
+
+add_folder_to_path("_internal/tesseract/")
+add_folder_to_path("_internal/poppler/poppler-24.02.0/Library/bin/")
+
+ensure_output_folder_exists()
+
+language = 'en'
+default_meta_keys_to_filter=["file_directory", "filetype"]
+default_element_types_to_filter = ['UncategorizedText', 'Header']
+
+
+def get_element_metadata(elements, prefix=""):
+    """Recursively retrieves element names and metadata in the desired format."""
+    result = []
+
+    for element in elements:
+        # print("Element metadata: ", element.metadata)
+        # print("Element metadata dict: ", element.metadata.__dict__)
+
+        if hasattr(element, 'metadata') and isinstance(element.metadata.__dict__, dict):
+            for key, value in element.metadata.__dict__.items():  # Iterate over key-value pairs in metadata dictionary
+                    new_prefix = f"{prefix}." if prefix else ""
+                    if isinstance(value, dict):  # Nested metadata
+                        result.extend(get_element_metadata([value], new_prefix))  # Recurse with the nested dictionary as a single-item list
+                    else:  # Leaf element
+                        meta_element_to_add = f"{new_prefix}{key}"
+                        if meta_element_to_add not in result:
+                            result.append(meta_element_to_add)
+        else:
+            print(f"Warning: Element {element} does not have a metadata dictionary.")  # Handle elements without metadata gracefully
+
+    return result
+
+def update_filter_dropdowns(elements_table:PandasDataFrame, elements:List[Element]):
+    if 'text' in elements_table.columns:
+        elements_table_filt = elements_table.drop('text', axis=1)
+    else:
+        elements_table_filt = elements_table
+
+    # Error handling for missing 'type' column
+    if 'type' not in elements_table_filt.columns:
+        print("Warning: 'type' column not found in the DataFrame.")
+        return gr.Dropdown(label="Element types (not available)"), gr.Dropdown(label="Metadata properties (not available)")
+
+    element_types_to_filter = elements_table_filt['type'].unique().tolist()
+    meta_keys_to_filter = get_element_metadata(elements)
+
+    #print("Element types:", element_types_to_filter)
+    #print("Meta keys:", meta_keys_to_filter)
+
+    element_types_to_filter_shortlist = [x for x in default_element_types_to_filter if x in element_types_to_filter]
+    meta_keys_to_filter_shortlist = [x for x in default_meta_keys_to_filter if x in meta_keys_to_filter]
+
+    return gr.Dropdown(
+        value=element_types_to_filter_shortlist, choices=element_types_to_filter, multiselect=True, interactive=True, label="Choose element types to exclude from element list"
+    ), gr.Dropdown(
+        value=meta_keys_to_filter_shortlist, choices=meta_keys_to_filter, multiselect=True, interactive=True, label="Choose metadata keys to filter out"
+    )
+
+# Create the gradio interface
+
+block = gr.Blocks(theme = gr.themes.Base())
+
+with block:
+
+    elements_state = gr.State([])
+    elements_table_state = gr.State(pd.DataFrame())
+    metadata_keys_state = gr.State([])
+    output_image_files_state = gr.State([])
+    output_file_list_state = gr.State([])
+    in_colnames_state = gr.State("text")
+
+    data_state = gr.State(pd.DataFrame())
+    embeddings_state = gr.State(np.array([]))
+    embeddings_type_state = gr.State("")
+    topic_model_state = gr.State()
+    assigned_topics_state = gr.State([])
+    custom_regex_state = gr.State(pd.DataFrame())
+    docs_state = gr.State()
+    data_file_name_no_ext_state = gr.State()
+    label_list_state = gr.State(pd.DataFrame())
+    output_name_state = gr.State("")
+
+    gr.Markdown(
+    """
+    # Document RAG preparation
+    Extract text from documents and convert into tabular format using the Unstructured package. The outputs can then be used downstream for e.g. RAG/other processes that require tabular data. Currently supports the following file types: .pdf, .docx, .odt, .pptx, .html, text files (.txt, .md., .rst), image files (.png, .jpg, .heic), email exports (.msg, .eml), tabular files (.csv, .xlsx), or code files (.py, .js, etc.). Outputs csvs and files in a 'Document' format commonly used as input to vector databases e.g. ChromaDB, or Langchain embedding datastore integrations. See [here](https://docs.unstructured.io/open-source/core-functionality/overview) for more details about what is going on under the hood.
+    """)
+
+    with gr.Tab("Partition document"):
+    
+        with gr.Accordion("Upload files - accepts .pdf, .docx, .odt, .pptx, .html, text files (.txt, .md., .rst), image files (.png, .jpg, .heic), email exports (.msg, .eml), tabular files (.csv, .xlsx),  or code files (.py, .js, etc.)", open = True):
+            in_file = gr.File(label="Choose file", file_count= "multiple", height=100)
+            in_pdf_partition_strategy = gr.Radio(label="PDF partition strategy", value = "fast", choices=["fast", "ocr_only", "hi_res"])
+        
+        partition_btn = gr.Button("Partition documents (outputs appear below)", variant='primary')
+
+        with gr.Accordion("Clean, anonymise, or filter text elements", open = False):
+            with gr.Accordion("Filter element types from text and information from metadata", open = False):
+                element_types_to_filter = gr.Dropdown(value=default_element_types_to_filter, choices=default_element_types_to_filter, multiselect=True, interactive=True, label = "Choose element types to exclude from element list")
+                meta_keys_to_filter = gr.Dropdown(value=default_meta_keys_to_filter, choices=default_meta_keys_to_filter, multiselect=True, interactive=True, label = "Choose metadata keys to filter out")                
+
+                filter_meta_btn = gr.Button("Filter elements/metadata")
+
+            with gr.Accordion("Clean/anonymise text", open = False):
+                with gr.Row():
+                    clean_options = gr.Dropdown(choices = ["Convert bytes to string","Replace quotes","Clean non ASCII","Clean ordered list", "Group paragraphs",
+                    "Remove trailing punctuation", "Remove all punctuation","Clean text","Remove extra whitespace", "Remove dashes","Remove bullets",
+                    "Make lowercase"],
+                    value=["Clean ordered list", "Group paragraphs", "Clean non ASCII", "Remove extra whitespace", "Remove dashes",  "Remove bullets"],
+                    label="Clean options", multiselect=True, interactive=True)                    
+
+                with gr.Accordion("Clean with custom regex", open = False):
+                    gr.Markdown("""Import custom regex - csv table with one column of regex patterns with header. Example pattern: (?i)roosevelt for case insensitive removal of this term.""")
+                    clean_text = gr.Dropdown(value = "No", choices=["Yes", "No"], multiselect=False, label="Remove custom regex.")
+                    with gr.Row():
+                        custom_regex = gr.UploadButton(label="Import custom regex file", file_count="multiple")
+                        custom_regex_text = gr.Textbox(label="Custom regex load status")
+
+                with gr.Accordion("Anonymise text", open = False):
+                    anonymise_drop = gr.Dropdown(value = "No", choices=["Yes", "No"], multiselect=False, label="Anonymise data. Personal details are redacted - not 100% effective. Please check results afterwards!")
+                    with gr.Row():
+                        anon_strat = gr.Dropdown(value = "redact", choices=["redact", "replace"], multiselect=False, label="Anonymisation strategy. Choose from redact (simply remove text), or replace with entity type (e.g. <PERSON>)")
+                        anon_entities_drop = gr.Dropdown(value=chosen_redact_entities, choices=full_entity_list, multiselect=True, label="Choose entities to find and anonymise in your open text")                        
+
+                unstructured_clean_btn = gr.Button("Clean data")       
+                
+        with gr.Accordion("Chunk text", open = False):
+            with gr.Row():
+                chunking_method_rad = gr.Radio(value = "Chunk within title", choices = ["Chunk within title", "Basic chunking"], interactive=True)
+                multipage_sections_drop =gr.Dropdown(choices=["Yes", "No"], value = "Yes", label = "Continue chunk over page breaks.", interactive=True)
+                overlap_all_drop =gr.Dropdown(choices=["Yes", "No"], value = "Yes", label="Overlap over adjacent element text if needed.", interactive=True)
+            with gr.Row():
+                minimum_chunk_length_slide = gr.Slider(value = minimum_chunk_length, minimum=100, maximum=10000, step = 100, label= "Minimum chunk character length. Chunk will overlap next title if character limit not reached.", interactive=True)
+                start_new_chunk_after_end_of_this_element_length_slide = gr.Slider(value = start_new_chunk_after_end_of_this_element_length, minimum=100, maximum=10000, step = 100, label = "'Soft' maximum chunk character length - chunk will continue until end of current element when length reached")
+                hard_max_character_length_chunks_slide = gr.Slider(value = hard_max_character_length_chunks, minimum=100, maximum=10000, step = 100, label = "'Hard' maximum chunk character length. Chunk will not be longer than this.", interactive=True)
+
+            chunk_btn = gr.Button("Chunk document")
+
+        # Save chunked data to file
+        with gr.Accordion("File outputs", open = True):
+            with gr.Row():
+                output_summary = gr.Textbox(label="Output summary")
+                output_file = gr.File(label="Output file")
+
+    # AWS functions not yet implemented in this app
+    # with gr.Tab(label="AWS data load"):
+    #     with gr.Accordion(label = "AWS data access", open = True):
+    #         aws_password_box = gr.Textbox(label="Password for AWS data access (ask the Data team if you don't have this)")
+    #         with gr.Row():
+    #             in_aws_file = gr.Dropdown(label="Choose file to load from AWS (only valid for API Gateway app)", choices=["None", "Lambeth borough plan"])
+    #             load_aws_data_button = gr.Button(value="Load data from AWS", variant="secondary")
+                
+    #         aws_log_box = gr.Textbox(label="AWS data load status")
+    
+    # Partition data, then Update filter dropdowns from loaded data
+    partition_btn.click(fn = partition_file, inputs=[in_file, in_pdf_partition_strategy],
+                    outputs=[output_summary, elements_state, output_file, output_name_state, elements_table_state], api_name="partition").\
+                    then(fn = update_filter_dropdowns, inputs=[elements_table_state, elements_state], outputs=[element_types_to_filter, meta_keys_to_filter])
+
+    # Clean data
+    ## Filter metadata
+    
+    filter_meta_btn.click(fn=filter_elements_and_metadata, inputs=[elements_state, element_types_to_filter, meta_keys_to_filter], outputs=[elements_state]).\
+    then(fn=export_elements_as_table_to_file, inputs=[elements_state, output_name_state], outputs=[output_summary, output_file])
+
+    ## General text clean and anonymisation
+
+    ### Custom regex load
+    custom_regex.upload(fn=custom_regex_load, inputs=[custom_regex], outputs=[custom_regex_text, custom_regex_state])
+
+    unstructured_clean_btn.click(fn=clean_elements, inputs=[elements_state, clean_options, output_name_state], outputs=[elements_state, output_summary, output_file, output_name_state]).\
+    then(fn=pre_clean, inputs=[elements_state, in_colnames_state, custom_regex_state, clean_text, output_name_state, anonymise_drop, anon_strat, anon_entities_drop], outputs=[output_summary, output_file, elements_state, output_name_state])
+
+    ## Chunk data
+    chunk_btn.click(fn = chunk_all_elements, inputs=[elements_state, output_name_state, chunking_method_rad, minimum_chunk_length_slide, start_new_chunk_after_end_of_this_element_length_slide, hard_max_character_length_chunks_slide, multipage_sections_drop, overlap_all_drop], outputs=[output_summary, output_file, output_name_state])
+    
+    # Loading AWS data - not yet implemented in this app
+    # load_aws_data_button.click(fn=load_data_from_aws, inputs=[in_aws_file, aws_password_box], outputs=[in_file, aws_log_box])
+    
+# Simple run
+block.queue().launch(ssl_verify=False) # root_path="/address-match", debug=True, server_name="0.0.0.0", server_port=7861

how_to_create_exe_dist.txt

@@ -0,0 +1,38 @@
+1. Create minimal environment to run the app in conda. E.g. 'conda create --name new_env'
+
+2. Activate the environment 'conda activate new_env'
+
+3. cd to this folder. Install packages from requirements.txt using 'pip install -r requirements.txt' 
+
+NOTE: for ensuring that spaCy models are loaded into the program correctly in requirements.txt, follow this guide: https://spacy.io/usage/models#models-download
+
+6. If necessary, create hook- files to tell pyinstaller to include specific packages in the exe build. Examples are provided for en_core_web_sm (a spaCy model). Put these in the build_deps\ subfolder
+
+7. pip install pyinstaller
+
+8. In command line, cd to the folder that contains app.py. 
+
+9.Run the following, assuming you want to make one single .exe file (This helped me: https://github.com/pyinstaller/pyinstaller/issues/8108):
+
+a) In command line: pyi-makespec --additional-hooks-dir="build_deps" --add-data "tesseract/:tesseract/" --add-data "poppler/poppler-24.02.0/:poppler/poppler-24.02.0/" --collect-data=gradio_client  --collect-data=gradio  --hidden-import pyarrow.vendored.version --hidden-import pydicom.encoders  --name DocRagPrepApp_0.1 app.py 
+
+# Add --onefile  to the above if you would like everything packaged as a single exe, although this will need to be extracted upon starting the app, slowing down initialisation time significantly.
+
+
+b) Open the created spec file in Notepad. Add the following to the end of the Analysis section then save:
+
+a = Analysis(
+    ...
+    module_collection_mode={
+        'gradio': 'py',  # Collect gradio package as source .py files
+    }
+)
+
+c) Back in command line, run this: pyinstaller --clean --noconfirm DocRagPrepApp_0.1.spec
+
+
+9. A 'dist' folder will be created with the executable inside along with all dependencies('dist\<app_name>'). 
+
+10. In 'dist\<app_name>' try double clicking on the .exe file. After a short delay, the command prompt should inform you about the IP address of the app that is now running. Copy the IP address. **Do not close this window!**
+
+11. In an Internet browser, navigate to the indicated IP address. The app should now be running in your browser window.

requirements.txt

@@ -0,0 +1,20 @@
+pandas==2.2.2
+spacy # Not specified as latest versions create a conflict with latest versions of gradio
+en_core_web_sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1.tar.gz
+gradio # Not specified as latest versions create a conflict with latest versions of spacy
+boto3==1.34.103
+unstructured
+unstructured[pdf]
+unstructured[docx]
+unstructured[pptx]
+unstructured[html]
+unstructured[text]
+unstructured[xlsx]
+unstructured[odt]
+unstructured[jpg]
+unstructured[msg]
+Faker==22.2.0
+presidio_analyzer==2.2.351
+presidio_anonymizer==2.2.351
+polars==0.20.6
+

tools/anonymiser.py

@@ -0,0 +1,296 @@
+from spacy.cli import download
+import spacy
+from tools.presidio_analyzer_custom import analyze_dict
+from tools.load_spacy_model_custom_recognisers import nlp_analyser
+from typing import List
+from unstructured.documents.elements import Element
+
+spacy.prefer_gpu()
+
+def spacy_model_installed(model_name):
+    try:
+        import en_core_web_sm
+        en_core_web_sm.load()
+        print("Successfully imported spaCy model")
+        #nlp = spacy.load("en_core_web_sm")
+        #print(nlp._path)
+    except:
+        download(model_name)
+        spacy.load(model_name)
+        print("Successfully imported spaCy model")
+    #print(nlp._path)
+
+
+#if not is_model_installed(model_name):
+#    os.system(f"python -m spacy download {model_name}")
+model_name = "en_core_web_sm"
+spacy_model_installed(model_name)
+
+#spacy.load(model_name)
+# Need to overwrite version of gradio present in Huggingface spaces as it doesn't have like buttons/avatars (Oct 2023)
+#os.system("pip uninstall -y gradio")
+#os.system("pip install gradio==3.50.0")
+#os.system("python -m spacy download en_core_web_lg")
+
+import re
+import secrets
+import base64
+import time
+
+import pandas as pd
+
+from faker import Faker
+
+from presidio_analyzer import AnalyzerEngine, BatchAnalyzerEngine, PatternRecognizer
+from presidio_anonymizer import AnonymizerEngine, BatchAnonymizerEngine
+from presidio_anonymizer.entities import OperatorConfig
+
+
+
+def anon_consistent_names(df):
+    # ## Pick out common names and replace them with the same person value
+    df_dict = df.to_dict(orient="list")
+
+    analyzer = AnalyzerEngine()
+    batch_analyzer = BatchAnalyzerEngine(analyzer_engine=analyzer)
+
+    analyzer_results = batch_analyzer.analyze_dict(df_dict, language="en")
+    analyzer_results = list(analyzer_results)
+
+    # + tags=[]
+    text = analyzer_results[3].value
+
+    # + tags=[]
+    recognizer_result = str(analyzer_results[3].recognizer_results)
+
+    # + tags=[]
+    recognizer_result
+
+    # + tags=[]
+    data_str = recognizer_result  # abbreviated for brevity
+
+    # Adjusting the parse_dict function to handle trailing ']'
+    # Splitting the main data string into individual list strings
+    list_strs = data_str[1:-1].split('], [')
+
+    def parse_dict(s):
+        s = s.strip('[]')  # Removing any surrounding brackets
+        items = s.split(', ')
+        d = {}
+        for item in items:
+            key, value = item.split(': ')
+            if key == 'score':
+                d[key] = float(value)
+            elif key in ['start', 'end']:
+                d[key] = int(value)
+            else:
+                d[key] = value
+        return d
+
+    # Re-running the improved processing code
+
+    result = []
+
+    for lst_str in list_strs:
+        # Splitting each list string into individual dictionary strings
+        dict_strs = lst_str.split(', type: ')
+        dict_strs = [dict_strs[0]] + ['type: ' + s for s in dict_strs[1:]]  # Prepending "type: " back to the split strings
+        
+        # Parsing each dictionary string
+        dicts = [parse_dict(d) for d in dict_strs]
+        result.append(dicts)
+
+    #result
+
+    # + tags=[]
+    names = []
+
+    for idx, paragraph in enumerate(text):
+        paragraph_texts = []
+        for dictionary in result[idx]:
+            if dictionary['type'] == 'PERSON':
+                paragraph_texts.append(paragraph[dictionary['start']:dictionary['end']])
+        names.append(paragraph_texts)
+
+    # + tags=[]
+    # Flatten the list of lists and extract unique names
+    unique_names = list(set(name for sublist in names for name in sublist))
+    
+    # + tags=[]
+    fake_names = pd.Series(unique_names).apply(fake_first_name)
+
+    # + tags=[]
+    mapping_df = pd.DataFrame(data={"Unique names":unique_names,
+                    "Fake names": fake_names})
+
+    # + tags=[]
+    # Convert mapping dataframe to dictionary
+    # Convert mapping dataframe to dictionary, adding word boundaries for full-word match
+    name_map = {r'\b' + k + r'\b': v for k, v in zip(mapping_df['Unique names'], mapping_df['Fake names'])}
+
+    # + tags=[]
+    name_map
+
+    # + tags=[]
+    scrubbed_df_consistent_names = df.replace(name_map, regex = True)
+
+    # + tags=[]
+    scrubbed_df_consistent_names
+
+    return scrubbed_df_consistent_names
+
+def detect_file_type(filename):
+    """Detect the file type based on its extension."""
+    if (filename.endswith('.csv')) | (filename.endswith('.csv.gz')) | (filename.endswith('.zip')):
+        return 'csv'
+    elif filename.endswith('.xlsx'):
+        return 'xlsx'
+    elif filename.endswith('.parquet'):
+        return 'parquet'
+    else:
+        raise ValueError("Unsupported file type.")
+
+def read_file(filename):
+    """Read the file based on its detected type."""
+    file_type = detect_file_type(filename)
+    
+    if file_type == 'csv':
+        return pd.read_csv(filename, low_memory=False)
+    elif file_type == 'xlsx':
+        return pd.read_excel(filename)
+    elif file_type == 'parquet':
+        return pd.read_parquet(filename)
+
+def anonymise_script(text_list:List[Element], anon_strat:str, nlp_analyser=None):
+
+    #print(df.shape)
+
+    #df_chosen_col_mask = (df[chosen_col].isnull()) | (df[chosen_col].str.strip() == "")
+    #print("Length of input series blank at start is: ", df_chosen_col_mask.value_counts())
+
+    # DataFrame to dict
+    df_dict = pd.DataFrame(data={"text":text_list}).to_dict(orient="list")   
+
+    if nlp_analyser:
+        analyzer = nlp_analyser
+    else:
+        analyzer = AnalyzerEngine()
+
+    # Add titles to analyzer list
+    titles_recognizer = PatternRecognizer(supported_entity="TITLE",
+                                      deny_list=["Mr","Mrs","Miss", "Ms", "mr", "mrs", "miss", "ms"])
+
+    analyzer.registry.add_recognizer(titles_recognizer)
+
+    batch_analyzer = BatchAnalyzerEngine(analyzer_engine=analyzer)
+
+    anonymizer = AnonymizerEngine()
+
+    batch_anonymizer = BatchAnonymizerEngine(anonymizer_engine = anonymizer)
+
+    print("Identifying personal data")
+    analyse_tic = time.perf_counter()
+    #analyzer_results = batch_analyzer.analyze_dict(df_dict, language="en")
+    analyzer_results = analyze_dict(batch_analyzer, df_dict, language="en")
+    #print(analyzer_results)
+    analyzer_results = list(analyzer_results)
+
+    analyse_toc = time.perf_counter()
+    analyse_time_out = f"Analysing the text took {analyse_toc - analyse_tic:0.1f} seconds."
+    print(analyse_time_out)
+
+    # Generate a 128-bit AES key. Then encode the key using base64 to get a string representation
+    key = secrets.token_bytes(16)  # 128 bits = 16 bytes 
+    key_string = base64.b64encode(key).decode('utf-8')
+
+    # Create faker function (note that it has to receive a value)
+    
+    fake = Faker("en_UK")
+
+    def fake_first_name(x):
+        return fake.first_name()
+
+    # Set up the anonymization configuration WITHOUT DATE_TIME
+    replace_config = eval('{"DEFAULT": OperatorConfig("replace")}')
+    redact_config = eval('{"DEFAULT": OperatorConfig("redact")}')
+    hash_config = eval('{"DEFAULT": OperatorConfig("hash")}')
+    mask_config = eval('{"DEFAULT": OperatorConfig("mask", {"masking_char":"*", "chars_to_mask":100, "from_end":True})}')
+    people_encrypt_config = eval('{"PERSON": OperatorConfig("encrypt", {"key": key_string})}') # The encryption is using AES cypher in CBC mode and requires a cryptographic key as an input for both the encryption and the decryption.
+    fake_first_name_config = eval('{"PERSON": OperatorConfig("custom", {"lambda": fake_first_name})}')
+
+
+    if anon_strat == "replace": chosen_mask_config = replace_config
+    if anon_strat == "redact": chosen_mask_config = redact_config
+    if anon_strat == "hash": chosen_mask_config = hash_config
+    if anon_strat == "mask": chosen_mask_config = mask_config
+    if anon_strat == "encrypt": chosen_mask_config = people_encrypt_config
+    elif anon_strat == "fake_first_name": chosen_mask_config = fake_first_name_config
+
+    # I think in general people will want to keep date / times - NOT FOR TOPIC MODELLING
+    #keep_date_config = eval('{"DATE_TIME": OperatorConfig("keep")}')
+
+    #combined_config = {**chosen_mask_config, **keep_date_config}
+    combined_config = {**chosen_mask_config}#, **keep_date_config}
+    combined_config
+
+    print("Anonymising personal data")
+    anonymizer_results = batch_anonymizer.anonymize_dict(analyzer_results, operators=combined_config)
+
+    #print(anonymizer_results)
+
+    scrubbed_df = pd.DataFrame(data={"text":anonymizer_results["text"]})
+
+    scrubbed_series = scrubbed_df["text"]
+
+    #print(scrubbed_series[0:6])
+
+    #print("Length of output series is: ", len(scrubbed_series))
+    #print("Length of input series at end is: ", len(df[chosen_col]))
+
+    
+    #scrubbed_values_mask = (scrubbed_series.isnull()) | (scrubbed_series.str.strip() == "")
+    #df_chosen_col_mask = (df[chosen_col].isnull()) | (df[chosen_col].str.strip() == "")
+
+    #print("Length of input series blank at end is: ", df_chosen_col_mask.value_counts())
+    #print("Length of output series blank is: ", scrubbed_values_mask.value_counts())
+    
+
+    # Create reporting message
+    out_message = "Successfully anonymised"
+    
+    if anon_strat == "encrypt":
+        out_message = out_message + ". Your decryption key is " + key_string + "."
+    
+    return scrubbed_series, out_message
+
+def do_anonymise(in_file:str, anon_strat:str, chosen_cols:List[str]):
+    
+    # Load file
+    
+    anon_df = pd.DataFrame()
+    
+    if in_file: 
+        for match_file in in_file:
+            match_temp_file = pd.read_csv(match_file.name, delimiter = ",", low_memory=False)#, encoding='cp1252')
+            anon_df = pd.concat([anon_df, match_temp_file])
+    
+    # Split dataframe to keep only selected columns
+    all_cols_original_order = list(anon_df.columns)
+    anon_df_part = anon_df[chosen_cols]
+    anon_df_remain = anon_df.drop(chosen_cols, axis = 1)
+    
+    # Anonymise the selected columns
+    anon_df_part_out, out_message = anonymise_script(anon_df_part, anon_strat)
+        
+    # Rejoin the dataframe together
+    anon_df_out = pd.concat([anon_df_part_out, anon_df_remain], axis = 1)
+    anon_df_out = anon_df_out[all_cols_original_order]
+    
+    # Export file
+    out_file_part = re.sub(r'\.csv', '', match_file.name)
+                
+    anon_export_file_name = out_file_part + "_anon_" + anon_strat + ".csv"
+    
+    anon_df_out.to_csv(anon_export_file_name, index = None)   
+    
+    return out_message, anon_export_file_name

tools/aws_functions.py

@@ -0,0 +1,164 @@
+from typing import Type
+import pandas as pd
+import boto3
+import tempfile
+import os
+
+PandasDataFrame = Type[pd.DataFrame]
+bucket_name = os.environ['DOCUMENT_REDACTION_BUCKET']
+
+try:
+    session = boto3.Session() # profile_name="default"
+except Exception as e:
+    print(e)
+
+# sts = session.client("sts")
+# Create a Session with the IAM role ARN
+# aws_role = os.environ['AWS_ROLE_DATA_TEXT_SEARCH']
+# response = sts.assume_role(
+#     RoleArn=aws_role,
+#     RoleSessionName="ecs-test-session"
+# )
+# print(response)
+
+
+def get_assumed_role_info():
+    sts = boto3.client('sts', region_name='eu-west-2', endpoint_url='https://sts.eu-west-2.amazonaws.com')
+    response = sts.get_caller_identity()
+
+    # Extract ARN of the assumed role
+    assumed_role_arn = response['Arn']
+    
+    # Extract the name of the assumed role from the ARN
+    assumed_role_name = assumed_role_arn.split('/')[-1]
+    
+    return assumed_role_arn, assumed_role_name
+
+try:
+    assumed_role_arn, assumed_role_name = get_assumed_role_info()
+
+    print("Assumed Role ARN:", assumed_role_arn)
+    print("Assumed Role Name:", assumed_role_name)
+except Exception as e:
+    print(e)
+
+# Download direct from S3 - requires login credentials
+def download_file_from_s3(bucket_name, key, local_file_path):
+
+    s3 = boto3.client('s3')
+    s3.download_file(bucket_name, key, local_file_path)
+    print(f"File downloaded from S3: s3://{bucket_name}/{key} to {local_file_path}")
+                         
+#download_file_from_s3(bucket_name, object_key, local_file_loc)
+
+def download_folder_from_s3(bucket_name, s3_folder, local_folder):
+    """
+    Download all files from an S3 folder to a local folder.
+    """
+    s3 = boto3.client('s3')
+
+    # List objects in the specified S3 folder
+    response = s3.list_objects_v2(Bucket=bucket_name, Prefix=s3_folder)
+
+    # Download each object
+    for obj in response.get('Contents', []):
+        # Extract object key and construct local file path
+        object_key = obj['Key']
+        local_file_path = os.path.join(local_folder, os.path.relpath(object_key, s3_folder))
+
+        # Create directories if necessary
+        os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
+
+        # Download the object
+        try:
+            s3.download_file(bucket_name, object_key, local_file_path)
+            print(f"Downloaded 's3://{bucket_name}/{object_key}' to '{local_file_path}'")
+        except Exception as e:
+            print(f"Error downloading 's3://{bucket_name}/{object_key}':", e)
+
+
+def download_files_from_s3(bucket_name, s3_folder, local_folder, filenames):
+    """
+    Download specific files from an S3 folder to a local folder.
+    """
+    s3 = boto3.client('s3')
+
+    print("Trying to download file: ", filenames)
+
+    if filenames == '*':
+        # List all objects in the S3 folder
+        print("Trying to download all files in AWS folder: ", s3_folder)
+        response = s3.list_objects_v2(Bucket=bucket_name, Prefix=s3_folder)
+
+        print("Found files in AWS folder: ", response.get('Contents', []))
+
+        filenames = [obj['Key'].split('/')[-1] for obj in response.get('Contents', [])]
+
+        print("Found filenames in AWS folder: ", filenames)
+
+    for filename in filenames:
+        object_key = os.path.join(s3_folder, filename)
+        local_file_path = os.path.join(local_folder, filename)
+
+        # Create directories if necessary
+        os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
+
+        # Download the object
+        try:
+            s3.download_file(bucket_name, object_key, local_file_path)
+            print(f"Downloaded 's3://{bucket_name}/{object_key}' to '{local_file_path}'")
+        except Exception as e:
+            print(f"Error downloading 's3://{bucket_name}/{object_key}':", e)
+
+
+
+def load_data_from_aws(in_aws_keyword_file, aws_password="", bucket_name=bucket_name):
+
+    temp_dir = tempfile.mkdtemp()
+    local_address_stub = temp_dir + '/doc-redaction/'
+    files = []
+
+    if not 'LAMBETH_BOROUGH_PLAN_PASSWORD' in os.environ:
+        out_message = "Can't verify password for dataset access. Do you have a valid AWS connection? Data not loaded."
+        return files, out_message
+    
+    if aws_password:
+        if "Lambeth borough plan" in in_aws_keyword_file and aws_password == os.environ['LAMBETH_BOROUGH_PLAN_PASSWORD']:
+
+            s3_folder_stub = 'example-data/lambeth-borough-plan/latest/'
+
+            local_folder_path = local_address_stub                
+
+            # Check if folder exists
+            if not os.path.exists(local_folder_path):
+                print(f"Folder {local_folder_path} does not exist! Making folder.")
+
+                os.mkdir(local_folder_path)
+
+            # Check if folder is empty
+            if len(os.listdir(local_folder_path)) == 0:
+                print(f"Folder {local_folder_path} is empty")
+                # Download data
+                download_files_from_s3(bucket_name, s3_folder_stub, local_folder_path, filenames='*')
+
+                print("AWS data downloaded")
+
+            else:
+                print(f"Folder {local_folder_path} is not empty")
+
+            #files = os.listdir(local_folder_stub)
+            #print(files)
+
+            files = [os.path.join(local_folder_path, f) for f in os.listdir(local_folder_path) if os.path.isfile(os.path.join(local_folder_path, f))]
+
+            out_message = "Data successfully loaded from AWS"
+            print(out_message)
+
+        else:
+            out_message = "Data not loaded from AWS"
+            print(out_message)
+    else:
+        out_message = "No password provided. Please ask the data team for access if you need this."
+        print(out_message)
+
+    return files, out_message

tools/clean_funcs.py

@@ -0,0 +1,194 @@
+import re
+import string
+import polars as pl
+import gradio as gr
+import time
+from datetime import datetime
+import tools.anonymiser as anon
+from unstructured.staging.base import convert_to_dataframe
+
+from typing import List
+from unstructured.documents.elements import Element
+
+from tools.unstructured_funcs import export_elements_as_table_to_file
+
+today_rev = datetime.now().strftime("%Y%m%d")
+
+chosen_redact_entities = ["TITLES", "PERSON", "PHONE_NUMBER", "EMAIL_ADDRESS", "STREETNAME", "UKPOSTCODE"]
+full_entity_list = ["TITLES", "PERSON", "PHONE_NUMBER", "EMAIL_ADDRESS", "STREETNAME", "UKPOSTCODE", 'CREDIT_CARD', 'CRYPTO', 'DATE_TIME', 'IBAN_CODE', 'IP_ADDRESS', 'NRP', 'LOCATION', 'MEDICAL_LICENSE', 'URL', 'UK_NHS']
+
+# Adding custom words to the stopwords
+custom_words = []
+my_stop_words = custom_words
+
+# #### Some of my cleaning functions
+html_pattern_regex = r'<.*?>|&([a-z0-9]+|#[0-9]{1,6}|#x[0-9a-f]{1,6});|\xa0|&nbsp;'
+html_start_pattern_end_dots_regex = r'<(.*?)\.\.'
+email_pattern_regex = r'\S*@\S*\s?'
+num_pattern_regex = r'[0-9]+'
+nums_two_more_regex = r'\b[0-9]{2,}\b|\b[0-9]+\s[0-9]+\b'
+postcode_pattern_regex = r'(\b(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]? ?[0-9][A-Z]{2})|((GIR ?0A{2})\b$)|(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]? ?[0-9]{1}?)$)|(\b(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]?)\b$)'
+multiple_spaces_regex = r'\s{2,}'
+
+def pre_clean(data:List[Element], in_colnames:str, custom_regex:List[str], clean_text:str, data_file_name_no_ext:str="combined_elements", anonymise_drop:List[str]="No", anon_strat:str = "redact", anon_entities:List[str]=chosen_redact_entities, progress=gr.Progress(track_tqdm=True)):
+    '''
+    Clean open text in tabular format with custom regex or anonymisation.
+    '''
+    
+    output_text = ""
+    output_list = []
+
+    progress(0, desc = "Cleaning data")
+
+    if not in_colnames:
+        error_message = "Please enter one column name to use for cleaning and finding topics."
+        print(error_message)
+        return error_message, None, data_file_name_no_ext, None, None
+
+    all_tic = time.perf_counter()
+
+    output_list = []
+    #file_list = [string.name for string in in_files]
+
+    in_colnames_list_first = in_colnames[0]
+
+    if clean_text == "Yes":
+        clean_tic = time.perf_counter()
+        print("Starting data clean.")
+
+        for element in data:
+            if not custom_regex.empty:
+                cleaned_data = initial_clean([element.text], custom_regex.iloc[:, 0].to_list())
+            else:
+                cleaned_data = initial_clean([element.text], [])
+
+            element.text = cleaned_data[0]
+            print(element.text)
+
+        clean_toc = time.perf_counter()
+        clean_time_out = f"Cleaning the text took {clean_toc - clean_tic:0.1f} seconds."
+        print(clean_time_out)
+
+    if anonymise_drop == "Yes":
+        progress(0.6, desc= "Anonymising data")
+
+        data_file_name_no_ext = data_file_name_no_ext + "_anon"
+
+        anon_tic = time.perf_counter()
+
+        data_list = []
+
+        for element in data:
+            data_list.append(element.text)
+    
+        data_anon_col, anonymisation_success = anon.anonymise_script(data_list, anon_strat=anon_strat)
+
+        for i, element in enumerate(data):
+            element.text = data_anon_col[i]
+
+        print(anonymisation_success)
+
+        anon_toc = time.perf_counter()
+        time_out = f"Anonymising text took {anon_toc - anon_tic:0.1f} seconds"
+
+    alt_out_message, out_files, output_file_base = export_elements_as_table_to_file(data, data_file_name_no_ext, file_name_suffix="_clean")
+
+    all_toc = time.perf_counter()
+    time_out = f"All processes took {all_toc - all_tic:0.1f} seconds."
+    print(time_out)
+
+    output_text = "Data clean completed."
+    
+    return output_text, out_files, data, output_file_base
+
+
+def initial_clean(texts, custom_regex, progress=gr.Progress()):
+    #texts = pl.Series(texts).str.strip_chars()
+    #text = texts.str.replace_all(html_pattern_regex, ' ')
+    #text = text.str.replace_all(html_start_pattern_end_dots_regex, ' ')
+    #text = text.str.replace_all(email_pattern_regex, ' ')
+    #text = text.str.replace_all(nums_two_more_regex, ' ')
+    #text = text.str.replace_all(postcode_pattern_regex, ' ')
+
+    texts = pl.Series(texts)
+
+    # Allow for custom regex patterns to be removed
+    if len(custom_regex) > 0:
+        for pattern in custom_regex:
+            raw_string_pattern = rf"{pattern}"  # Case-insensitive regex
+            #print(f"Removing regex pattern: {raw_string_pattern}")
+            text = text.str.replace_all(raw_string_pattern, " ")
+            #print("Text without pattern: ", text[0])
+    
+
+    #text = text.str.replace_all(multiple_spaces_regex, ' ')
+
+    text = text.to_list()
+    
+    return text
+
+def remove_hyphens(text_text):
+    return re.sub(r'(\w+)-(\w+)-?(\w)?', r'\1 \2 \3', text_text)
+
+
+def remove_characters_after_tokenization(tokens):
+    pattern = re.compile('[{}]'.format(re.escape(string.punctuation)))
+    filtered_tokens = filter(None, [pattern.sub('', token) for token in tokens])
+    return filtered_tokens
+
+def convert_to_lowercase(tokens):
+    return [token.lower() for token in tokens if token.isalpha()]
+
+def remove_short_tokens(tokens):
+    return [token for token in tokens if len(token) > 3]
+
+
+def remove_dups_text(data_samples_ready, data_samples_clean, data_samples):
+   # Identify duplicates in the data: https://stackoverflow.com/questions/44191465/efficiently-identify-duplicates-in-large-list-500-000
+    # Only identifies the second duplicate
+
+    seen = set()
+    dups = []
+
+    for i, doi in enumerate(data_samples_ready):
+        if doi not in seen:
+            seen.add(doi)
+        else:
+            dups.append(i) 
+    #data_samples_ready[dupes[0:]]
+    
+    # To see a specific duplicated value you know the position of
+    #matching = [s for s in data_samples_ready if data_samples_ready[83] in s]
+    #matching
+    
+    # Remove duplicates only (keep first instance)
+    #data_samples_ready = list( dict.fromkeys(data_samples_ready) ) # This way would keep one version of the duplicates
+    
+    ### Remove all duplicates including original instance
+    
+    # Identify ALL duplicates including initial values
+    # https://stackoverflow.com/questions/11236006/identify-duplicate-values-in-a-list-in-python
+
+    from collections import defaultdict
+    D = defaultdict(list)
+    for i,item in enumerate(data_samples_ready):
+        D[item].append(i)
+    D = {k:v for k,v in D.items() if len(v)>1}
+    
+    # https://stackoverflow.com/questions/952914/how-to-make-a-flat-list-out-of-a-list-of-lists
+    L = list(D.values())
+    flat_list_dups = [item for sublist in L for item in sublist]
+
+    # https://stackoverflow.com/questions/11303225/how-to-remove-multiple-indexes-from-a-list-at-the-same-time
+    for index in sorted(flat_list_dups, reverse=True):
+        del data_samples_ready[index]
+        del data_samples_clean[index]
+        del data_samples[index]
+    
+    # Remove blanks
+    data_samples_ready = [i for i in data_samples_ready if i]
+    data_samples_clean = [i for i in data_samples_clean if i]
+    data_samples = [i for i in data_samples if i]
+    
+    return data_samples_ready, data_samples_clean, flat_list_dups, data_samples
+

tools/file_conversion.py

@@ -0,0 +1,140 @@
+from pdf2image import convert_from_path, pdfinfo_from_path
+from tools.helper_functions import get_file_path_end
+from PIL import Image
+import os
+from gradio import Progress
+from typing import List
+
+def is_pdf_or_image(filename):
+    """
+    Check if a file name is a PDF or an image file.
+
+    Args:
+        filename (str): The name of the file.
+
+    Returns:
+        bool: True if the file name ends with ".pdf", ".jpg", or ".png", False otherwise.
+    """
+    if filename.lower().endswith(".pdf") or filename.lower().endswith(".jpg") or filename.lower().endswith(".jpeg") or filename.lower().endswith(".png"):
+        output = True
+    else:
+        output = False
+    return output
+
+def is_pdf(filename):
+    """
+    Check if a file name is a PDF.
+
+    Args:
+        filename (str): The name of the file.
+
+    Returns:
+        bool: True if the file name ends with ".pdf", False otherwise.
+    """
+    return filename.lower().endswith(".pdf")
+
+# %%
+## Convert pdf to image if necessary
+
+def convert_pdf_to_images(pdf_path:str, progress=Progress(track_tqdm=True)):
+
+    # Get the number of pages in the PDF
+    page_count = pdfinfo_from_path(pdf_path)['Pages']
+    print("Number of pages in PDF: ", str(page_count))
+
+    images = []
+
+    # Open the PDF file
+    for page_num in progress.tqdm(range(0,page_count), total=page_count, unit="pages", desc="Converting pages"):
+        
+        print("Current page: ", str(page_num))
+
+        # Convert one page to image
+        image = convert_from_path(pdf_path, first_page=page_num+1, last_page=page_num+1)
+        
+        # If no images are returned, break the loop
+        if not image:
+            break
+
+        images.extend(image)
+
+    print("PDF has been converted to images.")
+
+    return images
+
+
+# %% Function to take in a file path, decide if it is an image or pdf, then process appropriately.
+def process_file(file_path):
+    # Get the file extension
+    file_extension = os.path.splitext(file_path)[1].lower()
+
+    # Check if the file is an image type
+    if file_extension in ['.jpg', '.jpeg', '.png']:
+        print(f"{file_path} is an image file.")
+        # Perform image processing here
+        out_path = [Image.open(file_path)]
+
+    # Check if the file is a PDF
+    elif file_extension == '.pdf':
+        print(f"{file_path} is a PDF file. Converting to image set")
+        # Run your function for processing PDF files here
+        out_path = convert_pdf_to_images(file_path)
+
+    else:
+        print(f"{file_path} is not an image or PDF file.")
+        out_path = ['']
+
+    return out_path
+
+def prepare_image_or_text_pdf(file_path:str, in_redact_method:str, in_allow_list:List[List[str]]=None):
+
+    out_message = ''
+    out_file_paths = []
+
+    in_allow_list_flat = [item for sublist in in_allow_list for item in sublist]
+
+    if file_path:
+        file_path_without_ext = get_file_path_end(file_path)
+    else:
+        out_message = "No file selected"
+        print(out_message)
+        return out_message, out_file_paths
+
+    if in_redact_method == "Image analysis":
+        # Analyse and redact image-based pdf or image
+        if is_pdf_or_image(file_path) == False:
+            return "Please upload a PDF file or image file (JPG, PNG) for image analysis.", None
+        
+        out_file_path = process_file(file_path)
+
+    elif in_redact_method == "Text analysis":
+        if is_pdf(file_path) == False:
+            return "Please upload a PDF file for text analysis.", None
+        
+        out_file_path = file_path
+    
+    return out_message, out_file_path
+
+
+def convert_text_pdf_to_img_pdf(in_file_path:str, out_text_file_path:List[str]):
+    file_path_without_ext = get_file_path_end(in_file_path)
+
+    out_file_paths = out_text_file_path
+
+    # Convert annotated text pdf back to image to give genuine redactions
+    print("Creating image version of results")
+    pdf_text_image_paths = process_file(out_text_file_path[0])
+    out_text_image_file_path = "output/" + file_path_without_ext + "_result_as_text_back_to_img.pdf"
+    pdf_text_image_paths[0].save(out_text_image_file_path, "PDF" ,resolution=100.0, save_all=True, append_images=pdf_text_image_paths[1:])
+
+    out_file_paths.append(out_text_image_file_path)
+
+    out_message = "Image-based PDF successfully redacted and saved to text-based annotated file, and image-based file."
+
+    return out_message, out_file_paths
+
+        
+
+        
+    
+

tools/file_redaction.py

@@ -0,0 +1,236 @@
+from PIL import Image
+from typing import List
+import pandas as pd
+from presidio_image_redactor import ImageRedactorEngine, ImageAnalyzerEngine
+from pdfminer.high_level import extract_pages
+from tools.file_conversion import process_file
+from pdfminer.layout import LTTextContainer, LTChar, LTTextLine, LTAnno
+from pikepdf import Pdf, Dictionary, Name
+from gradio import Progress
+import time
+
+from tools.load_spacy_model_custom_recognisers import nlp_analyser, score_threshold
+from tools.helper_functions import get_file_path_end
+from tools.file_conversion import process_file, is_pdf, is_pdf_or_image
+import gradio as gr
+
+def choose_and_run_redactor(file_path:str, image_paths:List[str], language:str, chosen_redact_entities:List[str], in_redact_method:str, in_allow_list:List[List[str]]=None, progress=gr.Progress(track_tqdm=True)):
+
+    tic = time.perf_counter()
+
+    out_message = ''
+    out_file_paths = []
+
+    if in_allow_list:
+        in_allow_list_flat = [item for sublist in in_allow_list for item in sublist]
+
+    if file_path:
+         file_path_without_ext = get_file_path_end(file_path)
+    else:
+         out_message = "No file selected"
+         print(out_message)
+         return out_message, out_file_paths
+
+    if in_redact_method == "Image analysis":
+        # Analyse and redact image-based pdf or image
+        # if is_pdf_or_image(file_path) == False:
+        #     return "Please upload a PDF file or image file (JPG, PNG) for image analysis.", None
+
+        pdf_images = redact_image_pdf(file_path, image_paths, language, chosen_redact_entities, in_allow_list_flat)
+        out_image_file_path = "output/" + file_path_without_ext + "_result_as_img.pdf"
+        pdf_images[0].save(out_image_file_path, "PDF" ,resolution=100.0, save_all=True, append_images=pdf_images[1:])
+
+        out_file_paths.append(out_image_file_path)
+        out_message = "Image-based PDF successfully redacted and saved to file."
+
+    elif in_redact_method == "Text analysis":
+        if is_pdf(file_path) == False:
+            return "Please upload a PDF file for text analysis.", None
+
+        # Analyse text-based pdf
+        pdf_text = redact_text_pdf(file_path, language, chosen_redact_entities, in_allow_list_flat)
+        out_text_file_path = "output/" + file_path_without_ext + "_result_as_text.pdf"
+        pdf_text.save(out_text_file_path)
+
+        out_file_paths.append(out_text_file_path)
+
+        out_message = "Text-based PDF successfully redacted and saved to file."
+        
+    else:
+        out_message = "No redaction method selected"
+        print(out_message)
+        return out_message, out_file_paths
+    
+    toc = time.perf_counter()
+    out_time = f"Time taken: {toc - tic:0.1f} seconds."
+    print(out_time)
+
+    out_message = out_message + "\n\n" + out_time
+
+    return out_message, out_file_paths, out_file_paths
+
+
+def redact_image_pdf(file_path:str, image_paths:List[str], language:str, chosen_redact_entities:List[str], allow_list:List[str]=None, progress=Progress(track_tqdm=True)):
+    '''
+    take an path for an image of a document, then run this image through the Presidio ImageAnalyzer to get a redacted page back
+    '''
+
+    if not image_paths:
+
+        out_message = "PDF does not exist as images. Converting pages to image"
+        print(out_message)
+        progress(0, desc=out_message)
+
+        image_paths = process_file(file_path)
+
+    # Create a new PDF
+    #pdf = pikepdf.new()
+
+    images = []
+    number_of_pages = len(image_paths)
+
+    out_message = "Redacting pages"
+    print(out_message)
+    progress(0.1, desc=out_message)
+
+    for i in progress.tqdm(range(0,number_of_pages), total=number_of_pages, unit="pages", desc="Redacting pages"):
+
+        print("Redacting page ", str(i + 1))
+
+        # Get the image to redact using PIL lib (pillow)
+        image = image_paths[i] #Image.open(image_paths[i])
+
+        # %%
+        image_analyser = ImageAnalyzerEngine(nlp_analyser)
+        engine = ImageRedactorEngine(image_analyser)
+
+        if language == 'en':
+            ocr_lang = 'eng'
+        else: ocr_lang = language
+
+        # %%
+        # Redact the image with pink color
+        redacted_image = engine.redact(image,
+            fill=(0, 0, 0),
+            ocr_kwargs={"lang": ocr_lang},
+            allow_list=allow_list,
+            ad_hoc_recognizers= None,
+            **{
+                "language": language,
+                "entities": chosen_redact_entities,
+                "score_threshold": score_threshold
+            },
+            )
+        
+        images.append(redacted_image)
+
+    return images
+
+def redact_text_pdf(filename:str, language:str, chosen_redact_entities:List[str], allow_list:List[str]=None, progress=Progress(track_tqdm=True)):
+    '''
+    Redact chosen entities from a pdf that is made up of multiple pages that are not images.
+    '''
+    
+    combined_analyzer_results = []
+    analyser_explanations = []
+    annotations_all_pages = []
+    analyzed_bounding_boxes_df = pd.DataFrame()
+
+    pdf = Pdf.open(filename)
+
+    page_num = 0
+
+    for page in progress.tqdm(pdf.pages, total=len(pdf.pages), unit="pages", desc="Redacting pages"):
+
+
+        print("Page number is: ", page_num)
+
+        annotations_on_page = []
+        analyzed_bounding_boxes = []
+
+        for page_layout in extract_pages(filename, page_numbers = [page_num], maxpages=1):
+            analyzer_results = []
+
+            for text_container in page_layout:
+                if isinstance(text_container, LTTextContainer):
+                    text_to_analyze = text_container.get_text()
+
+                    analyzer_results = []
+                    characters = []
+
+                    analyzer_results = nlp_analyser.analyze(text=text_to_analyze,
+                                                            language=language, 
+                                                            entities=chosen_redact_entities,
+                                                            score_threshold=score_threshold,
+                                                            return_decision_process=False,
+                                                            allow_list=allow_list)
+
+                        #if analyzer_results:
+                        #    pass
+                        #explanation = analyzer_results[0].analysis_explanation.to_dict()
+                        #analyser_explanations.append(explanation)
+                    characters = [char                    # This is what we want to include in the list
+                            for line in text_container          # Loop through each line in text_container
+                            if isinstance(line, LTTextLine)    # Check if the line is an instance of LTTextLine
+                            for char in line]                   # Loop through each character in the line
+                            #if isinstance(char, LTChar)]  # Check if the character is not an instance of LTAnno #isinstance(char, LTChar) or
+                    
+                    # If any results found
+                    print(analyzer_results)
+
+                    if len(analyzer_results) > 0 and len(characters) > 0:
+                        analyzed_bounding_boxes.extend({"boundingBox": char.bbox, "result": result} for result in analyzer_results for char in characters[result.start:result.end] if isinstance(char, LTChar))
+                        combined_analyzer_results.extend(analyzer_results)
+
+            if len(analyzer_results) > 0:
+                # Create summary df of annotations to be made
+                analyzed_bounding_boxes_df_new = pd.DataFrame(analyzed_bounding_boxes)
+                analyzed_bounding_boxes_df_text = analyzed_bounding_boxes_df_new['result'].astype(str).str.split(",",expand=True).replace(".*: ", "", regex=True)
+                analyzed_bounding_boxes_df_text.columns = ["type", "start", "end", "score"]
+                analyzed_bounding_boxes_df_new = pd.concat([analyzed_bounding_boxes_df_new, analyzed_bounding_boxes_df_text], axis = 1)
+                analyzed_bounding_boxes_df_new['page'] = page_num + 1
+                analyzed_bounding_boxes_df = pd.concat([analyzed_bounding_boxes_df, analyzed_bounding_boxes_df_new], axis = 0)
+
+            for analyzed_bounding_box in analyzed_bounding_boxes:
+                bounding_box = analyzed_bounding_box["boundingBox"]
+                annotation = Dictionary(
+                    Type=Name.Annot,
+                    Subtype=Name.Highlight,
+                    QuadPoints=[bounding_box[0], bounding_box[3], bounding_box[2], bounding_box[3], bounding_box[0], bounding_box[1], bounding_box[2], bounding_box[1]],
+                    Rect=[bounding_box[0], bounding_box[1], bounding_box[2], bounding_box[3]],
+                    C=[0, 0, 0],
+                    CA=1, # Transparency
+                    T=analyzed_bounding_box["result"].entity_type
+                )
+                annotations_on_page.append(annotation)           
+
+            annotations_all_pages.extend([annotations_on_page])
+ 
+            print("For page number: ", page_num, " there are ", len(annotations_all_pages[page_num]), " annotations")
+            page.Annots = pdf.make_indirect(annotations_on_page)
+
+            page_num += 1
+
+        # Extracting data from dictionaries
+        # extracted_data = []
+        # for item in annotations_all_pages:
+        #     temp_dict = {}
+        #     #print(item)
+        #     for key, value in item.items():
+        #         if isinstance(value, Decimal):
+        #             temp_dict[key] = float(value)
+        #         elif isinstance(value, list):
+        #             temp_dict[key] = [float(v) if isinstance(v, Decimal) else v for v in value]
+        #         else:
+        #             temp_dict[key] = value
+        #     extracted_data.append(temp_dict)
+
+        # Creating DataFrame
+        # annotations_out = pd.DataFrame(extracted_data)
+        #print(df)
+
+        #annotations_out.to_csv("examples/annotations.csv")
+    
+    analyzed_bounding_boxes_df.to_csv("output/annotations_made.csv")
+
+    return pdf

tools/helper_functions.py

@@ -0,0 +1,126 @@
+import os
+import pandas as pd
+import gzip
+import pickle
+import numpy as np
+
+def get_file_path_end(file_path):
+    # First, get the basename of the file (e.g., "example.txt" from "/path/to/example.txt")
+    basename = os.path.basename(file_path)
+    
+    # Then, split the basename and its extension and return only the basename without the extension
+    filename_without_extension, _ = os.path.splitext(basename)
+
+    #print(filename_without_extension)
+    
+    return filename_without_extension
+
+def get_file_path_end_with_ext(file_path):
+    # First, get the basename of the file (e.g., "example.txt" from "/path/to/example.txt")
+    basename = os.path.basename(file_path)
+    
+    return basename
+
+def ensure_output_folder_exists():
+    """Checks if the 'output/' folder exists, creates it if not."""
+
+    folder_name = "output/"
+
+    if not os.path.exists(folder_name):
+        # Create the folder if it doesn't exist
+        os.makedirs(folder_name)
+        print(f"Created the 'output/' folder.")
+    else:
+        print(f"The 'output/' folder already exists.")
+
+def detect_file_type(filename):
+    """Detect the file type based on its extension."""
+    if (filename.endswith('.csv')) | (filename.endswith('.csv.gz')) | (filename.endswith('.zip')):
+        return 'csv'
+    elif filename.endswith('.xlsx'):
+        return 'xlsx'
+    elif filename.endswith('.parquet'):
+        return 'parquet'
+    elif filename.endswith('.pkl.gz'):
+        return 'pkl.gz'
+    elif filename.endswith('.pkl'):
+        return 'pkl'
+    elif filename.endswith('.npz'):
+        return 'npz'
+    else:
+        raise ValueError("Unsupported file type.")
+
+
+def read_file(filename, headers=0):
+    """Read the file based on its detected type."""
+    file_type = detect_file_type(filename)
+        
+    print("Loading in file")
+
+    if file_type == 'csv':
+        file = pd.read_csv(filename, low_memory=False, header=headers)#.reset_index().drop(["index", "Unnamed: 0"], axis=1, errors="ignore")
+    elif file_type == 'xlsx':
+        file = pd.read_excel(filename, header=headers)#.reset_index().drop(["index", "Unnamed: 0"], axis=1, errors="ignore")
+    elif file_type == 'parquet':
+        file = pd.read_parquet(filename, header = headers)#.reset_index().drop(["index", "Unnamed: 0"], axis=1, errors="ignore")
+    elif file_type == 'pkl.gz':
+        with gzip.open(filename, 'rb') as file:
+            file = pickle.load(file)
+            #file = pd.read_pickle(filename)
+    elif file_type == 'npz':
+        file = np.load(filename)['arr_0']
+
+        # If embedding files have 'super_compress' in the title, they have been multiplied by 100 before save
+        if "compress" in filename:
+            file /= 100
+
+    print("File load complete")
+
+    return file
+
+# Following function is only relevant for locally-created executable files based on this app (when using pyinstaller it creates a _internal folder that contains tesseract and poppler. These need to be added to the system path to enable the app to run)
+def add_folder_to_path(folder_path: str):
+    '''
+    Check if a folder exists on your system. If so, get the absolute path and then add it to the system Path variable if it doesn't already exist.
+    '''
+
+    if os.path.exists(folder_path) and os.path.isdir(folder_path):
+        print(folder_path, "folder exists.")
+
+        # Resolve relative path to absolute path
+        absolute_path = os.path.abspath(folder_path)
+
+        current_path = os.environ['PATH']
+        if absolute_path not in current_path.split(os.pathsep):
+            full_path_extension = absolute_path + os.pathsep + current_path
+            os.environ['PATH'] = full_path_extension
+            print(f"Updated PATH with: ", full_path_extension)
+        else:
+            print(f"Directory {folder_path} already exists in PATH.")
+    else:
+        print(f"Folder not found at {folder_path} - not added to PATH")
+
+def custom_regex_load(in_file, headers = None):
+    '''
+    When file is loaded, update the column dropdown choices and write to relevant data states.
+    '''
+
+    custom_regex = pd.DataFrame()
+
+    file_list = [string.name for string in in_file]
+
+    regex_file_names = [string for string in file_list if "csv" in string.lower()]
+    if regex_file_names:
+        regex_file_name = regex_file_names[0]
+        custom_regex = read_file(regex_file_name, headers)
+        #regex_file_name_no_ext = get_file_path_end(regex_file_name)
+
+        output_text = "Data file loaded."
+        print(output_text)
+    else:
+        error = "No regex file provided."
+        print(error)
+        output_text = error
+        return error, custom_regex
+       
+    return output_text, custom_regex

tools/load_spacy_model_custom_recognisers.py

@@ -0,0 +1,168 @@
+# %%
+from typing import List
+from presidio_analyzer import AnalyzerEngine, PatternRecognizer, EntityRecognizer, Pattern, RecognizerResult
+from presidio_analyzer.nlp_engine import SpacyNlpEngine, NlpArtifacts
+import spacy
+spacy.prefer_gpu()
+from spacy.cli.download import download
+import re
+
+# %%
+model_name = "en_core_web_sm" #"en_core_web_trf"
+score_threshold = 0.001
+
+# %% [markdown]
+# #### Custom recognisers
+
+# %%
+# Custom title recogniser
+import re
+titles_list = ["Sir", "Ma'am", "Madam", "Mr", "Mr.", "Mrs", "Mrs.", "Ms", "Ms.", "Miss", "Dr", "Dr.", "Professor"]
+titles_regex = '\\b' + ' \\b|\\b'.join(rf"{re.escape(street_type)}" for street_type in titles_list) + ' \\b'
+titles_pattern = Pattern(name="titles_pattern",regex=titles_regex, score = 1)
+titles_recogniser = PatternRecognizer(supported_entity="TITLES", patterns = [titles_pattern])
+
+# %%
+# Custom postcode recogniser
+
+# Define the regex pattern in a Presidio `Pattern` object:
+ukpostcode_pattern = Pattern(name="ukpostcode_pattern",regex="\\b(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]? ?[0-9][A-Z]{2}|GIR ?0A{2})\\b|(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]? ?[0-9]{1}?)$|\\b(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]?)\\b", score = 1)
+
+# Define the recognizer with one or more patterns
+ukpostcode_recogniser = PatternRecognizer(supported_entity="UKPOSTCODE", patterns = [ukpostcode_pattern])
+
+# %%
+# Examples for testing
+
+#text = "I live in 510 Broad st SE5 9NG ."
+
+#numbers_result = ukpostcode_recogniser.analyze(text=text, entities=["UKPOSTCODE"])
+#print("Result:")
+#print(numbers_result)
+
+# %%
+def extract_street_name(text:str) -> str:
+    """
+    Extracts the street name and preceding word (that should contain at least one number) from the given text.
+
+    """    
+   
+    street_types = [
+    'Street', 'St', 'Boulevard', 'Blvd', 'Highway', 'Hwy', 'Broadway', 'Freeway',
+    'Causeway', 'Cswy', 'Expressway', 'Way', 'Walk', 'Lane', 'Ln', 'Road', 'Rd',
+    'Avenue', 'Ave', 'Circle', 'Cir', 'Cove', 'Cv', 'Drive', 'Dr', 'Parkway', 'Pkwy',
+    'Park', 'Court', 'Ct', 'Square', 'Sq', 'Loop', 'Place', 'Pl', 'Parade', 'Estate',
+    'Alley', 'Arcade', 'Avenue', 'Ave', 'Bay', 'Bend', 'Brae', 'Byway', 'Close', 'Corner', 'Cove',
+    'Crescent', 'Cres', 'Cul-de-sac', 'Dell', 'Drive', 'Dr', 'Esplanade', 'Glen', 'Green', 'Grove', 'Heights', 'Hts',
+    'Mews', 'Parade', 'Path', 'Piazza', 'Promenade', 'Quay', 'Ridge', 'Row', 'Terrace', 'Ter', 'Track', 'Trail', 'View', 'Villas',
+    'Marsh', 'Embankment', 'Cut', 'Hill', 'Passage', 'Rise', 'Vale', 'Side'
+    ]
+
+    # Dynamically construct the regex pattern with all possible street types
+    street_types_pattern = '|'.join(rf"{re.escape(street_type)}" for street_type in street_types)
+
+    # The overall regex pattern to capture the street name and preceding word(s)
+
+    pattern = rf'(?P<preceding_word>\w*\d\w*)\s*'
+    pattern += rf'(?P<street_name>\w+\s*\b(?:{street_types_pattern})\b)'
+
+    # Find all matches in text
+    matches = re.finditer(pattern, text, re.IGNORECASE)
+
+    start_positions = []
+    end_positions = []
+
+    for match in matches:
+        preceding_word = match.group('preceding_word').strip()
+        street_name = match.group('street_name').strip()
+        start_pos = match.start()
+        end_pos = match.end()
+        print(f"Start: {start_pos}, End: {end_pos}")
+        print(f"Preceding words: {preceding_word}")
+        print(f"Street name: {street_name}")
+        print()
+
+        start_positions.append(start_pos)
+        end_positions.append(end_pos)
+
+    return start_positions, end_positions
+
+
+# %%
+# Some examples for testing
+
+#text = "1234 Main Street, 5678 Oak Rd, 9ABC Elm Blvd, 42 Eagle st."
+#text = "Roberto lives in Five 10 Broad st in Oregon"
+#text = "Roberto lives in 55 Oregon Square"
+#text = "There is 51a no way I will do that"
+#text = "I am writing to apply for"
+
+#extract_street_name(text)
+
+# %%
+class StreetNameRecognizer(EntityRecognizer):
+
+    def load(self) -> None:
+        """No loading is required."""
+        pass
+
+    def analyze(self, text: str, entities: List[str], nlp_artifacts: NlpArtifacts) -> List[RecognizerResult]:
+        """
+        Logic for detecting a specific PII
+        """
+
+        start_pos, end_pos = extract_street_name(text)
+
+        results = []
+
+        for i in range(0, len(start_pos)):
+
+            result = RecognizerResult(
+                        entity_type="STREETNAME",
+                        start = start_pos[i],
+                        end = end_pos[i],
+                        score= 1
+                    )
+        
+            results.append(result)
+        
+        return results
+    
+street_recogniser = StreetNameRecognizer(supported_entities=["STREETNAME"])
+
+# %%
+# Create a class inheriting from SpacyNlpEngine
+class LoadedSpacyNlpEngine(SpacyNlpEngine):
+    def __init__(self, loaded_spacy_model):
+        super().__init__()
+        self.nlp = {"en": loaded_spacy_model}
+
+# %%
+# Load spacy model
+try:
+	import en_core_web_lg
+	nlp = en_core_web_lg.load()
+	print("Successfully imported spaCy model")
+
+except:
+	download("en_core_web_lg")
+	nlp = spacy.load("en_core_web_lg")
+	print("Successfully downloaded and imported spaCy model")
+
+# Pass the loaded model to the new LoadedSpacyNlpEngine
+loaded_nlp_engine = LoadedSpacyNlpEngine(loaded_spacy_model = nlp)
+
+
+
+# %%
+nlp_analyser = AnalyzerEngine(nlp_engine=loaded_nlp_engine,
+                default_score_threshold=score_threshold,
+                supported_languages=["en"],
+                log_decision_process=True,
+                )
+
+# %%
+nlp_analyser.registry.add_recognizer(street_recogniser)
+nlp_analyser.registry.add_recognizer(ukpostcode_recogniser)
+nlp_analyser.registry.add_recognizer(titles_recogniser)
+

tools/presidio_analyzer_custom.py

@@ -0,0 +1,114 @@
+import gradio as gr
+from typing import List, Iterable, Dict, Union, Any, Optional, Iterator, Tuple
+from tqdm import tqdm
+
+from presidio_analyzer import DictAnalyzerResult, RecognizerResult, AnalyzerEngine
+from presidio_analyzer.nlp_engine import NlpArtifacts
+
+def analyze_iterator_custom(
+        self,
+        texts: Iterable[Union[str, bool, float, int]],
+        language: str,
+        list_length:int,
+        progress=gr.Progress(),
+        **kwargs,
+    ) -> List[List[RecognizerResult]]:
+        """
+        Analyze an iterable of strings.
+
+        :param texts: An list containing strings to be analyzed.
+        :param language: Input language
+        :param list_length: Length of the input list.
+        :param kwargs: Additional parameters for the `AnalyzerEngine.analyze` method.
+        """
+
+        # validate types
+        texts = self._validate_types(texts)
+
+        # Process the texts as batch for improved performance
+        nlp_artifacts_batch: Iterator[
+            Tuple[str, NlpArtifacts]
+        ] = self.analyzer_engine.nlp_engine.process_batch(
+            texts=texts, language=language
+        )
+
+        
+
+        list_results = []
+        for text, nlp_artifacts in progress.tqdm(nlp_artifacts_batch, total = list_length, desc = "Analysing text for personal information", unit = "rows"):
+            results = self.analyzer_engine.analyze(
+                text=str(text), nlp_artifacts=nlp_artifacts, language=language, **kwargs
+            )
+
+            list_results.append(results)
+
+        return list_results
+
+def analyze_dict(
+        self,
+        input_dict: Dict[str, Union[Any, Iterable[Any]]],
+        language: str,
+        keys_to_skip: Optional[List[str]] = None,
+        **kwargs,
+    ) -> Iterator[DictAnalyzerResult]:
+        """
+        Analyze a dictionary of keys (strings) and values/iterable of values.
+
+        Non-string values are returned as is.
+
+        :param input_dict: The input dictionary for analysis
+        :param language: Input language
+        :param keys_to_skip: Keys to ignore during analysis
+        :param kwargs: Additional keyword arguments
+        for the `AnalyzerEngine.analyze` method.
+        Use this to pass arguments to the analyze method,
+        such as `ad_hoc_recognizers`, `context`, `return_decision_process`.
+        See `AnalyzerEngine.analyze` for the full list.
+        """
+
+        context = []
+        if "context" in kwargs:
+            context = kwargs["context"]
+            del kwargs["context"]
+
+        if not keys_to_skip:
+            keys_to_skip = []
+
+            
+        for key, value in input_dict.items():
+            if not value or key in keys_to_skip:
+                yield DictAnalyzerResult(key=key, value=value, recognizer_results=[])
+                continue  # skip this key as requested
+
+            # Add the key as an additional context
+            specific_context = context[:]
+            specific_context.append(key)
+
+            if type(value) in (str, int, bool, float):
+                results: List[RecognizerResult] = self.analyzer_engine.analyze(
+                    text=str(value), language=language, context=[key], **kwargs
+                )
+            elif isinstance(value, dict):
+                new_keys_to_skip = self._get_nested_keys_to_skip(key, keys_to_skip)
+                results = self.analyze_dict(
+                    input_dict=value,
+                    language=language,
+                    context=specific_context,
+                    keys_to_skip=new_keys_to_skip,
+                    **kwargs,
+                )
+            elif isinstance(value, Iterable):
+                # Recursively iterate nested dicts
+                list_length = len(value)
+
+                results: List[List[RecognizerResult]] = analyze_iterator_custom(self,
+                    texts=value,
+                    language=language,
+                    context=specific_context,
+                    list_length=list_length,
+                    **kwargs,
+                )
+            else:
+                raise ValueError(f"type {type(value)} is unsupported.")
+
+            yield DictAnalyzerResult(key=key, value=value, recognizer_results=results)

tools/unstructured_funcs.py

@@ -0,0 +1,884 @@
+from unstructured.partition.auto import partition 
+from unstructured.chunking.title import chunk_by_title
+from unstructured.chunking.basic import chunk_elements 
+from unstructured.documents.elements import Element, Title, CompositeElement
+from unstructured.staging.base import convert_to_dataframe
+from typing import Type, List, Literal, Tuple
+
+from unstructured.cleaners.core import replace_unicode_quotes, clean_non_ascii_chars, clean_ordered_bullets, group_broken_paragraphs, replace_unicode_quotes, clean, clean_trailing_punctuation, remove_punctuation, bytes_string_to_string
+import gradio as gr
+import time
+import pandas as pd
+import re
+import gzip
+import pickle
+from pydantic import BaseModel, Field
+
+from tools.helper_functions import get_file_path_end, get_file_path_end_with_ext
+
+# Creating an alias for pandas DataFrame using Type
+PandasDataFrame = Type[pd.DataFrame]
+
+# %%
+# pdf partitioning strategy vars
+pdf_partition_strat = "ocr_only" # ["fast", "ocr_only", "hi_res"]
+
+# %%
+# Element metadata modification vars
+meta_keys_to_filter = ["file_directory", "filetype"]
+element_types_to_filter = ['UncategorizedText', 'Header']
+
+# %%
+# Clean function vars
+
+bytes_to_string=False
+replace_quotes=True 
+clean_non_ascii=False 
+clean_ordered_list=True 
+group_paragraphs=True
+trailing_punctuation=False
+all_punctuation=False
+clean_text=True 
+extra_whitespace=True 
+dashes=True 
+bullets=True 
+lowercase=False
+
+# %%
+# Chunking vars
+
+minimum_chunk_length = 2000
+start_new_chunk_after_end_of_this_element_length = 2000
+hard_max_character_length_chunks = 3000
+multipage_sections=True
+overlap_all=True
+include_orig_elements=True
+
+# %%
+class Document(BaseModel):
+    """Class for storing a piece of text and associated metadata. Implementation adapted from Langchain code: https://github.com/langchain-ai/langchain/blob/master/libs/core/langchain_core/documents/base.py"""
+
+    page_content: str
+    """String text."""
+    metadata: dict = Field(default_factory=dict)
+    """Arbitrary metadata about the page content (e.g., source, relationships to other
+        documents, etc.).
+    """
+    type: Literal["Document"] = "Document"
+
+# %%
+def create_title_id_dict(elements:List[Element]):
+
+    # Assuming the object is stored in a variable named 'elements_list'
+    titles = [item.text for item in elements if isinstance(item, Title)]
+
+    #### Get all elements under these titles
+    chapter_ids = {}
+    for element in elements:
+        for chapter in titles:
+            if element.text == chapter and element.category == "Title":
+                chapter_ids[element._element_id] = chapter
+                break
+
+    chapter_to_id = {v: k for k, v in chapter_ids.items()}
+
+    return chapter_ids, chapter_to_id
+
+# %%
+def filter_elements(elements:List[Element], excluded_elements: List[str] = ['']):
+    """
+    Filter out elements from a list based on their categories.
+
+    Args:
+        elements: The list of elements to filter.
+        excluded_elements: A list of element categories to exclude.
+
+    Returns:
+        A new list containing the filtered elements.
+    """
+    filtered_elements = []
+    for element in elements:
+        if element.category not in excluded_elements:
+            filtered_elements.append(element)
+    return filtered_elements
+
+# %%
+def remove_keys_from_meta(
+    elements: List[Element], 
+    meta_remove_keys: List[str], 
+    excluded_element_types: List[str] = []
+) -> List[Element]:
+    '''
+    Remove specified metadata keys from an Unstructured Element object
+    '''
+
+    for element in elements:
+        if element.category not in excluded_element_types:
+            for key in meta_remove_keys:
+                try:
+                    del element.metadata.__dict__[key]  # Directly modify metadata
+                except KeyError:
+                    print(f"Key '{key}' not found in element metadata.")
+
+    return elements
+
+def filter_elements_and_metadata(
+    elements: List[Element],
+    excluded_categories: List[str] = [],
+    meta_remove_keys: List[str] = [],
+) -> List[Element]:
+    """
+    Filters elements based on categories and removes specified metadata keys.
+
+    Args:
+        elements: The list of elements to process.
+        excluded_categories: A list of element categories to exclude.
+        meta_remove_keys: A list of metadata keys to remove.
+
+    Returns:
+        A new list containing the processed elements.
+    """
+
+    filtered_elements = []
+    for element in elements:
+        if element.category not in excluded_categories:
+            for key in meta_remove_keys:
+                try:
+                    del element.metadata.__dict__[key]
+                except KeyError:
+                    # Better logging/error handling instead of just printing
+                    # Use a proper logger or raise a warning/exception
+                    pass 
+            filtered_elements.append(element)
+
+    return filtered_elements
+
+# %%
+def add_parent_title_to_meta(elements:List[Element], chapter_ids:List[str], excluded_element_types:List[str]=['']) -> List[Element]:
+    '''
+    Add parent title to Unstructured metadata elements
+    
+    '''
+    for element in elements:
+        if element.category in excluded_element_types:
+            pass
+
+        else:
+            meta = element.metadata.to_dict()
+            
+            if "parent_id" in meta and meta["parent_id"] in chapter_ids and "title_name" not in meta:
+                title_name = chapter_ids[meta["parent_id"]]
+                # Directly modify the existing element metadata object
+                element.metadata.title_name = title_name
+
+    return elements
+
+
+def chunk_all_elements(elements:List[Element], file_name_base:str, chunk_type:str = "Basic_chunking",  minimum_chunk_length:int=minimum_chunk_length, start_new_chunk_after_end_of_this_element_length:int=start_new_chunk_after_end_of_this_element_length, hard_max_character_length_chunks:int=hard_max_character_length_chunks, multipage_sections:bool=multipage_sections, overlap_all:bool=overlap_all, include_orig_elements:bool=include_orig_elements):
+
+    '''
+    Use Unstructured.io functions to chunk an Element object by Title or across all elements.
+    '''
+    output_files = []
+    output_summary = ""
+
+    chapter_ids, chapter_to_id = create_title_id_dict(elements)
+    
+    ### Break text down into chunks
+
+    try:
+
+        if chunk_type == "Chunk within title":
+            chunks = chunk_by_title(
+                elements,
+                include_orig_elements=include_orig_elements,
+                combine_text_under_n_chars=minimum_chunk_length,
+                new_after_n_chars=start_new_chunk_after_end_of_this_element_length,
+                max_characters=hard_max_character_length_chunks,
+                multipage_sections=multipage_sections,
+                overlap_all=overlap_all
+            )
+
+        else:
+            chunks = chunk_elements(
+                elements,
+                include_orig_elements=include_orig_elements,
+                new_after_n_chars=start_new_chunk_after_end_of_this_element_length,
+                max_characters=hard_max_character_length_chunks,
+                overlap_all=overlap_all
+            )
+    
+    except Exception as output_summary:
+        print(output_summary)
+        return output_summary, output_files, file_name_base
+
+    chunk_sections, chunk_df, chunks_out = element_chunks_to_document(chunks, chapter_ids)
+
+    file_name_suffix = "_chunk"
+
+    # The new file name does not overwrite the old file name as the 'chunked' elements are only used as an output, and not an input to other functions
+    output_summary, output_files, file_name_base_new = export_elements_as_table_to_file(chunks_out, file_name_base, file_name_suffix, chunk_sections)
+
+    return output_summary, output_files, file_name_base
+
+# %%
+def element_chunks_to_document(chunks:CompositeElement, chapter_ids:List[str]) -> Tuple[List[Document], PandasDataFrame, List[str]]:
+    '''
+    Take an Unstructured.io chunk_by_title output with the original parsed document elements and turn it into a Document format commonly used by vector databases, and a Pandas dataframe. 
+    '''
+    chunk_sections = []
+    current_title_id = ''
+    current_title = ''
+    last_page = ''
+    chunk_df_list = []
+
+    for chunk in chunks:
+        chunk_meta = chunk.metadata.to_dict()
+        true_element_ids = []
+        element_categories = []
+        titles = []
+        titles_id = []        
+
+        if "page_number" in chunk_meta:
+            last_page = chunk_meta["page_number"]
+
+        chunk_text = chunk.text
+        #chunk_page_number = chunk.metadata.to_dict()["page_number"]
+
+        # If the same element text is found, add the element_id to the chunk (NOT PERFECT. THIS WILL FAIL IF THE SAME TEXT IS SEEN MULTIPL TIMES)
+        for element in chunk.metadata.orig_elements:
+            
+            #element_text = element.text
+            element_id = element._element_id
+            element_category = element.category
+            element_meta = element.metadata.to_dict()
+
+            if "page_number" in element_meta:
+                element_page_number = element_meta["page_number"]
+                last_page = element_page_number
+
+            true_element_ids.append(element_id)
+            element_categories.append(element_category)
+            
+
+        # Set new metadata for chunk
+        if "page_number" in element_meta:
+            chunk_meta["last_page_number"] = last_page
+        
+        chunk_meta["true_element_ids"] = true_element_ids        
+
+        for loop_id in chunk_meta['true_element_ids']:
+            if loop_id in chapter_ids:
+                current_title = chapter_ids[loop_id]
+                current_title_id = loop_id
+
+                titles.append(current_title)
+                titles_id.append(current_title_id)        
+                
+        chunk_meta['titles'] = titles
+        chunk_meta['titles_id'] = titles_id
+
+        # Remove original elements data for documents
+        chunk_meta.pop('orig_elements')
+
+        chunk_dict_for_df = chunk_meta.copy()
+        chunk_dict_for_df['text'] = chunk.text
+
+        chunk_df_list.append(chunk_dict_for_df)
+
+        
+        chunk_doc = [Document(page_content=chunk_text, metadata=chunk_meta)]
+        chunk_sections.extend(chunk_doc)
+
+        ## Write metadata back to elements
+        chunk.metadata.__dict__ = chunk_meta
+
+    chunk_df = pd.DataFrame(chunk_df_list)
+
+    # print("Doc format: ", chunk_sections)
+
+    return chunk_sections, chunk_df, chunks
+
+# %%
+def write_elements_to_documents(elements:List[Element]):
+    '''
+    Take Unstructured.io parsed elements and write it into a 'Document' format commonly used by vector databases
+    '''
+
+    doc_sections = []
+
+    for element in elements:
+        meta = element.metadata.to_dict()
+
+        meta["type"] = element.category
+        meta["element_id"] = element._element_id
+
+        element_doc = [Document(page_content=element.text, metadata= meta)]
+        doc_sections.extend(element_doc)
+
+        #print("Doc format: ", doc_sections)
+
+
+    return doc_sections
+
+# %%
+def clean_elements(elements:List[Element], dropdown_options: List[str] = [''], 
+                       output_name:str = "combined_elements",
+                       bytes_to_string:bool=False,
+                       replace_quotes:bool=True, 
+                       clean_non_ascii:bool=False, 
+                       clean_ordered_list:bool=True, 
+                       group_paragraphs:bool=True,
+                       trailing_punctuation:bool=False,
+                       all_punctuation:bool=False,
+                       clean_text:bool=True, 
+                       extra_whitespace:bool=True, 
+                       dashes:bool=True, 
+                       bullets:bool=True, 
+                       lowercase:bool=False) -> List[Element]:
+    
+    '''
+    Apply Unstructured cleaning processes to a list of parse elements.
+    '''
+
+    out_files = []
+    output_summary = ""
+
+    # Set variables to True based on dropdown selections
+    for option in dropdown_options:
+        if option == "Convert bytes to string":
+            bytes_to_string = True
+        elif option == "Replace quotes":
+            replace_quotes = True
+        elif option == "Clean non ASCII":
+            clean_non_ascii = True
+        elif option == "Clean ordered list":
+            clean_ordered_list = True
+        elif option == "Group paragraphs":
+            group_paragraphs = True
+        elif option == "Remove trailing punctuation":
+            trailing_punctuation = True
+        elif option == "Remove all punctuation":
+            all_punctuation = True
+        elif option == "Clean text":
+            clean_text = True
+        elif option == "Remove extra whitespace":
+            extra_whitespace = True
+        elif option == "Remove dashes":
+            dashes = True
+        elif option == "Remove bullets":
+            bullets = True
+        elif option == "Make lowercase":
+            lowercase = True
+           
+
+    cleaned_elements = elements.copy()
+
+    for element in cleaned_elements:
+
+        try:
+            if element:  # Check if element is not None or empty
+                if bytes_to_string:
+                    element.apply(bytes_string_to_string)
+                if replace_quotes:
+                    element.apply(replace_unicode_quotes)
+                if clean_non_ascii:
+                    element.apply(clean_non_ascii_chars)
+                if clean_ordered_list:
+                    element.apply(clean_ordered_bullets)
+                if group_paragraphs:
+                    element.apply(group_broken_paragraphs)
+                if trailing_punctuation:
+                    element.apply(clean_trailing_punctuation)
+                if all_punctuation:
+                    element.apply(remove_punctuation)
+                if group_paragraphs:
+                    element.apply(group_broken_paragraphs)
+                if clean_text:
+                    element.apply(lambda x: clean(x, extra_whitespace=extra_whitespace, dashes=dashes, bullets=bullets, lowercase=lowercase))
+        except Exception as e:
+            print(e)
+            element = element
+
+    alt_out_message, out_files, output_file_base = export_elements_as_table_to_file(cleaned_elements, output_name, file_name_suffix="_clean")
+
+    output_summary = "Text elements successfully cleaned."
+    print(output_summary)
+
+    return cleaned_elements, output_summary, out_files, output_file_base
+
+# %% [markdown]
+def export_elements_as_table_to_file(elements:List[Element], file_name_base:str, file_name_suffix:str="", chunk_documents:List[Document]=[]):
+    '''
+    Export elements as as a table.
+    '''
+    output_summary = ""
+    out_files = []
+
+    # Convert to dataframe format
+    out_table = convert_to_dataframe(elements)
+
+    # If the file suffix already exists in the output file name, don't add it again.
+    if file_name_suffix not in file_name_base:
+        out_file_name_base = file_name_base + file_name_suffix
+
+    else:
+        out_file_name_base = file_name_base
+        
+    out_file_name = "output/" + out_file_name_base + ".csv"
+
+    out_table.to_csv(out_file_name)
+    out_files.append(out_file_name)
+
+    # Convert to document format
+    if chunk_documents:
+        out_documents = chunk_documents
+    else:
+        out_documents = write_elements_to_documents(elements)
+
+    
+
+    out_file_name_docs = "output/" + out_file_name_base + "_docs.pkl.gz"
+    with gzip.open(out_file_name_docs, 'wb') as file:
+        pickle.dump(out_documents, file)
+
+    out_files.append(out_file_name_docs)
+
+    output_summary = "File successfully exported."
+
+    return output_summary, out_files, out_file_name_base
+
+# # Partition PDF
+
+def get_file_type(filename):
+    pattern = r"\.(\w+)$"  # Match a dot followed by one or more word characters at the end of the string
+
+    match = re.search(pattern, filename)
+    if match:
+        file_type = match.group(1)  # Extract the captured file type (without the dot)
+        print(file_type)  # Output: "png"
+    else:
+        print("No file type found.")
+
+    return file_type 
+
+# %%
+def partition_file(filenames:List[str], pdf_partition_strat:str = pdf_partition_strat, progress = gr.Progress()):
+    '''
+    Partition document files into text elements using the Unstructured package. Currently supports PDF, docx, pptx, html, several image file types, text document types, email messages, code files.
+    '''
+
+    out_message = ""
+    combined_elements = []
+    out_files = []
+
+    for file in progress.tqdm(filenames, desc="Partitioning files", unit="files"):
+
+        try:
+
+            tic = time.perf_counter()
+            print(file)
+
+            file_name = get_file_path_end_with_ext(file)
+            file_name_base = get_file_path_end(file)
+            file_type = get_file_type(file_name)
+
+            image_file_type_list = ["jpg", "jpeg", "png", "heic"]
+
+            if file_type in image_file_type_list:
+                print("File is an image. Using OCR method to partition.")
+                file_elements = partition(file, strategy="ocr_only")
+            else:
+                file_elements = partition(file, strategy=pdf_partition_strat)
+
+            toc = time.perf_counter()
+
+
+            new_out_message = f"Successfully partitioned file: {file_name} in {toc - tic:0.1f} seconds\n"
+            print(new_out_message)
+
+            out_message = out_message + new_out_message
+            combined_elements.extend(file_elements)
+
+        except Exception as e:
+            new_out_message = f"Failed to partition file:  {file_name} due to {e}. Partitioning halted."
+            print(new_out_message)
+            out_message = out_message + new_out_message
+            break
+
+    out_table = convert_to_dataframe(combined_elements)
+
+    # If multiple files, overwrite default file name for outputs
+    if len(filenames) > 1:
+        file_name_base = "combined_files"
+
+    alt_out_message, out_files, output_file_base = export_elements_as_table_to_file(combined_elements, file_name_base, file_name_suffix="_elements")
+
+    return out_message, combined_elements, out_files, output_file_base, out_table
+        
+# %%
+def modify_metadata_elements(elements_out_cleaned:List[Element], meta_keys_to_filter:List[str]=meta_keys_to_filter, element_types_to_filter:List[str]=element_types_to_filter) -> List[Element]:
+
+    '''
+    Take an element object, add parent title names to metadata. Remove specified metadata keys or element types from element list.
+    '''
+
+    chapter_ids, chapter_to_id = create_title_id_dict(elements_out_cleaned.copy())
+    elements_out_meta_mod = add_parent_title_to_meta(elements_out_cleaned.copy(), chapter_ids)
+    elements_out_meta_mod_meta_filt = remove_keys_from_meta(elements_out_meta_mod.copy(), meta_keys_to_filter)
+    elements_out_filtered_meta_mod = filter_elements(elements_out_meta_mod_meta_filt, element_types_to_filter)
+
+    return elements_out_filtered_meta_mod
+# %%
+# file_stub = "C:/Users/SPedrickCase/OneDrive - Lambeth Council/Apps/doc_rag_prep/examples/"
+# filenames = []
+# pdf_filename = [file_stub + "Lambeth_2030-Our_Future_Our_Lambeth_foreword.pdf"]
+# filenames.extend(pdf_filename)
+
+# html_filename = [file_stub + "transport-strategy.html"]
+# filenames.extend(html_filename)
+
+# docx_filename = [file_stub + "FINAL Policy and Procedure for Writing Housing Policies.docx"]
+# filenames.extend(docx_filename)
+
+# out_message, elements_parse = partition_file(filenames=filenames, pdf_partition_strat="ocr_only")
+
+# for element in elements_parse[:10]:
+#     print(f"{element.category.upper()}: {element.text} - Metadata: {element.metadata.to_dict()}")
+#     elements_out = elements_parse.copy()
+
+# %% [markdown]
+# ###  Process with document layout detection - fast strategy
+# 
+# The "fast" strategy will extract the text using pdfminer and process the raw text with partition_text. If the PDF text is not extractable, partition_pdf will fall back to "ocr_only". We recommend using the "fast" strategy in most cases where the PDF has extractable text.
+# elements_out_parse = partition_pdf(filename=filename, strategy="fast")
+# for element in elements_out_parse[:10]:
+#     print(f"{element.category.upper()}: {element.text} - Metadata: {element.metadata.to_dict()}")
+#  elements_out = elements_out_parse.copy()
+# ### OCR only
+# 
+# The "ocr_only" strategy runs the document through Tesseract for OCR and then runs the raw text through partition_text. Currently, "hi_res" has difficulty ordering elements for documents with multiple columns. If you have a document with multiple columns that does not have extractable text, we recommend using the "ocr_only" strategy. "ocr_only" falls back to "fast" if Tesseract is not available and the document has extractable text.
+#  elements_out_parse = partition_pdf(filename=filename, strategy="ocr_only")
+#  for element in elements_out_parse[:10]:
+#     print(f"{element.category.upper()}: {element.text} - Metadata: {element.metadata.to_dict()}")
+#     elements_out = elements_out_parse.copy()
+# ### Hi-res partitioning
+# 
+# The "hi_res" strategy will identify the layout of the document using detectron2. The advantage of “hi_res” is that it uses the document layout to gain additional information about document elements. We recommend using this strategy if your use case is highly sensitive to correct classifications for document elements. If detectron2 is not available, the "hi_res" strategy will fall back to the "ocr_only" strategy.
+# elements_out = partition_pdf(filename=filename, strategy="hi_res")
+# for element in elements_out[:10]:
+#     print(f"{element.category.upper()}: {element.text} - Metadata: {element.metadata.to_dict()}")
+
+# %% [markdown]
+# ## Clean data
+
+# %%
+# elements_out_cleaned = clean_elements(elements_out.copy(), bytes_to_string=False,
+# replace_quotes=True ,
+# clean_non_ascii=False, 
+# clean_ordered_list=True ,
+# group_paragraphs=True,
+# trailing_punctuation=False,
+# all_punctuation=False,
+# clean_text=True ,
+# extra_whitespace=True, 
+# dashes=True ,
+# bullets=True ,
+# lowercase=False)
+
+# %% [markdown]
+# ## Add/remove elements to/from metadata
+
+
+
+# %% [markdown]
+# ### Write to table, dictionary, document format
+
+# %%
+### Dataframe format
+
+# elements_out_filtered_df = convert_to_dataframe(elements_out_filtered_meta_mod)
+
+# elements_out_filtered_df.to_csv("table.csv")
+# elements_out_filtered_df.head(6)
+
+# # %%
+# ### Dictionary format
+
+# elements_out_filtered_dict = convert_to_dict(elements_out_filtered_meta_mod)
+# elements_out_filtered_dict[20]
+
+# # %% [markdown]
+# # ### Document format for embeddings
+
+# # %%
+# doc_sections = write_elements_to_documents(elements_out_filtered_meta_mod, element_types_to_filter)
+
+# doc_sections[0:10]
+
+# # %% [markdown]
+# # ### Break text down into chunks
+
+# # %%
+# chunks_by_title = chunk_by_title(
+#     elements_out_filtered_meta_mod,
+#     include_orig_elements=True,
+#     combine_text_under_n_chars=minimum_chunk_length,
+#     new_after_n_chars=start_new_chunk_after_end_of_this_element_length,
+#     max_characters=hard_max_character_length_chunks,
+#     multipage_sections=True,
+#     overlap_all=True
+# )
+
+# chunk_sections, chunk_df = element_chunks_to_document(chunks_by_title, chapter_ids)
+# chunk_df.to_csv("chunked_df.csv")
+# print(chunk_sections[2])
+
+# # %%
+# chunks_basic = chunk_elements(
+#     elements_out_filtered_meta_mod,
+#     include_orig_elements=True,
+#     new_after_n_chars=start_new_chunk_after_end_of_this_element_length,
+#     max_characters=hard_max_character_length_chunks,
+#     overlap_all=True
+# )
+
+# chunk_basic_sections, chunk_basic_df = element_chunks_to_document(chunks_basic, chapter_ids)
+# chunk_basic_df.to_csv("chunked_basic_df.csv")
+
+# %% [markdown]
+# # Partition Word document
+# 
+# You cannot get location metadata for bounding boxes from word documents
+
+# %%
+# word_filename = "../examples/FINAL Policy and Procedure for Writing Housing Policies.docx"
+
+# # %%
+# docx_elements = partition(filename=word_filename)
+# for element in docx_elements:
+#     print(f"{element.category.upper()}: {element.text} - Metadata: {element.metadata.to_dict()}")
+
+# # %%
+# docx_elements[5].text
+
+# # %%
+# docx_elements[5].category
+
+# # %%
+# docx_elements[5].metadata.to_dict()
+
+# # %% [markdown]
+# # ## Find elements associated with chapters
+
+# # %%
+# chapter_ids, chapter_to_id = create_title_id_dict(docx_elements)
+
+# chapter_ids
+
+# # %%
+# doc_sections = write_elements_to_documents(docx_elements.copy(), chapter_ids)
+
+# # %%
+# doc_sections
+
+# # %% [markdown]
+# # ### Chunk documents
+
+# # %%
+# chunks = chunk_by_title(
+#     docx_elements,
+#     include_orig_elements=False,
+#     combine_text_under_n_chars=0,
+#     new_after_n_chars=500,
+#     max_characters=1000,
+#     multipage_sections=True,
+#     overlap_all=True
+# )
+
+# # %%
+# print(chunks)
+
+# # %%
+# chunk_sections = element_chunks_to_document(chunks.copy(), docx_elements.copy(), chapter_ids)
+
+# # %%
+# chunk_sections[5].page_content
+
+# # %%
+# chunk_sections[5].metadata["true_element_ids"]
+
+# # %%
+# for element in docx_elements:
+#     if element._element_id in chunk_sections[5].metadata["true_element_ids"]:
+#         print(element.text)
+
+# # %% [markdown]
+# # # Partition PPTX document
+
+# # %%
+# pptx_filename = "../examples/LOTI presentation Jan 2024.pptx"
+
+# # %%
+# pptx_elements = partition(filename=pptx_filename)
+# for element in pptx_elements[:10]:
+#     print(f"{element.category.upper()}: {element.text} - Metadata: {element.metadata.to_dict()}")
+
+# # %%
+# chapter_ids, chapter_to_id = create_title_id_dict(pptx_elements)
+# chapter_ids
+
+# # %%
+# pptx_sections = write_elements_to_documents(pptx_elements.copy(), chapter_ids)
+
+# # %%
+# pptx_sections
+
+# # %%
+# pptx_chunks = chunk_by_title(
+#     pptx_elements,
+#     include_orig_elements=False,
+#     combine_text_under_n_chars=0,
+#     new_after_n_chars=500,
+#     max_characters=1000,
+#     multipage_sections=True,
+#     overlap_all=True
+# )
+
+# # %%
+# pptx_chunk_sections = element_chunks_to_document(pptx_chunks.copy(), pptx_elements.copy(), chapter_ids)
+
+# # %% [markdown]
+# # ### Load documents into a vectorDB (Not necessary)
+
+# # %%
+# import chromadb
+
+# # %%
+# client = chromadb.PersistentClient(path="chroma_tmp", settings=chromadb.Settings(allow_reset=True))
+# client.reset()
+
+# # %%
+# collection = client.create_collection(
+#     name="policy_statements",
+#     metadata={"hnsw:space": "cosine"}
+# )
+
+# # %%
+# chapter_ids
+
+# # %%
+# for element in docx_elements:
+#     parent_id = element.metadata.parent_id
+#     #print(element.text)
+#     #print(parent_id)
+#     #print(element.metadata.to_dict())
+#     if parent_id:
+#         try:
+#             print(parent_id)
+#             chapter = chapter_ids[parent_id]
+#             print(chapter)
+#         except KeyError:
+#             chapter = "None"
+#     else:
+#         chapter = "None"
+#     collection.add(
+#         documents=[element.text],
+#         ids=[element._element_id],
+#         metadatas=[{"chapter": chapter}]
+#     )
+
+# # %% [markdown]
+# # #### See the elements in the VectorDB and perform hybrid search
+
+# # %%
+# results = collection.peek()
+# print(results["documents"])
+
+# # %%
+# print(collection.metadata)
+
+# # %%
+# import json
+
+# result = collection.query(
+#     query_texts=["What should policies do?"],
+#     n_results=2,
+#     where={"chapter": '3.0  Policy Statements'},
+# )
+# print(json.dumps(result, indent=2))
+
+# # %%
+# collection = client.create_collection(
+#     name="policy_statements_chunk",
+#     metadata={"hnsw:space": "cosine"}
+# )
+
+# # %%
+# for element in chunks:
+#     parent_id = element.metadata.parent_id
+#     #print(element.text)
+#     #print(parent_id)
+#     #print(element.metadata.to_dict())
+#     if parent_id:
+#         try:
+#             print(parent_id)
+#             chapter = chapter_ids[parent_id]
+#             print(chapter)
+#         except KeyError:
+#             chapter = "None"
+#     else:
+#         chapter = "None"
+
+#     print(element._element_id)
+#     collection.add(
+#         documents=[element.text],
+#         ids=[element.orig_elements],
+#         metadatas=[{"chapter": chapter}]
+#     )
+
+# # %% [markdown]
+# # # Partition HTML
+
+# # %%
+# html_filename = "../examples/transport-strategy.html"
+
+# # %%
+# html_elements = partition(filename=html_filename)
+# for element in html_elements[:10]:
+#     print(f"{element.category.upper()}: {element.text} - Metadata: {element.metadata.to_dict()}")
+
+# # %% [markdown]
+# # # Partition image
+
+# # %%
+# img_filename = "../examples/example_complaint_letter.jpg"
+
+# # %%
+# img_elements = partition(filename=img_filename)
+# for element in img_elements[:10]:
+#     print(f"{element.category.upper()}: {element.text} - Metadata: {element.metadata.to_dict()}")
+
+# # %% [markdown]
+# # # Partition XLSX
+
+# # %%
+# xlsx_filename = "../examples/fuel-poverty-sub-regional-tables-2020-2018-data.xlsx"
+
+# # %%
+# xlsx_elements = partition(filename=xlsx_filename)
+# for element in xlsx_elements[:10]:
+#     print(f"{element.category.upper()}: {element.text} - Metadata: {element.metadata.to_dict()}")
+
+# # %% [markdown]
+# # # Partition .py
+
+# # %%
+# py_filename = "../examples/app.py"
+
+# # %%
+# py_elements = partition(filename=py_filename)
+# for element in py_elements[:10]:
+#     print(f"{element.category.upper()}: {element.text} - Metadata: {element.metadata.to_dict()}")
+
+