Dockerfile now loads models to local folder. Can use custom output folder. requrirements for GPU-enabled summarisation now in separate file to hopefully avoid HF space issues.

.dockerignore

@@ -0,0 +1,7 @@
+*.pyc
+*.ipynb
+*.csv
+*.parquet
+tests/*
+output/*
+model/*

.gitignore

@@ -1,3 +1,7 @@
 *.pyc
 *.ipynb
-*.csv
+*.csv
+*.parquet
+tests/*
+output/*
+model/*

Dockerfile

@@ -1,30 +1,66 @@
-FROM python:3.10
+# First stage: build dependencies
+FROM public.ecr.aws/docker/library/python:3.11.9-slim-bookworm
+
+# Install Lambda web adapter
+COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:0.8.3 /lambda-adapter /opt/extensions/lambda-adapter
+
+# Install wget, git, curl
+RUN apt-get update && \
+    apt-get install -y wget git curl && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /src
 
 COPY requirements.txt .
 
-RUN pip install --no-cache-dir -r requirements.txt
+# Optimized dependency installation
+RUN pip install --no-cache-dir -r requirements.txt && \
+    pip install --no-cache-dir gradio==4.36.0
 
-# Set up a new user named "user" with user ID 1000
-RUN useradd -m -u 1000 user
-# Switch to the "user" user
+# Create a directory for the models and switch to user
+RUN mkdir /model && \
+    useradd -m -u 1000 user && \
+    chown -R user:user /model
 USER user
-# Set home to the user's home directory
+
+WORKDIR /home/user
+
+# Download the GGUF model to local model/phi directory:
+ENV REPO_ID "QuantFactory/Phi-3-mini-128k-instruct-GGUF"
+ENV MODEL_FILE "Phi-3-mini-128k-instruct.Q4_K_M.gguf"
+
+RUN python -c "from huggingface_hub import hf_hub_download; \
+                hf_hub_download(repo_id='$REPO_ID', filename='$MODEL_FILE', local_dir='/model/phi')"
+
+# Download the transformers-based models
+RUN curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash && \
+    apt-get install -y git-lfs && \
+    git lfs install
+
+RUN git clone https://huggingface.co/stacked-summaries/flan-t5-large-stacked-samsum-1024 /model/stacked_t5 && \
+    rm -rf /model/stacked_t5/.git && \
+    git clone https://huggingface.co/pszemraj/long-t5-tglobal-base-16384-book-summary /model/long_t5 && \
+    rm -rf /model/long_t5/.git
+
+
 ENV HOME=/home/user \
-	PATH=/home/user/.local/bin:$PATH \
+    PATH=/home/user/.local/bin:$PATH \
     PYTHONPATH=$HOME/app \
-	PYTHONUNBUFFERED=1 \
-	GRADIO_ALLOW_FLAGGING=never \
-	GRADIO_NUM_PORTS=1 \
-	GRADIO_SERVER_NAME=0.0.0.0 \
-	GRADIO_THEME=huggingface \
-	SYSTEM=spaces
- 
-# Set the working directory to the user's home directory
-WORKDIR $HOME/app
+    PYTHONUNBUFFERED=1 \
+    GRADIO_ALLOW_FLAGGING=never \
+    GRADIO_NUM_PORTS=1 \
+    GRADIO_SERVER_NAME=0.0.0.0 \
+    GRADIO_SERVER_PORT=7860 \
+    GRADIO_THEME=huggingface \
+    SYSTEM=spaces
+
+# Switch back to root to copy the app files
+USER root
+WORKDIR /home/user/app
 
 # Copy the current directory contents into the container at $HOME/app setting the owner to the user
-COPY --chown=user . $HOME/app
+COPY --chown=user . $HOME/user/app
 
+# Switch back to the user to run the app
+USER user
 CMD ["python", "app.py"]

app.py

@@ -14,9 +14,11 @@ PandasDataFrame = Type[pd.DataFrame]
 import chatfuncs.chatfuncs as chatf
 import chatfuncs.summarise_funcs as sumf
 
-from chatfuncs.helper_functions import dummy_function, put_columns_in_df
+from chatfuncs.helper_functions import dummy_function, put_columns_in_df, output_folder, ensure_output_folder_exists
 from chatfuncs.summarise_funcs import summarise_text
 
+ensure_output_folder_exists(output_folder)
+
 # Disable cuda devices if necessary
 #os.environ['CUDA_VISIBLE_DEVICES'] = '-1' 
 
@@ -34,11 +36,28 @@ else:
 
 print("Device used is: ", torch_device)
 
-def create_hf_model(model_name):
+def create_hf_model(model_name, local_model_dir="model/t5_long"):
+
+    # Construct the expected local model path
+    local_model_path = os.path.join(local_model_dir, model_name)
+
+    # Check if the model directory exists
+    if os.path.exists(local_model_path):
+        print(f"Model '{model_name}' found locally at: {local_model_path}")
+        
+        # Load tokenizer and pipeline from local path
+        tokenizer = AutoTokenizer.from_pretrained(local_model_path, model_max_length=chatf.context_length) 
+        summariser = pipeline("summarization", model=local_model_path, tokenizer=tokenizer) 
 
-    tokenizer = AutoTokenizer.from_pretrained(model_name, model_max_length = chatf.context_length)
+    else:
+        print(f"Downloading model '{model_name}' from Hugging Face Hub...")
+        
+        # Download tokenizer and pipeline from Hugging Face Hub
+        tokenizer = AutoTokenizer.from_pretrained(model_name, model_max_length=chatf.context_length)
+        summariser = pipeline("summarization", model=model_name, tokenizer=tokenizer)
 
-    summariser = pipeline("summarization", model=model_name, tokenizer=tokenizer) # philschmid/bart-large-cnn-samsum    
+        # Save the model locally (optional, but recommended for future use)
+        #summariser.save_pretrained(local_model_path) 
 
     return summariser, tokenizer, model_name
 
@@ -53,7 +72,7 @@ def load_model(model_type, gpu_layers, gpu_config=None, cpu_config=None, torch_d
     if torch_device is None:
         torch_device = chatf.torch_device
 
-    if model_type == "Phi 3 128k (larger, slow)":
+    if model_type == "Phi 3 128k (24k tokens max)":
         if torch_device == "cuda":
             gpu_config.update_gpu(gpu_layers)
             print("Loading with", gpu_config.n_gpu_layers, "model layers sent to GPU.")
@@ -66,38 +85,43 @@ def load_model(model_type, gpu_layers, gpu_config=None, cpu_config=None, torch_d
         print(vars(gpu_config))
         print(vars(cpu_config))
 
+        def get_model_path():
+            repo_id = os.environ.get("REPO_ID", "QuantFactory/Phi-3-mini-128k-instruct-GGUF")
+            filename = os.environ.get("MODEL_FILE", "Phi-3-mini-128k-instruct.Q4_K_M.gguf")
+            model_dir = "model/phi"  # Assuming this is your intended directory
+
+            # Construct the expected local path
+            local_path = os.path.join(model_dir, filename)
+
+            if os.path.exists(local_path):
+                print(f"Model already exists at: {local_path}")
+                return local_path
+            else:
+                print(f"Checking default Hugging Face folder. Downloading model from Hugging Face Hub if not found")
+                return hf_hub_download(repo_id=repo_id, filename=filename)
+                                       
+        model_path = get_model_path()
+        
+
         try:
-            summariser = Llama(
-            model_path=hf_hub_download(
-            repo_id=os.environ.get("REPO_ID", "QuantFactory/Phi-3-mini-128k-instruct-GGUF"),# "QuantFactory/Phi-3-mini-128k-instruct-GGUF"), # "QuantFactory/Meta-Llama-3-8B-Instruct-GGUF-v2"), #"microsoft/Phi-3-mini-4k-instruct-gguf"),#"TheBloke/Mistral-7B-OpenOrca-GGUF"),
-            filename=os.environ.get("MODEL_FILE", "Phi-3-mini-128k-instruct.Q4_K_M.gguf") #"Phi-3-mini-128k-instruct.Q4_K_M.gguf")  #"Meta-Llama-3-8B-Instruct-v2.Q6_K.gguf") #"Phi-3-mini-4k-instruct-q4.gguf")#"mistral-7b-openorca.Q4_K_M.gguf"),
-        ),
-        **vars(gpu_config) # change n_gpu_layers if you have more or less VRAM 
-        )
+            summariser = Llama(model_path=model_path, **vars(gpu_config))
         
         except Exception as e:
             print("GPU load failed")
             print(e)
-            summariser = Llama(
-            model_path=hf_hub_download(
-            repo_id=os.environ.get("REPO_ID", "QuantFactory/Phi-3-mini-128k-instruct-GGUF"), #"QuantFactory/Phi-3-mini-128k-instruct-GGUF"), #, "microsoft/Phi-3-mini-4k-instruct-gguf"),#"QuantFactory/Meta-Llama-3-8B-Instruct-GGUF-v2"), #"microsoft/Phi-3-mini-4k-instruct-gguf"),#"TheBloke/Mistral-7B-OpenOrca-GGUF"),
-            filename=os.environ.get("MODEL_FILE", "Phi-3-mini-128k-instruct.Q4_K_M.gguf"), # "Phi-3-mini-128k-instruct.Q4_K_M.gguf") # , #"Meta-Llama-3-8B-Instruct-v2.Q6_K.gguf") #"Phi-3-mini-4k-instruct-q4.gguf"),#"mistral-7b-openorca.Q4_K_M.gguf"),
-        ),
-        **vars(cpu_config)
-        )
+            summariser = Llama(model_path=model_path, **vars(cpu_config))
 
         tokenizer = []
 
     if model_type == "Flan T5 Large Stacked Samsum 1k":
         # Huggingface chat model
-        hf_checkpoint = 'stacked-summaries/flan-t5-large-stacked-samsum-1024'#'declare-lab/flan-alpaca-base' # # #
-
-        summariser, tokenizer, model_type = create_hf_model(model_name = hf_checkpoint)
+        hf_checkpoint = 'stacked-summaries/flan-t5-large-stacked-samsum-1024'
+        summariser, tokenizer, model_type = create_hf_model(model_name = hf_checkpoint, local_model_dir="model/t5_stacked")
 
     if model_type == "Long T5 Global Base 16k Book Summary":
         # Huggingface chat model
-        hf_checkpoint = 'pszemraj/long-t5-tglobal-base-16384-book-summary' #'philschmid/flan-t5-small-stacked-samsum'#'declare-lab/flan-alpaca-base' # # #
-        summariser, tokenizer, model_type = create_hf_model(model_name = hf_checkpoint)
+        hf_checkpoint = 'pszemraj/long-t5-tglobal-base-16384-book-summary'
+        summariser, tokenizer, model_type = create_hf_model(model_name = hf_checkpoint, local_model_dir="model/t5_long")
 
     sumf.model = summariser
     sumf.tokenizer = tokenizer
@@ -109,7 +133,7 @@ def load_model(model_type, gpu_layers, gpu_config=None, cpu_config=None, torch_d
     return model_type, load_confirmation, model_type
 
 # Both models are loaded on app initialisation so that users don't have to wait for the models to be downloaded
-model_type = "Phi 3 128k (larger, slow)"
+model_type = "Phi 3 128k (24k tokens max)"
 load_model(model_type, chatf.gpu_layers, chatf.gpu_config, chatf.cpu_config, chatf.torch_device)
 
 model_type = "Flan T5 Large Stacked Samsum 1k"
@@ -133,7 +157,7 @@ with block:
     gr.Markdown(
     """
     # Text summariser
-    Enter open text below to get a summary. You can copy and paste text directly, or upload a file and specify the column that you want to summarise. The default small model will be able to summarise up to about 16,000 words, but the quality may not be great. The larger model around 900 words of better quality. Summarisation with Phi 3 128k works on up to around 4,000 words, and may give a higher quality summary, but will be slow, and it may not respect your desired maximum word count.
+    Enter open text below to get a summary. You can copy and paste text directly, or upload a file and specify the column that you want to summarise. The default small model will be able to summarise up to about 12,000 words, but the quality may not be great. The larger model around 800 words of better quality. Summarisation with Phi 3 128k works on up to around 20,000 words (suitable for a 12Gb graphics card without out of memory issues), and may give a higher quality summary, but will be slow, and it may not respect your desired maximum word count.
     """)    
     
     with gr.Tab("Summariser"):
@@ -149,7 +173,7 @@ with block:
         with gr.Row():
             summarise_btn = gr.Button("Summarise", variant="primary")
             stop = gr.Button(value="Interrupt processing", variant="secondary", scale=0)
-            length_slider = gr.Slider(minimum = 30, maximum = 500, value = 100, step = 10, label = "Maximum length of summary")
+            length_slider = gr.Slider(minimum = 30, maximum = 1000, value = 500, step = 10, label = "Maximum length of summary (in words)")
         
         with gr.Row():
             output_single_text = gr.Textbox(label="Output example (first example in dataset)")
@@ -157,12 +181,12 @@ with block:
 
     with gr.Tab("Advanced features"):
         with gr.Row():
-            model_choice = gr.Radio(label="Choose a summariser model", value="Long T5 Global Base 16k Book Summary", choices = ["Long T5 Global Base 16k Book Summary", "Flan T5 Large Stacked Samsum 1k", "Phi 3 128k (larger, slow)"])
+            model_choice = gr.Radio(label="Choose a summariser model", value="Long T5 Global Base 16k Book Summary", choices = ["Long T5 Global Base 16k Book Summary", "Flan T5 Large Stacked Samsum 1k", "Phi 3 128k (24k tokens max)"])
             change_model_button = gr.Button(value="Load model", scale=0)
         with gr.Accordion("Choose number of model layers to send to GPU (WARNING: please don't modify unless you are sure you have a GPU).", open = False):
             gpu_layer_choice = gr.Slider(label="Choose number of model layers to send to GPU.", value=0, minimum=0, maximum=100, step = 1, visible=True)
         with gr.Accordion("LLM parameters"):
-            temp_slide = gr.Slider(minimum=0.1, value = 0.5, maximum=1, step=0.1, label="Choose temperature setting for response generation.")
+            temp_slide = gr.Slider(minimum=0.1, value = 0.5, maximum=1, step=0.1, label="Choose temperature setting for response generation.", interactive=True)
 
         load_text = gr.Text(label="Load status")
 
@@ -172,7 +196,7 @@ with block:
     change_model_button.click(fn=load_model, inputs=[model_choice, gpu_layer_choice], outputs = [model_type_state, load_text, current_model])
 
     summarise_click = summarise_btn.click(fn=summarise_text, inputs=[in_text, data_state, length_slider, in_colname, model_type_state],
-                       outputs=[output_single_text, output_file], api_name="summarise_single_text")
+                       outputs=[output_single_text, output_file], api_name="summarise")
     # summarise_enter = summarise_btn.submit(fn=summarise_text, inputs=[in_text, data_state, length_slider, in_colname, model_type_state],
     #                    outputs=[output_single_text, output_file])
     
@@ -184,7 +208,7 @@ with block:
     # Dummy function to allow dropdown modification to work correctly (strange thing needed for Gradio 3.50, will be deprecated upon upgrading Gradio version)
     in_colname.change(dummy_function, in_colname, None)
 
-block.queue().launch()
+block.queue().launch(show_error=True)
 
 # def load_model(model_type, gpu_layers, gpu_config=None, cpu_config=None, torch_device=None):
 #     print("Loading model ", model_type)
@@ -197,7 +221,7 @@ block.queue().launch()
 #     if torch_device is None:
 #         torch_device = chatf.torch_device
 
-#     if model_type == "Phi 3 128k (larger, slow)":
+#     if model_type == "Phi 3 128k (24k tokens max)":
 #         hf_checkpoint = 'NousResearch/Nous-Capybara-7B-V1.9-GGUF'
 
 #         if torch_device == "cuda":

chatfuncs/chatfuncs.py

@@ -52,7 +52,7 @@ class CtransInitConfig_gpu:
                  seed=seed,
                  n_threads=threads,
                  n_batch=batch_size,
-                 n_ctx=32768,
+                 n_ctx=24576,
                  n_gpu_layers=gpu_layers):
 
         self.last_n_tokens = last_n_tokens

chatfuncs/helper_functions.py

@@ -12,6 +12,36 @@ import getpass
 import gzip
 import pickle
 
+def get_or_create_env_var(var_name, default_value):
+    # Get the environment variable if it exists
+    value = os.environ.get(var_name)
+    
+    # If it doesn't exist, set it to the default value
+    if value is None:
+        os.environ[var_name] = default_value
+        value = default_value
+    
+    return value
+
+# Retrieving or setting output folder
+env_var_name = 'GRADIO_OUTPUT_FOLDER'
+default_value = 'output/'
+
+output_folder = get_or_create_env_var(env_var_name, default_value)
+print(f'The value of {env_var_name} is {output_folder}')
+
+def ensure_output_folder_exists(output_folder):
+    """Checks if the output folder exists, creates it if not."""
+
+    folder_name = output_folder
+
+    if not os.path.exists(folder_name):
+        # Create the folder if it doesn't exist
+        os.makedirs(folder_name)
+        print(f"Created the output folder:", folder_name)
+    else:
+        print(f"The output folder already exists:", folder_name)
+
 # Attempt to delete content of gradio temp folder
 def get_temp_folder_path():
     username = getpass.getuser()

chatfuncs/summarise_funcs.py

@@ -3,158 +3,175 @@ import concurrent.futures
 import gradio as gr
 from chatfuncs.chatfuncs import model, CtransGenGenerationConfig, temperature
 from datetime import datetime
+from typing import Type
+
+from chatfuncs.helper_functions import output_folder
 
 today = datetime.now().strftime("%d%m%Y")
 today_rev = datetime.now().strftime("%Y%m%d")
 
-def summarise_text(text, text_df, length_slider, in_colname, model_type, progress=gr.Progress()):      
-         
-        if text_df.empty:
-            in_colname="text"
-            in_colname_list_first = in_colname
+PandasDataFrame = Type[pd.DataFrame]
 
-            in_text_df = pd.DataFrame({in_colname_list_first:[text]})
-            
-        else: 
-            in_text_df = text_df
-            in_colname_list_first = in_colname
+def summarise_text(text:str, text_df:PandasDataFrame, length_slider:int, in_colname:str, model_type:str, progress=gr.Progress()):
+    '''
+    Summarise a text or series of texts using Transformers of Llama.cpp
+    '''
 
-        print(model_type)
+    outputs = []
+    output_name = ""
+    output_name_parquet = ""     
+        
+    if text_df.empty:
+        in_colname="text"
+        in_colname_list_first = in_colname
 
-        texts_list = list(in_text_df[in_colname_list_first])
+        in_text_df = pd.DataFrame({in_colname_list_first:[text]})
+        
+    else: 
+        in_text_df = text_df
+        in_colname_list_first = in_colname
 
-        if model_type != "Phi 3 128k (larger, slow)":
-            summarised_texts = []
+    print(model_type)
 
-            for single_text in progress.tqdm(texts_list, desc = "Summarising texts", unit = "texts"):
+    texts_list = list(in_text_df[in_colname_list_first])
 
-                summarised_text = model(single_text, max_length=length_slider)
+    if model_type != "Phi 3 128k (24k tokens max)":
+        summarised_texts = []
 
-                #print(summarised_text)
+        for single_text in progress.tqdm(texts_list, desc = "Summarising texts", unit = "texts"):
 
-                summarised_text_str = summarised_text[0]['summary_text']
+            summarised_text = model(single_text, max_length=length_slider)
 
-                summarised_texts.append(summarised_text_str)
+            #print(summarised_text)
 
-                print(summarised_text_str)
+            summarised_text_str = summarised_text[0]['summary_text']
 
-                #pd.Series(summarised_texts).to_csv("summarised_texts_out.csv")
+            summarised_texts.append(summarised_text_str)
 
-            #print(summarised_texts)
+            print(summarised_text_str)
 
-        if model_type == "Phi 3 128k (larger, slow)":
+            #pd.Series(summarised_texts).to_csv("summarised_texts_out.csv")
 
-            gen_config = CtransGenGenerationConfig()
-            gen_config.update_temp(temperature)
+        #print(summarised_texts)
 
-            print(gen_config)
+    if model_type == "Phi 3 128k (24k tokens max)":
 
-            # Define a function that calls your model
-            # def call_model(formatted_string):#, vars):
-            #     return model(formatted_string)#, vars)
-            
-            def call_model(formatted_string, gen_config):
-                """
-                Calls your generation model with parameters from the CtransGenGenerationConfig object.
-
-                Args:
-                    formatted_string (str): The formatted input text for the model.
-                    gen_config (CtransGenGenerationConfig): An object containing generation parameters.
-                """
-                # Extracting parameters from the gen_config object
-                temperature = gen_config.temperature
-                top_k = gen_config.top_k
-                top_p = gen_config.top_p
-                repeat_penalty = gen_config.repeat_penalty
-                seed = gen_config.seed
-                max_tokens = gen_config.max_tokens
-                stream = gen_config.stream
-
-                # Now you can call your model directly, passing the parameters:
-                output = model(
-                    formatted_string, 
-                    temperature=temperature, 
-                    top_k=top_k, 
-                    top_p=top_p, 
-                    repeat_penalty=repeat_penalty, 
-                    seed=seed,
-                    max_tokens=max_tokens,
-                    stream=stream,
-                )
-
-                return output
-
-            # Set your timeout duration (in seconds)
-            timeout_duration = 300  # Adjust this value as needed
-
-            length = str(length_slider)
-
-            from chatfuncs.prompts import instruction_prompt_phi3
-
-            summarised_texts = []
-
-            for single_text in progress.tqdm(texts_list, desc = "Summarising texts", unit = "texts"):
-
-                formatted_string = instruction_prompt_phi3.format(length=length, text=single_text)
-
-                # Use ThreadPoolExecutor to enforce a timeout
-                with concurrent.futures.ThreadPoolExecutor() as executor:
-                    #future = executor.submit(call_model, formatted_string)#, **vars(gen_config))
-                    future = executor.submit(call_model, formatted_string, gen_config)
-                    try:
-                        output = future.result(timeout=timeout_duration)
-                        # Process the output here
-                    except concurrent.futures.TimeoutError:
-                        error_text = f"Timeout (five minutes) occurred for text: {single_text}. Consider using a smaller model."
-                        print(error_text)
-                        return error_text, None
-
-                print(output)
-
-                output_str = output['choices'][0]['text']
-
-                # Find the index of 'ASSISTANT: ' to select only text after this location
-                # index = output_str.find('ASSISTANT: ')
-
-                # # Check if 'ASSISTANT: ' is found in the string
-                # if index != -1:
-                #     # Add the length of 'ASSISTANT: ' to the index to start from the end of this substring
-                #     start_index = index + len('ASSISTANT: ')
-                    
-                #     # Slice the string from this point to the end
-                #     assistant_text = output_str[start_index:]
-                # else:
-                #     assistant_text = "ASSISTANT: not found in text"
+        gen_config = CtransGenGenerationConfig()
+        gen_config.update_temp(temperature)
+
+        print(gen_config)
+
+        # Define a function that calls your model
+        # def call_model(formatted_string):#, vars):
+        #     return model(formatted_string)#, vars)
+        
+        def call_model(formatted_string, gen_config):
+            """
+            Calls your generation model with parameters from the CtransGenGenerationConfig object.
+
+            Args:
+                formatted_string (str): The formatted input text for the model.
+                gen_config (CtransGenGenerationConfig): An object containing generation parameters.
+            """
+            # Extracting parameters from the gen_config object
+            temperature = gen_config.temperature
+            top_k = gen_config.top_k
+            top_p = gen_config.top_p
+            repeat_penalty = gen_config.repeat_penalty
+            seed = gen_config.seed
+            max_tokens = gen_config.max_tokens
+            stream = gen_config.stream
+
+            # Now you can call your model directly, passing the parameters:
+            output = model(
+                formatted_string, 
+                temperature=temperature, 
+                top_k=top_k, 
+                top_p=top_p, 
+                repeat_penalty=repeat_penalty, 
+                seed=seed,
+                max_tokens=max_tokens,
+                stream=stream,
+            )
+
+            return output
+
+        # Set your timeout duration (in seconds)
+        timeout_duration = 300  # Adjust this value as needed
+
+        length = str(length_slider)
 
-                # print(assistant_text)
+        from chatfuncs.prompts import instruction_prompt_phi3
 
-                #summarised_texts.append(assistant_text)
+        summarised_texts = []
 
-                summarised_texts.append(output_str)
+        for single_text in progress.tqdm(texts_list, desc = "Summarising texts", unit = "texts"):
 
-                #print(summarised_text)
+            formatted_string = instruction_prompt_phi3.format(length=length, text=single_text)
+
+            # Use ThreadPoolExecutor to enforce a timeout
+            with concurrent.futures.ThreadPoolExecutor() as executor:
+                #future = executor.submit(call_model, formatted_string)#, **vars(gen_config))
+                future = executor.submit(call_model, formatted_string, gen_config)
+                try:
+                    output = future.result(timeout=timeout_duration)
+                    # Process the output here
+                except concurrent.futures.TimeoutError:
+                    error_text = f"Timeout (five minutes) occurred for text: {single_text}. Consider using a smaller model."
+                    print(error_text)
+                    return error_text, None
+
+            print(output)
+
+            output_str = output['choices'][0]['text']
+
+            # Find the index of 'ASSISTANT: ' to select only text after this location
+            # index = output_str.find('ASSISTANT: ')
+
+            # # Check if 'ASSISTANT: ' is found in the string
+            # if index != -1:
+            #     # Add the length of 'ASSISTANT: ' to the index to start from the end of this substring
+            #     start_index = index + len('ASSISTANT: ')
                 
-                #pd.Series(summarised_texts).to_csv("summarised_texts_out.csv")
+            #     # Slice the string from this point to the end
+            #     assistant_text = output_str[start_index:]
+            # else:
+            #     assistant_text = "ASSISTANT: not found in text"
+
+            # print(assistant_text)
+
+            #summarised_texts.append(assistant_text)
+
+            summarised_texts.append(output_str)
+
+            #print(summarised_text)
+            
+            #pd.Series(summarised_texts).to_csv("summarised_texts_out.csv")
+
+    if text_df.empty:
+        #if model_type != "Phi 3 128k (24k tokens max)":
+        summarised_text_out = summarised_texts[0]#.values()
 
-        if text_df.empty:
-            #if model_type != "Phi 3 128k (larger, slow)":
-            summarised_text_out = summarised_texts[0]#.values()
+        #if model_type == "Phi 3 128k (24k tokens max)":
+        #    summarised_text_out = summarised_texts[0]
 
-            #if model_type == "Phi 3 128k (larger, slow)":
-            #    summarised_text_out = summarised_texts[0]
+    else: 
+        summarised_text_out = summarised_texts #[d['summary_text'] for d in summarised_texts] #summarised_text[0].values()
 
-        else: 
-            summarised_text_out = summarised_texts #[d['summary_text'] for d in summarised_texts] #summarised_text[0].values()
+    output_name = output_folder + "summarise_output_" + today_rev + ".csv"
+    output_name_parquet = output_folder + "summarise_output_" + today_rev + ".parquet"
+    output_df = pd.DataFrame({"Original text":in_text_df[in_colname_list_first],
+                                "Summarised text":summarised_text_out})
 
-        output_name = "summarise_output_" + today_rev + ".csv"
-        output_df = pd.DataFrame({"Original text":in_text_df[in_colname_list_first],
-                                    "Summarised text":summarised_text_out})
+    summarised_text_out_str = str(output_df["Summarised text"][0])#.str.replace("dict_values([","").str.replace("])",""))
 
-        summarised_text_out_str = str(output_df["Summarised text"][0])#.str.replace("dict_values([","").str.replace("])",""))
+    output_df.to_csv(output_name, index = None)
+    output_df.to_parquet(output_name_parquet, index = None)
 
-        output_df.to_csv(output_name, index = None)
+    outputs.append(output_name)
+    outputs.append(output_name_parquet)
 
-        return summarised_text_out_str, output_name
+    return summarised_text_out_str, outputs
 
 
 # def summarise_text(text, text_df, length_slider, in_colname, model_type, progress=gr.Progress()):      
@@ -173,7 +190,7 @@ def summarise_text(text, text_df, length_slider, in_colname, model_type, progres
 
 #         texts_list = list(in_text_df[in_colname_list_first])
 
-#         if model_type != "Phi 3 128k (larger, slow)":
+#         if model_type != "Phi 3 128k (24k tokens max)":
 #             summarised_texts = []
 
 #             for single_text in progress.tqdm(texts_list, desc = "Summarising texts", unit = "texts"):
@@ -191,7 +208,7 @@ def summarise_text(text, text_df, length_slider, in_colname, model_type, progres
 
 #             #print(summarised_texts)
 
-#         if model_type == "Phi 3 128k (larger, slow)":
+#         if model_type == "Phi 3 128k (24k tokens max)":
 
 
 #             # Define a function that calls your model
@@ -248,10 +265,10 @@ def summarise_text(text, text_df, length_slider, in_colname, model_type, progres
 #                 #pd.Series(summarised_texts).to_csv("summarised_texts_out.csv")
 
 #         if text_df.empty:
-#             #if model_type != "Phi 3 128k (larger, slow)":
+#             #if model_type != "Phi 3 128k (24k tokens max)":
 #             summarised_text_out = summarised_texts[0]#.values()
 
-#             #if model_type == "Phi 3 128k (larger, slow)":
+#             #if model_type == "Phi 3 128k (24k tokens max)":
 #             #    summarised_text_out = summarised_texts[0]
 
 #         else: 

requirements.txt

@@ -2,5 +2,5 @@ gradio==4.36.0
 transformers
 pyarrow
 openpyxl
-llama-cpp-python==0.2.77 --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu121
-torch==2.3.1 --extra-index-url https://download.pytorch.org/whl/cu121
+llama-cpp-python==0.2.77
+torch==2.3.1

requirements_gpu.txt

@@ -0,0 +1,6 @@
+gradio==4.36.0
+transformers
+pyarrow
+openpyxl
+llama-cpp-python==0.2.77 --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu121
+torch==2.3.1 --extra-index-url https://download.pytorch.org/whl/cu121