cleanup

.gitignore

@@ -0,0 +1,5 @@
+# .gitignore
+
+.env
+__pycache__/
+*.pyc

README.md

@@ -5,7 +5,7 @@ colorFrom: green
 colorTo: blue
 sdk: streamlit
 python_version: 3.10.11
-sdk_version: 1.21.0
+sdk_version: 1.38.0
 app_file: app.py
 pinned: false
 ---

app.py

@@ -1,289 +1,142 @@
+# app.py
 
 import streamlit as st
 import os
-import pkg_resources
-
-# # Using this wacky hack to get around the massively ridicolous managed env loading order
-# def is_installed(package_name, version):
-#     try:
-#         pkg = pkg_resources.get_distribution(package_name)
-#         return pkg.version == version
-#     except pkg_resources.DistributionNotFound:
-#         return False
-
-# @st.cache_resource 
-# def install_packages():
-#     install_commands = []
-
-#     if not is_installed("spaces", "0.12.0"):
-#         install_commands.append("pip install spaces==0.12.0")
-    
-#     if not is_installed("pydantic", "1.8.2"):
-#         install_commands.append("pip install pydantic==1.8.2")
-
-#     if install_commands:
-#         os.system(" && ".join(install_commands))
-
-# # install packages if necessary
-# # install_packages()
-
-
 import re
 import json
-from dotenv import load_dotenv
 import numpy as np
 import pandas as pd
 import getpass
-import os
 from dotenv import load_dotenv, find_dotenv
-from pinecone import Pinecone, ServerlessSpec
-from langchain_pinecone import PineconeVectorStore
-from langchain_huggingface import HuggingFaceEmbeddings
-# from langchain_core.output_parsers import StrOutputParser
-# from langchain_core.runnables import RunnablePassthrough
-# from langchain_openai import ChatOpenAI
 from langchain.docstore.document import Document
 from openai import OpenAI
+import utils  # Import the utils module
 
-client = OpenAI(
-  organization='org-x0YBcOjkdPyf6ExxWCkmFHAj',
-  project='proj_40oH22n9XudeKL2rgka1IQ5B',
-  api_key='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'
-)
-
-pinecone_api_key = os.environ.get("PINECONE_API_KEY")
-
-@st.cache_resource
-def initialize_embeddings(model_name: str = "all-mpnet-base-v2"):
-    embeddings = HuggingFaceEmbeddings(model_name=model_name)
-    return embeddings
-
-
-@st.cache_resource
-def initialize_vector_store(pinecone_api_key: str, index_name: str):
-    # Initialize Pinecone
-    pc = Pinecone(api_key=pinecone_api_key)
-    
-    # Access the index
-    index = pc.Index(index_name)
-    
-    # Use the cached embeddings
-    embeddings = initialize_embeddings()
-    
-    # Create the vector store
-    vector_store = PineconeVectorStore(index=index, embedding=embeddings, text_key='content')
-    
-    return vector_store, embeddings
-
-# Unpack the tuple into both vector_store and embeddings
-vector_store, embeddings = initialize_vector_store(pinecone_api_key, index_name="cpv-full-southern-africa-test")
-
-
-
-
-def get_docs(query, country = [], vulnerability_cat = []):
-
-    if not country:
-        country = "All Countries"
-    if not vulnerability_cat:
-        if country == "All Countries":
-            filters = None
-        else:
-            filters = {'country': {'$in': country}}
-    else:
-        if country == "All Countries":
-            filters = {'vulnerability_cat': {'$in': vulnerability_cat}}
-        else:
-            filters = {'country': {'$in': country},'vulnerability_cat': {'$in': vulnerability_cat}}
-
-
-    docs = vector_store.similarity_search_by_vector_with_score(
-        embeddings.embed_query(query),
-        k=20,
-        filter=filters,
-    )
+from dotenv import load_dotenv
+import os
 
-    # Break out the key fields and convert to pandas for filtering
-    docs_dict = [{**x[0].metadata,"score":x[1],"content":x[0].page_content} for x in docs]
-    df_docs = pd.DataFrame(docs_dict)
-    # Get ourselves an index setup from which to base the source reference number from (in the prompt and matching afterwards)
-    df_docs = df_docs.reset_index()
-    df_docs['ref_id'] = df_docs.index + 1 # start the index at 1
-    # Convert back to Document format
-    ls_dict = []
-    # Iterate over df and add relevant fields to the dict object
-    for index, row in df_docs.iterrows():
-        # Create a Document object for each row
-        doc = Document(
-            page_content = row['content'],
-            metadata={'country': row['country'],'document': row['document'], 'page': row['page'], 'file_name': row['file_name'], 'ref_id': row['ref_id'], 'vulnerability_cat': row['vulnerability_cat'], 'score': row['score']}
-        )
-        # Append the Document object to the documents list
-        ls_dict.append(doc)
+os.environ.pop("OPENAI_API_KEY", None)
+load_dotenv()
+# openai_api_key = os.getenv("OPENAI_API_KEY")
 
-    return ls_dict
 
+# Initialize OpenAI client
+client = utils.get_openai_client()
 
-prompt_template="Answer the given question using the following documents. \
-Formulate your answer in the style of an academic report. \
-Provide example quotes and citations using extracted text from the documents. \
-Use facts and numbers from the documents in your answer. \
-ALWAYS include references for information used from documents at the end of each applicable sentence using the format: '[ref. #]', where '[ref. #]' is included in the text provided at the start of each document (demarcated by the pattern '- &&& [ref. #] document_name &&&:')'. \
-Do not include page numbers in the references. \
-If no relevant information to answer the question is present in the documents, just say you don't have enough information to answer."
+# Get Pinecone API key from environment variables
+pinecone_api_key = os.getenv("PINECONE_API_KEY")
 
+# Initialize vector store and embeddings
+vector_store, embeddings = utils.initialize_vector_store(
+    pinecone_api_key, index_name="cpv-full-southern-africa-test"
+)
 
-# Create a list of options for the dropdown
-# model_options = ['chatGPT','Llama2']
+# Prompt template
+prompt_template = (
+    "Answer the given question using the following documents. "
+    "Formulate your answer in the style of an academic report. "
+    "Provide example quotes and citations using extracted text from the documents. "
+    "Use facts and numbers from the documents in your answer. "
+    "ALWAYS include references for information used from documents at the end of each applicable sentence "
+    "using the format: '[ref. #]', where '[ref. #]' is included in the text provided at the start of each document "
+    "(demarcated by the pattern '- &&& [ref. #] document_name &&&:')'. "
+    "Do not include page numbers in the references. "
+    "If no relevant information to answer the question is present in the documents, just say you don't have enough information to answer."
+)
 
-# Create a list of options for the dropdown
-country_options = ['All Countries','Angola','Botswana','Lesotho','Kenya','Malawi','Mozambique','Namibia','Rwanda','South Africa','Zambia','Zimbabwe']
+# Dropdown options
+country_options = [
+    'All Countries', 'Angola', 'Botswana', 'Lesotho', 'Kenya', 'Malawi',
+    'Mozambique', 'Namibia', 'Rwanda', 'South Africa', 'Zambia', 'Zimbabwe'
+]
 
-# Create a list of options for the dropdown
-vulnerability_options = ['All Categories','Agricultural communities', 'Children', 'Coastal communities', 'Ethnic, racial or other minorities', 'Fishery communities', 'Informal sector workers', 'Members of indigenous and local communities', 'Migrants and displaced persons', 'Older persons', 'Persons living in poverty', 'Persons with disabilities', 'Persons with pre-existing health conditions', 'Residents of drought-prone regions', 'Rural populations', 'Sexual minorities (LGBTQI+)', 'Urban populations', 'Women and other genders','Other']
+vulnerability_options = [
+    'All Categories', 'Agricultural communities', 'Children', 'Coastal communities',
+    'Ethnic, racial or other minorities', 'Fishery communities', 'Informal sector workers',
+    'Members of indigenous and local communities', 'Migrants and displaced persons',
+    'Older persons', 'Persons living in poverty', 'Persons with disabilities',
+    'Persons with pre-existing health conditions', 'Residents of drought-prone regions',
+    'Rural populations', 'Sexual minorities (LGBTQI+)', 'Urban populations',
+    'Women and other genders', 'Other'
+]
 
-# List of examples
 examples = [
     "-",
     "What specific initiatives are presented in the context to address the needs of groups such as women and children to the effects climate change?",
     "In addition to gender, children, and youth, is there any mention of other groups facing disproportional impacts from climate change due to their geographic location, socio-economic status, age, gender, health, and occupation?"
 ]
 
-
-def get_refs(docs, res):
-  '''
-  Parse response for engineered reference ids (refer to prompt template)
-  Extract documents using reference ids
-  '''
-  res = res.lower()  # Convert to lowercase for matching
-  # This pattern should be returned by gpt3.5
-  # pattern = r'ref\. (\d+)\]\.'
-  pattern = r'ref\. (\d+)'
-  ref_ids = [int(match) for match in re.findall(pattern, res)]
-  # extract
-  result_str = ""  # Initialize an empty string to store the result
-  for i in range(len(docs)):
-      ref_id = docs[i].metadata['ref_id']
-      if ref_id in ref_ids:
-        if docs[i].metadata['document'] == "Supplementary":
-            result_str += "**Ref. " + str(ref_id) + " [" + docs[i].metadata['country'] + " " + docs[i].metadata['document'] + ':' + docs[i].metadata['file_name'] + ' p' + str(docs[i].metadata['page']) + '; vulnerabilities: ' + docs[i].metadata['vulnerability_cat'] + "]:** " + "*'" + docs[i].page_content + "'*<br> <br>" # Add <br> for a line break
-        else:
-            result_str += "**Ref. " + str(ref_id) + " [" + docs[i].metadata['country'] + " " + docs[i].metadata['document'] + ' p' + str(docs[i].metadata['page']) + '; vulnerabilities: ' + docs[i].metadata['vulnerability_cat'] + "]:** " + "*'" + docs[i].page_content + "'*<br> <br>" # Add <br> for a line break
-  
-  return result_str
-
-# define a special function for putting the prompt together (as we can't use haystack)
-def get_prompt(docs, input_query):
-  base_prompt=prompt_template
-  # Add the metadata data for references
-  context = ' - '.join(['&&& [ref. '+str(d.metadata['ref_id'])+'] '+d.metadata['document']+' &&&: '+d.page_content for d in docs])
-  prompt = base_prompt+"; Context: "+context+"; Question: "+input_query+"; Answer:"
-  return(prompt)
-
-def run_query(query, country, model_sel):
-    # first call the retriever function using selected filters
-    docs = get_docs(query, country=country,vulnerability_cat=vulnerabilities_cat)
-    # model selector (not currently being used)
-    if model_sel == "chatGPT":
-        # instantiate ChatCompletion as a generator object (stream is set to True)
-        # response = openai.ChatCompletion.create(model="gpt-4o-mini-2024-07-18", messages=[{"role": "user", "content": get_prompt(docs, query)}], stream=True)
-
-
-        stream = client.chat.completions.create(
-            model="gpt-4o-mini-2024-07-18",
-            messages=[{"role": "user", "content": get_prompt(docs, query)}],
-            stream=True,
-        )
-        # iterate through the streamed output
-        report = []
-
-        for chunk in stream:
-            if chunk.choices[0].delta.content is not None:
-                # print(chunk.choices[0].delta.content, end="")
-                report.append(chunk.choices[0].delta.content)
-                result = "".join(report).strip()
-                res_box.success(result) # output to response text box
-
-        references = get_refs(docs, result) # extract references from the generated text
-    # Llama2 selection (was running on HF)
-    # else:
-    #     res = client.text_generation(get_prompt(docs, query=input_query), max_new_tokens=4000, temperature=0.01, model=model)
-    #     output = res
-    #     references = get_refs(docs, res)
-    
-    st.markdown("----")
-    st.markdown('**REFERENCES:**')
-    st.markdown('References are based on text automatically extracted from climate policy documents. These extracts may contain non-legible characters or disjointed text as an artifact of the extraction procedure')
-    st.markdown(references, unsafe_allow_html=True)
-
-
-#___________________________________________________________________________________________________________
-
-# Sidebar (filters)
+# Sidebar Filters
 with st.sidebar:
-    country = st.sidebar.multiselect('Filter by country:', country_options)
-    vulnerabilities_cat = st.sidebar.multiselect('Filter by vulnerabilities category:', vulnerability_options)
+    country = st.multiselect('Filter by country:', country_options)
+    vulnerabilities_cat = st.multiselect('Filter by vulnerabilities category:', vulnerability_options)
     with st.expander("ℹ️ - About filters", expanded=False):
         st.markdown(
             """
-            * *These selections will filter the data matched against your query*
-            * *For a comparative analysis of multiple countries or vulnerability categories, select the items you require or select **'All Countries'** or **'All Categories'***
-            * *Be careful in using the vulnerabilities category filter, as many of the categories are not well represented in the documents. Therefore, this will severly limit the data available for analysis*
+            * *These selections will filter the data matched against your query.*
+            * *For a comparative analysis of multiple countries or vulnerability categories, select the items you require or select **'All Countries'** or **'All Categories'***.
+            * *Be careful in using the vulnerabilities category filter, as many of the categories are not well represented in the documents. Therefore, this will severely limit the data available for analysis.*
             """
         )
 
-# Main window title
-with st.container():
-        st.markdown("<h2 style='text-align: center;'> Climate Policy Documents: Vulnerabilities Analysis Q&A </h2>", unsafe_allow_html=True)
-        st.write(' ')
+# Main Window Title
+st.markdown("<h2 style='text-align: center;'> Climate Policy Documents: Vulnerabilities Analysis Q&A </h2>", unsafe_allow_html=True)
+st.write(' ')
 
-# Main window instructions
+# Main Window Instructions
 with st.expander("ℹ️ - About this app", expanded=False):
     st.write(
         """
-        This tool seeks to provide an interface for quering national climate policy documents (NDCs, LTS etc.). The current version is powered by chatGPT (3.5). The document store is limited to 10 Southern African countries (Angola, Botswana, Eswatini, Lesotho, Malawi, Mozambique, Namibia, South Africa, Zambia, Zimbabwe), as well as Kenya and Rwanda. The intended use case is to allow users to interact with the documents and obtain valuable insights on various vulnerable groups affected by climate change.
+        This tool seeks to provide an interface for querying national climate policy documents (NDCs, LTS etc.). The current version is powered by chatGPT (3.5). The document store is limited to 10 Southern African countries (Angola, Botswana, Eswatini, Lesotho, Malawi, Mozambique, Namibia, South Africa, Zambia, Zimbabwe), as well as Kenya and Rwanda. The intended use case is to allow users to interact with the documents and obtain valuable insights on various vulnerable groups affected by climate change.
         
         **DISCLAIMER:** *This prototype tool based on LLMs (Language Models) is provided "as is" for experimental and exploratory purposes only, and should not be used for critical or production applications. Users are advised that the tool may contain errors, bugs, or limitations and should be used with caution and awareness of potential risks, and the developers make no warranties or guarantees regarding its performance, reliability, or suitability for any specific purpose.*
-        """)
-    # Display the text passages as radio buttons
+        """
+    )
     selected_example = st.radio("Example questions", examples)
     st.write(
-    """
-    You can request comparative analyses between countries by filtering by country, and using more advanced prompts. For example:
+        """
+        You can request comparative analyses between countries by filtering by country, and using more advanced prompts. For example:
+        
+        *Provide a comparative analysis between Angola and Kenya with regard to specific initiatives presented in the context to address the needs of groups such women and children to the effects climate change.*
     
-    *Provide a comparative analysis between Angola and Kenya with regard to specific initiatives presented in the context to address the needs of groups such women and children to the effects climate change.*
-
-    Make sure your filters match the countries you have specified for the analysis!
-    """)
-
+        Make sure your filters match the countries you have specified for the analysis!
+        """
+    )
 
-# Dropdown selectbox: model (currently not used)
-# model_sel = st.selectbox('Select an LLM:', model_options)
+# Model Selection (Currently fixed to chatGPT)
 model_sel = "chatGPT"
 
-#----Model Select logic-------
+# Prompt Logic
 if model_sel == "chatGPT":
     model_name = "gpt-3.5-turbo"
 
-# else:
-#     model = "meta-llama/Llama-2-70b-chat-hf"
-#     # Instantiate the inference client
-#     client = InferenceClient()
-
-# get prompt from user or example prompt
-if selected_example == "-": #hyphen used as a work around (st won't allow null selection)
+# Input Text Area
+if selected_example == "-":
     text = st.text_area('Enter your question in the text box below using natural language or select an example from above:')
 else:
     text = st.text_area('Enter your question in the text box below using natural language or select an example from above:', value=selected_example)
 
+# Submit Button
 if st.button('Submit'):
     st.markdown("----")
     st.markdown('**RESPONSE:**')
     res_box = st.empty()
-    run_query(text, country=country, model_sel=model_sel)
-
-
-
-
-
-
+    
+    # Fetch documents based on user input and filters
+    docs = utils.get_docs(vector_store, embeddings, text, country=country, vulnerability_cat=vulnerabilities_cat)
+    
+    # Construct the prompt
+    prompt = utils.get_prompt(prompt_template, docs, text)
+    
+    # Run the query and get references
+    references = utils.run_query(client, prompt, docs, res_box)
+    
+    # Display references
+    st.markdown("----")
+    st.markdown('**REFERENCES:**')
+    st.markdown(
+        'References are based on text automatically extracted from climate policy documents. '
+        'These extracts may contain non-legible characters or disjointed text as an artifact of the extraction procedure',
+        unsafe_allow_html=True
+    )
+    st.markdown(references, unsafe_allow_html=True)

cpv_full.db

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:30511f3207929443a0ba546dd10de94c4d9f8d73dfe6cec34e0bcf2de8367862
-size 12734464

cpv_full.faiss

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b1711d36c1fbb1425b9b257ce1b8a7dd5f40159e02f95a8170a006478f627b3a
-size 25543725

cpv_full.json

@@ -1 +0,0 @@
-{"faiss_index_factory_str": "Flat", "sql_url": "sqlite:///cpv_full.db"}

cpv_full_southern_africa_kenya.db

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a31dd7681c0ec71efd469fe89329d1bb2138e8fb8e7019741dfca484242f6990
-size 18247680

cpv_full_southern_africa_kenya.faiss

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:53130abde470d7a3a63b543dbb4b1ae44eea230662ee7d484123e345a2ec50f9
-size 26173485

cpv_full_southern_africa_kenya.json

@@ -1 +0,0 @@
-{"faiss_index_factory_str": "Flat", "sql_url": "sqlite:///cpv_full_southern_africa_kenya.db"}

env

@@ -1,5 +0,0 @@
-OPENAI_API_KEY="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"
-HF_API_KEY="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"
-PINECONE_API_KEY="c3f5717c-f43a-46d0-893e-02b44dbcf13b"
-USER1_HASH="$2b$12$hZbOi6zKmQQWvvpcllds9uAB3ili66N0aQyPzuDctl7IkNhl226oG"
-USER2_HASH="$2b$12$kWnArbA.2QTkpMv2yvE2J.7UJw0Fgc/3FH1k5JRqhjg.cvytriGt2"

requirements.txt

@@ -1,3 +1,8 @@
-farm-haystack[colab, inference, pinecone]
-python-dotenv==1.0.0
-openai==0.27.8
+streamlit==1.38.0
+numpy==1.26.4
+pandas==2.2.3
+python-dotenv==1.0.1
+pinecone-client==5.0.1
+langchain-pinecone==0.2.0
+langchain-huggingface==0.1.0
+openai==1.50.2

utils.py

@@ -0,0 +1,129 @@
+# utils.py
+
+import streamlit as st
+import os
+import re
+import pandas as pd
+from langchain_pinecone import PineconeVectorStore
+from langchain_huggingface import HuggingFaceEmbeddings
+from langchain.docstore.document import Document
+from dotenv import load_dotenv
+from pinecone import Pinecone
+from openai import OpenAI
+
+# Load environment variables
+load_dotenv()
+
+# Initialize OpenAI client
+def get_openai_client():
+    return OpenAI(
+        organization=os.getenv('OPENAI_ORG_ID'),
+        project=os.getenv('OPENAI_PROJECT_ID'),
+        api_key=os.getenv('OPENAI_API_KEY')
+    )
+
+# Initialize embeddings
+@st.cache_resource
+def initialize_embeddings(model_name: str = "all-mpnet-base-v2"):
+    embeddings = HuggingFaceEmbeddings(model_name=model_name)
+    return embeddings
+
+# Initialize vector store
+@st.cache_resource
+def initialize_vector_store(pinecone_api_key: str, index_name: str):
+    pc = Pinecone(api_key=pinecone_api_key)
+    index = pc.Index(index_name)
+    embeddings = initialize_embeddings()
+    vector_store = PineconeVectorStore(index=index, embedding=embeddings, text_key='content')
+    return vector_store, embeddings
+
+# Fetch documents based on query and filters
+def get_docs(vector_store, embeddings, query, country=[], vulnerability_cat=[]):
+    if not country:
+        country = "All Countries"
+    if not vulnerability_cat:
+        filters = None if country == "All Countries" else {'country': {'$in': country}}
+    else:
+        if country == "All Countries":
+            filters = {'vulnerability_cat': {'$in': vulnerability_cat}}
+        else:
+            filters = {
+                'country': {'$in': country},
+                'vulnerability_cat': {'$in': vulnerability_cat}
+            }
+
+    docs = vector_store.similarity_search_by_vector_with_score(
+        embeddings.embed_query(query),
+        k=20,
+        filter=filters,
+    )
+
+    docs_dict = [{**x[0].metadata, "score": x[1], "content": x[0].page_content} for x in docs]
+    df_docs = pd.DataFrame(docs_dict).reset_index()
+    df_docs['ref_id'] = df_docs.index + 1
+
+    ls_dict = [
+        Document(
+            page_content=row['content'],
+            metadata={
+                'country': row['country'],
+                'document': row['document'],
+                'page': row['page'],
+                'file_name': row['file_name'],
+                'ref_id': row['ref_id'],
+                'vulnerability_cat': row['vulnerability_cat'],
+                'score': row['score']
+            }
+        )
+        for _, row in df_docs.iterrows()
+    ]
+
+    return ls_dict
+
+# Extract references from the response
+def get_refs(docs, res):
+    res = res.lower()
+    pattern = r'ref\. (\d+)'
+    ref_ids = [int(match) for match in re.findall(pattern, res)]
+    result_str = ""
+    for doc in docs:
+        ref_id = doc.metadata['ref_id']
+        if ref_id in ref_ids:
+            metadata = doc.metadata
+            if metadata['document'] == "Supplementary":
+                result_str += (
+                    f"**Ref. {ref_id} [{metadata['country']} {metadata['document']}: {metadata['file_name']} p{metadata['page']}; "
+                    f"vulnerabilities: {metadata['vulnerability_cat']}]:** *'{doc.page_content}'*<br><br>"
+                )
+            else:
+                result_str += (
+                    f"**Ref. {ref_id} [{metadata['country']} {metadata['document']} p{metadata['page']}; "
+                    f"vulnerabilities: {metadata['vulnerability_cat']}]:** *'{doc.page_content}'*<br><br>"
+                )
+    return result_str
+
+# Construct the prompt for the model
+def get_prompt(prompt_template, docs, input_query):
+    context = ' - '.join([
+        f"&&& [ref. {d.metadata['ref_id']}] {d.metadata['document']} &&&: {d.page_content}"
+        for d in docs
+    ])
+    prompt = f"{prompt_template}; Context: {context}; Question: {input_query}; Answer:"
+    return prompt
+
+# Execute the query and generate the response
+def run_query(client, prompt, docs, res_box):
+    stream = client.chat.completions.create(
+        model="gpt-4o-mini-2024-07-18",
+        messages=[{"role": "user", "content": prompt}],
+        stream=True,
+    )
+    report = []
+    for chunk in stream:
+        if chunk.choices[0].delta.content is not None:
+            report.append(chunk.choices[0].delta.content)
+            result = "".join(report).strip()
+            res_box.success(result)
+
+    references = get_refs(docs, result)
+    return references