# set path import glob, os, sys; sys.path.append('../utils') #import needed libraries import seaborn as sns import matplotlib.pyplot as plt import numpy as np import pandas as pd import streamlit as st from utils.target_classifier import load_targetClassifier, target_classification import logging logger = logging.getLogger(__name__) from utils.config import get_classifier_params from utils.preprocessing import paraLengthCheck from io import BytesIO import xlsxwriter import plotly.express as px from utils.target_classifier import label_dict from appStore.rag import run_query from math import exp import re import json import nltk from nltk.corpus import stopwords nltk.download('stopwords') import openai openai_api_key = os.environ["OPEN_AI_KEY"] # Declare all the necessary variables classifier_identifier = 'target' params = get_classifier_params(classifier_identifier) @st.cache_data def to_excel(df,sectorlist): len_df = len(df) output = BytesIO() writer = pd.ExcelWriter(output, engine='xlsxwriter') df.to_excel(writer, index=False, sheet_name='Sheet1') workbook = writer.book worksheet = writer.sheets['Sheet1'] worksheet.data_validation('S2:S{}'.format(len_df), {'validate': 'list', 'source': ['No', 'Yes', 'Discard']}) worksheet.data_validation('X2:X{}'.format(len_df), {'validate': 'list', 'source': sectorlist + ['Blank']}) worksheet.data_validation('T2:T{}'.format(len_df), {'validate': 'list', 'source': sectorlist + ['Blank']}) worksheet.data_validation('U2:U{}'.format(len_df), {'validate': 'list', 'source': sectorlist + ['Blank']}) worksheet.data_validation('V2:V{}'.format(len_df), {'validate': 'list', 'source': sectorlist + ['Blank']}) worksheet.data_validation('W2:U{}'.format(len_df), {'validate': 'list', 'source': sectorlist + ['Blank']}) writer.save() processed_data = output.getvalue() return processed_data def app(): ### Main app code ### with st.container(): if 'key1' in st.session_state: # Load the existing dataset df = st.session_state.key1 # Filter out all paragraphs that do not have a reference to groups df = df[df['Vulnerability Label'].apply(lambda x: len(x) > 0 and 'Other' not in x)] # Load the classifier model classifier = load_targetClassifier(classifier_name=params['model_name']) st.session_state['{}_classifier'.format(classifier_identifier)] = classifier df = target_classification(haystack_doc=df, threshold= params['threshold']) # Rename column df.rename(columns={'Target Label': 'Specific action/target/measure mentioned'}, inplace=True) st.session_state.key2 = df vc_prompt=""" You are assessing the accuracy of a multi-label classifier. The classifier seeks to assess the relevance of a given passage of context to any of 18 classes: 'Agricultural communities', 'Children', 'Coastal communities', 'Ethnic, racial or other minorities', 'Fishery communities', 'Informal sector workers', 'Members of indigenous and local communities', 'Migrants and displaced persons', 'Older persons', 'Other', 'Persons living in poverty', 'Persons with disabilities', 'Persons with pre-existing health conditions', 'Residents of drought-prone regions', 'Rural populations', 'Sexual minorities (LGBTQI+)', 'Urban populations', 'Women and other genders' If there is a semantic relevance or keyword(s) match between labels and context, then assess accuracy as a boolean True. Assessing class relevance may be tricky in some cases as the context can use technical language which is sometimes ambiguous. Please take your time to ensure a robust assessment. If you can't decide, err on the side of the classifier, and assume it is correct. Use the examples below for reference: EXAMPLE 1 LABEL: ['Agricultural communities', 'Residents of drought-prone regions'] CONTEXT: "Future climatic predictions for Kenya indicate possible temperature increase of 1C by 2020 and 2.3C by 2050. These changes unless effectively mitigated, will likely result to erosion of the productive assets and the weakening of coping strategies and resilience of rain-fed farming systems, especially in the arid and semi-arid lands." RESPONSE: True EXAMPLE 2 LABEL: ['Fishery communities'] CONTEXT: "The reduced water availability resulting from frequent droughts also limits aquaculture development. Forests and agroforestry The farmed fisheries resources include the trout fish in cold water high altitude areas and tilapia, catfish, common carp for warmer water low altitude areas. Figure 5 shows the quantities and monetary value of fish produced in Kenya between 2005 and 2016." RESPONSE: True EXAMPLE 3 LABEL: ['Persons with disabilities'] CONTEXT: "In addressing climate change issues, public entities are required to undertake public awareness and consultations, and ensure gender mainstreaming, in line with the Constitution and the Climate Change Bill (2014). 5. Means of implementation Kenya's contribution will be implemented with both domestic and international support." RESPONSE: False EXAMPLE 4 LABEL: ['Children', 'Women and other genders'] CONTEXT: "Enhance quality control and food safety by relevant institutions along crop, livestock and fisheries value chains. Enhance use of low greenhouse gas emitting fish production technologies and practices. Promote integrated farming systems comprising crops, livestock, aquaculture and farm forestry. Create awareness and capacity build women, youth and venerable groups (WY&VG) on CSA." RESPONSE: True EXAMPLE 5 LABEL: ['Ethnic, racial or other minorities'] CONTEXT: "Harmonize livestock vaccinations across the bordering counties and across the international borders. Facilitate management of veterinary drug residues, carcasses and agrochemicals. Promote efficient use of farm mechanization. Promote mechanized and animal powered conservation tillage practices as compared to conventional tillage. Promote value addition of farm produce through cottage industries." RESPONSE: False EXAMPLE 6 LABEL: ['Agricultural communities'] CONTEXT: "There is also no traceability mechanism for produce and products from farm to folk. Value addition will ensure longer shelf life, reduced transaction costs and higher incomes. Summary of Actions: Identify and promote existing value addition technologies. Incentivize the private sector to invest in agricultural value addition." RESPONSE: False EXAMPLE 7 LABEL: ['Agricultural communities', 'Rural populations'] CONTEXT: "Kenya's total greenhouse gas (GHG) emissions are relatively low, standing at 73 MtCO2eq in 2010, out of which 75%/ are from the land use, land-use change and forestry (LULUCF) and agriculture sectors. This may be explained by the reliance on wood fuel by a large proportion of the population coupled with the increasing demand for agricultural land and urban development." RESPONSE: True Return the assessment as a boolean True or False. Return only the boolean, and nothing else. Now assess the following sample: """ tma_prompt=""" You are assessing the accuracy of a binary ('YES'/'NO') classifier. The classifier classifies a given passage of text as to whether it contains reference to a target, measure, action, and plans in the context of the United Nations Framework Convention on Climate Change (UNFCCC) and the Paris Agreement. The text is extracted from Nationally Determined Contributions (NDCs) documents. The concepts of targets, measures, actions, and plans are defined below: 1. Targets • Definition: Targets in the NDCs refer to the specific, quantified objectives that each country sets for itself to reduce greenhouse gas (GHG) emissions and mitigate climate change. These targets reflect the level of ambition a country is willing to commit to in its climate action. • Example in NDCs: A common form of a target is a percentage reduction in GHG emissions by a certain year, such as “reducing emissions by 50% by 2030 compared to 1990 levels.” Targets can also be sector-specific, such as setting renewable energy capacity goals. 2. Measures • Definition: Measures are the policies, regulations, and actions that are implemented to achieve the targets set in the NDCs. These are the instruments through which a country can ensure that it is on the right path to meet its climate goals. • Example in NDCs: Measures can include implementing a carbon tax, introducing renewable energy incentives, or regulations to improve energy efficiency in buildings or transportation sectors. These could also involve reforestation or land-use changes to enhance carbon sinks. 3. Actions • Definition: Actions refer to the specific activities, projects, or steps that are undertaken to implement the measures and meet the set targets. Actions are the tangible efforts that contribute to reducing emissions or adapting to climate change impacts. • Example in NDCs: Actions might include building solar or wind power plants, electrifying transportation systems, or retrofitting existing infrastructure to make it more energy efficient. Actions are often the ground-level, operational steps that translate plans into reality. If you agree with the classifier, then assess accuracy as a boolean True. Note - assessing targets, measures and actions may be tricky in some cases as the text can use technical language which is sometimes ambiguous. Please take your time to ensure a robust assessment. If you can't decide, err on the side of the classifier, and assume it is correct. EXAMPLE 1: LABEL: 'YES' CONTEXT: "This will lead to more climate related vulnerabilities thereby predisposing farming communities to food insecurity and more poverty. In response to this scenario, the Government has been exploring innovative and transformative measures to assist stakeholders across the agricultural value chains to manage the effects of current and projected change of climate patterns." RESPONSE: True EXAMPLE 2: LABEL: 'NO' CONTEXT: "Kenya's total greenhouse gas (GHG) emissions are relatively low, standing at 73 MtCO2eq in 2010, out of which 75%/ are from the land use, land-use change and forestry (LULUCF) and agriculture sectors. This may be explained by the reliance on wood fuel by a large proportion of the population coupled with the increasing demand for agricultural land and urban development." RESPONSE: True EXAMPLE 3: LABEL: 'YES' CONTEXT: "1.1 National Circumstances Kenya is located in the Greater Horn of Africa region, which is highly vulnerable to the impacts of climate change. More than 80% of the country‚Äôs landmass is arid and semi-arid land (ASAL) with poor infrastructure, and other developmental challenges." RESPONSE: True Return the assessment as a boolean True or False. Return only the boolean, and nothing else. Now assess the following sample: """ def send_to_chatgpt_api(context, label, prompt, openai_api_key, logprobs_flag=None, logprobs_n=None): # Combine the result object and context with the new prompt combined_message = f""" {prompt} LABEL: {label} CONTEXT: {context} RESPONSE: """ # Set up the OpenAI API openai.api_key = openai_api_key # Send the combined message to the ChatGPT API response = openai.ChatCompletion.create( # model="gpt-4o-mini", model="gpt-4o-mini-2024-07-18", # model="gpt-4o-2024-08-06", messages=[ {"role": "system", "content": "You are ChatGPT."}, {"role": "user", "content": combined_message} ], logprobs=logprobs_flag, # whether to return log probabilities of the output tokens or not. If true, returns the log probabilities of each output token returned in the content of message.. top_logprobs=logprobs_n, ) # Get the response from the API if logprobs_flag: gpt_response = response.choices[0].logprobs.content[0].top_logprobs[0] else: gpt_response = response.choices[0].message['content'] return gpt_response # Fuzzy matching def levenshtein_distance(a, b): # Initialize the matrix dp = [[0 for _ in range(len(b) + 1)] for _ in range(len(a) + 1)] # Base cases for i in range(len(a) + 1): dp[i][0] = i for j in range(len(b) + 1): dp[0][j] = j # Fill the matrix for i in range(1, len(a) + 1): for j in range(1, len(b) + 1): if a[i - 1] == b[j - 1]: cost = 0 else: cost = 1 dp[i][j] = min(dp[i - 1][j] + 1, # Deletion dp[i][j - 1] + 1, # Insertion dp[i - 1][j - 1] + cost) # Substitution # Return the Levenshtein distance return dp[-1][-1] def similarity_score(a, b): max_len = max(len(a), len(b)) if max_len == 0: return 1.0 return (max_len - levenshtein_distance(a, b)) / max_len def remove_stopwords(text): stop_words = set(stopwords.words('english')) # Tokenize the string and filter out stopwords return ' '.join([word for word in text.split() if word.lower() not in stop_words]) def fuzzy_match_sequence(sequence, long_string, threshold=0.4): # If the sequence is a single string, split it into phrases (based on commas or similar punctuation) if isinstance(sequence, str): sequence = re.split(r',\s*|\s+', sequence) # Remove stopwords from both the sequence and the long string sequence = [remove_stopwords(phrase) for phrase in sequence] long_string = remove_stopwords(long_string) # Ensure that the input is now a list or tuple if not isinstance(sequence, (list, tuple)): sequence = list(sequence) # Split the long string into words long_string_words = long_string.split() # Perform Levenshtein-based fuzzy matching and calculate overall similarity score total_score = 0 matches = [] count_high_prob_matches = 0 for word in sequence: # Find the best match for the current word in the long string best_match_score, best_match_word = max((similarity_score(word, ls_word), ls_word) for ls_word in long_string_words) # Only count matches with a similarity score above the threshold if best_match_score >= threshold: count_high_prob_matches += 1 # Cap the total score at 1 and calculate contribution total_score += min(best_match_score, 1 - total_score) # Ensure the score doesn't go above 1 matches.append(f"Keyword '{word}' matched '{best_match_word}' with a similarity score of {best_match_score:.2f}") # The total score should never exceed 1, ensure it is capped at 1 total_score = round(min(total_score, 1),2) return total_score def target_display(model_sel_name, doc_name): ### TABLE Output ### # Assign dataframe a name df = st.session_state['key2'] st.write(df) ### RAG Output by group ## # Expand the DataFrame df_expand = ( df.query("`Specific action/target/measure mentioned` == 'YES'") .explode('Vulnerability Label') ) # Group by 'Vulnerability Label' and concatenate 'text' df_agg = df_expand.groupby('Vulnerability Label')['text'].agg('; '.join).reset_index() # st.write(df_agg) st.markdown("----") st.markdown('**SUMMARY OF GOALS BY VULNERABILITY LABEL:**') # Check if the results are already in session state if 'results_df' not in st.session_state: # Initialize an empty list to store the results summary_list = [] results_list = [] # Process the data in the loop for i in range(0, len(df_agg)): st.write(df_agg['Vulnerability Label'].iloc[i]) # Run query to get the result result = run_query( context=df_agg['text'].iloc[i], label=df_agg['Vulnerability Label'].iloc[i], model_sel_name=model_sel_name ) # Store the Vulnerability Label and the response in a list of dictionaries summary_list.append({ 'document': doc_name, 'text': df_agg['text'].iloc[i], 'label': df_agg['Vulnerability Label'].iloc[i], 'summary': result.get_full_content() }) # Process the data in the loop for i in range(0, len(df)): # Send the result to the ChatGPT API and get the labeled response vc_response = send_to_chatgpt_api( context = df['text'].iloc[i], label = df['Vulnerability Label'].iloc[i], prompt = vc_prompt, openai_api_key=openai_api_key, logprobs_flag=True, logprobs_n=1) tma_response = send_to_chatgpt_api( context = df['text'].iloc[i], label = df['Specific action/target/measure mentioned'].iloc[i], prompt = tma_prompt, openai_api_key=openai_api_key, logprobs_flag=True, logprobs_n=1) # Convert logprobs to % scale vc_prob = np.round(np.exp(vc_response.logprob),2) vc_token = vc_response.token # Convert contrary predictions to probability of positive prediction (inverse) if vc_token == 'False': vc_prob_cnv = round(1 - vc_prob,2) else: vc_prob_cnv = vc_prob # Do some fuzzy matching to check for class-related keywords in the text vc_keywords = fuzzy_match_sequence(str(df['Vulnerability Label'].iloc[i]), str(df['text'].iloc[i])) # Compute vulnerability classifciation eval vc_eval = False if vc_prob_cnv > 0.5 or vc_keywords > 0: vc_eval = True # Convert logprobs to % scale tma_prob = np.round(np.exp(tma_response.logprob),2) tma_token = tma_response.token # Convert contrary predictions to probability of positive prediction (inverse) if tma_token == 'False': tma_prob_cnv = round(1 - tma_prob,2) else: tma_prob_cnv = tma_prob # Compute TMA classification eval tma_eval = False if tma_prob_cnv > 0.5: tma_eval = True # Store the Vulnerability Label and the response in a list of dictionaries results_list.append({ 'document': doc_name, 'text': df['text'].iloc[i], 'page': df['page'].iloc[i], 'label': df['Vulnerability Label'].iloc[i], 'target': df['Specific action/target/measure mentioned'].iloc[i], 'VC_prob': vc_prob_cnv, 'VC_keywords': vc_keywords, 'VC_eval': vc_eval, 'TMA_prob': tma_prob_cnv, 'TMA_eval': tma_eval, 'VC_check': None, 'TMA_check': None, }) # Once the loop is done, convert results to a DataFrame and store in session state st.session_state['results_df'] = pd.DataFrame(results_list) st.session_state['summary_df'] = pd.DataFrame(summary_list) df_full = st.session_state['key1'] num_paragraphs = len(df_full['Vulnerability Label']) num_references = len(df['Vulnerability Label']) meta_list = [] # Store the Vulnerability Label and the response in a list of dictionaries meta_list.append({ 'document': doc_name, 'paragraphs': num_paragraphs, 'references': num_references, }) st.session_state['meta_df'] = pd.DataFrame(meta_list) # Retrieve the results from session state meta_df = st.session_state['meta_df'] summary_df = st.session_state['summary_df'] results_df = st.session_state['results_df'] # Use an in-memory buffer to hold the Excel file excel_buffer = BytesIO() # Create an Excel writer and write each DataFrame to a separate sheet with pd.ExcelWriter(excel_buffer, engine='xlsxwriter') as writer: meta_df.to_excel(writer, sheet_name='Meta', index=False) summary_df.to_excel(writer, sheet_name='Summary', index=False) results_df.to_excel(writer, sheet_name='Results', index=False) # Ensure the buffer is ready for downloading excel_buffer.seek(0) # Create a download button for the Excel file st.download_button( label="Download LLM Evaluation", data=excel_buffer, file_name='eval_' + str.split(doc_name,".")[0] + '.xlsx', mime='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', )