Spaces:
Sleeping
Sleeping
| import gradio as gr | |
| import pandas as pd | |
| def data_pre_processing(file_responses): | |
| # Financial Weights can be anything (ultimately the row-wise weights are aggregated and the corresponding fractions are obtained from that rows' total tax payed) | |
| try: # Define the columns to be processed | |
| # Developing Numeric Columns | |
| # Convert columns to numeric and fill NaN values with 0 | |
| file_responses['Personal_TaxDirection_1_TaxWeightageAllocated'] = pd.to_numeric(file_responses['Personal_TaxDirection_1_TaxWeightageAllocated'], errors='coerce').fillna(0) | |
| file_responses['Personal_TaxDirection_2_TaxWeightageAllocated'] = pd.to_numeric(file_responses['Personal_TaxDirection_2_TaxWeightageAllocated'], errors='coerce').fillna(0) | |
| file_responses['Personal_TaxDirection_3_TaxWeightageAllocated'] = pd.to_numeric(file_responses['Personal_TaxDirection_3_TaxWeightageAllocated'], errors='coerce').fillna(0) | |
| file_responses['Latest estimated Tax payment?'] = pd.to_numeric(file_responses['Latest estimated Tax payment?'], errors='coerce').fillna(0) | |
| # Adding a new column 'TotalWeightageAllocated' by summing specific columns by their names | |
| file_responses['TotalWeightageAllocated'] = file_responses['Personal_TaxDirection_1_TaxWeightageAllocated'] + file_responses['Personal_TaxDirection_2_TaxWeightageAllocated'] + file_responses['Personal_TaxDirection_3_TaxWeightageAllocated'] | |
| # Creating Datasets (we assume everything has been provided to us in English, or the translations have been done already) | |
| # Renaming the datasets into similar column headings | |
| initial_dataset_1 = file_responses.rename(columns={ | |
| 'Personal_TaxDirection_1_Wish': 'Problem_Description', | |
| 'Personal_TaxDirection_1_GeographicalLocation': 'Geographical_Location', | |
| 'Personal_TaxDirection_1_TaxWeightageAllocated': 'Financial_Weight' | |
| })[['Problem_Description', 'Geographical_Location', 'Financial_Weight']] | |
| initial_dataset_2 = file_responses.rename(columns={ | |
| 'Personal_TaxDirection_2_Wish': 'Problem_Description', | |
| 'Personal_TaxDirection_2_GeographicalLocation': 'Geographical_Location', | |
| 'Personal_TaxDirection_2_TaxWeightageAllocated': 'Financial_Weight' | |
| })[['Problem_Description', 'Geographical_Location', 'Financial_Weight']] | |
| initial_dataset_3 = file_responses.rename(columns={ | |
| 'Personal_TaxDirection_3_Wish': 'Problem_Description', | |
| 'Personal_TaxDirection_3_GeographicalLocation': 'Geographical_Location', | |
| 'Personal_TaxDirection_3_TaxWeightageAllocated': 'Financial_Weight' | |
| })[['Problem_Description', 'Geographical_Location', 'Financial_Weight']] | |
| # Calculating the actual TaxAmount to be allocated against each WISH (by overwriting the newly created columns) | |
| initial_dataset_1['Financial_Weight'] = file_responses['Personal_TaxDirection_1_TaxWeightageAllocated'] * file_responses['Latest estimated Tax payment?'] / file_responses['TotalWeightageAllocated'] | |
| initial_dataset_2['Financial_Weight'] = file_responses['Personal_TaxDirection_2_TaxWeightageAllocated'] * file_responses['Latest estimated Tax payment?'] / file_responses['TotalWeightageAllocated'] | |
| initial_dataset_3['Financial_Weight'] = file_responses['Personal_TaxDirection_3_TaxWeightageAllocated'] * file_responses['Latest estimated Tax payment?'] / file_responses['TotalWeightageAllocated'] | |
| # Removing useless rows | |
| # Drop rows where Problem_Description is NaN or an empty string | |
| initial_dataset_1 = initial_dataset_1.dropna(subset=['Problem_Description'], axis=0) | |
| initial_dataset_2 = initial_dataset_2.dropna(subset=['Problem_Description'], axis=0) | |
| initial_dataset_3 = initial_dataset_3.dropna(subset=['Problem_Description'], axis=0) | |
| # Convert 'Problem_Description' column to string type | |
| initial_dataset_1['Problem_Description'] = initial_dataset_1['Problem_Description'].astype(str) | |
| initial_dataset_2['Problem_Description'] = initial_dataset_2['Problem_Description'].astype(str) | |
| initial_dataset_3['Problem_Description'] = initial_dataset_3['Problem_Description'].astype(str) | |
| # Merging the Datasets | |
| # Vertically concatenating (merging) the 3 DataFrames | |
| merged_dataset = pd.concat([initial_dataset_1, initial_dataset_2, initial_dataset_3], ignore_index=True) | |
| # Different return can be used to check the processing | |
| # return file_responses | |
| return merged_dataset | |
| except Exception as e: | |
| return str(e) | |
| import spacy | |
| from transformers import AutoTokenizer, AutoModel | |
| import torch | |
| # Load SpaCy model | |
| # Install the 'en_core_web_sm' model if it isn't already installed | |
| try: | |
| nlp = spacy.load('en_core_web_sm') | |
| except OSError: | |
| # Instead of this try~catch, we could also include this < https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.5.0/en_core_web_sm-3.5.0.tar.gz > in the requirements.txt to directly load it | |
| from spacy.cli import download | |
| download('en_core_web_sm') | |
| nlp = spacy.load('en_core_web_sm') | |
| # Load Hugging Face Transformers model | |
| tokenizer = AutoTokenizer.from_pretrained("sentence-transformers/all-mpnet-base-v2") | |
| model = AutoModel.from_pretrained("sentence-transformers/all-mpnet-base-v2") | |
| # def combined_text_processing(text): | |
| # # Basic NLP processing using SpaCy | |
| # doc = nlp(text) | |
| # lemmatized_text = ' '.join([token.lemma_ for token in doc]) | |
| # # Advanced text representation using Hugging Face Transformers | |
| # inputs = tokenizer(lemmatized_text, return_tensors="pt", truncation=False, padding=True) | |
| # with torch.no_grad(): | |
| # outputs = model(**inputs) | |
| # return outputs.last_hidden_state.mean(dim=1).squeeze().numpy() | |
| import re | |
| import nltk | |
| from nltk.corpus import stopwords | |
| from nltk.tokenize import word_tokenize | |
| # Download necessary NLTK data | |
| nltk.download('punkt') | |
| nltk.download('stopwords') | |
| nltk.download('averaged_perceptron_tagger') | |
| # def combined_text_processing(text): | |
| # # Remove punctuation, numbers, URLs, and special characters | |
| # text = re.sub(r'[^\w\s]', '', text) # Remove punctuation and special characters | |
| # text = re.sub(r'\d+', '', text) # Remove numbers | |
| # text = re.sub(r'http\S+', '', text) # Remove URLs | |
| # # Tokenize and remove stopwords | |
| # tokens = word_tokenize(text.lower()) # Convert to lowercase | |
| # stop_words = set(stopwords.words('english')) | |
| # tokens = [word for word in tokens if word not in stop_words] | |
| # # Lemmatize tokens using SpaCy | |
| # doc = nlp(' '.join(tokens)) | |
| # lemmatized_text = ' '.join([token.lemma_ for token in doc]) | |
| # # Apply Hugging Face Transformers | |
| # inputs = tokenizer(lemmatized_text, return_tensors="pt", truncation=False, padding=True) | |
| # with torch.no_grad(): | |
| # outputs = model(**inputs) | |
| # return outputs.last_hidden_state.mean(dim=1).squeeze().numpy() | |
| def text_processing_for_domain(text): | |
| # Text Cleaning | |
| text = re.sub(r'[^\w\s]', '', text) | |
| text = re.sub(r'\d+', '', text) | |
| text = re.sub(r'http\S+', '', text) # Remove https URLs | |
| text = re.sub(r'www\.\S+', '', text) # Remove www URLs | |
| # Tokenize and remove stopwords | |
| tokens = word_tokenize(text.lower()) | |
| stop_words = set(stopwords.words('english')) | |
| custom_stopwords = {'example', 'another'} # Add custom stopwords | |
| tokens = [word for word in tokens if word not in stop_words and word not in custom_stopwords] | |
| # NER - Remove named entities | |
| doc = nlp(' '.join(tokens)) | |
| tokens = [token.text for token in doc if not token.ent_type_] | |
| # POS Tagging (optional) | |
| pos_tags = nltk.pos_tag(tokens) | |
| tokens = [word for word, pos in pos_tags if pos in ['NN', 'NNS']] # Filter nouns | |
| # Lemmatize tokens using SpaCy | |
| doc = nlp(' '.join(tokens)) | |
| lemmatized_text = ' '.join([token.lemma_ for token in doc]) | |
| # Apply Hugging Face Transformers | |
| inputs = tokenizer(lemmatized_text, return_tensors="pt", truncation=False, padding=True) | |
| with torch.no_grad(): | |
| outputs = model(**inputs) | |
| return outputs.last_hidden_state.mean(dim=1).squeeze().numpy() | |
| # # 2. Clustering from ChatGPT | |
| # # Libraries: scikit-learn, sentence-transformers | |
| # # Use sentence embeddings and clustering algorithms to group similar project proposals. | |
| # from bertopic import BERTopic | |
| # def perform_clustering(texts, n_clusters): | |
| # topic_model = BERTopic(n_topics=n_clusters) | |
| # topics, _ = topic_model.fit_transform(texts) | |
| # return topics, topic_model | |
| # # Clustering function call | |
| # clustered_df, cluster_centers = clustering(processed_df) | |
| # Method 1: Sentence Transformers + KMeans | |
| # # 2. Clustering: from Claude | |
| # # Use BERTopic for advanced topic modeling and clustering. | |
| # from bertopic import BERTopic | |
| # def perform_clustering(texts, n_clusters): | |
| # topic_model = BERTopic(n_topics=n_clusters) | |
| # topics, _ = topic_model.fit_transform(texts) | |
| # return topics, topic_model | |
| # # Clustering function call | |
| # problem_clusters, problem_model = perform_clustering(processed_df['Problem_Description'], n_clusters=10) | |
| # location_clusters, location_model = perform_clustering(processed_df['Geographical_Location'], n_clusters=5) | |
| # After this Method 2: BERTopic function, the following need to be done: | |
| # processed_df['Problem_Cluster'] = problem_clusters | |
| # 2. Meta AI Function: Sentence Transformers + Hierarchical Clustering + Silhouette Analysis | |
| # Now this also includes: | |
| # Topic Modeling using BERTopic: Integrated BERTopic to extract representative words for each cluster. | |
| # Cluster Visualization: Added a simple visualization to display the top words in each cluster. | |
| # Hyperparameter Tuning: Include a parameter to adjust the number of top words to display for each cluster. | |
| from sentence_transformers import SentenceTransformer | |
| from sklearn.cluster import AgglomerativeClustering, KMeans | |
| from sklearn.feature_extraction.text import TfidfVectorizer | |
| from sklearn.metrics import silhouette_score | |
| from bertopic import BERTopic | |
| from collections import Counter | |
| import numpy as np | |
| def extract_problem_domains(df, | |
| text_column='Problem_Description', | |
| cluster_range=(10, 50), | |
| top_words=17, | |
| # method='sentence_transformers' | |
| method='tfidf_kmeans' | |
| ): | |
| if method == 'sentence_transformers': | |
| # Sentence Transformers approach | |
| model = SentenceTransformer('all-mpnet-base-v2') | |
| embeddings = model.encode(df[text_column].tolist()) | |
| # Perform hierarchical clustering with Silhouette Analysis | |
| silhouette_scores = [] | |
| for n_clusters in range(cluster_range[0], cluster_range[1] + 1): | |
| clustering = AgglomerativeClustering(n_clusters=n_clusters) | |
| cluster_labels = clustering.fit_predict(embeddings) | |
| silhouette_avg = silhouette_score(embeddings, cluster_labels) | |
| silhouette_scores.append(silhouette_avg) | |
| # Determine the optimal number of clusters | |
| optimal_n_clusters = cluster_range[0] + silhouette_scores.index(max(silhouette_scores)) | |
| # Perform clustering with the optimal number of clusters | |
| clustering = AgglomerativeClustering(n_clusters=optimal_n_clusters) | |
| cluster_labels = clustering.fit_predict(embeddings) | |
| elif method == 'tfidf_kmeans': | |
| # TF-IDF Vectorization and K-Means approach | |
| vectorizer = TfidfVectorizer(stop_words='english', max_features=5000) | |
| X = vectorizer.fit_transform(df[text_column]) | |
| # Perform K-Means clustering with Silhouette Analysis | |
| silhouette_scores = [] | |
| for n_clusters in range(cluster_range[0], cluster_range[1] + 1): | |
| kmeans = KMeans(n_clusters=n_clusters, random_state=42) | |
| cluster_labels = kmeans.fit_predict(X) | |
| silhouette_avg = silhouette_score(X, cluster_labels) | |
| silhouette_scores.append(silhouette_avg) | |
| # Determine the optimal number of clusters | |
| optimal_n_clusters = cluster_range[0] + silhouette_scores.index(max(silhouette_scores)) | |
| # Perform final clustering with optimal number of clusters | |
| kmeans = KMeans(n_clusters=optimal_n_clusters, random_state=42) | |
| cluster_labels = kmeans.fit_predict(X) | |
| # # BERTopic approach (commented out) | |
| # topic_model = BERTopic() | |
| # topics, _ = topic_model.fit_transform(df[text_column].tolist()) | |
| # topic_model.reduce_topics(df[text_column].tolist(), nr_topics=optimal_n_clusters) | |
| # cluster_labels = topics | |
| # Get representative words for each cluster | |
| if method == 'sentence_transformers': | |
| cluster_representations = {} | |
| for i in range(optimal_n_clusters): | |
| cluster_words = df.loc[cluster_labels == i, text_column].str.cat(sep=' ').split() | |
| cluster_representations[i] = [word for word, _ in Counter(cluster_words).most_common(top_words)] | |
| elif method == 'tfidf_kmeans': | |
| feature_names = vectorizer.get_feature_names_out() | |
| cluster_representations = {} | |
| for i in range(optimal_n_clusters): | |
| center = kmeans.cluster_centers_[i] | |
| top_word_indices = center.argsort()[-top_words:][::-1] | |
| top_words = [feature_names[index] for index in top_word_indices] | |
| cluster_representations[i] = top_words | |
| # Map cluster labels to representative words | |
| df["Problem_Cluster"] = cluster_labels | |
| df['Problem_Category_Words'] = [cluster_representations[label] for label in cluster_labels] | |
| return df, optimal_n_clusters | |
| # Usage | |
| # clustered_df, optimal_n_clusters = optimal_Problem_clustering(processed_df) | |
| # print(f'Optimal number of clusters: {optimal_n_clusters}') | |
| def nlp_pipeline(original_df): | |
| # Data Preprocessing | |
| processed_df = data_pre_processing(original_df) # merged_dataset | |
| # Starting the Pipeline for Domain Extraction | |
| # Apply the text_processing_for_domain function to the DataFrame | |
| processed_df['Processed_ProblemDescription_forDomainExtraction'] = processed_df['Problem_Description'].apply(text_processing_for_domain) | |
| # Domain Clustering | |
| domain_df, optimal_n_clusters = extract_problem_domains(processed_df) | |
| # problem_clusters, problem_model = perform_clustering(processed_df['Problem_Description'], n_clusters=10) | |
| # location_clusters, location_model = perform_clustering(processed_df['Geographical_Location'], n_clusters=5) | |
| # return processed_df | |
| return domain_df | |
| def process_excel(file): | |
| try: | |
| # Ensure the file path is correct | |
| file_path = file.name if hasattr(file, 'name') else file | |
| # Read the Excel file | |
| df = pd.read_excel(file_path) | |
| # Process the DataFrame | |
| result_df = nlp_pipeline(df) | |
| # output_file = "Output_ProjectProposals.xlsx" | |
| output_file = "Output_Proposals.xlsx" | |
| result_df.to_excel(output_file, index=False) | |
| return output_file # Return the processed DataFrame as Excel file | |
| except Exception as e: | |
| # return str(e) # Return the error message | |
| error_message = f"Error processing file: {str(e)}" | |
| print(error_message) # Log the error | |
| return error_message # Return the error message to the user | |
| # example_files = ['#TaxDirection (Responses)_BasicExample.xlsx', | |
| # '#TaxDirection (Responses)_IntermediateExample.xlsx', | |
| # '#TaxDirection (Responses)_UltimateExample.xlsx' | |
| # ] | |
| example_files = ['a.xlsx',] | |
| import random | |
| a_random_object = random.choice(["⇒", "↣", "↠", "→"]) | |
| # Define the Gradio interface | |
| interface = gr.Interface( | |
| fn=process_excel, # The function to process the uploaded file | |
| inputs=gr.File(type="filepath", label="Upload Excel File here. \t Be sure to check that the column headings in your upload are the same as in the Example files below. \t (Otherwise there will be Error during the processing)"), # File upload input | |
| examples=example_files, # Add the example files | |
| # outputs=gr.File(label="Download Processed Excel File"), # File download output | |
| outputs=gr.File(label="Download the processed Excel File containing the ** Project Proposals ** for each Location~Problem paired combination"), # File download output | |
| # title="Excel File Uploader", | |
| # title="Upload Excel file containing #TaxDirections → Download HyperLocal Project Proposals\n", | |
| title = ( | |
| "<p style='font-weight: bold; font-size: 25px; text-align: center;'>" | |
| "<span style='color: blue;'>Upload Excel file containing #TaxDirections</span> " | |
| # "<span style='color: brown; font-size: 35px;'>→ </span>" | |
| # "<span style='color: brown; font-size: 35px;'>⇒ ↣ ↠ </span>" | |
| "<span style='color: brown; font-size: 35px;'> " +a_random_object +" </span>" | |
| "<span style='color: green;'>Download HyperLocal Project Proposals</span>" | |
| "</p>\n" | |
| ), | |
| description=( | |
| "<p style='font-size: 12px; color: gray; text-align: center'>This tool allows for the systematic evaluation and proposal of solutions tailored to specific location-problem pairs, ensuring efficient resource allocation and project planning. For more information, visit <a href='https://santanban.github.io/TaxDirection/' target='_blank'>#TaxDirection weblink</a>.</p>" | |
| "<p style='font-weight: bold; font-size: 16px; color: blue;'>Upload an Excel file to process and download the result or use the Example files:</p>" | |
| "<p style='font-weight: bold; font-size: 15px; color: blue;'>(click on any of them to directly process the file and Download the result)</p>" | |
| "<p style='font-weight: bold; font-size: 14px; color: green; text-align: right;'>Processed output contains a Project Proposal for each Location~Problem paired combination (i.e. each cell).</p>" | |
| "<p style='font-weight: bold; font-size: 13px; color: green; text-align: right;'>Corresponding Budget Allocation and estimated Project Completion Time are provided in different sheets.</p>" | |
| "<p style='font-size: 12px; color: gray; text-align: center'>Note: The example files provided above are for demonstration purposes. Feel free to upload your own Excel files to see the results. If you have any questions, refer to the documentation-links or contact <a href='https://www.change.org/p/democracy-evolution-ensuring-humanity-s-eternal-existence-through-taxdirection' target='_blank'>support</a>.</p>" | |
| ) # Solid description with right-aligned second sentence | |
| ) | |
| # Launch the interface | |
| if __name__ == "__main__": | |
| interface.launch() |