Update app.py

app.py

@@ -68,8 +68,89 @@ def data_pre_processing(file_responses):
     except Exception as e:
         return str(e)
 
+
+
+
+
+
+
+
+import spacy
+from transformers import AutoTokenizer, AutoModel
+import torch
+
+# Load SpaCy model
+nlp = spacy.load('en_core_web_sm')
+
+# Load Hugging Face Transformers model
+tokenizer = AutoTokenizer.from_pretrained("sentence-transformers/all-mpnet-base-v2")
+model = AutoModel.from_pretrained("sentence-transformers/all-mpnet-base-v2")
+
+# def combined_text_processing(text):
+#     # Basic NLP processing using SpaCy
+#     doc = nlp(text)
+#     lemmatized_text = ' '.join([token.lemma_ for token in doc])
+    
+#     # Advanced text representation using Hugging Face Transformers
+#     inputs = tokenizer(lemmatized_text, return_tensors="pt", truncation=False, padding=True)
+#     with torch.no_grad():
+#         outputs = model(**inputs)
+    
+#     return outputs.last_hidden_state.mean(dim=1).squeeze().numpy()
+
+
+import re
+from nltk.corpus import stopwords
+from nltk.tokenize import word_tokenize
+
+def combined_text_processing(text):
+    # Remove punctuation, numbers, URLs, and special characters
+    text = re.sub(r'[^\w\s]', '', text)  # Remove punctuation and special characters
+    text = re.sub(r'\d+', '', text)  # Remove numbers
+    text = re.sub(r'http\S+', '', text)  # Remove URLs
+    
+    # Tokenize and remove stopwords
+    tokens = word_tokenize(text.lower())  # Convert to lowercase
+    stop_words = set(stopwords.words('english'))
+    tokens = [word for word in tokens if word not in stop_words]
+    
+    # Lemmatize tokens using SpaCy
+    doc = nlp(' '.join(tokens))
+    lemmatized_text = ' '.join([token.lemma_ for token in doc])
+    
+    # Apply Hugging Face Transformers
+    inputs = tokenizer(lemmatized_text, return_tensors="pt", truncation=False, padding=True)
+    with torch.no_grad():
+        outputs = model(**inputs)
+    
+    return outputs.last_hidden_state.mean(dim=1).squeeze().numpy()
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 def nlp_pipeline(original_df):
+    # Data Preprocessing
     processed_df = data_pre_processing(original_df)
+    
+    
+    # Apply the combined function to your DataFrame
+    processed_df['Processed_ProblemDescription'] = processed_df['Problem_Description'].apply(combined_text_processing)
+    
+    
+    
     return processed_df
 
 def process_excel(file):
@@ -138,10 +219,10 @@ interface = gr.Interface(
         "<p style='font-size: 12px; color: gray; text-align: center'>This tool allows for the systematic evaluation and proposal of solutions tailored to specific location-problem pairs, ensuring efficient resource allocation and project planning. For more information, visit <a href='https://santanban.github.io/TaxDirection/' target='_blank'>#TaxDirection weblink</a>.</p>"
         
         "<p style='font-weight: bold; font-size: 16px; color: blue;'>Upload an Excel file to process and download the result or use the Example files:</p>"
-        "<p style='font-weight: bold; font-size: 16px; color: blue;'>(click on any of them to directly process the file and Download the result)</p>"
+        "<p style='font-weight: bold; font-size: 15px; color: blue;'>(click on any of them to directly process the file and Download the result)</p>"
         
         "<p style='font-weight: bold; font-size: 14px; color: green; text-align: right;'>Processed output contains a Project Proposal for each Location~Problem paired combination (i.e. each cell).</p>"
-        "<p style='font-weight: bold; font-size: 14px; color: green; text-align: right;'>Corresponding Budget Allocation and estimated Project Completion Time are provided in different sheets.</p>"
+        "<p style='font-weight: bold; font-size: 13px; color: green; text-align: right;'>Corresponding Budget Allocation and estimated Project Completion Time are provided in different sheets.</p>"
 
         
         "<p style='font-size: 12px; color: gray; text-align: center'>Note: The example files provided above are for demonstration purposes. Feel free to upload your own Excel files to see the results. If you have any questions, refer to the documentation-links or contact <a href='https://www.change.org/p/democracy-evolution-ensuring-humanity-s-eternal-existence-through-taxdirection' target='_blank'>support</a>.</p>"