Update app.py

app.py

@@ -340,20 +340,37 @@ def process_and_summarize(df, bert_tokenizer, bert_model, emotion_classifier, to
         texts = [clean_arabic_text(poem) for poem in group['poem'].dropna()]
         all_emotions = []
         
-        # Get embeddings with proper output handling
+        # Enhanced embedding generation
         embeddings = []
         for i, text in enumerate(texts):
-            embedding = get_embedding_for_text(text, bert_tokenizer, bert_model)
-            embeddings.append(embedding)
+            text_chunks = [text[i:i+512] for i in range(0, len(text), 512)]
+            chunk_embeddings = []
+            
+            for chunk in text_chunks:
+                chunk_embedding = get_embedding_for_text(chunk, bert_tokenizer, bert_model)
+                chunk_embeddings.append(chunk_embedding)
+            
+            full_embedding = np.mean(chunk_embeddings, axis=0)
+            embeddings.append(full_embedding)
+            
             progress = (i + 1) / len(texts) * 0.4
             progress_bar.progress(progress, text=f"Generated embeddings for {i+1}/{len(texts)} poems...")
         
         embeddings = np.array(embeddings)
         
-        # Process emotions with correct output structure
+        # Process emotions with proper chunking
         for i, text in enumerate(texts):
-            emotion = emotion_classifier(text)[0]['label']  # Access the label directly
-            all_emotions.append(emotion)
+            text_chunks = [text[i:i+512] for i in range(0, len(text), 512)]
+            chunk_emotions = []
+            
+            for chunk in text_chunks:
+                emotion = emotion_classifier(chunk)[0]['label']
+                chunk_emotions.append(emotion)
+            
+            # Use most common emotion for the full text
+            final_emotion = max(set(chunk_emotions), key=chunk_emotions.count)
+            all_emotions.append(final_emotion)
+            
             progress = 0.4 + ((i + 1) / len(texts) * 0.3)
             progress_bar.progress(progress, text=f"Classified emotions for {i+1}/{len(texts)} poems...")