Update tools/quran_search.py

tools/quran_search.py

@@ -5,6 +5,7 @@ from sentence_transformers import SentenceTransformer
 from sklearn.metrics.pairwise import cosine_similarity
 from config import MODEL_NAME, CHUNK_SIZE
 import time
+import sys
 
 class QuranSearchEngine:
     def __init__(self):
@@ -13,18 +14,23 @@ class QuranSearchEngine:
         self.surahs = None
         self.all_verses = []  # List of {'surah_id': int, 'verse_num': int, 'text': str}
         self.verse_embeddings = None
-        self.model = None
-        print("Starting QuranSearchEngine initialization...")  # Debug
-        self._load_full_quran()
-        print(f"Surahs loaded: {len(self.surahs) if self.surahs else 0}")  # Debug
-        self._load_all_verses_and_embeddings()
-        print(f"Verses loaded: {len(self.all_verses)}")  # Debug
-        
+        self.model = None  # Deferred loading
+        print("Starting QuranSearchEngine initialization at", time.ctime(), file=sys.stderr)  # Debug
+        try:
+            self._load_full_quran()
+            print(f"Surahs loaded: {len(self.surahs) if self.surahs else 0}", file=sys.stderr)  # Debug
+            self._load_all_verses_and_embeddings()
+            print(f"Verses loaded: {len(self.all_verses)}", file=sys.stderr)  # Debug
+        except Exception as e:
+            self.logger.error(f"Initialization failed: {e}", exc_info=True)
+            print(f"Initialization error: {e}", file=sys.stderr)
+            self._load_fallback_data()  # Ensure minimal startup
+
     def _load_full_quran(self):
-        max_retries = 3
+        max_retries = 5  # Increased retries
         for attempt in range(max_retries):
             try:
-                response = requests.get(f"{self.api_url}surah.json", timeout=10)
+                response = requests.get(f"{self.api_url}surah.json", timeout=15)  # Increased timeout
                 response.raise_for_status()
                 self.surahs = response.json()
                 for i, s in enumerate(self.surahs):
@@ -33,15 +39,14 @@ class QuranSearchEngine:
             except Exception as e:
                 self.logger.error(f"Attempt {attempt + 1}/{max_retries} failed to fetch surahs: {e}")
                 if attempt == max_retries - 1:
-                    self.surahs = self._load_fallback_surahs()
+                    self._load_fallback_data()
                 time.sleep(2 ** attempt)
-        
-        # Load verses
+
         if self.surahs:
             for surah in self.surahs:
                 surah_id = surah['id']
                 try:
-                    response = requests.get(f"{self.api_url}{surah_id}.json", timeout=10)
+                    response = requests.get(f"{self.api_url}{surah_id}.json", timeout=15)
                     response.raise_for_status()
                     data = response.json()
                     verses = data['arabic1']
@@ -54,18 +59,22 @@ class QuranSearchEngine:
                 except Exception as e:
                     self.logger.error(f"Failed to fetch verses for surah {surah_id}: {e}")
 
-        if not self.all_verses:
-            self.all_verses = [
-                {'surah_id': 1, 'verse_num': 1, 'text': "بِسْمِ ٱللَّهِ ٱلرَّحْمَـٰنِ ٱلرَّحِيمِ"},
-                # Add more if needed
-            ]
-    
+    def _load_fallback_data(self):
+        self.logger.warning("Falling back to minimal data due to API failure")
+        self.surahs = self._load_fallback_surahs()
+        self.all_verses = [
+            {'surah_id': 1, 'verse_num': 1, 'text': "بِسْمِ ٱللَّهِ ٱلرَّحْمَـٰنِ ٱلرَّحِيمِ"},
+            {'surah_id': 1, 'verse_num': 2, 'text': "ٱلْحَمْدُ لِلَّهِ رَبِّ ٱلْعَٰلَمِينَ"}
+        ]
+
     def _load_all_verses_and_embeddings(self):
         if not self.all_verses:
             return
         
         try:
+            print("Attempting to load model...", file=sys.stderr)  # Debug
             self.model = SentenceTransformer(MODEL_NAME)
+            print("Model loaded successfully", file=sys.stderr)  # Debug
             verse_texts = [v['text'] for v in self.all_verses]
             self.verse_embeddings = []
             for i in range(0, len(verse_texts), CHUNK_SIZE):
@@ -73,23 +82,22 @@ class QuranSearchEngine:
                 embeddings = self.model.encode(chunk, convert_to_tensor=False)
                 self.verse_embeddings.append(embeddings)
             self.verse_embeddings = np.vstack(self.verse_embeddings)
+            print("Embeddings computed successfully", file=sys.stderr)  # Debug
         except Exception as e:
-            self.logger.error(f"Failed to compute embeddings: {e}")
+            self.logger.error(f"Failed to compute embeddings: {e}", exc_info=True)
             self.verse_embeddings = None
-    
+            self.logger.warning("Falling back to keyword-based search due to embedding failure")
+
     def get_surahs(self):
         if self.surahs:
-            return [
-                (s['surahNameArabicLong'], s['id'])
-                for s in self.surahs
-            ]
+            return [(s['surahNameArabicLong'], s['id']) for s in self.surahs]
         return self._load_fallback_surahs()
-    
+
     def get_surah_text(self, surah_id):
         max_retries = 3
         for attempt in range(max_retries):
             try:
-                response = requests.get(f"{self.api_url}{surah_id}.json", timeout=10)
+                response = requests.get(f"{self.api_url}{surah_id}.json", timeout=15)
                 response.raise_for_status()
                 data = response.json()
                 verses = data['arabic1']
@@ -99,12 +107,29 @@ class QuranSearchEngine:
                 if attempt == max_retries - 1:
                     return self._load_fallback_verse()
                 time.sleep(2 ** attempt)
-    
+
     def search_verses(self, query, top_k=5):
+        if self.model is None:
+            try:
+                print("Loading model on demand...", file=sys.stderr)
+                self.model = SentenceTransformer(MODEL_NAME)
+                print("Model loaded successfully", file=sys.stderr)
+                verse_texts = [v['text'] for v in self.all_verses]
+                self.verse_embeddings = []
+                for i in range(0, len(verse_texts), CHUNK_SIZE):
+                    chunk = verse_texts[i:i + CHUNK_SIZE]
+                    embeddings = self.model.encode(chunk, convert_to_tensor=False)
+                    self.verse_embeddings.append(embeddings)
+                self.verse_embeddings = np.vstack(self.verse_embeddings)
+            except Exception as e:
+                self.logger.error(f"Failed to load model on demand: {e}", exc_info=True)
+                self.verse_embeddings = None
+
         if self.verse_embeddings is None or not self.all_verses:
             return self._keyword_fallback_search(query, top_k)
         
         try:
+            print(f"Encoding query: {query}", file=sys.stderr)  # Debug
             query_embedding = self.model.encode([query], convert_to_tensor=False)
             similarities = cosine_similarity(query_embedding, self.verse_embeddings)[0]
             top_indices = np.argsort(similarities)[-top_k:][::-1]
@@ -113,12 +138,10 @@ class QuranSearchEngine:
             for idx in top_indices:
                 verse = self.all_verses[idx]
                 surah_name = self.surahs[verse['surah_id'] - 1]['surahNameArabicLong']
-                results.append(
-                    f"سورة {surah_name} - آية {verse['verse_num']}:\n{verse['text']}"
-                )
+                results.append(f"سورة {surah_name} - آية {verse['verse_num']}:\n{verse['text']}")
             return "\n\n".join(results)
         except Exception as e:
-            self.logger.error(f"Search failed: {e}")
+            self.logger.error(f"Search failed: {e}", exc_info=True)
             return "حدث خطأ أثناء البحث. جرب مرة أخرى."
 
     def _keyword_fallback_search(self, query, top_k=5):
@@ -131,11 +154,7 @@ class QuranSearchEngine:
         return "\n\n".join(matches[:top_k]) or "لا توجد نتائج مطابقة."
 
     def _load_fallback_surahs(self):
-        return [
-            ("الفاتحة", 1),
-            ("البقرة", 2),
-            ("آل عمران", 3)
-        ]
+        return [("الفاتحة", 1), ("البقرة", 2), ("آل عمران", 3)]
 
     def _load_fallback_verse(self):
         return "بسم الله الرحمن الرحيم\nالله لا إله إلا هو الحي القيوم"