Update tools/quran_search.py

tools/quran_search.py

@@ -5,43 +5,55 @@ import numpy as np
 from sentence_transformers import SentenceTransformer
 from sklearn.metrics.pairwise import cosine_similarity
 from config import QURAN_DATA_SOURCES, MODEL_NAME, CHUNK_SIZE
+import time
 
 class QuranSearchEngine:
     def __init__(self):
-        self.api_url = "https://quranapi.pages.dev/api/"
+        self.full_quran_url = "https://cdn.jsdelivr.net/npm/[email protected]/dist/quran.json"
         self.logger = logging.getLogger(__name__)
+        self.full_quran = None
         self.surahs = None
         self.all_verses = []  # List of {'surah_id': int, 'verse_num': int, 'text': str}
         self.verse_embeddings = None
         self.model = None
-        self._load_surahs()
+        self._load_full_quran()
+        print(f"Surahs loaded: {len(self.surahs) if self.surahs else 0}")  # Debug
         self._load_all_verses_and_embeddings()
+        print(f"Verses loaded: {len(self.all_verses)}")  # Debug
         
-    def _load_surahs(self):
-        try:
-            response = requests.get(f"{self.api_url}surah.json", timeout=5)
-            response.raise_for_status()
-            self.surahs = response.json()  # List of surah dicts
-        except Exception as e:
-            self.logger.error(f"Failed to fetch surahs: {e}")
-            self.surahs = self._load_fallback_surahs()
+    def _load_full_quran(self):
+        max_retries = 3
+        for attempt in range(max_retries):
+            try:
+                response = requests.get(self.full_quran_url, timeout=10)
+                response.raise_for_status()
+                self.full_quran = response.json()  # Array of surah dicts
+                self.surahs = self.full_quran
+                break
+            except Exception as e:
+                self.logger.error(f"Attempt {attempt + 1}/{max_retries} failed to fetch full Quran: {e}")
+                if attempt == max_retries - 1:
+                    self.surahs = self._load_fallback_surahs()
+                time.sleep(2 ** attempt)
     
     def _load_all_verses_and_embeddings(self):
-        """Load all verses and precompute embeddings"""
+        if not self.full_quran:
+            self.logger.error("No full Quran loaded, skipping verse loading")
+            self.all_verses = [
+                {'surah_id': 1, 'verse_num': 1, 'text': "بِسْمِ ٱللَّهِ ٱلرَّحْمَـٰنِ ٱلرَّحِيمِ"},
+            ]
+            return
+        
+        for surah in self.full_quran:
+            surah_id = surah['id']
+            for verse in surah['verses']:
+                self.all_verses.append({
+                    'surah_id': surah_id,
+                    'verse_num': verse['id'],
+                    'text': verse['text']
+                })
+        
         try:
-            for surah_id in range(1, 115):  # 1 to 114
-                response = requests.get(f"{self.api_url}{surah_id}.json", timeout=10)
-                response.raise_for_status()
-                data = response.json()
-                verses = data.get('arabic1', [])  # Arabic with tashkeel
-                for verse_num, text in enumerate(verses, start=1):
-                    self.all_verses.append({
-                        'surah_id': surah_id,
-                        'verse_num': verse_num,
-                        'text': text
-                    })
-            
-            # Precompute embeddings in chunks
             self.model = SentenceTransformer(MODEL_NAME)
             verse_texts = [v['text'] for v in self.all_verses]
             self.verse_embeddings = []
@@ -50,39 +62,44 @@ class QuranSearchEngine:
                 embeddings = self.model.encode(chunk, convert_to_tensor=False)
                 self.verse_embeddings.append(embeddings)
             self.verse_embeddings = np.vstack(self.verse_embeddings)
-            
         except Exception as e:
-            self.logger.error(f"Failed to load verses/embeddings: {e}")
-            # Fallback: Use limited hardcoded data (no embeddings)
-            self.all_verses = [
-                {'surah_id': 1, 'verse_num': 1, 'text': "بِسْمِ ٱللَّهِ ٱلرَّحْمَـٰنِ ٱلرَّحِيمِ"},
-                # Add more if needed, but limited
-            ]
-            self.verse_embeddings = None  # Will use keyword fallback in search
+            self.logger.error(f"Failed to compute embeddings: {e}")
+            self.verse_embeddings = None
     
     def get_surahs(self):
         if self.surahs:
             return [
-                (f"{s['surahNameArabicLong']} ({s['surahNameTranslation']})", i + 1)
-                for i, s in enumerate(self.surahs)
+                (s['name'], s['id'])
+                for s in self.surahs
             ]
         return self._load_fallback_surahs()
     
     def get_surah_text(self, surah_id):
-        try:
-            response = requests.get(f"{self.api_url}{surah_id}.json", timeout=10)
-            response.raise_for_status()
-            data = response.json()
-            verses = data['arabic1']
-            return "\n\n".join(f"آية {i + 1}: {v}" for i, v in enumerate(verses))
-        except Exception as e:
-            self.logger.error(f"Failed to fetch surah {surah_id}: {e}")
-            return self._load_fallback_verse()
-
+        if self.full_quran:
+            try:
+                surah = self.full_quran[surah_id - 1]
+                verses = surah['verses']
+                return "\n\n".join(f"آية {v['id']}: {v['text']}" for v in verses)
+            except IndexError:
+                self.logger.error(f"Surah {surah_id} not found in cached data")
+        
+        # Fallback if cache failed
+        max_retries = 3
+        for attempt in range(max_retries):
+            try:
+                response = requests.get(f"https://quranapi.pages.dev/api/{surah_id}.json", timeout=10)
+                response.raise_for_status()
+                data = response.json()
+                verses = data['arabic1']
+                return "\n\n".join(f"آية {i + 1}: {v}" for i, v in enumerate(verses))
+            except Exception as e:
+                self.logger.error(f"Attempt {attempt + 1}/{max_retries} failed to fetch surah {surah_id}: {e}")
+                if attempt == max_retries - 1:
+                    return self._load_fallback_verse()
+                time.sleep(2 ** attempt)
+    
     def search_verses(self, query, top_k=5):
-        """Semantic search for verses based on meaning"""
-        if self.verse_embeddings is None:
-            # Fallback to simple keyword search if embeddings failed
+        if self.verse_embeddings is None or not self.all_verses:
             return self._keyword_fallback_search(query, top_k)
         
         try:
@@ -93,9 +110,9 @@ class QuranSearchEngine:
             results = []
             for idx in top_indices:
                 verse = self.all_verses[idx]
-                surah_name = self.surahs[verse['surah_id'] - 1]['surahNameArabicLong']
+                surah_name = self.surahs[verse['surah_id'] - 1]['name']
                 results.append(
-                    f"سورة {surah_name} - آية {verse['verse_num']}:\n{verse['text']}\n(تشابه: {similarities[idx]:.2f})"
+                    f"سورة {surah_name} - آية {verse['verse_num']}:\n{verse['text']}"
                 )
             return "\n\n".join(results)
         except Exception as e:
@@ -103,38 +120,37 @@ class QuranSearchEngine:
             return "حدث خطأ أثناء البحث. جرب مرة أخرى."
 
     def _keyword_fallback_search(self, query, top_k=5):
-        """Simple keyword fallback if semantic fails"""
         query_lower = query.lower()
         matches = []
         for verse in self.all_verses:
             if query_lower in verse['text'].lower():
-                surah_name = self.surahs[verse['surah_id'] - 1]['surahNameArabicLong'] if self.surahs else f"سورة {verse['surah_id']}"
+                surah_name = self.surahs[verse['surah_id'] - 1]['name'] if self.surahs else f"سورة {verse['surah_id']}"
                 matches.append(f"سورة {surah_name} - آية {verse['verse_num']}:\n{verse['text']}")
         return "\n\n".join(matches[:top_k]) or "لا توجد نتائج مطابقة."
 
-    # Existing fallback methods (unchanged)
+    # Fallback methods (unchanged)
     def _load_fallback_surahs(self):
         try:
             for source in QURAN_DATA_SOURCES:
                 try:
                     df = pd.read_csv(source)
                     return [
-                        (f"{row['name_arabic']} ({row['name_english']})", row['surah_id'])
+                        (row['name_arabic'], row['surah_id'])
                         for _, row in df.drop_duplicates(subset=['surah_id']).iterrows()
                     ]
                 except:
                     continue
             return [
-                ("سورة الفاتحة (The Opening)", 1),
-                ("سورة البقرة (The Cow)", 2),
-                ("سورة آل عمران (The Family of Imran)", 3)
+                ("الفاتحة", 1),
+                ("البقرة", 2),
+                ("آل عمران", 3)
             ]
         except Exception as e:
             self.logger.error(f"Failed to load fallback surahs: {e}")
             return [
-                ("سورة الفاتحة (The Opening)", 1),
-                ("سورة البقرة (The Cow)", 2),
-                ("سورة آل عمران (The Family of Imran)", 3)
+                ("الفاتحة", 1),
+                ("البقرة", 2),
+                ("آل عمران", 3)
             ]
 
     def _load_fallback_verse(self):