Update app.py

app.py

@@ -121,33 +121,40 @@ class pleiasBot:
             traceback.print_exc()
             return None, None, None
 
-def format_references(text):
-    ref_pattern = r'<ref name="([^"]+)">"([^"]+)"</ref>\.\s*'
+def hybrid_search(text):
+    results = table.search(text, query_type="hybrid").limit(5).to_pandas()
     
-    parts = []
-    current_pos = 0
-    ref_number = 1
-
-    for match in re.finditer(ref_pattern, text):
-        # Add text before the reference
-        text_before = text[current_pos:match.start()].rstrip()
-        parts.append(text_before)
-        
-        # Extract reference components
-        ref_id = match.group(1)
-        ref_text = match.group(2).strip()
-        
-        # Add the reference with sequential numbering
-        tooltip_html = f'<span class="tooltip"><strong>[{ref_number}]</strong><span class="tooltiptext"><strong>{ref_id}</strong>: {ref_text}</span></span>.<br>'
-        parts.append(tooltip_html)
+    # Use a list to maintain order
+    seen_hashes = []
+    
+    document = []
+    document_html = []
+    for _, row in results.iterrows():
+        hash_id = str(row['hash'])
         
-        current_pos = match.end()
-        ref_number += 1
+        # Skip if we've already seen this hash
+        if hash_id in seen_hashes:
+            continue
+            
+        seen_hashes.append(hash_id)  # append instead of add to maintain order
+        title = row['section']
+        content = row['text']
+
+        source_text = f"<|source_start|><|source_id_start|>{hash_id}<|source_id_end|>{title}\n{content}<|source_end|>"
+        document.append(source_text)
+        document_html.append(f'<div class="source" id="{hash_id}"><p><b>{hash_id}</b> : {title}<br>{content}</div>')
+
+        # Print for debugging
+        print(f"Added source {hash_id}")
+        print(f"Length of source text: {len(source_text)}")
+
+    document = "\n".join(document)
+    document_html = '<div id="source_listing">' + "".join(document_html) + "</div>"
     
-    # Add any remaining text
-    parts.append(text[current_pos:])
+    # Print total length for debugging
+    print(f"Total length of document: {len(document)}")
     
-    return ''.join(parts)
+    return document, document_html
 
 # Initialize the pleiasBot
 pleias_bot = pleiasBot()