File size: 3,834 Bytes
e904f34 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 |
# services/faq_service.py
from typing import List, Dict, Any, Optional
import aiohttp
from bs4 import BeautifulSoup
import faiss
import logging
from config.config import settings
logger = logging.getLogger(__name__)
class FAQService:
def __init__(self, model_service):
self.embedder = model_service.embedder
self.faiss_index = None
self.faq_data = []
async def fetch_faq_pages(self) -> List[Dict[str, Any]]:
async with aiohttp.ClientSession() as session:
try:
async with session.get(f"{settings.FAQ_ROOT_URL}sitemap.xml", timeout=settings.TIMEOUT) as response:
if response.status == 200:
sitemap = await response.text()
soup = BeautifulSoup(sitemap, 'xml')
faq_urls = [loc.text for loc in soup.find_all('loc') if "/faq/" in loc.text]
tasks = [self.fetch_faq_content(url, session) for url in faq_urls]
return await asyncio.gather(*tasks)
except Exception as e:
logger.error(f"Error fetching FAQ sitemap: {e}")
return []
async def fetch_faq_content(self, url: str, session: aiohttp.ClientSession) -> Optional[Dict[str, Any]]:
try:
async with session.get(url, timeout=settings.TIMEOUT) as response:
if response.status == 200:
content = await response.text()
soup = BeautifulSoup(content, 'html.parser')
faq_title = soup.find('h1').text.strip() if soup.find('h1') else "Unknown Title"
faqs = []
sections = soup.find_all(['div', 'section'], class_=['faq-item', 'faq-section'])
for section in sections:
question = section.find(['h2', 'h3']).text.strip() if section.find(['h2', 'h3']) else None
answer = section.find(['p']).text.strip() if section.find(['p']) else None
if question and answer:
faqs.append({"question": question, "answer": answer})
return {"url": url, "title": faq_title, "faqs": faqs}
except Exception as e:
logger.error(f"Error fetching FAQ content from {url}: {e}")
return None
async def index_faqs(self):
faq_pages = await self.fetch_faq_pages()
faq_pages = [page for page in faq_pages if page]
self.faq_data = []
all_texts = []
for faq_page in faq_pages:
for item in faq_page['faqs']:
combined_text = f"{item['question']} {item['answer']}"
all_texts.append(combined_text)
self.faq_data.append({
"question": item['question'],
"answer": item['answer'],
"source": faq_page['url']
})
embeddings = self.embedder.encode(all_texts, convert_to_tensor=True).cpu().detach().numpy()
dimension = embeddings.shape[1]
self.faiss_index = faiss.IndexFlatL2(dimension)
self.faiss_index.add(embeddings)
async def search_faqs(self, query: str, top_k: int = 5) -> List[Dict[str, Any]]:
if not self.faiss_index:
await self.index_faqs()
query_embedding = self.embedder.encode([query], convert_to_tensor=True).cpu().detach().numpy()
distances, indices = self.faiss_index.search(query_embedding, top_k)
results = []
for i, idx in enumerate(indices[0]):
if idx < len(self.faq_data):
result = self.faq_data[idx].copy()
result["score"] = float(distances[0][i])
results.append(result)
return results |