Spaces:
Sleeping
Sleeping
| import faiss | |
| import numpy as np | |
| import gradio as gr | |
| from sentence_transformers import SentenceTransformer | |
| import fitz # PyMuPDF für die Textextraktion aus PDFs | |
| from transformers import pipeline | |
| import logging | |
| # Logging konfigurieren | |
| logging.basicConfig(level=logging.INFO) | |
| # Modelle laden | |
| model = SentenceTransformer('all-mpnet-base-v2') | |
| qa_model = pipeline("question-answering", model="deepset/roberta-base-squad2") | |
| # FAISS-Index erstellen | |
| def create_faiss_index(documents): | |
| document_embeddings = model.encode(documents) | |
| dimension = len(document_embeddings[0]) | |
| index = faiss.IndexFlatL2(dimension) | |
| document_embeddings = np.array(document_embeddings).astype('float32') | |
| index.add(document_embeddings) | |
| return index, documents | |
| # Text aus PDF extrahieren | |
| def extract_text_from_pdf(pdf_path): | |
| doc = fitz.open(pdf_path) | |
| text_chunks = [] | |
| for page_num in range(len(doc)): | |
| page = doc.load_page(page_num) | |
| text = page.get_text("text") | |
| chunks = text.split('\n\n') # Unterteilen nach Absätzen | |
| text_chunks.extend(chunks) | |
| return text_chunks | |
| # Suche nach mehreren passenden Abschnitten | |
| def search_documents(query, index, documents, k=10): | |
| query_embedding = model.encode([query])[0].astype('float32') | |
| D, I = index.search(np.array([query_embedding]), k=k) # Suche nach den Top k Treffern | |
| results = [documents[i] for i in I[0]] | |
| return results # Liste von Kontexten zurückgeben | |
| # Kombiniere mehrere Antworten | |
| def generate_detailed_answer(contexts, question): | |
| detailed_answer = [] | |
| for context in contexts: | |
| try: | |
| result = qa_model(question=question, context=context) | |
| detailed_answer.append(result['answer']) | |
| except Exception as e: | |
| logging.warning(f"Fehler im QA-Modell: {e}") | |
| return " ".join(detailed_answer) | |
| # Gesamtprozess | |
| def chatbot_response(pdf_path, question): | |
| logging.info(f"Frage: {question}") | |
| # Text extrahieren | |
| text_chunks = extract_text_from_pdf(pdf_path) | |
| # FAISS-Index erstellen | |
| index, documents = create_faiss_index(text_chunks) | |
| # Suche nach Kontexten | |
| contexts = search_documents(question, index, documents, k=10) | |
| logging.info(f"Gefundene Kontexte: {[context[:100] for context in contexts]}") # Logge die ersten 100 Zeichen jedes Kontexts | |
| # Antwort generieren | |
| answer = generate_detailed_answer(contexts, question) | |
| return answer | |
| # Gradio-Interface | |
| pdf_input = gr.File(label="PDF-Datei hochladen", type="filepath") | |
| question_input = gr.Textbox(label="Frage eingeben", placeholder="Stelle eine Frage zu dem PDF-Dokument") | |
| response_output = gr.Textbox(label="Antwort") | |
| interface = gr.Interface( | |
| fn=chatbot_response, | |
| inputs=[pdf_input, question_input], | |
| outputs=response_output, | |
| title="PDF-Fragebeantwortung mit FAISS und Transformers", | |
| description="Lade eine PDF-Datei hoch und stelle Fragen zu ihrem Inhalt. Das System verwendet FAISS, Transformers und ein QA-Modell, um detaillierte Antworten zu liefern." | |
| ) | |
| if __name__ == "__main__": | |
| interface.launch() | |