switched back to langchain

app.py

@@ -1,29 +1,236 @@
-from backend import handle_query
+import os
+from threading import Thread
+from typing import Iterator
+from backend2 import load_documents, prepare_documents, get_context_sources
 import gradio as gr
-
+import spaces
+import torch
+from transformers import AutoModelForCausalLM, GemmaTokenizerFast, TextIteratorStreamer
 
 DESCRIPTION = """\
-# <div style="text-align: center;">Odi, l'assistente ricercatore degli Osservatori</div>
+# La Chatbot degli Osservatori
+"""
+MAX_MAX_NEW_TOKENS = 2048
+DEFAULT_MAX_NEW_TOKENS = 1024
+os.environ["MAX_INPUT_TOKEN_LENGTH"] = "8192"
+MAX_INPUT_TOKEN_LENGTH = int(os.getenv("MAX_INPUT_TOKEN_LENGTH"))
 
 
-👉 Retrieval-Augmented Generation - Ask me anything about the research carried out at the Osservatori.
-"""
+# Force usage of CPU
+#device = torch.device("cpu")
+
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+model_id = "google/gemma-2-2b-it"
+model = AutoModelForCausalLM.from_pretrained(
+    model_id,
+    device_map="auto",
+    torch_dtype= torch.float16 if torch.cuda.is_available() else torch.float32,
+)
+tokenizer = GemmaTokenizerFast.from_pretrained(model_id)
+#tokenizer = AutoTokenizer.from_pretrained(model_id)
+tokenizer.use_default_system_prompt = False
+model.config.sliding_window = 4096
+#model = model.to(device)
+model.eval()
+
+###------####
+# rag
+documents_paths = {
+    'blockchain': 'documents/blockchain',
+    'metaverse': 'documents/metaverso',
+    'payment': 'documents/payment'
+}
+
+session_state = {"documents_loaded": False, 
+                 "document_db": None, 
+                 "original_message": None, 
+                 "clarification": False}
+
+INSTRUCTION_1 = 'In italiano, chiedi molto brevemente se la domanda si riferisce agli "Osservatori Blockchain", "Osservatori Payment" oppure "Osservatori Metaverse".'
+INSTRUCTION_2 = 'Sei un assistente che risponde in italiano alle domande basandosi solo sulle informazioni fornite nel contesto. Se non trovi informazioni, rispondi "Puoi chiedere maggiori informazioni all\'ufficio di riferimento.". Se invece la domanda è completamente fuori contesto, non rispondere e rammenta il topic del contesto'
+
+default_error_response = (
+    'Non sono sicuro che tu voglia indirizzare la tua ricerca su una di queste opzioni: '
+    '"Blockchain", "Metaverse", "Payment". '
+    'Per favore utilizza il nome corretto.'
+)
 
 
-chat_interface =gr.ChatInterface(
-        fn=handle_query, 
-        chatbot=gr.Chatbot(height=500),
-        textbox=gr.Textbox(placeholder="Chiedimi qualasiasi cosa relativa agli Osservatori", container=False, scale=7),
-        #examples=[["Ciao, in cosa puoi aiutarmi?"],["Dimmi i risultati e le modalità di conduzione del censimento per favore"]]
-        )
+@spaces.GPU(duration=90)
+def generate(
+    message: str,
+    chat_history: list[tuple[str, str]],
+    max_new_tokens: int = 1024,
+    temperature: float = 0.6,
+    top_p: float = 0.9,
+    top_k: int = 50,
+    repetition_penalty: float = 1.2,
+) -> Iterator[str]:
+    
+    global context, sources, conversation, session_state
+    
+    if not (session_state["documents_loaded"]) and not (session_state["clarification"]):
+        
+        conversation = []
+        for user, assistant in chat_history:
+            conversation.extend(
+                [
+                    {"role": "user", "content": user},
+                    {"role": "assistant", "content": assistant},
+                ]
+            )
+        conversation.append({"role": "user", "content": f"Domanda: {message} . Comando: {INSTRUCTION_1}" })
+        conversation.append({"role": "assistant", "content": "Ok."})   
+        print("debug - CONV1", conversation)
+        
+        session_state["original_message"] = message
+        session_state["clarification"] = True       
+
+        
+    elif session_state["clarification"]:
+
+        message = message.lower()
+        matched_path = None
+        
+        for key, path in documents_paths.items():
+             if key in message:
+                matched_path = path
+                break
+            
+        if matched_path:
+            yield "Fammi cercare tra i miei documenti..." 
+            documents = load_documents(matched_path)
+            session_state["document_db"] = prepare_documents(documents)
+            session_state["documents_loaded"] = True
+            yield f"Ecco, ho raccolto informazioni dagli Osservatori {key.capitalize()}. Ora sto elaborando una risposta per te..."
+            context, sources = get_context_sources(session_state["original_message"], session_state["document_db"])
+            
+            #conversation = []
+            conversation.append({"role": "user", "content": f"{INSTRUCTION_2}"})
+            for user, assistant in chat_history:
+                conversation.extend(
+                    [
+                        #{"role": "user", "content": user },
+                        {"role": "assistant", "content": assistant},
+                    ]
+                )      
+            conversation.append({"role": "user", "content": f"Contesto: {context}\n\n Domanda iniziale: {session_state["original_message"]} . Rispondi solo in italiano."})
+            session_state["clarification"] = False 
+            print("debug - CONV2", conversation)
+         
+        else:
+             print(default_error_response)
+        
+    else:
+        conversation = []
+        conversation.append({"role": "user", "content": f"Comandi: {INSTRUCTION_2}"})
+        conversation.append({"role": "assistant", "content": "Va bene."})
+        for user, assistant in chat_history:
+            conversation.extend(
+                [
+                    {"role": "user", "content": user},
+                    {"role": "assistant", "content": assistant},
+                ]
+            ) 
+        conversation.append({"role": "user", "content": f"Contesto: {context}\n\n Nuova domanda: {message} . Rispondi in italiano e seguendo i comandi che ti ho dato prima"})
+        print("debug - CONV3", conversation)
+        
+        """ retriever = db.as_retriever()
+        qa = RetrievalQA.from_chain_type(llm=model, chain_type="refine", retriever=retriever, return_source_documents=False)
+        question = "Cosa sono i RWA?"
+        result = qa.run({"query": question})
+        print(result["result"]) """
+
+    # Iterate model output
+    input_ids = tokenizer.apply_chat_template(conversation, add_generation_prompt=True, return_tensors="pt")
+    if input_ids.shape[1] > MAX_INPUT_TOKEN_LENGTH:
+        input_ids = input_ids[:, -MAX_INPUT_TOKEN_LENGTH:]
+        gr.Warning(f"Trimmed input from conversation as it was longer than {MAX_INPUT_TOKEN_LENGTH} tokens.")
+    input_ids = input_ids.to(model.device)
 
+    streamer = TextIteratorStreamer(tokenizer, timeout=None, skip_prompt=True, skip_special_tokens=True)
+    generate_kwargs = dict(
+        {"input_ids": input_ids},
+        streamer=streamer,
+        max_new_tokens=max_new_tokens,
+        do_sample=True,
+        top_p=top_p,
+        top_k=top_k,
+        temperature=temperature,
+        num_beams=1,
+        repetition_penalty=repetition_penalty,
+    )
+    t = Thread(target=model.generate, kwargs=generate_kwargs)
+    t.start()
     
-with gr.Blocks(css=".gradio-container {background-color: #B9D9EB}") as demo:
-    gr.Markdown(DESCRIPTION)
-    #gr.DuplicateButton(value="Duplicate Space for private use", elem_id="duplicate-button")
-    chat_interface.render() 
+
+    outputs = []
+    for text in streamer:
+        outputs.append(text)
+        yield "".join(outputs)
     
+    if session_state["documents_loaded"]:
+        outputs.append(f"Fonti utilizzate: {sources}")
+        yield "".join(outputs) 
+        
+    sources = []    
+    print("debug - CHATHISTORY", chat_history)    
+
+chat_interface = gr.ChatInterface(
+    fn=generate,
+    additional_inputs=[
+        gr.Slider(
+            label="Max new tokens",
+            minimum=1,
+            maximum=MAX_MAX_NEW_TOKENS,
+            step=1,
+            value=DEFAULT_MAX_NEW_TOKENS,
+        ),
+        gr.Slider(
+            label="Temperature",
+            minimum=0.1,
+            maximum=4.0,
+            step=0.1,
+            value=0.6,
+        ),
+        gr.Slider(
+            label="Top-p (nucleus sampling)",
+            minimum=0.05,
+            maximum=1.0,
+            step=0.05,
+            value=0.9,
+        ),
+        gr.Slider(
+            label="Top-k",
+            minimum=1,
+            maximum=1000,
+            step=1,
+            value=50,
+        ),
+        gr.Slider(
+            label="Repetition penalty",
+            minimum=1.0,
+            maximum=2.0,
+            step=0.05,
+            value=1.2,
+        ),
+    ],
+    stop_btn=None,
+    examples=[
+        ["Ciao, in cosa puoi aiutarmi?"],
+        ["Ciao, in cosa consiste un piatto di spaghetti?"],
+        ["Ciao, quali sono le aziende che hanno iniziato ad integrare le stablecoins? Fammi un breve sommario."],
+        ["Spiegami la differenza tra mondi virtuali pubblici o privati"],
+        ["Trovami un esempio di progetto B2B"],
+        ["Quali sono le regole europee sui bonifici istantanei?"],
+    ],
+    cache_examples=False,
+)
+
+with gr.Blocks(css="style.css", fill_height=True) as demo:
+    gr.Markdown(DESCRIPTION, elem_classes="centered")
+    chat_interface.render()
+
 if __name__ == "__main__":
-    #progress = gr.Progress(track_tqdm=True)
-    demo.launch()
-       
+    demo.queue(max_size=20).launch()

appLlama.py

@@ -0,0 +1,29 @@
+from backend import handle_query
+import gradio as gr
+
+
+DESCRIPTION = """\
+# <div style="text-align: center;">Odi, l'assistente ricercatore degli Osservatori</div>
+
+
+👉 Retrieval-Augmented Generation - Ask me anything about the research carried out at the Osservatori.
+"""
+
+
+chat_interface =gr.ChatInterface(
+        fn=handle_query, 
+        chatbot=gr.Chatbot(height=500),
+        textbox=gr.Textbox(placeholder="Chiedimi qualasiasi cosa relativa agli Osservatori", container=False, scale=7),
+        #examples=[["Ciao, in cosa puoi aiutarmi?"],["Dimmi i risultati e le modalità di conduzione del censimento per favore"]]
+        )
+
+    
+with gr.Blocks(css=".gradio-container {background-color: #B9D9EB}") as demo:
+    gr.Markdown(DESCRIPTION)
+    #gr.DuplicateButton(value="Duplicate Space for private use", elem_id="duplicate-button")
+    chat_interface.render() 
+    
+if __name__ == "__main__":
+    #progress = gr.Progress(track_tqdm=True)
+    demo.launch()
+       

backend.py

@@ -76,7 +76,6 @@ def build_index(path: str):
     nodes = parser.get_nodes_from_documents(documents)
     # Build the vector store index from the nodes
     index = VectorStoreIndex(nodes)
-    
     #storage_context = StorageContext.from_defaults()
     #index.storage_context.persist(persist_dir=PERSIST_DIR)
     
@@ -106,7 +105,6 @@ def handle_query(query_str: str,
                     ]
                 )
     
-
     try:
         
         memory = ChatMemoryBuffer.from_defaults(token_limit=1500)

backend2.py

@@ -0,0 +1,95 @@
+import os
+import logging
+from concurrent.futures import ThreadPoolExecutor
+from pypdf import PdfReader
+from langchain.text_splitter import CharacterTextSplitter, RecursiveCharacterTextSplitter
+from langchain_community.vectorstores import FAISS
+#from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain_huggingface import HuggingFaceEmbeddings
+import time
+import torch
+from dotenv import load_dotenv
+
+logging.basicConfig(
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    level=logging.DEBUG
+)
+logger = logging.getLogger(__name__)
+logging.getLogger('matplotlib').setLevel(logging.WARNING)  # Suppress Matplotlib debug messages
+
+load_dotenv()
+
+logger.debug("Environment variables loaded.")
+
+def load_single_document(filepath):
+    if filepath.endswith('.pdf'):
+        with open(filepath, 'rb') as file:
+            pdf_reader = PdfReader(file)
+            text = " ".join([page.extract_text() for page in pdf_reader.pages])
+    elif filepath.endswith('.txt'):
+        with open(filepath, 'r', encoding='utf-8') as file:
+            text = file.read()
+    else:
+        logger.warning("Unsupported file type: %s", filepath)
+        return {"content": "", "source": filepath}
+    
+    return {"content": text, "source": filepath}
+
+def load_documents(directory):
+    logger.debug("Loading documents from directory: %s", directory)
+    start_time = time.time()
+    filepaths = [os.path.join(directory, filename) for filename in os.listdir(directory) if filename.endswith('.pdf') or filename.endswith('.txt')]
+    
+    if not filepaths:
+        logger.error("No documents found in the directory.")
+    else:
+        logger.debug("Found %d documents", len(filepaths))
+    
+    documents = []
+    with ThreadPoolExecutor() as executor:
+        documents = list(executor.map(load_single_document, filepaths))
+    
+    end_time = time.time()
+    logger.debug("Loaded %d documents in %.2f seconds.", len(documents), end_time - start_time)
+    return documents
+
+
+def prepare_documents(documents):
+    logger.debug("Preparing documents for embedding.")
+    start_time = time.time()
+    
+    text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
+    # It splits text into chunks of 1000 characters each with a 150-character overlap.
+    #text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
+    texts = text_splitter.create_documents([doc["content"] for doc in documents], metadatas=[{"source": os.path.basename(doc["source"])} for doc in documents])  
+    if not texts:
+        logger.error("No texts to embed.")
+        return None
+    
+    modelPath = "sentence-transformers/all-MiniLM-l6-v2"
+    model_kwargs = {'device':'mps'}
+    encode_kwargs = {'normalize_embeddings': False}
+    embeddings = HuggingFaceEmbeddings(model_name=modelPath, model_kwargs=model_kwargs, encode_kwargs=encode_kwargs )
+
+    try:
+        db = FAISS.from_documents(texts, embeddings)
+    except Exception as e:
+        logger.error("Error creating FAISS index: %s", e)
+        return None
+    
+    end_time = time.time()
+    logger.debug("Documents prepared in %.2f seconds.", end_time - start_time)
+    return db
+
+
+def get_context_sources(question, db):
+    start_time = time.time()
+
+    docs = db.similarity_search(question, k=3)
+    context = " ".join([doc.page_content for doc in docs])
+    sources = ", ".join(set([doc.metadata['source'] for doc in docs]))
+    
+    end_time = time.time()
+    logger.debug("Similarity search done in %.2f seconds.", end_time - start_time)
+
+    return context, sources

requirements.txt

@@ -3,7 +3,7 @@ llama-index
 llama-index-embeddings-huggingface
 llama-index-llms-huggingface
 llama-index-embeddings-instructor
-sentence-transformers==2.2.2
+sentence-transformers #==2.2.2
 llama-index-readers-web
 llama-index-readers-file
 gradio
@@ -13,9 +13,8 @@ setuptools
 spaces
 pydantic
 ipython
-keras
-keras-nlp
-tensorflow
-#langchain
-#langchain-community
-#langchain_huggingface
+#keras
+#keras-nlp
+#tensorflow
+langchain-community
+langchain-huggingface