Upload 10 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+embeddings/embeddings.xlsx filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,19 @@
+import gradio as gr
+import functions as fn
+import json
+
+data = fn.load_embeddings()
+num_documents = data['num_documents']
+num_segment_contents = data['num_segment_contents']
+
+with open("gradio.json", encoding='utf-8') as f:
+    config = json.load(f)
+    config['description'] = config['description'].format(num_documents=num_documents, num_segment_contents=num_segment_contents)
+
+def on_submit(query, history):
+    response = fn.rag_response(query, data=data, detailed_response=False)
+    return gr.HTML(response.replace("\n", "<br>"))
+
+demo = gr.ChatInterface(fn=on_submit, **config)
+    
+demo.launch()

app_details.py

@@ -0,0 +1,20 @@
+import gradio as gr
+import functions as fn
+import json
+
+data = fn.load_embeddings()
+num_documents = data['num_documents']
+num_segment_contents = data['num_segment_contents']
+
+with open("gradio.json") as f:
+    config = json.load(f)
+    config['description'] = config['description'].format(num_documents=num_documents, num_segment_contents=num_segment_contents)
+    config['title'] += " - Interface de Respostas Detalhadas"
+
+def on_submit(query, history):
+    response = fn.rag_response(query, data=data, detailed_response=True)
+    return gr.HTML(response.replace("\n", "<br>"))
+
+demo = gr.ChatInterface(fn=on_submit, **config)
+    
+demo.launch()

documents_names.json

@@ -0,0 +1,3 @@
+{   "dar_normas_academicas.txt": ["DAR - Normas Acadêmicas", "https://www.puc-rio.br/sobrepuc/depto/dar/download/dar_normas_academicas.pdf"],
+    "posgraduacao_stritosensu_regulamento.txt": ["Regulamento dos Programas de Pós-Graduação da PUC-Rio", "https://www.puc-rio.br/ensinopesq/ccpg/download/posgraduacao_stritosensu_regulamento.pdf"]
+}

embeddings.py

@@ -0,0 +1,56 @@
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from sentence_transformers import SentenceTransformer
+import os
+import sys
+import glob
+import torch
+import pandas as pd
+from tqdm import tqdm
+ 
+parent_dir = os.path.abspath(os.path.join(os.getcwd(), os.pardir))
+sys.path.append(parent_dir)
+ 
+import functions as fn
+ 
+def get_embeddings(chunk_size, chunk_overlap, model_name, input_path='docs/*.txt', output_path='embeddings/embeddings.xlsx'):
+
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap,
+        length_function=len,
+        is_separator_regex=False,
+    )
+ 
+    all_splitted_text = []
+    file_names = []
+ 
+    for file in glob.glob(input_path):
+        text = fn.load_text(file)
+        splitted_text = text_splitter.create_documents([text])
+        all_splitted_text.extend(splitted_text)
+        file_names.extend([os.path.basename(file)] * len(splitted_text))
+ 
+    model = SentenceTransformer(model_name)
+ 
+    embeddings_list = []
+    content_list = []
+    file_name_list = []
+    model_name_list = []
+ 
+    for segment, file_name in tqdm(zip(all_splitted_text, file_names), desc="Procesando segmentos"):
+        embeddings = model.encode(segment.page_content)
+        embeddings_list.append(embeddings)
+        content_list.append(segment.page_content)
+        file_name_list.append(file_name)
+        model_name_list.append(model_name)
+ 
+    embeddings_df = pd.DataFrame(embeddings_list)
+    embeddings_df['segment_content'] = content_list
+    embeddings_df['file_name'] = file_name_list
+    embeddings_df['model_name'] = model_name_list
+ 
+    embeddings_df.to_excel(output_path, index=False)
+ 
+if __name__ == "__main__":
+    current_dir = os.getcwd()
+    get_embeddings(chunk_size=512, chunk_overlap=100, model_name='intfloat/multilingual-e5-large')

embeddings/embeddings.xlsx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03f5000178f061609ae23a245440a8cd7638769dbc9ee642b25f413b7c088664
+size 7187184

functions.py

@@ -0,0 +1,116 @@
+import chardet
+import torch
+from langchain_openai import ChatOpenAI, OpenAI
+from langchain_core.prompts import PromptTemplate
+from langchain.prompts import PromptTemplate
+from sentence_transformers import SentenceTransformer
+import os
+import pandas as pd
+import json
+
+current_dir = os.getcwd()
+
+def load_api_key(file_path):
+    with open(file_path, 'r', encoding='utf-8') as file:
+        data = json.load(file)
+        return data.get('api_key')
+    
+def load_dictionary(json_path):
+    with open(json_path, 'r', encoding='utf-8') as file:
+        return json.load(file)
+    
+def detect_encoding(file_path):
+    with open(file_path, 'rb') as file:
+        raw_data = file.read()
+    result = chardet.detect(raw_data)
+    return result['encoding']
+
+def load_text(file_path):
+    encoding = detect_encoding(file_path)
+    with open(file_path, 'r', encoding=encoding) as file:
+        return file.read()
+    
+def search_query(query, embeddings_tensor, model, segment_contents, file_names, k=5):
+    query_embedding = torch.tensor(model.encode(query)).unsqueeze(0)
+    similarities = torch.mm(query_embedding, embeddings_tensor.t()).squeeze(0)
+    topk_similarities, topk_indices = torch.topk(similarities, k)
+
+    top_segments = [segment_contents[idx] for idx in topk_indices]
+    top_file_names = [file_names[idx] for idx in topk_indices]
+    top_similarities = topk_similarities.tolist()
+    
+    return top_segments, top_file_names, top_similarities
+
+def load_embeddings(file_path="embeddings/embeddings.xlsx"):
+    embeddings_df = pd.read_excel(os.path.join(current_dir, file_path))
+    embeddings = embeddings_df.iloc[:, :-3].values
+    segment_contents = embeddings_df['segment_content'].values
+    num_segment_contents = len(segment_contents)
+    num_documents = embeddings_df['file_name'].nunique()
+    file_names = embeddings_df['file_name'].values
+    model_name = embeddings_df['model_name'].values[0]
+
+    return {
+        "embeddings": embeddings,
+        "segment_contents": segment_contents,
+        "num_documents": num_documents, 
+        "num_segment_contents": num_segment_contents,         
+        "file_names": file_names,
+        "model_name": model_name,
+        }
+
+def generate_answer_with_references(query, data):
+    embeddings = data["embeddings"]
+    segment_contents = data["segment_contents"]
+    model_name = data["model_name"]
+    file_names = data["file_names"]
+    embeddings_tensor = torch.tensor(embeddings, dtype=torch.float32)
+    model = SentenceTransformer(model_name)
+    dictionary_path = os.path.join(current_dir, 'documents_names.json')
+    file_name_dict = load_dictionary(dictionary_path)
+    file_names = [file_name_dict.get(name, name) for name in file_names]
+
+    top_segments, top_file_names, top_similarities = search_query(query, embeddings_tensor, model, segment_contents, file_names, k=5)
+    context = "\n----\n".join(top_segments)
+    prompt_template = """
+        Você é um assistente de inteligência artificial que responde a perguntas baseadas nos documentos de forma detalhada na forma culta da língua portuguesa.
+        Não é possível gerar informações ou fornecer informações que não estejam contidas nos documentos recuperados. 
+        Se a informação não se encontra nos documentos, responda com: Não foi possível encontrar a informação requerida nos documentos.
+        
+        Contexto:
+        
+        {context}
+
+        Pergunta: {query}
+
+        Resposta:""".format(context=context, query=query)
+        
+    qa_prompt = PromptTemplate.from_template(prompt_template)
+    api_key = load_api_key('api_key.json')
+    
+    llm = ChatOpenAI(api_key=api_key, model="gpt-3.5-turbo")
+    response = llm.invoke(qa_prompt.template)
+    resposta = response.content
+    total_tokens  = response.response_metadata['token_usage']['total_tokens']
+    prompt_tokens = response.response_metadata['token_usage']['prompt_tokens']
+        
+    return resposta, total_tokens, prompt_tokens, top_segments, top_file_names, top_similarities, prompt_template
+
+def rag_response(query, data, detailed_response):
+    resposta, total_tokens, prompt_tokens, top_segments, top_file_names, top_similarities, prompt_template = generate_answer_with_references(query, data)
+    file_names = [x[0] for x in top_file_names]
+    file_links = {x[0]: x[1] for x in top_file_names}
+
+    if detailed_response==True:
+        references_detail = "\n\n".join([
+        f"* Segmento: {segment}\nArquivo: <a href='{file_links[file_name]}' target='_blank'>{file_name}</a>\nSimilaridade: {similarity:.4f}"
+        for segment, file_name, similarity in zip(top_segments, file_names, top_similarities)])
+ 
+        formatted_detailed_response = f"Resposta:\n\n{resposta}\n\nPrompt:\n{prompt_template}\n\nPrompt Tokens: {prompt_tokens}\nTotal Tokens: {total_tokens}\n\n{references_detail}"
+
+        return formatted_detailed_response 
+    else: 
+        file_set = set(file_name for file_name in file_names)
+        references = "\n".join("<a href='{}' target='_blank'>{}</a>".format(file_links[file_name], file_name) for file_name in file_set)
+        formatted_response = f"{resposta}\n\n----\n{references}"
+        return formatted_response 

gradio.json

@@ -0,0 +1,16 @@
+{
+    "title": "Chatbot PUC-Rio",
+    "description": "<center>O assistente tem acesso a {num_documents} documentos ({num_segment_contents} parágrafos)</center>",
+    "examples": [
+        ["Quando deve ser renovada a matrícula?"],
+        ["O que é o histórico escolar?"], ["Como posso virar aluno da PUC-Rio?"],
+        ["Quais são os requisitos de proficiência linguística para os alunos de mestrado no programa?"],
+        ["Como faço para cancelar uma disciplina?"], ["A PUC-Rio tem curso de medicina?"]
+    ],
+    "theme": "gradio/default",
+    "submit_btn": "Enviar",
+    "stop_btn": "Parar",
+    "retry_btn": "🔄 Tentar novamente",
+    "undo_btn": "↩️ Desfazer",
+    "clear_btn": "🗑️ Limpar"
+}

requirements.txt

@@ -0,0 +1,11 @@
+chardet==5.2.0
+torch==2.3.0
+langchain_text_splitters
+sentence-transformers==3.0.1
+pandas
+tqdm
+openpyxl
+gradio==4.37.1
+langchain-openai
+langchain-core
+langchain