feat: tentando melhorar espaçamento da resposta final

_utils/utils.py

@@ -16,68 +16,74 @@ import openai
 import pandas as pd
 import markdown
 
-os.environ["LANGCHAIN_TRACING_V2"]="true"
-os.environ["LANGCHAIN_ENDPOINT"]="https://api.smith.langchain.com"
+os.environ["LANGCHAIN_TRACING_V2"] = "true"
+os.environ["LANGCHAIN_ENDPOINT"] = "https://api.smith.langchain.com"
 os.environ.get("LANGCHAIN_API_KEY")
-os.environ["LANGCHAIN_PROJECT"]="VELLA"
+os.environ["LANGCHAIN_PROJECT"] = "VELLA"
 os.environ.get("OPENAI_API_KEY")
 os.environ.get("HUGGINGFACEHUB_API_TOKEN")
-embeddings_model = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2")
+embeddings_model = HuggingFaceEmbeddings(
+    model_name="sentence-transformers/all-mpnet-base-v2"
+)
 
 allIds = []
 
+
 def getPDF(file_paths):
-  documentId = 0
-  text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
-  pages = []
-  for file in file_paths:
-    loader = PyPDFLoader(file, extract_images=False)
-    pagesDoc = loader.load_and_split(text_splitter)
-    pages = pages + pagesDoc
-    
-  for page in pages:
-    documentId = str(uuid4())
-    allIds.append(documentId)
-    page.id = documentId
-  return pages
+    documentId = 0
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
+    pages = []
+    for file in file_paths:
+        loader = PyPDFLoader(file, extract_images=False)
+        pagesDoc = loader.load_and_split(text_splitter)
+        pages = pages + pagesDoc
+
+    for page in pages:
+        documentId = str(uuid4())
+        allIds.append(documentId)
+        page.id = documentId
+    return pages
+
 
 def create_retriever(documents, vectorstore):
-  print('\n\n')
-  print('documents: ', documents[:2])
+    print("\n\n")
+    print("documents: ", documents[:2])
 
-  vectorstore.add_documents(documents=documents)
+    vectorstore.add_documents(documents=documents)
+
+    retriever = vectorstore.as_retriever(
+        # search_type="similarity",
+        # search_kwargs={"k": 3},
+    )
+
+    return retriever
 
-  retriever = vectorstore.as_retriever(
-      # search_type="similarity",
-      # search_kwargs={"k": 3},
-  )
-  
-  return retriever
 
 def create_prompt_llm_chain(system_prompt, modelParam):
-  model = create_llm(modelParam)
-
-  system_prompt = system_prompt + "\n\n" + "{context}"
-  prompt = ChatPromptTemplate.from_messages(
-      [
-          ("system", system_prompt),
-          ("human", "{input}"),
-      ]
-  )
-  question_answer_chain = create_stuff_documents_chain(model, prompt)
-  return question_answer_chain
+    model = create_llm(modelParam)
+
+    system_prompt = system_prompt + "\n\n" + "{context}"
+    prompt = ChatPromptTemplate.from_messages(
+        [
+            ("system", system_prompt),
+            ("human", "{input}"),
+        ]
+    )
+    question_answer_chain = create_stuff_documents_chain(model, prompt)
+    return question_answer_chain
+
 
 def create_llm(modelParam):
-  if modelParam == default_model:
-    return ChatOpenAI(model=modelParam, max_tokens=16384)
-  else:
-    return HuggingFaceEndpoint(
-        repo_id=modelParam,
-        task="text-generation",
-        max_new_tokens=1100,
-        do_sample=False,
-        huggingfacehub_api_token=os.environ.get("HUGGINGFACEHUB_API_TOKEN")
-    )
+    if modelParam == default_model:
+        return ChatOpenAI(model=modelParam, max_tokens=16384)
+    else:
+        return HuggingFaceEndpoint(
+            repo_id=modelParam,
+            task="text-generation",
+            max_new_tokens=1100,
+            do_sample=False,
+            huggingfacehub_api_token=os.environ.get("HUGGINGFACEHUB_API_TOKEN"),
+        )
 
 
 class Resumo(BaseModel):
@@ -87,46 +93,49 @@ class Resumo(BaseModel):
     doutrina: str = Field()
     palavras_chave: List[str] = Field()
 
+
 def create_prompt_llm_chain_summary(system_prompt, model_param):
-  prompt_and_llm = create_prompt_and_llm(system_prompt, model_param)
+    prompt_and_llm = create_prompt_and_llm(system_prompt, model_param)
+
+    question_answer_chain = create_stuff_documents_chain(
+        prompt_and_llm["model"], prompt_and_llm["prompt"]
+    )
+    final_chain = question_answer_chain | JsonOutputParser(pydantic_object=Resumo)
+    return final_chain
 
-  question_answer_chain = create_stuff_documents_chain(prompt_and_llm["model"], prompt_and_llm["prompt"])
-  final_chain = question_answer_chain | JsonOutputParser(pydantic_object=Resumo)
-  return final_chain
 
 def process_embedding_summary(system_prompt, model_param, full_text):
-  prompt_and_llm = create_prompt_and_llm(system_prompt, model_param)
-  
-  text_splitter=RecursiveCharacterTextSplitter(chunk_size=2000, chunk_overlap=200)
-  docs = text_splitter.create_documents([full_text])
-  embeddings=get_embeddings([doc.page_content for doc in docs])
-  
-  content_list = [doc.page_content for doc in docs]
-  df = pd.DataFrame(content_list, columns=['page_content'])
-  vectors = [embedding.embedding for embedding in embeddings]
-  array = np.array(vectors)
-  embeddings_series = pd.Series(list(array))
-  df['embeddings'] = embeddings_series
+    prompt_and_llm = create_prompt_and_llm(system_prompt, model_param)
+
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=2000, chunk_overlap=200)
+    docs = text_splitter.create_documents([full_text])
+    embeddings = get_embeddings([doc.page_content for doc in docs])
+
+    content_list = [doc.page_content for doc in docs]
+    df = pd.DataFrame(content_list, columns=["page_content"])
+    vectors = [embedding.embedding for embedding in embeddings]
+    array = np.array(vectors)
+    embeddings_series = pd.Series(list(array))
+    df["embeddings"] = embeddings_series
 
 
 def get_embeddings(text):
-  response = openai.embeddings.create(
-      model="text-embedding-3-small",
-      input=text
-  )
-  return response.data
+    response = openai.embeddings.create(model="text-embedding-3-small", input=text)
+    return response.data
+
 
 def create_prompt_and_llm(system_prompt, model_param):
-  model = create_llm(model_param)
-  
-  system_prompt = system_prompt + "\n\n" + "{context}"
-  prompt = ChatPromptTemplate.from_messages(
-      [
-          ("system", system_prompt),
-          ("human", "{input}"),
-      ]
-  )
-  return {"model": model, "prompt": prompt}
+    model = create_llm(model_param)
+
+    system_prompt = system_prompt + "\n\n" + "{context}"
+    prompt = ChatPromptTemplate.from_messages(
+        [
+            ("system", system_prompt),
+            ("human", "{input}"),
+        ]
+    )
+    return {"model": model, "prompt": prompt}
+
 
 DEFAULT_SYSTEM_PROMPT = """
 
@@ -201,6 +210,10 @@ def convert_markdown_to_HTML(text: str):
         .replace("<diagnostico_processual>", "")
         .replace("</diagnostico_processual>", "")
         .replace("xml", "")
+        .replace("<li>\n", "<li>")
+        .replace("<ol>\n<li>", "<ol><li>")
+        .replace("</li>\n</ol>", "</li></ol>")
+        .replace("</li>\n<li>", "</li><li>")
         .replace("\n", "\n\n")
     )
     html = markdown.markdown(texto_inicial)