Update chatpdf.py

chatpdf.py

@@ -127,7 +127,7 @@ class Rag:
             self,
             similarity_model: SimilarityABC = None,
             generate_model_type: str = "auto",
-            generate_model_name_or_path: str = "LenguajeNaturalAI/leniachat-qwen2-1.5B-v0",
+            generate_model_name_or_path: str = "Qwen/Qwen2-0.5B-Instruct",
             lora_model_name_or_path: str = None,
             corpus_files: Union[str, List[str]] = None,
             save_corpus_emb_dir: str = "corpus_embs/",
@@ -176,7 +176,7 @@ class Rag:
         if similarity_model is not None:
             self.sim_model = similarity_model
         else:
-            m1 = BertSimilarity(model_name_or_path="shibing624/text2vec-base-multilingual", device=self.device)
+            m1 = BertSimilarity(model_name_or_path="hiiamsid/sentence_similarity_spanish_es", device=self.device)
             m2 = BM25Similarity()
             default_sim_model = EnsembleSimilarity(similarities=[m1, m2], weights=[0.5, 0.5], c=2)
             self.sim_model = default_sim_model
@@ -193,7 +193,7 @@ class Rag:
             self.add_corpus(corpus_files)
         self.save_corpus_emb_dir = save_corpus_emb_dir
         if rerank_model_name_or_path is None:
-            rerank_model_name_or_path = "BAAI/bge-reranker-base"
+            rerank_model_name_or_path = "BAAI/bge-reranker-large"
         if rerank_model_name_or_path:
             self.rerank_tokenizer = AutoTokenizer.from_pretrained(rerank_model_name_or_path)
             self.rerank_model = AutoModelForSequenceClassification.from_pretrained(rerank_model_name_or_path)
@@ -341,6 +341,8 @@ class Rag:
                 raw_text = [text.strip() for text in page_text.splitlines() if text.strip()]
                 new_text = ''
                 for text in raw_text:
+                    if new_text:
+                        new_text += ' '
                     new_text += text
                     if text[-1] in ['.', '!', '?', '。', '！', '？', '…', ';', '；', ':', '：', '”', '’', '）', '】', '》', '」',
                                     '』', '〕', '〉', '》', '〗', '〞', '〟', '»', '"', "'", ')', ']', '}']:
@@ -521,12 +523,12 @@ class Rag:
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
-    parser.add_argument("--sim_model_name", type=str, default="shibing624/text2vec-base-multilingual")
+    parser.add_argument("--sim_model_name", type=str, default="hiiamsid/sentence_similarity_spanish_es")
     parser.add_argument("--gen_model_type", type=str, default="auto")
-    parser.add_argument("--gen_model_name", type=str, default="LenguajeNaturalAI/leniachat-qwen2-1.5B-v0")
+    parser.add_argument("--gen_model_name", type=str, default="Qwen/Qwen2-0.5B-Instruct")
     parser.add_argument("--lora_model", type=str, default=None)
     parser.add_argument("--rerank_model_name", type=str, default="")
-    parser.add_argument("--corpus_files", type=str, default="data/sample.pdf")
+    parser.add_argument("--corpus_files", type=str, default="Acuerdo009.pdf")
     parser.add_argument("--device", type=str, default=None)
     parser.add_argument("--int4", action='store_true', help="use int4 quantization")
     parser.add_argument("--int8", action='store_true', help="use int8 quantization")