Update djezzy.py

djezzy.py

@@ -0,0 +1,58 @@
+%%capture
+!pip install -U bitsandbytes
+!pip install -U peft
+!pip install -U accelerate
+!pip install -U trl
+!pip install datasets==2.16.0
+!pip install sentencepiece
+!pip install -q -U git+https://github.com/huggingface/transformers.git
+!pip install "torch>=2.1.1" -U
+!pip install  accelerate
+!pip install -q python-dotenv==1.0.0
+
+!pip install -q pandas==1.5.3
+!pip install -q unstructured==0.7.12
+!pip install -q wikipedia==1.4.0
+!pip install -q pypdf==3.12.0
+!pip install -q jq==1.4.1
+!pip install -q nltk==3.8.1
+!pip install -q tiktoken==0.4.0
+!pip install -q  sentencepiece==0.1.99
+!pip install -q sentence-transformers==2.2.2
+!pip install -q cohere==4.11.2
+!pip install -q faiss-cpu==1.7.4
+!pip install -q chromadb==0.3.26
+!pip install transformers
+!pip install langchain
+!pip install langchain-community
+from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig,HfArgumentParser,TrainingArguments,pipeline, logging
+from peft import LoraConfig, PeftModel, prepare_model_for_kbit_training, get_peft_model
+import os,torch
+from datasets import load_dataset
+from trl import SFTTrainer
+import pandas as pd
+import pyarrow as pa
+import pyarrow.dataset as ds
+from datasets import Dataset
+import re
+import pandas as pd
+import os
+
+with open('/content/mots_clés.txt', 'r') as fichier:
+    contenu = fichier.read()
+
+# Séparer les mots en utilisant la virgule comme séparateur
+mots = contenu.split(',')
+
+# Afficher les mots pour vérifier
+for mot in mots:
+    print(mot.strip())
+
+# stocker les mots dans un tableau (une liste)
+tableau_de_mots = [mot.strip() for mot in mots]
+
+from langchain.embeddings import SentenceTransformerEmbeddings
+
+model_name = "sentence-transformers/all-MiniLM-L6-v2"
+
+embedding_llm = SentenceTransformerEmbeddings(model_name=model_name)