Update rag

rag.py

@@ -14,7 +14,7 @@ from langchain_huggingface import HuggingFaceEndpoint
 
 from typing import Dict, Any
 from tqdm import tqdm
-from transformers import LlamaTokenizer
+from transformers import AutoTokenizer
 
 NEO4J_URI = os.getenv("NEO4J_URI")
 NEO4J_USERNAME = os.getenv("NEO4J_USERNAME")
@@ -28,7 +28,7 @@ chat_llm = HuggingFaceEndpoint(
     do_sample=False,
 )
 
-global_tokenizer = LlamaTokenizer.from_pretrained("meta-llama/Meta-Llama-3-8B-Instruct")
+global_tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-8B-Instruct")
 
 
 def local_retriever(query: str):
@@ -277,7 +277,7 @@ def global_retriever(query: str, level: int, response_type: str):
 
     ###Debug####
 
-    tokens = global_tokenizer.encode(intermediate_results)
+    tokens = global_tokenizer(intermediate_results)
     print(f"Number of input tokens: {len(tokens)}")
     ###Debug###
     final_response = reduce_chain.invoke(