Update app.py

app.py

@@ -125,6 +125,11 @@ def PDFChunkerWithSeparator(filepath, separator):
 
 def loadKB(fileprovided, urlProvided, uploads_dir, request):
     documents = []
+    global tokenizer
+    BASE_MODEL = "LLAMA-TOKENIZER"
+    savedModelPath = "./model/" + BASE_MODEL
+    #tokenizer = LlamaTokenizer.from_pretrained(savedModelPath)
+    tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
     if fileprovided:
         # Delete Files
         clearKBUploadDirectory(uploads_dir)
@@ -134,11 +139,8 @@ def loadKB(fileprovided, urlProvided, uploads_dir, request):
             file.save(os.path.join(uploads_dir, secure_filename(file.filename)))
             #loader = PyPDFLoader(os.path.join(uploads_dir, secure_filename(file.filename)))
             #documents.extend(loader.load())
-            BASE_MODEL = "LLAMA-TOKENIZER"
-            savedModelPath = "./model/" + BASE_MODEL
-            global tokenizer
-            #tokenizer = LlamaTokenizer.from_pretrained(savedModelPath)
-            tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
+            
+            
             separator = "</Q>"
             documents.extend(PDFChunkerWithSeparator(os.path.join(uploads_dir, secure_filename(file.filename)),separator))
     else: