Update app.py

app.py

@@ -8,13 +8,13 @@ import time
 #import torch
 import pandas as pd
 
-from transformers import AutoTokenizer, AutoModelForCausalLM
+#from transformers import AutoTokenizer, AutoModelForCausalLM
 #from transformers import AutoModelForCausalLM, AutoModel
 from transformers import TextIteratorStreamer
 from threading import Thread
 #from transformers import LlamaForCausalLM, LlamaTokenizer
 #git lfs install
-#from ctransformers import AutoModelForCausalLM, AutoConfig, Config, AutoTokenizer
+from ctransformers import AutoModelForCausalLM, AutoConfig, Config, AutoTokenizer
 
 #from huggingface_hub import InferenceClient
 from huggingface_hub import Repository, upload_file
@@ -29,12 +29,31 @@ historylog = [{
         "Output": ''
 }]
 
-llm_model = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
+i_temperature = 0.30 
+i_max_new_tokens=1100
+i_repetitionpenalty = 1.2
+i_contextlength=12048
+logfile = 'TinyLlama.1B.txt'
+
+print("loading model...")
+modelfile="TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF"
+
+
+conf = AutoConfig(Config(temperature=i_temperature, 
+                         repetition_penalty=i_repetitionpenalty, 
+                         batch_size=64,
+                         max_new_tokens=i_max_new_tokens, 
+                         context_length=i_contextlength))
+llm_model = AutoModelForCausalLM.from_pretrained(modelfile,
+                                           model_type="llama",
+                                           config=conf)
+
+
 # TheBloke/Llama-2-7B-Chat-GGML , TinyLlama/TinyLlama-1.1B-Chat-v1.0 , microsoft/Phi-3-mini-4k-instruct, health360/Healix-1.1B-V1-Chat-dDPO
 # TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF and tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf not working
 
 model = AutoModelForCausalLM.from_pretrained(llm_model)
-tokenizer = AutoTokenizer.from_pretrained(llm_model)
+#tokenizer = AutoTokenizer.from_pretrained(llm_model)
 #initiate model and tokenizer
 
 data = load_dataset("Namitg02/Test", split='train', streaming=False)
@@ -64,10 +83,10 @@ print("check2")
 # memory = ConversationBufferMemory(return_messages=True)
 
 
-terminators = [
-    tokenizer.eos_token_id, # End-of-Sequence Token that indicates where the model should consider the text sequence to be complete
-    tokenizer.convert_tokens_to_ids("<|eot_id|>") # Converts a token strings in a single/ sequence of integer id using the vocabulary
-]
+#terminators = [
+#    tokenizer.eos_token_id, # End-of-Sequence Token that indicates where the model should consider the text sequence to be complete
+#    tokenizer.convert_tokens_to_ids("<|eot_id|>") # Converts a token strings in a single/ sequence of integer id using the vocabulary
+#]
 # indicates the end of a sequence