Update app.py

app.py

@@ -1,4 +1,3 @@
-from unsloth import FastLanguageModel
 import torch
 import gradio as gr
 import os
@@ -7,8 +6,12 @@ from typing import Iterator
 from transformers import (
     AutoModelForCausalLM,
     BitsAndBytesConfig,
+    GenerationConfig,
+    AutoTokenizer,
     TextIteratorStreamer,
 )
+from peft import AutoPeftModelForCausalLM
+
 
 
 #deklarasi
@@ -23,21 +26,23 @@ load_in_4bit = True # Use 4bit quantization to reduce memory usage. Can be False
 ### Response:
 #{}"""
 
-if True:
-    from unsloth import FastLanguageModel
-    model, tokenizer = FastLanguageModel.from_pretrained(
-        model_name = "abdfajar707/llama3_8B_lora_model_rkp_pn2025_v3", # YOUR MODEL YOU USED FOR TRAINING
-        max_seq_length = max_seq_length,
-        dtype = dtype,
-        load_in_4bit = load_in_4bit,
-    )
-    FastLanguageModel.for_inference(model) # Enable native 2x faster inference
-
-
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 MAX_MAX_NEW_TOKENS = 2048
 DEFAULT_MAX_NEW_TOKENS = 1024
-MAX_INPUT_TOKEN_LENGTH = 4096 #int(os.getenv("MAX_INPUT_TOKEN_LENGTH", "4096"))
-
+MAX_INPUT_TOKEN_LENGTH = int(os.getenv("MAX_INPUT_TOKEN_LENGTH", "4096"))
+
+model_id = "abdfajar707/llama3_8B_lora_model_rkp_pn2025_v3"
+#tokenizer = LlamaTokenizer.from_pretrained(model_id)
+#model, tokenizer = AutoModelForCausalLM.from_pretrained(
+#    model_id,
+#    device_map="auto",
+#   quantization_config=BitsAndBytesConfig(load_in_8bit=True),
+#)
+model = AutoPeftModelForCausalLM.from_pretrained(
+        model_id, # YOUR MODEL YOU USED FOR TRAINING
+        load_in_4bit = load_in_4bit,
+    )
+tokenizer = AutoTokenizer.from_pretrained(model_id)
 
 model.config.sliding_window = 4096
 model.eval()
@@ -128,7 +133,7 @@ chatbot=gr.Chatbot(height=450, placeholder=PLACEHOLDER, label='Interlinked Sytem
 
 chat_interface = gr.ChatInterface(
     fn=generate,
-    #chatbot=chatbot,
+    chatbot=chatbot,
     additional_inputs=[
         gr.Slider(
             label="Max new tokens",
@@ -183,5 +188,4 @@ with gr.Blocks(css=css, fill_height=True) as demo:
     chat_interface.render()
 
 if __name__ == "__main__":
-    demo.queue(max_size=20).launch()
-    
+    demo.queue(max_size=20).launch()