Update app.py

app.py

@@ -1,60 +1,145 @@
 from unsloth import FastLanguageModel
 import torch
 import gradio as gr
+import os
+from threading import Thread
+from typing import Iterator
+from transformers import (
+    AutoModelForCausalLM,
+    BitsAndBytesConfig,
+    GemmaTokenizerFast,
+    TextIteratorStreamer,
+)
+
 
 #deklarasi
 max_seq_length = 2048 # Choose any! We auto support RoPE Scaling internally!
 dtype = None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+
 load_in_4bit = True # Use 4bit quantization to reduce memory usage. Can be False.
-alpaca_prompt = """Berikut adalah instruksi yang deskripsikan tugas dan sepasang input dan konteksnya. Tulis response sesuai dengan permintaan.
+#alpaca_prompt = """Berikut adalah instruksi yang deskripsikan tugas dan sepasang input dan konteksnya. Tulis response sesuai dengan permintaan.
 ### Instruction:
 {}
 ### Input:
 {}
 ### Response:
-{}"""
+#{}"""
 
 if True:
-    from app import FastLanguageModel
+    from unsloth import FastLanguageModel
     model, tokenizer = FastLanguageModel.from_pretrained(
-        model_name = "abdfajar707/llama3_8B_lora_model_rkp_pn2025_v2", # YOUR MODEL YOU USED FOR TRAINING
+        model_name = "abdfajar707/llama3_8B_lora_model_rkp_pn2025_v3", # YOUR MODEL YOU USED FOR TRAINING
         max_seq_length = max_seq_length,
         dtype = dtype,
         load_in_4bit = load_in_4bit,
-        temperature = 0.1,
     )
     FastLanguageModel.for_inference(model) # Enable native 2x faster inference
 
 
+MAX_MAX_NEW_TOKENS = 2048
+DEFAULT_MAX_NEW_TOKENS = 1024
+MAX_INPUT_TOKEN_LENGTH = int(os.getenv("MAX_INPUT_TOKEN_LENGTH", "4096"))
+
+
+model.config.sliding_window = 4096
+model.eval()
+
+
+#@spaces.GPU(duration=90)
+def generate(
+    message: str,
+    chat_history: list[tuple[str, str]],
+    max_new_tokens: int = 1024,
+    temperature: float = 0.6,
+    top_p: float = 0.9,
+    top_k: int = 50,
+    repetition_penalty: float = 1.2,
+) -> Iterator[str]:
+    conversation = []
+    for user, assistant in chat_history:
+        conversation.extend(
+            [
+                {"role": "user", "content": user},
+                {"role": "assistant", "content": assistant},
+            ]
+        )
+    conversation.append({"role": "user", "content": message})
+
+    input_ids = tokenizer.apply_chat_template(conversation, add_generation_prompt=True, return_tensors="pt")
+    if input_ids.shape[1] > MAX_INPUT_TOKEN_LENGTH:
+        input_ids = input_ids[:, -MAX_INPUT_TOKEN_LENGTH:]
+        gr.Warning(f"Trimmed input from conversation as it was longer than {MAX_INPUT_TOKEN_LENGTH} tokens.")
+    input_ids = input_ids.to(model.device)
 
-# Fungsi untuk menghasilkan respons
-def generate_response(prompt, max_length=1024):
-    inputs = tokenizer(
-[
-    alpaca_prompt.format(
-        prompt, # instruction
-        "", # input
-        "", # output - leave this blank for generation!
+    streamer = TextIteratorStreamer(tokenizer, timeout=20.0, skip_prompt=True, skip_special_tokens=True)
+    generate_kwargs = dict(
+        {"input_ids": input_ids},
+        streamer=streamer,
+        max_new_tokens=max_new_tokens,
+        do_sample=True,
+        top_p=top_p,
+        top_k=top_k,
+        temperature=temperature,
+        num_beams=1,
+        repetition_penalty=repetition_penalty,
     )
-], return_tensors = "pt").to("cuda")
-    outputs = model.generate(**inputs, max_length=max_length, pad_token_id=tokenizer.eos_token_id)
-    response = tokenizer.decode(outputs[0], skip_special_tokens=True)
-    return response
-
-history = []
-def wrapper_chat_history(chat_history, history):
-    chat_history = history[1:]
-    return chat_history
-
-def converse(message, chat_history):
-    response = generate_response(message)
-    print(response)
-    user_msg = {"role": "user", "content": message}
-    history.append(user_msg)
-    ai_msg = {"role": "assistant", "content": response}
-    history.append(ai_msg)
-    return history[-1]["content"]
-    
+    t = Thread(target=model.generate, kwargs=generate_kwargs)
+    t.start()
+
+    outputs = []
+    for text in streamer:
+        outputs.append(text)
+        yield "".join(outputs)
+
+
+chat_interface = gr.ChatInterface(
+    fn=generate,
+    chatbot=chatbot,
+    additional_inputs=[
+        gr.Slider(
+            label="Max new tokens",
+            minimum=1,
+            maximum=MAX_MAX_NEW_TOKENS,
+            step=1,
+            value=DEFAULT_MAX_NEW_TOKENS,
+        ),
+        gr.Slider(
+            label="Temperature",
+            minimum=0.1,
+            maximum=4.0,
+            step=0.1,
+            value=0.6,
+        ),
+        gr.Slider(
+            label="Top-p (nucleus sampling)",
+            minimum=0.05,
+            maximum=1.0,
+            step=0.05,
+            value=0.9,
+        ),
+        gr.Slider(
+            label="Top-k",
+            minimum=1,
+            maximum=1000,
+            step=1,
+            value=50,
+        ),
+        gr.Slider(
+            label="Repetition penalty",
+            minimum=1.0,
+            maximum=2.0,
+            step=0.05,
+            value=1.2,
+        ),
+    ],
+    stop_btn=None,
+    examples=[
+        ["Apa yang dimaksud dengan RPJMN"],
+        ["Jelaskan tentang RPJMN 2020-2024"],
+        ["Apa peran RKP 2021 dan 20211 dalam RPJM 2020-2024"],
+        ["Apa saja program prioritas RPJMN 2020-2024"],
+    ],
+)
+
 DESCRIPTION = '''
 <div style="padding: 5px; text-align: left; display: flex; flex-direction: column; align-items: left;">
    <img src="https://sdgs.bappenas.go.id/repository/assets/bappenas_logo_square.png" style="width: 40%; max-width: 200px; height: auto; opacity: 0.55;  "> 
@@ -71,7 +156,7 @@ Dibangun dari Meta Llama 3
 PLACEHOLDER = """
 <div style="padding: 100px; text-align: center; display: flex; flex-direction: column; align-items: center;">
    <img src="https://cdn3.iconfinder.com/data/icons/human-resources-flat-3/48/150-4096.png" style="width: 1000; max-width: 200px; height: auto; opacity: 0.55;  "> 
-   <h2 style="font-size: 20px; margin-bottom: 2px; opacity: 0.55;">Perencana-GPT</h2>
+   <h2 style="font-size: 20px; margin-bottom: 2px; opacity: 0.55;">Asisten Virtual Perencana</h2>
     <p style="font-size: 18px; margin-bottom: 2px; opacity: 0.65;">Silakan mulai tanya...</p>
 </div>
 """
@@ -89,19 +174,12 @@ h1 {
   border-radius: 100vh;
 }
 """
-# Gradio block
 chatbot=gr.Chatbot(height=450, placeholder=PLACEHOLDER, label='Interlinked Sytem ChatInterface')
 
-with gr.Blocks(fill_height=True, css=css) as demo:
-    
+with gr.Blocks(css=css, fill_height=True) as demo:
     gr.Markdown(DESCRIPTION)
-    gr.ChatInterface(
-        fn=converse,
-        chatbot=chatbot,
-        fill_height=True,
-                     )
-    
-    gr.Markdown(LICENSE)
-    
+    #gr.DuplicateButton(value="Duplicate Space for private use", elem_id="duplicate-button")
+    chat_interface.render()
+
 if __name__ == "__main__":
-    demo.launch()
+    demo.queue(max_size=20).launch()