app.py

import torch
import gradio as gr
import os
from threading import Thread
from typing import Iterator
from transformers import (
    AutoModelForCausalLM,
    BitsAndBytesConfig,
    GenerationConfig,
    AutoTokenizer,
    TextIteratorStreamer,
)
from peft import AutoPeftModelForCausalLM



#deklarasi
max_seq_length = 2048 # Choose any! We auto support RoPE Scaling internally!
dtype = None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+
load_in_4bit = True # Use 4bit quantization to reduce memory usage. Can be False.
#alpaca_prompt = """Berikut adalah instruksi yang deskripsikan tugas dan sepasang input dan konteksnya. Tulis response sesuai dengan permintaan.
### Instruction:
{}
### Input:
{}
### Response:
#{}"""

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
MAX_MAX_NEW_TOKENS = 2048
DEFAULT_MAX_NEW_TOKENS = 1024
MAX_INPUT_TOKEN_LENGTH = int(os.getenv("MAX_INPUT_TOKEN_LENGTH", "4096"))

model_id = "abdfajar707/llama3_8B_lora_model_rkp_pn2025_v3"
#tokenizer = LlamaTokenizer.from_pretrained(model_id)
#model, tokenizer = AutoModelForCausalLM.from_pretrained(
#    model_id,
#    device_map="auto",
#   quantization_config=BitsAndBytesConfig(load_in_8bit=True),
#)
model = AutoPeftModelForCausalLM.from_pretrained(
        model_id, # YOUR MODEL YOU USED FOR TRAINING
        load_in_4bit = load_in_4bit,
    )
tokenizer = AutoTokenizer.from_pretrained(
            model_id,
            quantization_config=BitsAndBytesConfig(load_in_8bit=True)
    )

model.config.sliding_window = 4096
model.eval()


#@spaces.GPU(duration=90)
def generate(
    message: str,
    chat_history: list[tuple[str, str]],
    max_new_tokens: int = 1024,
    temperature: float = 0.6,
    top_p: float = 0.9,
    top_k: int = 50,
    repetition_penalty: float = 1.2,
) -> Iterator[str]:
    conversation = []
    for user, assistant in chat_history:
        conversation.extend(
            [
                {"role": "user", "content": user},
                {"role": "assistant", "content": assistant},
            ]
        )
    conversation.append({"role": "user", "content": message})

    input_ids = tokenizer.apply_chat_template(conversation, add_generation_prompt=True, return_tensors="pt")
    if input_ids.shape[1] > MAX_INPUT_TOKEN_LENGTH:
        input_ids = input_ids[:, -MAX_INPUT_TOKEN_LENGTH:]
        gr.Warning(f"Trimmed input from conversation as it was longer than {MAX_INPUT_TOKEN_LENGTH} tokens.")
    input_ids = input_ids.to(model.device)

    streamer = TextIteratorStreamer(tokenizer, timeout=20.0, skip_prompt=True, skip_special_tokens=True)
    generate_kwargs = dict(
        {"input_ids": input_ids},
        streamer=streamer,
        max_new_tokens=max_new_tokens,
        do_sample=True,
        top_p=top_p,
        top_k=top_k,
        temperature=temperature,
        num_beams=1,
        repetition_penalty=repetition_penalty,
    )
    t = Thread(target=model.generate, kwargs=generate_kwargs)
    t.start()

    outputs = []
    for text in streamer:
        outputs.append(text)
        yield "".join(outputs)

DESCRIPTION = '''
<div style="padding: 5px; text-align: left; display: flex; flex-direction: column; align-items: left;">
   <img src="https://sdgs.bappenas.go.id/repository/assets/bappenas_logo_square.png" style="width: 40%; max-width: 200px; height: auto; opacity: 0.55;  "> 
   <h2 style="font-size: 28px; margin-bottom: 2px; opacity: 0.55;">AI-Interlinked System/Bappenas GPT</h2>
</div>
'''

LICENSE = """
<p/>
---
Dibangun dari Meta Llama 3
"""

PLACEHOLDER = """
<div style="padding: 100px; text-align: center; display: flex; flex-direction: column; align-items: center;">
   <img src="https://cdn3.iconfinder.com/data/icons/human-resources-flat-3/48/150-4096.png" style="width: 1000; max-width: 200px; height: auto; opacity: 0.55;  "> 
   <h2 style="font-size: 20px; margin-bottom: 2px; opacity: 0.55;">Asisten Virtual Perencana</h2>
    <p style="font-size: 18px; margin-bottom: 2px; opacity: 0.65;">Silakan mulai tanya...</p>
</div>
"""


css = """
h1 {
  text-align: center;
  display: block;
}
#duplicate-button {
  margin: auto;
  color: white;
  background: #1565c0;
  border-radius: 100vh;
}
"""
chatbot=gr.Chatbot(height=450, placeholder=PLACEHOLDER, label='Interlinked Sytem ChatInterface')


chat_interface = gr.ChatInterface(
    fn=generate,
    chatbot=chatbot,
    additional_inputs=[
        gr.Slider(
            label="Max new tokens",
            minimum=1,
            maximum=MAX_MAX_NEW_TOKENS,
            step=1,
            value=DEFAULT_MAX_NEW_TOKENS,
        ),
        gr.Slider(
            label="Temperature",
            minimum=0.1,
            maximum=4.0,
            step=0.1,
            value=0.6,
        ),
        gr.Slider(
            label="Top-p (nucleus sampling)",
            minimum=0.05,
            maximum=1.0,
            step=0.05,
            value=0.9,
        ),
        gr.Slider(
            label="Top-k",
            minimum=1,
            maximum=1000,
            step=1,
            value=50,
        ),
        gr.Slider(
            label="Repetition penalty",
            minimum=1.0,
            maximum=2.0,
            step=0.05,
            value=1.2,
        ),
    ],
    stop_btn=None,
    examples=[
        ["Apa yang dimaksud dengan RPJMN"],
        ["Jelaskan tentang RPJMN 2020-2024"],
        ["Apa peran RKP 2021 dan 20211 dalam RPJM 2020-2024"],
        ["Apa saja program prioritas RPJMN 2020-2024"],
    ],
)



with gr.Blocks(css=css, fill_height=True) as demo:
    gr.Markdown(DESCRIPTION)
    #gr.DuplicateButton(value="Duplicate Space for private use", elem_id="duplicate-button")
    chat_interface.render()

if __name__ == "__main__":
    demo.queue(max_size=20).launch()