Update app.py

app.py

@@ -1,128 +1,28 @@
-import os
-import torch
-import spaces
-import gradio as gr
+# Use a pipeline as a high-level helper
+from transformers import pipeline
 
-from threading import Thread
-from huggingface_hub import login
-from transformers import AutoTokenizer, AutoModelForCausalLM, TextIteratorStreamer
+pipe = pipeline("text-generation", model="tiiuae/falcon-40b-instruct")
 
-login(os.environ.get("HF_TOKEN"))
 
-model_id = "meta-llama/Meta-Llama-3-8B-Instruct"
+# Function to interact with the chatbot
+def chat_with_bot():
+    while True:
+        # Take user input
+        question = input("You: ")
 
-tokenizer = AutoTokenizer.from_pretrained(model_id)
-model = AutoModelForCausalLM.from_pretrained(
-    model_id,
-    device_map="auto"
-)
+        # Exit if user inputs 'exit'
+        if question.lower() == "exit":
+            print("Chatbot: Goodbye!")
+            break
 
-@spaces.GPU()
-def generate(
-    message: str,
-    chat_history: list[tuple[str, str]],
-    system_prompt: str,
-    max_new_tokens: int,
-    temperature: float,
-    top_p: float,
-    top_k: int,
-    repetition_penalty: int
-    ):
+        # Generate answer using the model
+        answer = qa_pipeline(question=question, context="")
 
-    conversation = []
-    if system_prompt:
-        conversation.append({"role": "system", "content": system_prompt})
+        # Print the answer
+        print("Chatbot:", answer['answer'])
 
-    for user, assistant in chat_history:
-        conversation.append({"role": "user", "content": user})
-        conversation.append({"role": "assistant", "content": assistant})
 
-    conversation.append({"role": "user", "content": message})
-
-    streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
-    input_ids, attention_mask = tokenizer.apply_chat_template(
-        conversation,
-        add_generation_prompt=True,
-        return_tensors="pt",
-        return_dict=True
-    ).to(model.device).values()
-
-    terminators = [
-        tokenizer.eos_token_id,
-        tokenizer.convert_tokens_to_ids("<|eot_id|>")
-    ]
-
-    generate_kwargs = dict(
-        {"input_ids": input_ids, "attention_mask": attention_mask},
-        streamer=streamer,
-        do_sample=True,
-        temperature=temperature,
-        max_new_tokens=max_new_tokens,
-        eos_token_id=terminators,
-        top_k=top_k,
-        repetition_penalty=repetition_penalty,
-        top_p=top_p
-    )
-
-    t = Thread(target=model.generate, kwargs=generate_kwargs)
-    t.start()
-    outputs = []
-    for new_token in streamer:
-        outputs.append(new_token)
-        yield "".join(outputs)
-
-
-gr.ChatInterface(
-    fn=generate,
-    title="🦙 Llama-3 8B Chat",
-    description="",
-    additional_inputs=[
-        gr.Textbox(
-            label="System prompt", 
-            lines=5, 
-            value="Anda adalah asisten cerdas yang mahir berbahasa Indonesia. Anda dapat memahami dan merespons pertanyaan dalam berbagai bahasa, tetapi selalu menggunakan bahasa Indonesia yang baik dan benar dalam merespons. Anda ramah, sopan, dan berusaha memberikan jawaban yang jelas dan bermanfaat bagi pengguna. Jangan merespon dengan bahasa selain bahasa Indonesia!"
-        ),
-        gr.Slider(
-            label="Max new tokens",
-            minimum=1,
-            maximum=2048,
-            step=1,
-            value=1024,
-        ),
-        gr.Slider(
-            label="Temperature",
-            minimum=0.1,
-            maximum=4.0,
-            step=0.1,
-            value=0.6,
-        ),
-        gr.Slider(
-            label="Top-p (nucleus sampling)",
-            minimum=0.05,
-            maximum=1.0,
-            step=0.05,
-            value=0.9,
-        ),
-        gr.Slider(
-            label="Top-k",
-            minimum=1,
-            maximum=1000,
-            step=1,
-            value=50,
-        ),
-        gr.Slider(
-            label="Repetition penalty",
-            minimum=1.0,
-            maximum=2.0,
-            step=0.05,
-            value=1.2,
-        ),
-    ],
-    stop_btn=None,
-    examples=[
-        ["Halo apa kabar?"],
-        ["Apa manfaat berolahraga secara teratur?"],
-        ["Jika Budi berjalan sejauh 5 meter, berapa jumlah anak ayam bapaknya Budi?"],
-        ["Siapa presiden pertama Indonesia?"]
-    ],
-).queue().launch()
+# Start chatting
+if __name__ == "__main__":
+    print("Welcome to the chatbot! Type 'exit' to end the conversation.")
+    chat_with_bot()