Create app.py

app.py

@@ -0,0 +1,57 @@
+from threading import Thread
+from transformers import TextIteratorStreame
+from unsloth import FastLanguageModel
+import torch
+import gradio as gr
+
+max_seq_length = 2048 # Choose any! We auto support RoPE Scaling internally!
+dtype = None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+
+load_in_4bit = True # Use 4bit quantization to reduce memory usage. Can be False.
+
+model_name = "Danielrahmai1991/llama32_ganjoor_adapt_basic_model_16bit_v1"
+model, tokenizer = FastLanguageModel.from_pretrained(
+    model_name = model_name,
+    max_seq_length = max_seq_length,
+    dtype = dtype,
+    load_in_4bit = load_in_4bit,
+    trust_remote_code=True,
+    # token = "hf_...", # use one if using gated models like meta-llama/Llama-2-7b-hf
+)
+
+print("model loaded")
+
+
+streamer = TextIteratorStreamer(tokenizer, timeout=10.0, skip_prompt=True, skip_special_tokens = True)
+
+def generate_text(prompt, max_length, top_p, top_k):
+    inputs = tokenizer([prompt], return_tensors="pt")
+
+    generate_kwargs = dict(
+        inputs,
+        max_length=int(max_length),top_p=float(top_p), do_sample=True, top_k=int(top_k), streamer=streamer
+    )
+
+    t = Thread(target=model.generate, kwargs=generate_kwargs)
+    t.start()
+
+    generated_text=[]
+
+    for text in streamer:
+        generated_text.append(text)
+        yield "".join(generated_text)
+
+
+description = """
+# Deploy our LLM
+"""
+inputs = [
+    gr.Textbox(label="Prompt text"),
+    gr.Textbox(label="max-lenth generation", value=100),
+    gr.Slider(0.0, 1.0, label="top-p value", value=0.95),
+    gr.Textbox(label="top-k", value=50,),
+]
+outputs = [gr.Textbox(label="Generated Text")]
+
+demo = gr.Interface(fn=generate_text, inputs=inputs, outputs=outputs, allow_flagging=False, description=description)
+
+demo.launch(debug=True, share=True)