final

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.json filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,79 @@
+import os
+import torch
+import gradio as gr
+from peft import PeftModel, PeftConfig
+from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
+
+
+def load_model(model_path):
+    config = PeftConfig.from_pretrained(model_path)
+
+    bnb_config = BitsAndBytesConfig(
+        load_in_4bit=True,
+        bnb_4bit_quant_type="nf4",
+        bnb_4bit_compute_dtype=torch.float16,
+    )
+
+    model = AutoModelForCausalLM.from_pretrained(
+        config.base_model_name_or_path,
+        quantization_config=bnb_config,
+        trust_remote_code=True
+    )
+    model.config.use_cache = False
+    tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)
+    tokenizer.pad_token = tokenizer.eos_token
+
+    # Load the Lora model
+    model = PeftModel.from_pretrained(model, model_path)
+
+    return model, tokenizer
+
+
+def generate_text(prompt):
+    prompt = "<user>: " + prompt + " <bot>:"
+
+    batch = tokenizer(
+        prompt,
+        padding=True,
+        truncation=True,
+        return_tensors='pt'
+    )
+    batch = batch.to(device)
+
+    with torch.amp.autocast(device):
+        output_tokens = model.generate(
+            input_ids = batch.input_ids, 
+            max_new_tokens=200,
+            temperature=0.7,
+            top_p=0.7,
+            num_return_sequences=1,
+            pad_token_id=tokenizer.eos_token_id,
+            eos_token_id=tokenizer.eos_token_id,
+        )
+
+    generated_text = tokenizer.decode(output_tokens[0], skip_special_tokens=True)
+    
+    return generated_text.split("<user>: ")[1].split("<bot>: ")[-1]
+
+
+device = 'cuda' if torch.cuda.is_available() else 'cpu'
+model, tokenizer = load_model(os.path.join(os.getcwd(), "weights"))
+
+
+iface = gr.Interface(
+    fn=generate_text,
+    inputs=[
+        gr.Textbox(label="Prompt", placeholder="Enter your prompt here..."),
+    ],
+    outputs=gr.Textbox(label="Generated Text"),
+    title="LLaMA-3.2-3B-Instruct-QLoRA",
+    description="-3.2-3B-Instruct Finetuned using QLoRA on OpenAssistant/oasst1",
+    examples=[
+        ["can you describe winter?"],
+        ["How about we play a fun game?"],
+    ],
+)
+
+
+if __name__ == "__main__":
+    iface.launch()

requirements.txt

@@ -0,0 +1,4 @@
+torch
+transformers
+peft
+gradio

weights/adapter_config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5edf17dcd1994fbdb99ea77f16640f8ff15b9535da8cc77a7b498689cc77aad2
+size 805

weights/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb09470a5968c342cba08e1b53f26df379cb13ccdc1dfa073aa31407e2989b92
+size 97307544

weights/special_tokens_map.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b1835caa5b4d70acaa210fa222b0036f1882f9525c4660fd4810fb3e1e40ff8
+size 325

weights/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

weights/tokenizer_config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e992b5c11bc0255aa43b342a3943bdf08f88a8ebed72667d290f5a51bb59db8a
+size 54583