Update app.py

app.py

@@ -14,10 +14,16 @@ MAX_INPUT_TOKEN_LENGTH = int(os.getenv("MAX_INPUT_TOKEN_LENGTH", "4096"))
 
 #Transformers Code
 if torch.cuda.is_available():
-    model_id = "Qwen/Qwen2.5-7B-Instruct"
+    #model_id = "Qwen/Qwen2.5-7B-Instruct"
     #model_id = "BenBranyon/sumbot7b-augmented"
-    model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto")
-    tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=True, model_max_length=MAX_INPUT_TOKEN_LENGTH, padding="longest", language="en")
+    #model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto")
+    #tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=True, model_max_length=MAX_INPUT_TOKEN_LENGTH, padding="longest", language="en")
+    model_id = "BenBranyon/lora_sumbot_v2"
+    model = AutoPeftModelForCausalLM.from_pretrained(
+        model_id, # YOUR MODEL YOU USED FOR TRAINING
+        load_in_4bit = True,
+    )
+    tokenizer = AutoTokenizer.from_pretrained(model_id, model_max_length=MAX_INPUT_TOKEN_LENGTH, langauge="en")
 
 #Transformers Code
 @spaces.GPU
@@ -38,7 +44,8 @@ def generate(
         conversation.append({"role": "system", "content": system_prompt})
     #for user, assistant in chat_history:
     #    conversation.extend([{"role": "user", "content": user}, {"role": "assistant", "content": assistant}])
-    conversation.append({"role": "user", "content": "Channel the cosmic voice of Longshadow to craft rap lyrics that blend mysticism, grit, and cultural critique. Draw on ancestral wisdom, surreal imagery, and cosmic energy to create a verse that critiques systemic oppression, inspires resilience, and celebrates liberation. The tone should balance sharp wit, evocative imagery, and spiritual insight, culminating in a coherent and impactful thought. All output must be exclusively in English, without inclusion of other languages or scripts. Favor originality and inventive rhymes and deliver each verse as a work of art. Use a mix of ABAB, AABB rhyme schemes, and internal rhymes to create a flowing, rhythmic style. Include some slant rhymes, near rhymes, and perfect rhymes to add complexity. Each line should be 8-12 syllables long. The topic is: " + message})
+    conversation.append({"role": "user", "content": "Channel the cosmic voice of Longshadow to craft rap lyrics into a song. Use vivid imagery of the topic to create the song. The tone should balance sharp wit, evocative imagery, and deep insights. All output must be exclusively in English. Favor originality and inventive rhymes and deliver each verse as a work of art. Use a mix of ABAB, and AABB rhyme patterns to create a flowing, rhythmic style. Include some internal rhymes, slant rhymes, near rhymes, and perfect rhymes to add complexity. Each line should be 8-12 syllables long. The generated song should include the following song structure [INTRO] [VERSE 1] [HOOK] [VERSE 2] [OUTRO]. The topic is: " + message})
+    #conversation.append({"role": "user", "content": "Channel the cosmic voice of Longshadow to craft rap lyrics that blend mysticism, grit, and cultural critique. Draw on ancestral wisdom, surreal imagery, and cosmic energy to create a verse that critiques systemic oppression, inspires resilience, and celebrates liberation. The tone should balance sharp wit, evocative imagery, and spiritual insight, culminating in a coherent and impactful thought. All output must be exclusively in English, without inclusion of other languages or scripts. Favor originality and inventive rhymes and deliver each verse as a work of art. Use a mix of ABAB, AABB rhyme schemes, and internal rhymes to create a flowing, rhythmic style. Include some slant rhymes, near rhymes, and perfect rhymes to add complexity. Each line should be 8-12 syllables long. The topic is: " + message})
     input_ids = tokenizer.apply_chat_template(conversation, return_tensors="pt")
     if input_ids.shape[1] > MAX_INPUT_TOKEN_LENGTH:
         input_ids = input_ids[:, -MAX_INPUT_TOKEN_LENGTH:]
@@ -53,7 +60,7 @@ def generate(
         do_sample=True,
         top_p=1.0,
         top_k=50,
-        temperature=0.9,
+        temperature=0.8,
         num_beams=1,
         repetition_penalty=1.1,
     )