Update app.py

app.py

@@ -32,10 +32,10 @@ if not torch.cuda.is_available():
 # Model and Tokenizer Configuration
 model_id = "meta-llama/Llama-2-7b-hf"
 bnb_config = BitsAndBytesConfig(
-    load_in_4bit=True,
+    load_in_8bit=True,
     bnb_4bit_use_double_quant=False,
-    bnb_4bit_quant_type="nf4",
-    bnb_4bit_compute_dtype=torch.bfloat16
+    bnb_8bit_quant_type="nf8",
+    bnb_8bit_compute_dtype=torch.bfloat16
 )
 base_model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto", quantization_config=bnb_config)
 model = PeftModel.from_pretrained(base_model, "ranamhamoud/storytell")
@@ -58,11 +58,6 @@ def make_prompt(entry):
     # f"TELL A STORY, RELATE TO COMPUTER SCIENCE, INCLUDE ASSESMENTS. MAKE IT REALISTIC AND AROUND 800 WORDS: {entry}"
 
 def process_text(text):
-    # Bold specific bracketed phrases
-    text = re.sub(r'\[(assessment.*?)]', r'**[\1]**', text, flags=re.IGNORECASE)
-    text = re.sub(r'\[(answer.*?)]', r'**[\1]**', text, flags=re.IGNORECASE)
-    
-    # Remove all other bracketed content
     text = re.sub(r'\[.*?\]', '', text)  # Removes any text in square brackets
     text = re.sub(r'\(.*?\)', '', text)  # Removes any text in round brackets
     text = re.sub(r'\{.*?\}', '', text)  # Removes any text in curly brackets
@@ -75,7 +70,7 @@ def generate(
     message: str,
     chat_history: list[tuple[str, str]],
     max_new_tokens: int = DEFAULT_MAX_NEW_TOKENS,
-    temperature: float = 0.3,
+    temperature: float = 0.6,
     top_p: float = 0.7,
     top_k: int = 20,
     repetition_penalty: float = 1.0,