Update app.py

app.py

@@ -1,9 +1,15 @@
 import os
-from typing import Generator, Optional
 import gradio as gr
-from llama_cpp import Llama, LlamaGrammar
+from llama_cpp import Llama
 from huggingface_hub import hf_hub_download
 
+model = Llama(
+    model_path=hf_hub_download(
+        repo_id=os.environ.get("REPO_ID", "SimpleBerry/LLaMA-O1-Supervised-1129-Q2_K-GGUF"),
+        filename=os.environ.get("MODEL_FILE", "LLaMA-O1-Supervised-1129-q2_k.gguf"),
+    )
+)
+
 DESCRIPTION = '''
 # SimpleBerry/LLaMA-O1-Supervised-1129 | Duplicate the space and set it to private for faster & personal inference for free.
 SimpleBerry/LLaMA-O1-Supervised-1129: an experimental research model developed by the SimpleBerry.  
@@ -20,118 +26,136 @@ LICENSE = """
 
 template = "<start_of_father_id>-1<end_of_father_id><start_of_local_id>0<end_of_local_id><start_of_thought><problem>{content}<end_of_thought><start_of_rating><positive_rating><end_of_rating>\n<start_of_father_id>0<end_of_father_id><start_of_local_id>1<end_of_local_id><start_of_thought><expansion>"
 
-class OptimizedLLMInterface:
-    _model_instance = None  # Singleton pattern
-
-    def __init__(
-        self,
-        model_repo_id: str = "SimpleBerry/LLaMA-O1-Supervised-1129-Q2_K-GGUF",
-        model_filename: str = "LLaMA-O1-Supervised-1129-q2_k.gguf",
-    ):
-        if OptimizedLLMInterface._model_instance is None:
-            model_path = hf_hub_download(repo_id=model_repo_id, filename=model_filename)
-            OptimizedLLMInterface._model_instance = Llama(
-                model_path=model_path,
-                n_ctx=256,            # Minimal context for speed
-                n_threads=4,          # Fixed thread count
-                n_batch=1,            # Single batch for low latency
-                verbose=False,        # Disable logging
-                seed=-1,             # Disable random seed
-                logits_all=False,    # Disable logits
-                embedding=False,     # Disable embeddings
-                tensor_split=None,   # No tensor splitting
-                rope_freq_base=10000,  # Default RoPE settings
-                rope_freq_scale=1.0,
-                main_gpu=0,
-            )
-        self.model = OptimizedLLMInterface._model_instance
-        
-        # Pre-tokenize template parts
-        template_parts = template.split("{content}")
-        self._prefix_tokens = self.model.tokenize(template_parts[0].encode())
-        self._suffix_tokens = self.model.tokenize(template_parts[1].encode())
-
-    def generate_response(
-        self,
-        message: str,
-        history: Optional[list] = None,
-        max_tokens: int = 128,    # Reduced max tokens
-        temperature: float = 0.7,
-        top_p: float = 0.95,
-    ) -> Generator[str, None, None]:
-        try:
-            # Fast token preparation
-            message_tokens = self.model.tokenize(message.encode())
-            input_tokens = []
-            input_tokens.extend(self._prefix_tokens)
-            input_tokens.extend(message_tokens)
-            input_tokens.extend(self._suffix_tokens)
-            
-            output = ""
-            batch = []
-            batch_size = 4  # Small batch size for faster responses
-            
-            for token in self.model.generate(
-                input_tokens,
-                top_p=top_p,
-                temp=temperature,
-                top_k=1,           # Minimal top_k
-                repeat_penalty=1.0, # No repeat penalty
-                mirostat_mode=0,   # Disable mirostat
-                min_p=0.05,        # Allow more diversity
-                typical_p=1.0,     # Disable typical sampling
-                presence_penalty=0,
-                frequency_penalty=0,
-            ):
-                batch.append(token)
-                if len(batch) >= batch_size:
-                    text = self.model.detokenize(batch).decode('utf-8', errors='ignore')
-                    output += text
-                    yield output
-                    batch = []
-            
-            if batch:
-                text = self.model.detokenize(batch).decode('utf-8', errors='ignore')
-                output += text
-                yield output
-                
-        except Exception as e:
-            yield f"Error: {str(e)}"
-
-def create_demo(llm_interface: OptimizedLLMInterface) -> gr.Blocks:
-    with gr.Blocks() as demo:
-        gr.Markdown(DESCRIPTION)
-
-        chatbot = gr.ChatInterface(
-            llm_interface.generate_response,
-            title="SimpleBerry/LLaMA-O1-Supervised-1129 | GGUF Demo",
-            description="Edit Settings below if needed.",
-            examples=[
-                ["How many r's are in the word strawberry?"],
-                ['If Diana needs to bike 10 miles to reach home and she can bike at a speed of 3 mph for two hours before getting tired, and then at a speed of 1 mph until she reaches home, how long will it take her to get home?'],
-                ['Find the least odd prime factor of $2019^8+1$.'],
-            ],
-            cache_examples=False,
-            fill_height=True
-        )
-
-        with gr.Accordion("Adjust Parameters", open=False):
-            gr.Slider(minimum=64, maximum=512, value=128, step=64, label="Max Tokens")
-            gr.Slider(minimum=0.1, maximum=1.0, value=0.7, step=0.1, label="Temperature")
-            gr.Slider(minimum=0.05, maximum=1.0, value=0.95, step=0.05, label="Top-p")
-
-        gr.Markdown(LICENSE)
-    
-    return demo
+def llama_o1_template(data):
+    #query = data['query']
+    text = template.format(content=data)
+    return text
 
-def main():
-    llm = OptimizedLLMInterface()
-    demo = create_demo(llm)
-    
-    demo.launch(
-        share=False,
-        quiet=True
+def generate_text(message, history, max_tokens=512, temperature=0.9, top_p=0.95):
+    temp = ""
+    input_texts = [llama_o1_template(message)]
+    input_texts = [input_text.replace('<|end_of_text|>','') for input_text in input_texts]
+    #print(f"input_texts[0]: {input_texts[0]}")
+    inputs = model.tokenize(input_texts[0].encode('utf-8'))
+    for token in model.generate(inputs, top_p=top_p, temp=temperature):
+        #print(f"token: {token}")
+        text = model.detokenize([token])
+        #print(f"text detok: {text}")
+        temp += text.decode('utf-8')
+        yield temp
+
+with gr.Blocks() as demo:
+    gr.Markdown(DESCRIPTION)
+
+    chatbot = gr.ChatInterface(
+        generate_text,
+        title="SimpleBerry/LLaMA-O1-Supervised-1129 | GGUF Demo",
+        description="Edit Settings below if needed.",
+        examples=[
+            ["How many r's are in the word strawberry?"],
+            ['If Diana needs to bike 10 miles to reach home and she can bike at a speed of 3 mph for two hours before getting tired, and then at a speed of 1 mph until she reaches home, how long will it take her to get home?'],
+            ['Find the least odd prime factor of $2019^8+1$.'],
+        ],
+        cache_examples=False,
+        fill_height=True
     )
 
+    with gr.Accordion("Adjust Parameters", open=False):
+        gr.Slider(minimum=128, maximum=8192, value=512, step=1, label="Max Tokens")
+        gr.Slider(minimum=0.1, maximum=1.5, value=0.7, step=0.1, label="Temperature")
+        gr.Slider(minimum=0.05, maximum=1.0, value=0.95, step=0.01, label="Top-p (nucleus sampling)")
+
+    gr.Markdown(LICENSE)
+
 if __name__ == "__main__":
-    main()
+    demo.launch()
+# # import spaces
+
+# import os
+# import gradio as gr
+# from transformers import AutoTokenizer, AutoModelForCausalLM
+# from huggingface_hub import hf_hub_download, snapshot_download
+# import accelerate
+
+# accelerator = accelerate.Accelerator()
+
+# # Load the model and tokenizer from Hugging Face
+# model_path = snapshot_download(
+#     repo_id=os.environ.get("REPO_ID", "SimpleBerry/LLaMA-O1-Supervised-1129")
+# )
+
+# tokenizer = AutoTokenizer.from_pretrained(model_path)
+# model = AutoModelForCausalLM.from_pretrained(model_path,device_map='auto')
+
+# DESCRIPTION = '''
+# # SimpleBerry/LLaMA-O1-Supervised-1129 | Duplicate the space and set it to private for faster & personal inference for free.
+# SimpleBerry/LLaMA-O1-Supervised-1129: an experimental research model developed by the SimpleBerry.  
+# Focused on advancing AI reasoning capabilities.  
+
+# ## This Space was designed by Lyte/LLaMA-O1-Supervised-1129-GGUF, Many Thanks!
+
+# **To start a new chat**, click "clear" and start a new dialogue.
+# '''
+
+# LICENSE = """
+# --- MIT License ---
+# """
+
+# template = "<start_of_father_id>-1<end_of_father_id><start_of_local_id>0<end_of_local_id><start_of_thought><problem>{content}<end_of_thought><start_of_rating><positive_rating><end_of_rating>\n<start_of_father_id>0<end_of_father_id><start_of_local_id>1<end_of_local_id><start_of_thought><expansion>"
+
+# def llama_o1_template(data):
+#     #query = data['query']
+#     text = template.format(content=data)
+#     return text
+
+# def format_response(response):
+#     response = response.replace('<start_of_father_id>','')
+#     response = response.replace('<end_of_father_id><start_of_local_id>','👉')
+#     response = response.replace('<end_of_local_id><start_of_thought>',', ')
+#     response = response.replace('<end_of_thought><start_of_rating>','')
+#     response = response.replace('<end_of_rating>','')
+#     response = response.replace('<positive_rating>','👍')
+#     response = response.replace('<negative_rating>','👎')
+    
+# # @spaces.GPU
+# def generate_text(message, history, max_tokens=512, temperature=0.9, top_p=0.95):
+#     input_text = llama_o1_template(message)
+#     inputs = tokenizer(input_text, return_tensors="pt").to(accelerator.device)
+
+#     # Generate the text with the model
+#     output = model.generate(
+#         **inputs,
+#         max_length=max_tokens,
+#         temperature=temperature,
+#         top_p=top_p,
+#         do_sample=True,
+#     )
+
+#     response = tokenizer.decode(output[0], skip_special_tokens=False)
+#     yield response
+
+# with gr.Blocks() as demo:
+#     gr.Markdown(DESCRIPTION)
+
+#     chatbot = gr.ChatInterface(
+#         generate_text,
+#         title="SimpleBerry/LLaMA-O1-Supervised-1129 | GGUF Demo",
+#         description="Edit Settings below if needed.",
+#         examples=[
+#             ["How many r's are in the word strawberry?"],
+#             ['If Diana needs to bike 10 miles to reach home and she can bike at a speed of 3 mph for two hours before getting tired, and then at a speed of 1 mph until she reaches home, how long will it take her to get home?'],
+#             ['Find the least odd prime factor of $2019^8+1$.'],
+#         ],
+#         cache_examples=True,
+#         fill_height=True,
+#     )
+
+#     with gr.Accordion("Adjust Parameters", open=False):
+#         gr.Slider(minimum=1024, maximum=8192, value=2048, step=1, label="Max Tokens")
+#         gr.Slider(minimum=0.1, maximum=1.5, value=0.7, step=0.1, label="Temperature")
+#         gr.Slider(minimum=0.05, maximum=1.0, value=0.95, step=0.01, label="Top-p (nucleus sampling)")
+
+#     gr.Markdown(LICENSE)
+
+# if __name__ == "__main__":
+#     demo.launch()