Add dictalm2.0-instruct folder

app.py

@@ -5,10 +5,9 @@ import openai
 from typing import Generator, List, Optional, Tuple, Dict
 from urllib.error import HTTPError
 
-API_URL = os.getenv('API_URL')
-API_KEY = os.getenv('API_KEY')
-CUSTOM_JS = os.getenv('CUSTOM_JS', None)
-oai_client = openai.OpenAI(api_key=API_KEY, base_url=API_URL)
+# Load the model and tokenizer
+tokenizer = AutoTokenizer.from_pretrained("./dictalm2.0-instruct")
+model = AutoModelForCausalLM.from_pretrained("./dictalm2.0-instruct")
 
 History = List[Tuple[str, str]]
 Messages = List[Dict[str, str]]
@@ -38,20 +37,27 @@ def model_chat(query: Optional[str], history: Optional[History]) -> Generator[Tu
         return
     messages = history_to_messages(history)
     messages.append({'role': 'user', 'content': query.strip()})
-    gen = oai_client.chat.completions.create(
-        model='dicta-il/dictalm2.0-instruct',
-        messages=messages,
-        temperature=0.7,
-        max_tokens=1024,
-        top_p=0.9,
-        stream=True
-    )
-    full_response = ''
-    for completion in gen:
-        text = completion.choices[0].delta.content
-        full_response += text or ''
-        yield full_response
-        
+    
+    # Combine all messages into one formatted input text
+    formatted_text = "<s>" + "".join(f"[INST] {m['content']} [/INST]" for m in messages if m['role'] == 'user')
+    inputs = tokenizer(formatted_text, return_tensors='pt')
+
+    # Generate the output
+    outputs = model.generate(inputs['input_ids'], max_length=1024, temperature=0.7, top_p=0.9)
+    full_response = tokenizer.decode(outputs[0], skip_special_tokens=True)
+    
+    # Simulate streaming by yielding the response in chunks
+    chunk_size = 20  # You can adjust the chunk size
+    for i in range(0, len(full_response), chunk_size):
+        yield full_response[i:i+chunk_size]
+
+def run_flask():
+    app.run(host='0.0.0.0', port=5000)
+
+# Run Flask in a separate thread
+threading.Thread(target=run_flask).start()
+
+
 with gr.Blocks(css='''
     .gr-group {direction: rtl;}
     .chatbot{text-align:right;}
@@ -118,4 +124,4 @@ with gr.Blocks(css='''
     interface.textbox.text_align = 'right'
     interface.theme_css += '.gr-group {direction: rtl !important;}'
 
-demo.queue(api_open=False).launch(max_threads=20, share=False, allowed_paths=['dicta-logo.jpg'])
+demo.queue(api_open=False).launch(max_threads=20, share=False, allowed_paths=['logo111.png'])