Update run.py

run.py

@@ -186,17 +186,6 @@ def extend_prompt(message="", history=None, system=None, RAGAddon=None, system2=
 def response(
     message, history, temperature=0.9, max_new_tokens=500, top_p=0.95, repetition_penalty=1.0,
 ):
-    temperature = float(temperature)
-    if temperature < 1e-2: temperature = 1e-2
-    top_p = float(top_p)
-    generate_kwargs = dict(
-        temperature=temperature,
-        max_new_tokens=max_new_tokens,
-        top_p=top_p,
-        repetition_penalty=repetition_penalty,
-        do_sample=True,
-        seed=42,
-    )
     addon=""
     first_message = history[0][0] if history else message #[-1][0]
     results=collection.query(
@@ -222,17 +211,81 @@ def response(
       system,                   # system prompt
       addon,                    # RAG-component added to the system prompt
       None,                     # fictive first words of the AI (neither displayed nor stored)
-      historylimit=4,           # number of past messages to consider for response to current message
+      historylimit=0,           # number of past messages to consider for response to current message
       removeHTML=True           # remove HTML-components from History (to prevent bugs with Markdown)
     )
+  
+  ## Request response from model
+  #------------------------------
+  
+  print("AI running on prem!" if(onPrem) else "AI running HFHub!")
+  print(prompt)
+  if(onPrem==False):
+    temperature=float(0.9) 
+    max_new_tokens=1000 
+    top_p=0.95 
+    repetition_penalty=1.0
+    if temperature < 1e-2: temperature = 1e-2
+    top_p = float(top_p)
+    generate_kwargs = dict(
+        temperature=temperature,
+        max_new_tokens=max_new_tokens,
+        top_p=top_p,
+        repetition_penalty=repetition_penalty,
+        do_sample=True,
+        seed=42,
+    )
     stream = client.text_generation(prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
-    output = ""
-    for response in stream:
-        output += response.token.text
-        yield output
-    if(len(history)==0):
-      output=output+"\n\n<br><details open><summary><strong>Sources</strong></summary><br><ul>"+ "".join(["<li>" + s + "</li>" for s in combination])+"</ul></details>"
-    yield output
+    response = ""
+    #print("User: "+message+"\nAI: ")
+    for text in stream:
+        part=text.token.text
+        #print(part, end="", flush=True)
+        response += part
+        if removeHTML==True: response = re.sub("<(.*?)>","\n", response) # remove HTML-components in general (may cause bugs with markdown-rendering)
+        yield response
+    if(True): #len(history)==0):
+      response=response+"\n\n<br><details open><summary><strong>Sources</strong></summary><br><ul>"+ "".join(["<li>" + s + "</li>" for s in combination])+"</ul></details>"
+    yield response
+ 
+  if(onPrem==True):
+    # url="https://afischer1985-wizardlm-13b-v1-2-q4-0-gguf.hf.space/v1/completions"
+    url="http://0.0.0.0:2600/v1/completions"  
+    body={"prompt":prompt,"max_tokens":None, "echo":"False","stream":"True"}      # e.g. Mixtral-Instruct
+    if("Discolm_german_7b" in modelPath): body.update({"stop": ["<|im_end|>"]})   # fix stop-token of DiscoLM
+    if("Gemma-" in modelPath): body.update({"stop": ["<|im_end|>","</end_of_turn>"]})   # fix stop-token of Gemma
+    response="" #+"("+myType+")\n"
+    buffer=""
+    #print("URL: "+url)
+    #print("User: "+message+"\nAI: ")
+    for text in requests.post(url, json=body, stream=True):  #-H 'accept: application/json' -H 'Content-Type: application/json'
+      if buffer is None: buffer=""
+      buffer=str("".join(buffer))
+      # print("*** Raw String: "+str(text)+"\n***\n")
+      text=text.decode('utf-8')
+      if((text.startswith(": ping -")==False) & (len(text.strip("\n\r"))>0)): buffer=buffer+str(text)
+      # print("\n*** Buffer: "+str(buffer)+"\n***\n") 
+      buffer=buffer.split('"finish_reason": null}]}')
+      if(len(buffer)==1):
+        buffer="".join(buffer)
+        pass
+      if(len(buffer)==2):
+        part=buffer[0]+'"finish_reason": null}]}'  
+        if(part.lstrip('\n\r').startswith("data: ")): part=part.lstrip('\n\r').replace("data: ", "")
+        try: 
+          part = str(json.loads(part)["choices"][0]["text"])
+          #print(part, end="", flush=True)
+          response=response+part
+          buffer="" # reset buffer
+        except Exception as e:
+          print("Exception:"+str(e))
+          pass
+      if removeHTML==True: response = re.sub("<(.*?)>","\n", response) # remove HTML-components in general (may cause bugs with markdown-rendering)
+      yield response
+    if(True): #len(history)==0):
+      response=response+"\n\n<br><details open><summary><strong>Sources</strong></summary><br><ul>"+ "".join(["<li>" + s + "</li>" for s in combination])+"</ul></details>"
+    yield response
+    #history.append((message, response)) # add current dialog to history
 
 gr.ChatInterface(response, chatbot=gr.Chatbot(value=[[None,"Herzlich willkommen! Ich bin ein KI-basiertes Assistenzsystem, das für jede Anfrage die am besten geeigneten KI-Tools empfiehlt.<br>Aktuell bin ich wenig mehr als eine Tech-Demo und kenne nur 7 KI-Modelle - also sei bitte nicht zu streng mit mir.<br>Was ist dein Anliegen?"]],render_markdown=True),title="German AI-RAG-Interface to the Hugging Face Hub").queue().launch(share=True) #False, server_name="0.0.0.0", server_port=7864)
 print("Interface up and running!")