Spaces:
Sleeping
Sleeping
File size: 3,765 Bytes
56f16a2 96e4b44 b8d58df 724c4ca 56f16a2 504a6c1 fed089e 504a6c1 a857851 56f16a2 1883988 56f16a2 a857851 56f16a2 1883988 56f16a2 edb8f75 d0b0ea1 56f16a2 b8d58df 65d6ec0 b8d58df 56f16a2 b8d58df 56f16a2 b8d58df 56f16a2 c1b11f0 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 |
from huggingface_hub import InferenceClient, login
from transformers import AutoTokenizer
from langchain.chat_models import ChatOpenAI
import os, sys, json
import gradio as gr
from langchain.evaluation import load_evaluator
from pprint import pprint as print
# access token with permission to access the model and PRO subscription
#HUGGINGFACEHUB_API_TOKEN = os.getenv("HF_ACCESS_READ")
OAI_API_KEY=os.getenv("OPENAI_API_KEY")
login(token=os.environ["HF_ACCESS_READ"])
# tokenizer for generating prompt
print ("Tokenizer")
tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-70b-chat-hf")
# inference client
print ("Inf.Client")
client = InferenceClient("https://api-inference.huggingface.co/models/meta-llama/Llama-2-70b-chat-hf")
# generate function
def generate(text, history):
#mit RAG
#später entsprechend mit Vektorstore...
context="Nuremberg is the second-largest city of the German state of Bavaria after its capital Munich, and its 541,000 inhabitants make it the 14th-largest city in Germany. On the Pegnitz River (from its confluence with the Rednitz in Fürth onwards: Regnitz, a tributary of the River Main) and the Rhine–Main–Danube Canal, it lies in the Bavarian administrative region of Middle Franconia, and is the largest city and the unofficial capital of Franconia. Nuremberg forms with the neighbouring cities of Fürth, Erlangen and Schwabach a continuous conurbation with a total population of 812,248 (2022), which is the heart of the urban area region with around 1.4 million inhabitants,[4] while the larger Nuremberg Metropolitan Region has approximately 3.6 million inhabitants. The city lies about 170 kilometres (110 mi) north of Munich. It is the largest city in the East Franconian dialect area."
prompt = f"""Use the following pieces of context to answer the question at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer.
{context} Question: {question}"""
#zusammengesetzte Anfrage an Modell...
payload = tokenizer.apply_chat_template([{"role":"user","content":prompt}],tokenize=False)
res = client.text_generation(
payload,
do_sample=True,
return_full_text=False,
max_new_tokens=2048,
top_p=0.9,
temperature=0.6,
)
#zum Evaluieren:
# custom eli5 criteria
custom_criterion = {"eli5": "Is the output explained in a way that a 5 yeard old would unterstand it?"}
eval_result = evaluator.evaluate_strings(prediction=res.strip(), input=text, criteria=custom_criterion, requires_reference=True)
print ("eval_result:............ ")
print(eval_result)
return res.strip()
########################################
#Evaluation
########################################
evaluation_llm = ChatOpenAI(model="gpt-4")
# create evaluator
evaluator = load_evaluator("criteria", criteria="conciseness", llm=evaluation_llm)
################################################
#GUI
###############################################
#Beschreibung oben in GUI
################################################
chatbot_stream = gr.Chatbot()
chat_interface_stream = gr.ChatInterface(fn=generate,
title = "ChatGPT vom LI",
theme="soft",
chatbot=chatbot_stream,
retry_btn="🔄 Wiederholen",
undo_btn="↩️ Letztes löschen",
clear_btn="🗑️ Verlauf löschen",
submit_btn = "Abschicken",
)
with gr.Blocks() as demo:
with gr.Tab("Chatbot"):
#chatbot_stream.like(vote, None, None)
chat_interface_stream.queue().launch()
|