|
import streamlit as st |
|
from huggingface_hub import InferenceClient |
|
|
|
""" |
|
For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference |
|
""" |
|
client = InferenceClient() |
|
|
|
def respond(message, history, system_message, max_tokens, temperature, top_p): |
|
messages = [{"role": "system", "content": system_message}] |
|
for user_msg, assistant_msg in history: |
|
if user_msg: |
|
messages.append({"role": "user", "content": user_msg}) |
|
if assistant_msg: |
|
messages.append({"role": "assistant", "content": assistant_msg}) |
|
messages.append({"role": "user", "content": message}) |
|
|
|
response = "" |
|
try: |
|
for message in client.chat_completion( |
|
model="mistralai/Codestral-22B-v0.1", |
|
messages=messages, |
|
max_tokens=max_tokens, |
|
stream=True, |
|
temperature=temperature, |
|
top_p=top_p, |
|
): |
|
token = message.choices[0].delta.content |
|
response += token |
|
yield response |
|
except Exception as e: |
|
yield f"Error: {e}" |
|
|
|
|
|
st.title("Chat with Codestral Model") |
|
system_message = st.text_input("System message", value="You are an expert python coder with in depth knowledge of langchain.") |
|
max_tokens = st.slider("Max new tokens", min_value=1, max_value=2048, value=2048, step=1) |
|
temperature = st.slider("Temperature", min_value=0.0, max_value=1.0, value=0.6, step=0.1) |
|
top_p = st.slider("Top-p (nucleus sampling)", min_value=0.1, max_value=1.0, value=0.95, step=0.05) |
|
|
|
history = [] |
|
|
|
if "history" not in st.session_state: |
|
st.session_state.history = [] |
|
|
|
def get_response(): |
|
user_input = st.session_state.user_input |
|
if user_input: |
|
st.session_state.history.append((user_input, "")) |
|
response_generator = respond(user_input, st.session_state.history, system_message, max_tokens, temperature, top_p) |
|
response = "" |
|
for r in response_generator: |
|
response = r |
|
st.session_state.history[-1] = (user_input, response) |
|
|
|
st.text_area("Chat History", value="\n".join([f"User: {h[0]}\nAssistant: {h[1]}" for h in st.session_state.history]), height=300) |
|
|
|
st.text_input("Your message:", key="user_input", on_change=get_response) |
|
|
