app.py

import gradio as gr
from huggingface_hub import InferenceClient

client = InferenceClient("devagonal/flan-t5-rouge-durga-2")


def respond(
    message,
):

    messages.append({"role": "user", "content": message})

    response = ""

    for message in client.chat_completion(
        messages,
    ):
        token = message.choices[0].delta.content

        response += token
        yield response


"""
For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
"""
demo = gr.ChatInterface(
    respond,
)


if __name__ == "__main__":
    demo.launch()