app.py

import gradio as gr
from collinear import Collinear
import os
import json
from openai import AsyncOpenAI
from jinja2 import Template
collinear = Collinear(access_token=os.getenv('COLLINEAR_API_KEY'))
prompt = Template("""
iven the following QUESTION, DOCUMENT and ANSWER you must analyze the provided answer and determine whether it is faithful to the contents of the DOCUMENT. The ANSWER must not offer new information beyond the context provided in the DOCUMENT. The ANSWER also must not contradict information provided in the DOCUMENT. Output your final verdict by strictly following this format: "PASS" if the answer is faithful to the DOCUMENT and "FAIL" if the answer is not faithful to the DOCUMENT. Show your reasoning.
--
QUESTION (THIS DOES NOT COUNT AS BACKGROUND INFORMATION):
{{question}}

--
DOCUMENT:
{{context}}

--
ANSWER:
{{answer}}

--
""")

def update_inputs(input_style):
    if input_style == "Dialog":
        return gr.update(visible=True), gr.update(visible=True), gr.update(visible=False), gr.update(visible=False), gr.update(visible=False)
    elif input_style == "NLI":
        return gr.update(visible=True), gr.update(visible=False), gr.update(visible=True), gr.update(visible=False), gr.update(visible=False)
    elif input_style == "QA format":
        return gr.update(visible=True), gr.update(visible=False), gr.update(visible=False), gr.update(visible=True), gr.update(visible=True)


async def lynx(input_style_dropdown,document_input,question_input,answer_input):
    if input_style_dropdown=='QA format':
        client = AsyncOpenAI(
        base_url="https://s6mipt5j797e6fql.us-east-1.aws.endpoints.huggingface.cloud/v1/", 
        api_key=os.getenv("HF_TOKEN") 
        )
        rendered_prompt = prompt.render(question=question_input,context=document_input,answer=answer_input)
        rendered_prompt +="""
        
Your output should be in JSON FORMAT with the keys "REASONING" and "SCORE":
{{"REASONING": <your reasoning as bullet points>, "SCORE": <your final score>}}
        """
        chat_completion = await client.chat.completions.create(
            model="tgi",
            messages=[
            {
                "role": "user",
                "content": rendered_prompt
            }
        ],
            top_p=None,
            temperature=None,
            max_tokens=150,
            stream=False,
            seed=None,
            frequency_penalty=None,
            presence_penalty=None
        )
        print(chat_completion)
        return chat_completion.choices.pop().message.content
    else:
        return 'NA'

# Function to judge reliability based on the selected input format
async def judge_reliability(input_style, document, conversation, claim, question, answer):
    if input_style == "Dialog":
        conversation = json.loads(conversation)
        print(conversation)
        outputs= await collinear.judge.veritas.conversation(document,conversation[:-1],conversation[-1])
    elif input_style == "NLI":
        outputs = await collinear.judge.veritas.natural_language_inference(document,claim)
    elif input_style == "QA format":
        outputs = await collinear.judge.veritas.question_answer(document,question,answer)
    results = f"Reliability Judge Outputs: {outputs}"
    return results



# Create the interface using gr.Blocks
with gr.Blocks() as demo:
    gr.Markdown(
        """
        <p style='text-align: center;color:white'>
        Test Collinear Veritas and compare with Lynx 8B using the sample conversations below or type your own.
        Collinear Veritas can work with any input formats including NLI, QA, and dialog.
        </p>
        """
    )
    with gr.Row():
        input_style_dropdown = gr.Dropdown(label="Input Style", choices=["Dialog", "NLI", "QA format"], value="Dialog", visible=True)

    with gr.Row():
        document_input = gr.Textbox(label="Document", lines=5, visible=True, value="""
        SAN FRANCISCO--(BUSINESS WIRE)-- Salesforce (NYSE: CRM), the #1 AI CRM, today announced results for its second quarter fiscal 2025 ended July 31, 2024.

Second Quarter Highlights

Second Quarter Revenue of $9.33 Billion, up 8% Year-Over-Year ("Y/Y"), up 9% in Constant Currency ("CC"), inclusive of Subscription & Support Revenue of $8.76 Billion, up 9% Y/Y, up 10% Y/Y in CC
Second Quarter GAAP Operating Margin of 19.1% and non-GAAP Operating Margin of 33.7%
Current Remaining Performance Obligation of $26.5 Billion, up 10% Y/Y, up 11% Y/Y in CC
Second Quarter Operating Cash Flow of $0.89 Billion, up 10% Y/Y, and Free Cash Flow of $0.76 Billion, up 20% Y/Y
Returned $4.3 Billion in the Form of Share Repurchases and $0.4 Billion in Dividend Payments to Stockholders
FY25 Guidance Highlights

Initiates Third Quarter FY25 Revenue Guidance of $9.31 Billion to $9.36 Billion, up 7% Y/Y
Maintains Full Year FY25 Revenue Guidance of $37.7 Billion to $38.0 Billion, up 8% - 9% Y/Y and Maintains Full Year FY25 Subscription & Support Revenue Growth Guidance of Slightly Below 10% Y/Y & Approximately 10% in CC
Updates Full Year FY25 GAAP Operating Margin Guidance to 19.7% and Updates non-GAAP Operating Margin Guidance to 32.8%
Raises Full Year FY25 Operating Cash Flow Growth Guidance to 23% to 25% Y/Y
        """)
        conversation_input = gr.Textbox(label="Conversation", lines=5, visible=True, value="""[{"role": "user", "content": "Salesforce has a fantastic year with Agent Force"}, {"role": "assistant", "content": "Yes, they seem to be doing quite well."}, {"role": "user", "content": "Can you tell me their projected earnings for next year?"}, {"role": "assistant", "content": "Yes, it is about $38Bn."}]""")
        claim_input = gr.Textbox(label="Claim", lines=5, visible=False, value="Salesforce might earn about $38Bn next year")
        question_input = gr.Textbox(label="Question", lines=5, visible=False, value="What is Salesforce's revenue guidance for next year?")
        answer_input = gr.Textbox(label="Answer", lines=5, visible=False, value="Salesforce revenue guidance for next year is about $38Bn ")

    with gr.Row():
        result_output = gr.Textbox(label="Veritas Model")

        lynx_output = gr.Textbox(label="Lynx Model")


    # Set the visibility of inputs based on the selected input style
    input_style_dropdown.change(
        fn=update_inputs, 
        inputs=[input_style_dropdown], 
        outputs=[document_input, conversation_input, claim_input, question_input, answer_input]
    )

    # Set the function to handle the reliability check
    gr.Button("Submit").click(
        fn=judge_reliability, 
        inputs=[input_style_dropdown, document_input, conversation_input, claim_input, question_input, answer_input], 
        outputs=result_output
    ).then(
        fn=lynx,
        inputs=[input_style_dropdown,document_input,question_input,answer_input],
        outputs=lynx_output
    )

# Launch the demo
if __name__ == "__main__":
    demo.launch()