File size: 6,472 Bytes
5b45d10 297437a e760939 5b45d10 297437a fa0faa4 e760939 fa0faa4 e760939 fa0faa4 e760939 fa0faa4 e760939 fa0faa4 e760939 fa0faa4 e760939 fa0faa4 e760939 88f7b26 dc99d7c 235cd2f 46cd776 dc99d7c e760939 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 |
import gradio as gr
import requests
import os
def check_hallucination(assertion, citation):
api_url = "https://api-inference.huggingface.co/models/vectara/hallucination_evaluation_model"
header = {"Authorization": f"Bearer {hf_token}"}
payload = {"inputs": f"{assertion} [SEP] {citation}"}
response = requests.post(api_url, headers=header, json=payload, timeout=120)
output = response.json()
output = output[0][0]["score"]
return f"**hallucination score:** {output}"
# Define the API parameters
vapi_url = "https://api-inference.huggingface.co/models/vectara/hallucination_evaluation_model"
headers = {"Authorization": f"Bearer {hf_token}"}
# Function to query the API
def query(payload):
response = requests.post(vapi_url, headers=headers, json=payload)
return response.json()
# Function to evaluate hallucination
def evaluate_hallucination(input1, input2):
# Combine the inputs
combined_input = f"{input1}. {input2}"
# Make the API call
output = query({"inputs": combined_input})
# Extract the score from the output
score = output[0][0]['score']
# Generate a label based on the score
if score < 0.5:
label = f"🔴 High risk. Score: {score:.2f}"
else:
label = f"🟢 Low risk. Score: {score:.2f}"
return label
def query_vectara(text):
user_message = text
# Read authentication parameters from the .env file
customer_id = os.getenv('CUSTOMER_ID')
corpus_id = os.getenv('CORPUS_ID')
api_key = os.getenv('API_KEY')
# Define the headers
api_key_header = {
"customer-id": customer_id,
"x-api-key": api_key
}
# Define the request body in the structure provided in the example
request_body = {
"query": [
{
"query": user_message,
"queryContext": "",
"start": 1,
"numResults": 25,
"contextConfig": {
"charsBefore": 0,
"charsAfter": 0,
"sentencesBefore": 2,
"sentencesAfter": 2,
"startTag": "%START_SNIPPET%",
"endTag": "%END_SNIPPET%",
},
"rerankingConfig": {
"rerankerId": 272725718,
"mmrConfig": {
"diversityBias": 0.35
}
},
"corpusKey": [
{
"customerId": customer_id,
"corpusId": corpus_id,
"semantics": 0,
"metadataFilter": "",
"lexicalInterpolationConfig": {
"lambda": 0
},
"dim": []
}
],
"summary": [
{
"maxSummarizedResults": 5,
"responseLang": "auto",
"summarizerPromptName": "vectara-summary-ext-v1.2.0"
}
]
}
]
}
# Make the API request using Gradio
response = requests.post(
"https://api.vectara.io/v1/query",
json=request_body, # Use json to automatically serialize the request body
verify=True,
headers=api_key_header
)
if response.status_code == 200:
query_data = response.json()
if query_data:
sources_info = []
# Extract the summary.
summary = query_data['responseSet'][0]['summary'][0]['text']
# Iterate over all response sets
for response_set in query_data.get('responseSet', []):
# Extract sources
# Limit to top 5 sources.
for source in response_set.get('response', [])[:5]:
source_metadata = source.get('metadata', [])
source_info = {}
for metadata in source_metadata:
metadata_name = metadata.get('name', '')
metadata_value = metadata.get('value', '')
if metadata_name == 'title':
source_info['title'] = metadata_value
elif metadata_name == 'author':
source_info['author'] = metadata_value
elif metadata_name == 'pageNumber':
source_info['page number'] = metadata_value
if source_info:
sources_info.append(source_info)
result = {"summary": summary, "sources": sources_info}
return f"{json.dumps(result, indent=2)}"
else:
return "No data found in the response."
else:
return f"Error: {response.status_code}"
# Create the Gradio interface
iface = gr.Interface(
fn=evaluate_hallucination,
inputs=[gr.Textbox(label="Assertion"), gr.Textbox(label="Citation")],
outputs=[gr.Label(), gr.Textbox(label="Explanation")],
live=False,
title="👋🏻Welcome to 🌟Tonic's 🧠🌈Hallucination Tester 🔴🟢",
description="How To Use 🌈Hallucination tester: 🗣️📝add any assertion from an LLM or a human 🗣️😷 add any citation from a RAG retriever or a source 👇🏻📩 Press send 🔴red means a 🌈hallucination, 🟢 green means a 🧠credible assertion. Check out the model [vectara/hallucination_evaluation_model](https://huggingface.co/vectara/hallucination_evaluation_model) You can also use 🥒🍆🫑Vectara - Hallucination Tester 🗣️😷 via API below or way by cloning this space. 🧬🔬🔍 Simply click here: Join us : 🌟TeamTonic🌟 is always making cool demos! Join our active builder's🛠️community 👻 [](https://discord.gg/GWpVpekp) On 🤗Huggingface: [TeamTonic](https://huggingface.co/TeamTonic) & [MultiTransformer](https://huggingface.co/MultiTransformer) On 🌐Github: [Tonic-AI](https://github.com/tonic-ai) & contribute to 🌟 [DataTonic](https://github.com/Tonic-AI/DataTonic) 🤗Big thanks to Yuvi Sharma and all the folks at huggingface for the community grant 🤗",
theme='ParityError/Anime',
)
# Launch the interface
iface.launch() |