Spaces:
Running
on
Zero
Running
on
Zero
Commit
·
c77435d
1
Parent(s):
c3e4586
add requirements
Browse files- app.py +102 -0
- requirements.txt +8 -0
app.py
CHANGED
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import gradio as gr
|
2 |
+
import random
|
3 |
+
import json
|
4 |
+
import os
|
5 |
+
from datetime import datetime
|
6 |
+
|
7 |
+
# This would be replaced with your actual SLM integration
|
8 |
+
def generate_response(query, context, model_name):
|
9 |
+
"""Placeholder function to generate response from an SLM"""
|
10 |
+
return f"This is a placeholder response from {model_name} based on query: {query} and context: {context}"
|
11 |
+
|
12 |
+
def save_evaluation(query, context, model_a, model_b, response_a, response_b, preference):
|
13 |
+
"""Save evaluation results to a JSON file"""
|
14 |
+
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
15 |
+
evaluation = {
|
16 |
+
"timestamp": timestamp,
|
17 |
+
"query": query,
|
18 |
+
"context": context,
|
19 |
+
"models": {
|
20 |
+
"left": model_a,
|
21 |
+
"right": model_b
|
22 |
+
},
|
23 |
+
"responses": {
|
24 |
+
"left": response_a,
|
25 |
+
"right": response_b
|
26 |
+
},
|
27 |
+
"preference": preference
|
28 |
+
}
|
29 |
+
|
30 |
+
# Create directory if it doesn't exist
|
31 |
+
os.makedirs("evaluations", exist_ok=True)
|
32 |
+
|
33 |
+
# Save to a file
|
34 |
+
with open(f"evaluations/eval_{timestamp.replace(' ', '_').replace(':', '-')}.json", "w") as f:
|
35 |
+
json.dump(evaluation, f, indent=2)
|
36 |
+
|
37 |
+
return "Evaluation saved successfully!"
|
38 |
+
|
39 |
+
def process_query(query, context, model_a="SLM-A", model_b="SLM-B"):
|
40 |
+
"""Process query and generate responses from two models"""
|
41 |
+
# Generate responses
|
42 |
+
response_a = generate_response(query, context, model_a)
|
43 |
+
response_b = generate_response(query, context, model_b)
|
44 |
+
|
45 |
+
# Randomly swap to avoid position bias
|
46 |
+
if random.random() > 0.5:
|
47 |
+
return response_a, response_b, model_a, model_b
|
48 |
+
else:
|
49 |
+
return response_b, response_a, model_b, model_a
|
50 |
+
|
51 |
+
def submit_evaluation(query, context, response_left, response_right, preference, model_left, model_right):
|
52 |
+
"""Submit and save the evaluation"""
|
53 |
+
if not preference:
|
54 |
+
return "Please select a preference before submitting."
|
55 |
+
|
56 |
+
save_evaluation(query, context, model_left, model_right, response_left, response_right, preference)
|
57 |
+
return "Thank you for your evaluation!"
|
58 |
+
|
59 |
+
with gr.Blocks(title="SLM-RAG Arena") as app:
|
60 |
+
gr.Markdown("# SLM-RAG Arena")
|
61 |
+
gr.Markdown("Compare responses from different models for RAG tasks.")
|
62 |
+
|
63 |
+
with gr.Row():
|
64 |
+
with gr.Column():
|
65 |
+
query_input = gr.Textbox(label="Query", placeholder="Enter your query here...")
|
66 |
+
context_input = gr.Textbox(label="Context", placeholder="Enter context information here...", lines=5)
|
67 |
+
generate_btn = gr.Button("Generate Responses")
|
68 |
+
|
69 |
+
# Hidden state variables
|
70 |
+
model_left = gr.State("")
|
71 |
+
model_right = gr.State("")
|
72 |
+
|
73 |
+
with gr.Row():
|
74 |
+
with gr.Column():
|
75 |
+
gr.Markdown("### Response A")
|
76 |
+
response_left = gr.Textbox(label="", lines=10, interactive=False)
|
77 |
+
with gr.Column():
|
78 |
+
gr.Markdown("### Response B")
|
79 |
+
response_right = gr.Textbox(label="", lines=10, interactive=False)
|
80 |
+
|
81 |
+
with gr.Row():
|
82 |
+
preference = gr.Radio(
|
83 |
+
choices=["Prefer Left", "Tie", "Prefer Right", "Neither"],
|
84 |
+
label="Which response do you prefer?"
|
85 |
+
)
|
86 |
+
|
87 |
+
submit_btn = gr.Button("Submit Evaluation")
|
88 |
+
result = gr.Textbox(label="Result")
|
89 |
+
|
90 |
+
generate_btn.click(
|
91 |
+
process_query,
|
92 |
+
inputs=[query_input, context_input],
|
93 |
+
outputs=[response_left, response_right, model_left, model_right]
|
94 |
+
)
|
95 |
+
|
96 |
+
submit_btn.click(
|
97 |
+
submit_evaluation,
|
98 |
+
inputs=[query_input, context_input, response_left, response_right, preference, model_left, model_right],
|
99 |
+
outputs=[result]
|
100 |
+
)
|
101 |
+
|
102 |
+
app.launch()
|
requirements.txt
ADDED
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
python>=3.10
|
2 |
+
transformers>=4.51.0
|
3 |
+
pandas>=2.2.3
|
4 |
+
accelerate>=1.6.0
|
5 |
+
numpy==1.26.4
|
6 |
+
openai>=1.60.2
|
7 |
+
torch>=2.5.1
|
8 |
+
tqdm==4.67.1
|