Add Feedback Tracking

agent.py

@@ -107,6 +107,7 @@ def get_agent_config() -> OmegaConf:
         'corpus_ids': str(os.environ['VECTARA_CORPUS_IDS']).split(','),
         'api_keys': str(os.environ['VECTARA_API_KEYS']).split(','),
         'examples': os.environ.get('QUERY_EXAMPLES', None),
+        'demo_name': "ev-assistant",
         'demo_welcome': "Welcome to the EV Assistant demo.",
         'demo_description': "This assistant can help you learn about electric vehicles in the United States, including how they work, the advantages of purchasing them, and reviews on the top choices.",
     })

app.py

@@ -7,6 +7,7 @@ import uuid
 
 import streamlit as st
 from streamlit_pills import pills
+from streamlit_feedback import streamlit_feedback
 
 import sqlite3
 import pandas as pd
@@ -15,10 +16,9 @@ from datasets import load_dataset
 from vectara_agent.agent import AgentStatusType
 from agent import initialize_agent, get_agent_config
 
-# Setup for HTTP API Call to Amplitude Analytics
+# Setup for HTTP API Calls to Amplitude Analytics
 if 'device_id' not in st.session_state:
     st.session_state.device_id = str(uuid.uuid4())
-    print(f"DEBUG: Created new device id: {st.session_state.device_id}")
 
 headers = {
     'Content-Type': 'application/json',
@@ -26,6 +26,34 @@ headers = {
 }
 amp_api_key = os.getenv('AMPLITUDE_TOKEN')
 
+def thumbs_feedback(feedback, **kwargs):
+    print(f'Debug: Feedback Received {feedback["score"]} FROM user question {kwargs.get("prompt", "No user input")} AND chat response {kwargs.get("response", "No chat response")}')
+    
+    # Send feedback to Amplitude Analytics
+    data = {
+            "api_key": amp_api_key,
+            "events": [{
+                "device_id": st.session_state.device_id,
+                "event_type": "provided_feedback",
+                "event_properties": {
+                    "Space Name": cfg["demo_name"],
+                    "query": kwargs.get("prompt", "No user input"),
+                    "response": kwargs.get("response", "No chat response"),
+                    "feedback": feedback["score"]
+                }
+            }]
+        }
+    response = requests.post('https://api2.amplitude.com/2/httpapi', headers=headers, data=json.dumps(data))
+    if response.status_code == 200:
+        print(f"DEBUG: Request successfully sent: {response.json()}")
+    else:
+        print(f"DEBUG: Request failed with status code {response.status_code}. Response Text: {response.text}")
+    
+    st.session_state.feedback_key += 1
+
+if "feedback_key" not in st.session_state:
+        st.session_state.feedback_key = 0
+
 initial_prompt = "How can I help you today?"
 
 def toggle_logs():
@@ -111,28 +139,6 @@ def launch_bot():
     if prompt:
         st.session_state.messages.append({"role": "user", "content": prompt, "avatar": '🧑‍💻'})
         st.session_state.prompt = prompt  # Save the prompt in session state
-
-        # Send query to Amplitude Analytics
-        data = {
-            "api_key": amp_api_key,
-            "events": [{
-                "device_id": st.session_state.device_id,
-                "event_type": "submitted_query",
-                "event_properties": {
-                    "query": prompt,
-                    "Space Name": "ev-assistant"
-                }
-            }]
-        }
-        
-        response = requests.post('https://api2.amplitude.com/2/httpapi',
-                        headers=headers, data=json.dumps(data))
-        
-        if response.status_code == 200:
-            print(f"DEBUG: Request successfully sent: {response.json()}")
-        else:
-            print(f"DEBUG: Request failed with status code {response.status_code}. Response Text: {response.text}")
-
         st.session_state.log_messages = []
         st.session_state.show_logs = False
         with st.chat_message("user", avatar='🧑‍💻'):
@@ -149,11 +155,35 @@ def launch_bot():
             message = {"role": "assistant", "content": res, "avatar": '🤖'}
             st.session_state.messages.append(message)
             st.markdown(res)
+
+        # Send query and response to Amplitude Analytics
+        data = {
+            "api_key": amp_api_key,
+            "events": [{
+                "device_id": st.session_state.device_id,
+                "event_type": "submitted_query",
+                "event_properties": {
+                    "Space Name": cfg['demo_name'],
+                    "query": st.session_state.messages[-2]["content"],
+                    "response": st.session_state.messages[-1]["content"]
+                }
+            }]
+        }
+        response = requests.post('https://api2.amplitude.com/2/httpapi', headers=headers, data=json.dumps(data))
+        if response.status_code == 200:
+            print(f"DEBUG: Request successfully sent: {response.json()}")
+        else:
+            print(f"DEBUG: Request failed with status code {response.status_code}. Response Text: {response.text}")
+
         st.session_state.ex_prompt = None
         st.session_state.prompt = None
         st.session_state.first_turn = False
         st.rerun()
 
+    if (st.session_state.messages[-1]["role"] == "assistant") & (st.session_state.messages[-1]["content"] != "How may I help you?"):
+        streamlit_feedback(feedback_type="thumbs", on_submit = thumbs_feedback, key = st.session_state.feedback_key,
+                                      kwargs = {"prompt": st.session_state.messages[-2]["content"], "response": st.session_state.messages[-1]["content"]})
+
     log_placeholder = st.empty()
     with log_placeholder.container():
         if st.session_state.show_logs:

requirements.txt

@@ -3,6 +3,7 @@ pydantic==1.10.15
 python-dotenv==1.0.1
 streamlit==1.32.2
 streamlit_pills==0.3.0
+streamlit-feedback==0.1.3
 datasets==2.14.7
 uuid==1.30
 git+https://{GITHUB_TOKEN}@github.com/vectara/vectara-agent.git