Merge branch 'main' into main

README.md

@@ -0,0 +1,99 @@
+# Search Arena
+
+**Search Arena** is a comprehensive platform designed to rigorously evaluate and compare search-based web agents. Leveraging a variety of metrics, Search Arena ensures that users can identify the most effective solutions for their needs.
+
+## Key Features
+
+- **Output Evaluations:** Analyze the quality and relevance of search results.
+- **Perplexity:** Measure the predictive uncertainty of language models used by the agents.
+- **Exa (Exhaustiveness Analysis):** Assess the breadth and depth of search coverage.
+- **Multi-Agent Comparison:** Compare multiple agents side-by-side.
+- **Customizable Benchmarks:** Define specific benchmarks and criteria for evaluation.
+- **User Feedback Integration:** Incorporate user feedback to improve agent performance.
+- **Performance Metrics:** Detailed reports on response time, precision, recall, and F1 score.
+
+## Benefits
+
+- **Enhanced Decision-Making:** Make informed decisions with clear, data-driven evaluations.
+- **Optimization:** Help developers optimize their search agents.
+- **Innovation:** Foster innovation by promoting the best-performing search technologies.
+
+## Getting Started
+
+Follow these steps to set up the project on your local machine.
+
+### Prerequisites
+
+- Git
+- Python 3.x
+- Virtualenv (optional but recommended)
+
+### Installation
+
+1. **Clone the Repository:**
+
+   ```bash
+   git clone https://github.com/leowalker89/SearchArena
+   ```
+
+2. **Navigate to the Project Directory:**
+
+   ```bash
+   cd search-arena
+   ```
+
+3. **Create a Virtual Environment:**
+
+   ```bash
+   python -m venv env
+   ```
+
+4. **Activate the Virtual Environment:**
+
+   - On Windows:
+
+     ```bash
+     .\env\Scripts\activate
+     ```
+
+   - On macOS and Linux:
+
+     ```bash
+     source env/bin/activate
+     ```
+
+5. **Install the Required Dependencies:**
+
+   ```bash
+   pip install -r requirements.txt
+   ```
+
+### Running the Project
+
+1. **Start the Development Server:**
+
+   ```bash
+   streamlit run app.py
+   ```
+
+2. **Open your Browser:**
+
+   Navigate to `http://localhost:5000` to view the platform.
+
+### Usage
+
+- Follow the on-screen instructions to evaluate and compare search-based web agents.
+- Customize benchmarks and criteria as needed.
+- Analyze detailed reports and visualizations to make informed decisions.
+
+### Contributing
+
+We welcome contributions! Please read our [Contributing Guide](CONTRIBUTING.md) to get started.
+
+### License
+
+This project is licensed under the MIT License. See the [LICENSE](LICENSE) file for more details.
+
+### Contact
+
+If you have any questions, feel free to open an issue or contact us at [[email protected]](mailto:[email protected]).

agents.py

@@ -32,6 +32,7 @@ class Perplexity(BaseLLM):
     A class to interact with the Perplexity API using OpenAI's interface.
     """
 
+
     def __init__(self, api_key: str = perplexity_api_key(), *args, **kwargs):
         """
         Initialize the Perplexity class with an API key.
@@ -322,11 +323,11 @@ task = "What is the swarmms framework"
 
 # Run all of the agents
 agents = [
-    Perplexity,
+    # Perplexity,
     ExaAgent,
     # ResearchAgent,
     TavilyWrapper,
-    # YouSearchAgent,
+    YouSearchAgent,
 ]
 
 # Run each agent with the given task

app.py

@@ -1,14 +1,20 @@
 import streamlit as st
+import random
+from helpers import query_you_com, query_tavily, query_perplexity
 
 # Set Streamlit to wide mode
 st.set_page_config(layout="wide")
 
 # Define the function to process the question
 def ProcessQuestion(question):
-    # Placeholder for the actual implementation
-    # This should return answers from two models, A and B
-    answer_a = "Answer from Model A"
-    answer_b = "Answer from Model B"
+    # Randomly select two out of the three functions
+    functions = [query_you_com, query_tavily, query_perplexity]
+    selected_functions = random.sample(functions, 2)
+    
+    # Get answers from the selected functions
+    answer_a = selected_functions[0](question)
+    answer_b = selected_functions[1](question)
+    
     return answer_a, answer_b
 
 # Initialize session state if not already done
@@ -18,8 +24,6 @@ if 'answer_a' not in st.session_state:
     st.session_state['answer_a'] = ""
 if 'answer_b' not in st.session_state:
     st.session_state['answer_b'] = ""
-if 'selected_model' not in st.session_state:
-    st.session_state['selected_model'] = ""
 if 'question' not in st.session_state:
     st.session_state['question'] = ""
 
@@ -46,24 +50,23 @@ if submit_button:
             # Save answers and state to session state
             st.session_state['answer_a'] = answer_a
             st.session_state['answer_b'] = answer_b
-            # st.session_state['selected_model'] = selected_model
             st.session_state['question'] = question
             st.session_state['results_displayed'] = True
         else:
             st.error("Your question exceeds the 1,000 character limit. Please shorten your question.")
     else:
-        st.error("Please enter a question and select a model.")
+        st.error("Please enter a question.")
 
 # Display results if available in session state
 if st.session_state['results_displayed']:
     col1, col2 = st.columns(2)
 
     with col1:
-        st.write(f"### Output A from {st.session_state['selected_model']}")
+        st.write("### Output A")
         st.write(st.session_state['answer_a'])
 
     with col2:
-        st.write(f"### Output B from {st.session_state['selected_model']}")
+        st.write("### Output B")
         st.write(st.session_state['answer_b'])
     
     feedback_col = st.columns([1, 1, 1, 1])

helpers.py

@@ -0,0 +1,156 @@
+import requests
+from dotenv import load_dotenv
+import os
+
+# Load environment variables from .env file
+load_dotenv()
+
+# Get API keys from environment variables
+YOU_COM_API_KEY = os.getenv('YOU_API_KEY')
+TAVILY_API_KEY = os.getenv('TAVILY_API_KEY')
+PERPLEXITY_API_KEY = os.getenv('PPLX_API_KEY')
+BRAVE_API_KEY = os.getenv('BRAVESEARCH_API_KEY')
+
+def query_you_com(query):
+    headers = {"X-API-Key": YOU_COM_API_KEY}
+    params = {"query": query}
+    try:
+        response = requests.get(
+            "https://api.ydc-index.io/rag",  # Verify the correctness of the API endpoint
+            params=params,
+            headers=headers,
+        )
+        response.raise_for_status()  # Raises an HTTPError if the response code was unsuccessful
+        resp = response.json()
+        return resp['answer']
+    except requests.exceptions.HTTPError as http_err:
+        return f"HTTP error occurred: {http_err}"
+    except Exception as err:
+        return f"An error occurred: {err}"
+
+
+def query_tavily(query):
+    payload = {
+        "api_key": TAVILY_API_KEY,
+        "query": query,
+        "search_depth": "basic",
+        "include_answer": True,
+        "include_images": False,
+        "include_raw_content": False,
+        "max_results": 1,
+        "include_domains": [],
+        "exclude_domains": []
+    }
+    response = requests.post("https://api.tavily.com/search", json=payload)
+    if response.status_code == 200:
+        resp = response.json()
+        return resp['answer']
+    else:
+        return f"Request failed with status code: {response.status_code}"
+
+def query_perplexity(query):
+    url = 'https://api.perplexity.ai/chat/completions'
+    headers = {
+        'Accept': 'application/json',
+        'Content-Type': 'application/json',
+        'Authorization': f'Bearer {PERPLEXITY_API_KEY}'
+    }
+    data = {
+        "model": "llama-3-sonar-large-32k-online",
+        "stream": False,
+        "max_tokens": 1024,
+        "frequency_penalty": 1,
+        "temperature": 0.0,
+        "messages": [
+            {
+                "role": "system", 
+                "content": "Be precise and concise in your responses."
+            },
+            {
+                "role": "user",
+                "content": query
+            }
+        ]
+    }
+    response = requests.post(url, headers=headers, json=data)
+    if response.status_code == 200:
+        result = response.json()
+        return result['choices'][0]['message']['content']
+    else:
+        return f"Request failed with status code: {response.status_code}"
+
+# def query_brave(query):
+#     headers = {"X-API-Key": BRAVE_API_KEY}
+#     params = {
+#         "q": query,
+#         "count": 1,
+#         "summary": True
+#     }
+#     response = requests.get("https://api.search.brave.com/res/v1/web/search", params=params, headers=headers)
+#     if response.status_code == 200:
+#         return response.json().get("summary", "No summary available.")
+#     else:
+#         return f"Request failed with status code: {response}"
+    
+
+# def brave_search_summarization(query):
+#     # Endpoint for web search with summary
+#     web_search_url = "https://api.search.brave.com/res/v1/web/search"
+#     summarizer_url = "https://api.search.brave.com/res/v1/summarizer/search"
+    
+#     # Headers for the requests
+#     headers = {
+#         "Accept": "application/json",
+#         "Accept-Encoding": "gzip",
+#         "X-Subscription-Token": BRAVE_API_KEY
+#     }
+    
+#     # Parameters for the initial web search request
+#     web_search_params = {
+#         "q": query,
+#         "summary": 1
+#     }
+    
+#     # Make the initial request to the web search endpoint
+#     web_search_response = requests.get(web_search_url, headers=headers, params=web_search_params)
+    
+#     # Check if the request was successful
+#     if web_search_response.status_code != 200:
+#         raise Exception(f"Web search request failed with status code {web_search_response.status_code}")
+    
+#     web_search_data = web_search_response.json()
+    
+#     # Extract the summarizer key from the response
+#     summarizer_key = web_search_data.get('summarizer', {}).get('key')
+#     if not summarizer_key:
+#         raise Exception("No summarizer key found in the web search response")
+    
+#     # Parameters for the summarizer request
+#     summarizer_params = {
+#         "key": summarizer_key,
+#         "entity_info": 1
+#     }
+    
+#     # Make the request to the summarizer endpoint
+#     summarizer_response = requests.get(summarizer_url, headers=headers, params=summarizer_params)
+    
+#     # Check if the request was successful
+#     if summarizer_response.status_code != 200:
+#         raise Exception(f"Summarizer request failed with status code {summarizer_response.status_code}")
+    
+#     summarizer_data = summarizer_response.json()
+    
+#     # Return the summarized content
+#     return summarizer_data
+
+def ProcessQuestion(question, model):
+    if model == "You.com":
+        return query_you_com(question)
+    elif model == "Tavily.com":
+        return query_tavily(question)
+    elif model == "Perplexity.ai":
+        return query_perplexity(question)
+    elif model == "Brave.com":
+        return query_brave(question)
+    else:
+        return "Model not supported"

requirements.txt

@@ -2,3 +2,4 @@ swarms
 exa_py
 tavily-python
 openai
+exa_py

test_func.py

@@ -0,0 +1,24 @@
+from helpers import query_you_com, query_tavily, query_perplexity #, brave_search_summarization,
+
+def test_queries():
+    test_query = "How is the weather in Palo Alto, CA?"
+
+    print("Testing You.com API:")
+    # you_com_result = query_you_com(test_query)
+    you_com_result = query_you_com(test_query)
+    print(you_com_result['answer'])
+
+    print("\nTesting Tavily.com API:")
+    tavily_result = query_tavily(test_query)
+    print(tavily_result['answer'])
+
+    print("\nTesting Perplexity.ai API:")
+    perplexity_result = query_perplexity(test_query)
+    print(perplexity_result)
+
+    # print("\nTesting Brave.com API:")
+    # brave_result = brave_search_summarization(test_query)
+    # print(brave_result)
+
+if __name__ == "__main__":
+    test_queries()