Spaces:
Running
Running
| import logging | |
| from typing import Optional | |
| import requests | |
| from requests.auth import HTTPDigestAuth | |
| from open_webui.retrieval.web.main import SearchResult, get_filtered_results | |
| from open_webui.env import SRC_LOG_LEVELS | |
| log = logging.getLogger(__name__) | |
| log.setLevel(SRC_LOG_LEVELS["RAG"]) | |
| def search_yacy( | |
| query_url: str, | |
| username: Optional[str], | |
| password: Optional[str], | |
| query: str, | |
| count: int, | |
| filter_list: Optional[list[str]] = None, | |
| ) -> list[SearchResult]: | |
| """ | |
| Search a Yacy instance for a given query and return the results as a list of SearchResult objects. | |
| The function accepts username and password for authenticating to Yacy. | |
| Args: | |
| query_url (str): The base URL of the Yacy server. | |
| username (str): Optional YaCy username. | |
| password (str): Optional YaCy password. | |
| query (str): The search term or question to find in the Yacy database. | |
| count (int): The maximum number of results to retrieve from the search. | |
| Returns: | |
| list[SearchResult]: A list of SearchResults sorted by relevance score in descending order. | |
| Raise: | |
| requests.exceptions.RequestException: If a request error occurs during the search process. | |
| """ | |
| # Use authentication if either username or password is set | |
| yacy_auth = None | |
| if username or password: | |
| yacy_auth = HTTPDigestAuth(username, password) | |
| params = { | |
| "query": query, | |
| "contentdom": "text", | |
| "resource": "global", | |
| "maximumRecords": count, | |
| "nav": "none", | |
| } | |
| # Check if provided a json API URL | |
| if not query_url.endswith("yacysearch.json"): | |
| # Strip all query parameters from the URL | |
| query_url = query_url.rstrip("/") + "/yacysearch.json" | |
| log.debug(f"searching {query_url}") | |
| response = requests.get( | |
| query_url, | |
| auth=yacy_auth, | |
| headers={ | |
| "User-Agent": "Open WebUI (https://github.com/open-webui/open-webui) RAG Bot", | |
| "Accept": "text/html", | |
| "Accept-Encoding": "gzip, deflate", | |
| "Accept-Language": "en-US,en;q=0.5", | |
| "Connection": "keep-alive", | |
| }, | |
| params=params, | |
| ) | |
| response.raise_for_status() # Raise an exception for HTTP errors. | |
| json_response = response.json() | |
| results = json_response.get("channels", [{}])[0].get("items", []) | |
| sorted_results = sorted(results, key=lambda x: x.get("ranking", 0), reverse=True) | |
| if filter_list: | |
| sorted_results = get_filtered_results(sorted_results, filter_list) | |
| return [ | |
| SearchResult( | |
| link=result["link"], | |
| title=result.get("title"), | |
| snippet=result.get("description"), | |
| ) | |
| for result in sorted_results[:count] | |
| ] | |