File size: 2,347 Bytes
b9fe2b4 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 |
#
# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import logging
from tavily import TavilyClient
from api.utils import get_uuid
from rag.nlp import rag_tokenizer
class Tavily:
def __init__(self, api_key: str):
self.tavily_client = TavilyClient(api_key=api_key)
def search(self, query):
try:
response = self.tavily_client.search(
query=query,
search_depth="advanced",
max_results=6
)
return [{"url": res["url"], "title": res["title"], "content": res["content"], "score": res["score"]} for res in response["results"]]
except Exception as e:
logging.exception(e)
return []
def retrieve_chunks(self, question):
chunks = []
aggs = []
logging.info("[Tavily]Q: " + question)
for r in self.search(question):
id = get_uuid()
chunks.append({
"chunk_id": id,
"content_ltks": rag_tokenizer.tokenize(r["content"]),
"content_with_weight": r["content"],
"doc_id": id,
"docnm_kwd": r["title"],
"kb_id": [],
"important_kwd": [],
"image_id": "",
"similarity": r["score"],
"vector_similarity": 1.,
"term_similarity": 0,
"vector": [],
"positions": [],
"url": r["url"]
})
aggs.append({
"doc_name": r["title"],
"doc_id": id,
"count": 1,
"url": r["url"]
})
logging.info("[Tavily]R: "+r["content"][:128]+"...")
return {"chunks": chunks, "doc_aggs": aggs} |