Upload free_ask_internet.py

free_ask_internet.py

@@ -8,7 +8,6 @@ import trafilatura
 from trafilatura import bare_extraction
 from concurrent.futures import ThreadPoolExecutor
 import concurrent
-import requests
 import openai
 import time 
 from datetime import datetime
@@ -19,24 +18,27 @@ import urllib.parse
 
  
 def extract_url_content(url):
-    downloaded = trafilatura.fetch_url(url)
-    content =  trafilatura.extract(downloaded)
+    downloaded = requests.get(url, headers={
+        'Accept-Language': 'en-US,en;q=0.9'  # 强制英文内容
+    }).content
+    content = trafilatura.extract(downloaded)
     
-    return {"url":url, "content":content}
-
+    return {"url": url, "content": content}
 
- 
 
-def search_web_ref(query:str, debug=False):
- 
+def search_web_ref(query: str, lang="zh-CN", debug=False):
     content_list = []
 
     try:
-
         safe_string = urllib.parse.quote_plus(":all !general " + query)
 
         searxng_url = os.environ.get('SEARXNG_URL')
-        response = requests.get(searxng_url + '?q=' + safe_string + '&format=json')
+        params = {
+            "q": safe_string,
+            "language": "zh-CN" if lang.startswith("zh") else "en-US",
+            "time_range": "day"  # 限制当天结果
+        }
+        response = requests.get(searxng_url, params=params)
         response.raise_for_status()
         search_results = response.json()
  
@@ -44,7 +46,6 @@ def search_web_ref(query:str, debug=False):
             print("JSON Response:")
             pprint(search_results)
         pedding_urls = []
-
         conv_links = []
 
         if search_results.get('results'):
@@ -57,15 +58,15 @@ def search_web_ref(query:str, debug=False):
                 if url:
                     url_parsed = urlparse(url)
                     domain = url_parsed.netloc
-                    icon_url =  url_parsed.scheme + '://' + url_parsed.netloc + '/favicon.ico'
+                    icon_url = url_parsed.scheme + '://' + url_parsed.netloc + '/favicon.ico'
                     site_name = tldextract.extract(url).domain
  
                 conv_links.append({
-                    'site_name':site_name,
-                    'icon_url':icon_url,
-                    'title':name,
-                    'url':url,
-                    'snippet':snippet
+                    'site_name': site_name,
+                    'icon_url': icon_url,
+                    'title': name,
+                    'url': url,
+                    'snippet': snippet
                 })
 
             results = []
@@ -73,143 +74,113 @@ def search_web_ref(query:str, debug=False):
 
             executor = ThreadPoolExecutor(max_workers=10) 
             for url in pedding_urls:
-                futures.append(executor.submit(extract_url_content,url))
+                futures.append(executor.submit(extract_url_content, url))
             try:
                 for future in futures:
                     res = future.result(timeout=5)
                     results.append(res)
             except concurrent.futures.TimeoutError:
                 print("任务执行超时")
-                executor.shutdown(wait=False,cancel_futures=True)
+                executor.shutdown(wait=False, cancel_futures=True)
 
             for content in results:
                 if content and content.get('content'):
-                    
                     item_dict = {
-                        "url":content.get('url'),
+                        "url": content.get('url'),
                         "content": content.get('content'),
-                        "length":len(content.get('content'))
+                        "length": len(content.get('content'))
                     }
                     content_list.append(item_dict)
                 if debug:
                     print("URL: {}".format(url))
                     print("=================")
  
-        return  content_list
+        return content_list
     except Exception as ex:
         raise ex
 
 
-def gen_prompt(question,content_list, lang="zh-CN", context_length_limit=11000,debug=False):
-    
+def gen_prompt(question, content_list, lang="zh-CN", context_length_limit=11000, debug=False):
     limit_len = (context_length_limit - 2000)
     if len(question) > limit_len:
         question = question[0:limit_len]
     
-    ref_content = [ item.get("content") for item in content_list]
+    ref_content = [item.get("content") for item in content_list]
     
-    answer_language = ' Simplified Chinese '
+    answer_language = 'Simplified Chinese'
     if lang == "zh-CN":
-        answer_language = ' Simplified Chinese '
+        answer_language = 'Simplified Chinese'
     if lang == "zh-TW":
-        answer_language = ' Traditional Chinese '
+        answer_language = 'Traditional Chinese'
     if lang == "en-US":
-        answer_language = ' English '
-
+        answer_language = 'Professional English'
 
     if len(ref_content) > 0:
-        
-        if False:
-            prompts = '''
-            您是一位由 nash_su 开发的大型语言人工智能助手。您将被提供一个用户问题，并需要撰写一个清晰、简���且准确的答案。提供了一组与问题相关的上下文，每个都以 [[citation:x]] 这样的编号开头，x 代表一个数字。请在适当的情况下在句子末尾引用上下文。答案必须正确、精确，并以专家的中立和职业语气撰写。请将答案限制在 2000 个标记内。不要提供与问题无关的信息，也不要重复。如果给出的上下文信息不足，请在相关主题后写上“信息缺失：”。请按照引用编号 [citation:x] 的格式在答案中对应部分引用上下文。如果一句话源自多个上下文，请列出所有相关的引用编号，例如 [citation:3][citation:5]，不要将引用集中在最后返回，而是在答案对应部分列出。除非是代码、特定的名称或引用编号，答案的语言应与问题相同。以下是上下文的内容集：
-            '''  + "\n\n" + "```" 
-            ref_index = 1
-
-            for ref_text in ref_content:
-                
-                prompts = prompts + "\n\n" + " [citation:{}]  ".format(str(ref_index)) +  ref_text
-                ref_index += 1
-
-            if len(prompts) >= limit_len:
-                prompts = prompts[0:limit_len]        
-            prompts = prompts + '''
-    ```
-    记住，不要一字不差的重复上下文内容。回答必须使用简体中文，如果回答很长，请尽量结构化、分段落总结。请按照引用编号 [citation:x] 的格式在答案中对应部分引用上下文。如果一句话源自多个上下文，请列出所有相关的引用编号，例如 [citation:3][citation:5]，不要将引用集中在最后返回，而是在答案对应部分列出。下面是用户问题：
-    ''' + question  
-        else:
-            prompts = '''
-            You are a large language AI assistant develop by nash_su. You are given a user question, and please write clean, concise and accurate answer to the question. You will be given a set of related contexts to the question, each starting with a reference number like [[citation:x]], where x is a number. Please use the context and cite the context at the end of each sentence if applicable.
-            Your answer must be correct, accurate and written by an expert using an unbiased and professional tone. Please limit to 1024 tokens. Do not give any information that is not related to the question, and do not repeat. Say "information is missing on" followed by the related topic, if the given context do not provide sufficient information.
-
-            Please cite the contexts with the reference numbers, in the format [citation:x]. If a sentence comes from multiple contexts, please list all applicable citations, like [citation:3][citation:5]. Other than code and specific names and citations, your answer must be written in the same language as the question.
-            Here are the set of contexts:
-            '''  + "\n\n" + "```" 
-            ref_index = 1
-
-            for ref_text in ref_content:
-                
-                prompts = prompts + "\n\n" + " [citation:{}]  ".format(str(ref_index)) +  ref_text
-                ref_index += 1
-
-            if len(prompts) >= limit_len:
-                prompts = prompts[0:limit_len]        
-            prompts = prompts + '''
-            ```
-            Above is the reference contexts. Remember, don't repeat the context word for word. Answer in ''' + answer_language + '''. If the response is lengthy, structure it in paragraphs and summarize where possible. Cite the context using the format [citation:x] where x is the reference number. If a sentence originates from multiple contexts, list all relevant citation numbers, like [citation:3][citation:5]. Don't cluster the citations at the end but include them in the answer where they correspond.
-            Remember, don't blindly repeat the contexts verbatim. And here is the user question:
-            ''' + question  
- 
-     
+        prompts = '''
+        You are a large language AI assistant developed by nash_su. You are given a user question, and please write a clean, concise and accurate answer to the question. You will be given a set of related contexts to the question, each starting with a reference number like [[citation:x]], where x is a number. Please use the context and cite the context at the end of each sentence if applicable.
+        Your answer must be correct, accurate and written by an expert using an unbiased and professional tone. Please limit to 1024 tokens. Do not give any information that is not related to the question, and do not repeat. Say "information is missing on" followed by the related topic, if the given context does not provide sufficient information.
+
+        Please cite the contexts with the reference numbers, in the format [citation:x]. If a sentence comes from multiple contexts, please list all applicable citations, like [citation:3][citation:5]. Other than code and specific names and citations, your answer must be written in the same language as the question.
+        Here are the set of contexts:
+        ''' + "\n\n" + "```" 
+        ref_index = 1
+
+        for ref_text in ref_content:
+            prompts = prompts + "\n\n" + " [citation:{}]  ".format(str(ref_index)) + ref_text
+            ref_index += 1
+
+        if len(prompts) >= limit_len:
+            prompts = prompts[0:limit_len]        
+        prompts = prompts + '''
+        ```
+        Above is the reference contexts. Remember, don't repeat the context word for word. Answer in ''' + answer_language + '''. If the response is lengthy, structure it in paragraphs and summarize where possible. Cite the context using the format [citation:x] where x is the reference number. If a sentence originates from multiple contexts, list all relevant citation numbers, like [citation:3][citation:5]. Don't cluster the citations at the end but include them in the answer where they correspond.
+        Remember, don't blindly repeat the contexts verbatim. And here is the user question:
+        ''' + question  
     else:
         prompts = question
 
     if debug:
         print(prompts)
-        print("总长度："+ str(len(prompts)))
+        print("总长度：" + str(len(prompts)))
     return prompts
 
 
-def defaultchat(message, model:str, stream=True, debug=False):
+def defaultchat(message, model: str, stream=True, debug=False):
     openai.base_url = os.environ.get('OPENAI_BASE_URL')
     openai.api_key = os.environ.get('OPENAI_API_KEY')
     total_content = ""
-    #print(message)
     for chunk in openai.chat.completions.create(
         model=model,
         messages=message,
         stream=True,
-        max_tokens=3072,temperature=0.2
+        max_tokens=3072, temperature=0.2
     ):
         stream_resp = chunk.dict()
-        #print(stream_resp)
         token = stream_resp["choices"][0]["delta"].get("content", "")
-        #print(token)
         if token:
             total_content += token
             yield token
 
 def ask_gpt(message, model_id, debug=False):
-    #print(message)
     total_token = ""
     for token in defaultchat(message, model_id):
         if token:
             total_token += token
             yield token
             
-def summary_gpt(message,  model:str, debug=False):
-    #message = '\n'.join([msg.content for msg in message])
+def summary_gpt(message, model: str, debug=False):
     msgs = []
-    msgs.append({"role": "system", "content": '作为一位专业的问题审核专家，你的任务是确保每一个提问都是清晰、具体并且没有模糊歧义的，不需要在根据额外的内容就可以理解你的提问。在审阅提问时，请遵循以下规则进行优化：替换模糊的代名词，确保所有的人称和名词都有明确的指代，不允许出现"你我他这那"等这种类似的代名词;如果提问中包含泛指的名词，请根据上下文明确的定语，补充具体的细节以提供完整的信息;最后，只允许输出经过你精确优化的问题，不要有任何多余的文字。举例说明，1-当提问者问：他在做什么？,你根据上下文你可以得知他是"小明",那么你优化问题后输出"小明在干什么？"2-当提问者问：他们乐队都有谁？,你根据上下文可以得知乐队是"小强乐队",那么你优化问题后输出"小强乐队都有谁？"'})
-    msgs.append({"role": "user", "content":str(message)})
+    msgs.append({"role": "system", "content": '作为一位专业的问题审核专家，你的任务是确保每一个提问都是清晰、具体并且没有模糊歧义的，不需要在根据额外的内容就可以理解你的提问。在审阅提问时，请遵循以下规则进行优化：替换模糊的代名词，确保所有的人称和名词都有明确的指代，不允许出现"你我他这那"等这种类似的代名词;如果提问中包含泛指的名词，请根据上下文明确的定语，补充具体的细节以提供完整的信息;最后，只允许输出经过你精确优化的问题，不要有任何多余的文字。'})
+    msgs.append({"role": "user", "content": str(message)})
     json_data = {
-            "model":model,
-            "messages":msgs,
-            "temperature":0.8,
-            "max_tokens":2560,
-            "top_p":1,
-            "frequency_penalty":0,
-            "presence_penalty":0,
-            "stop":None
+        "model": model,
+        "messages": msgs,
+        "temperature": 0.8,
+        "max_tokens": 2560,
+        "top_p": 1,
+        "frequency_penalty": 0,
+        "presence_penalty": 0,
+        "stop": None
     }
     apiurl = os.environ.get('OPENAI_BASE_URL')
     pooltoken = os.environ.get('OPENAI_API_KEY')
@@ -217,12 +188,11 @@ def summary_gpt(message,  model:str, debug=False):
         'Content-Type': 'application/json',
         'Authorization': 'Bearer {}'.format(pooltoken),
     }
-    response = requests.post( apiurl + 'chat/completions', headers=headers, json=json_data )
+    response = requests.post(apiurl + 'chat/completions', headers=headers, json=json_data)
     res = json.loads(response.text)['choices'][0]['message']['content']
-    #print(res)
     return res
 
-def chat(prompt, model:str, stream=True, debug=False):
+def chat(prompt, model: str, stream=True, debug=False):
     openai.base_url = os.environ.get('OPENAI_BASE_URL')
     openai.api_key = os.environ.get('OPENAI_API_KEY')
     total_content = ""
@@ -233,26 +203,21 @@ def chat(prompt, model:str, stream=True, debug=False):
             "content": prompt
         }],
         stream=True,
-        max_tokens=3072,temperature=0.2
+        max_tokens=3072, temperature=0.2
     ):
         stream_resp = chunk.dict()
         token = stream_resp["choices"][0]["delta"].get("content", "")
         if token:
-            
             total_content += token
             yield token
     if debug:
         print(total_content)
- 
 
- 
-    
-def ask_internet(query:str, model:str, debug=False):
-  
-    content_list = search_web_ref(query,debug=debug)
+def ask_internet(query: str, model: str, debug=False):
+    content_list = search_web_ref(query, lang="en-US", debug=debug)  # 确保使用英文
     if debug:
         print(content_list)
-    prompt = gen_prompt(query,content_list,context_length_limit=6000,debug=debug)
+    prompt = gen_prompt(query, content_list, context_length_limit=6000, debug=debug)
     total_token = ""
     
     # 收集所有回答内容
@@ -261,16 +226,9 @@ def ask_internet(query:str, model:str, debug=False):
         if token:
             total_token += token
             response_content += token
-            #yield token #原始回答
             
     # 处理引用链接
     if content_list:
-        # 使用更灵活的引用识别方式
-        import re
-        
-    # 处理引用链接
-    if content_list:
-        # 创建引用到URL的映射
         citation_map = {f"citation:{i+1}": content_list[i].get('url') for i in range(len(content_list))}
         
         # 替换所有引用为链接
@@ -279,8 +237,6 @@ def ask_internet(query:str, model:str, debug=False):
             if url:
                 modified_content = modified_content.replace(citation, f"[{citation}]({url})")
         
-        # 输出修改后的内容
-        #yield "\n\n修改引用链接后的内容：\n"
         yield modified_content
     
     yield "\n\n"
@@ -291,8 +247,7 @@ def ask_internet(query:str, model:str, debug=False):
         yield "参考资料:\n"
         count = 1
         for url_content in content_list:
-           url = url_content.get('url')
-           yield "*[{}. {}]({})*".format(str(count),url,url )  
-           yield "\n"
-           count += 1
- 
+            url = url_content.get('url')
+            yield "*[{}. {}]({})*".format(str(count), url, url)  
+            yield "\n"
+            count += 1