Upload 5 files

Dockerfile

@@ -0,0 +1,10 @@
+FROM python:3.9.15
+WORKDIR /app
+COPY requirements.txt /app
+RUN pip3 install  -i https://mirrors.aliyun.com/pypi/simple/  --trusted-host mirrors.aliyun.com  -r requirements.txt --no-cache-dir
+COPY . /app
+RUN mkdir /.cache
+RUN chmod -R 777 /.cache
+EXPOSE 8000
+ENTRYPOINT ["python3"] 
+CMD ["server.py"]

README.md

@@ -1,10 +1,10 @@
----
-title: Net
-emoji: 🌖
-colorFrom: purple
-colorTo: red
-sdk: docker
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---
+title: net
+emoji: 👩‍🎨
+colorFrom: red
+colorTo: yellow
+sdk: docker
+pinned: false
+app_port: 8000
+---
+

free_ask_internet.py

@@ -0,0 +1,251 @@
+import json
+import os
+from pprint import pprint
+import requests
+import trafilatura
+from trafilatura import bare_extraction
+from concurrent.futures import ThreadPoolExecutor
+import concurrent
+import requests
+import openai
+import time 
+from datetime import datetime
+from urllib.parse import urlparse
+import tldextract
+import platform
+import urllib.parse
+
+ 
+def extract_url_content(url):
+    downloaded = trafilatura.fetch_url(url)
+    content =  trafilatura.extract(downloaded)
+    
+    return {"url":url, "content":content}
+
+
+ 
+
+def search_web_ref(query:str, debug=False):
+ 
+    content_list = []
+
+    try:
+
+        safe_string = urllib.parse.quote_plus(":all !general " + query)
+
+        searxng_url = os.environ.get('SEARXNG_URL')
+        response = requests.get(searxng_url + '?q=' + safe_string + '&format=json')
+        response.raise_for_status()
+        search_results = response.json()
+ 
+#        if debug:
+#            print("JSON Response:")
+#            pprint(search_results)
+        pedding_urls = []
+
+        conv_links = []
+
+        if search_results.get('results'):
+            for item in search_results.get('results')[0:9]:
+                name = item.get('title')
+                snippet = item.get('content')
+                url = item.get('url')
+                pedding_urls.append(url)
+
+                if url:
+                    url_parsed = urlparse(url)
+                    domain = url_parsed.netloc
+                    icon_url =  url_parsed.scheme + '://' + url_parsed.netloc + '/favicon.ico'
+                    site_name = tldextract.extract(url).domain
+ 
+                conv_links.append({
+                    'site_name':site_name,
+                    'icon_url':icon_url,
+                    'title':name,
+                    'url':url,
+                    'snippet':snippet
+                })
+
+            results = []
+            futures = []
+
+            executor = ThreadPoolExecutor(max_workers=10) 
+            for url in pedding_urls:
+                futures.append(executor.submit(extract_url_content,url))
+            try:
+                for future in futures:
+                    res = future.result(timeout=5)
+                    results.append(res)
+            except concurrent.futures.TimeoutError:
+                print("任务执行超时")
+                executor.shutdown(wait=False,cancel_futures=True)
+
+            for content in results:
+                if content and content.get('content'):
+                    
+                    item_dict = {
+                        "url":content.get('url'),
+                        "content": content.get('content'),
+                        "length":len(content.get('content'))
+                    }
+                    content_list.append(item_dict)
+#                if debug:
+#                    print("URL: {}".format(url))
+#                    print("=================")
+ 
+        return  content_list
+    except Exception as ex:
+        raise ex
+
+
+def gen_prompt(question,content_list, lang="zh-CN", context_length_limit=11000,debug=False):
+    
+    limit_len = (context_length_limit - 2000)
+    if len(question) > limit_len:
+        question = question[0:limit_len]
+    
+    ref_content = [ item.get("content") for item in content_list]
+    
+    answer_language = ' Simplified Chinese '
+    if lang == "zh-CN":
+        answer_language = ' Simplified Chinese '
+    if lang == "zh-TW":
+        answer_language = ' Traditional Chinese '
+    if lang == "en-US":
+        answer_language = ' English '
+
+
+    if len(ref_content) > 0:
+        prompts = '''
+        You are a large language AI assistant develop by nash_su. You are given a user question, and please write clean, concise and accurate answer to the question. You will be given a set of related contexts to the question. Please use the context to provide accurate information.
+        Your answer must be correct, accurate and written by an expert using an unbiased and professional tone. Please limit to 1024 tokens. Do not give any information that is not related to the question, and do not repeat. Say "information is missing on" followed by the related topic, if the given context do not provide sufficient information.
+
+        Other than code and specific names, your answer must be written in the same language as the question.
+        Here are the set of contexts:
+        '''  + "\n\n" + "```" 
+        ref_index = 1
+
+        for ref_text in ref_content:
+            prompts = prompts + "\n\n" + " [Context {}]  ".format(str(ref_index)) +  ref_text
+            ref_index += 1
+
+        if len(prompts) >= limit_len:
+            prompts = prompts[0:limit_len]        
+        prompts = prompts + '''
+        ```
+        Above is the reference contexts. Remember, don't repeat the context word for word. Answer in ''' + answer_language + '''. If the response is lengthy, structure it in paragraphs and summarize where possible. Don't mention the context numbers in your response.
+        Remember, don't blindly repeat the contexts verbatim. And here is the user question:
+        ''' + question 
+
+    else:
+        prompts = question
+
+#    if debug:
+#        print(prompts)
+#        print("总长度："+ str(len(prompts)))
+    return prompts
+
+
+def defaultchat(message, model:str, stream=True, debug=False):
+    openai.base_url = os.environ.get('OPENAI_BASE_URL')
+    openai.api_key = os.environ.get('OPENAI_API_KEY')
+    total_content = ""
+    #print(message)
+    for chunk in openai.chat.completions.create(
+        model=model,
+        messages=message,
+        stream=True,
+        max_tokens=3072,temperature=0.2
+    ):
+        stream_resp = chunk.dict()
+        #print(stream_resp)
+        token = stream_resp["choices"][0]["delta"].get("content", "")
+        #print(token)
+        if token:
+            total_content += token
+            yield token
+
+def ask_gpt(message, model_id, debug=False):
+    #print(message)
+    total_token = ""
+    for token in defaultchat(message, model_id):
+        if token:
+            total_token += token
+            yield token
+            
+def summary_gpt(message,  model:str, debug=False):
+    #message = '\n'.join([msg.content for msg in message])
+    msgs = []
+    msgs.append({"role": "system", "content": '作为一位专业的问题审核专家，你的任务是确保每一个提问都是清晰、具体并且没有模糊歧义的，不需要在根据额外的内容就可以理解你的提问。在审阅提问时，请遵循以下规则进行优化：替换模糊的代名词，确保所有的人称和名词都有明确的指代，不允许出现"你我他这那"等这种类似的代名词;如果提问中包含泛指的名词，请根据上下文明确的定语，补充具体的细节以提供完整的信息;最后，只允许输出经过你精确优化的问题，不要有任何多余的文字。举例说明，1-当提问者问：他在做什么？,你根据上下文你可以得知他是"小明",那么你优化问题后输出"小明在干什么？"2-当提问者问：他们乐队都有谁？,你根据上下文可以得知乐队是"小强乐队",那么你优化问题后输出"小强乐队都有谁？"'})
+    msgs.append({"role": "user", "content":str(message)})
+    json_data = {
+            "model":model,
+            "messages":msgs,
+            "temperature":0.8,
+            "max_tokens":2560,
+            "top_p":1,
+            "frequency_penalty":0,
+            "presence_penalty":0,
+            "stop":None
+    }
+    apiurl = os.environ.get('OPENAI_BASE_URL')
+    pooltoken = os.environ.get('OPENAI_API_KEY')
+    headers = {
+        'Content-Type': 'application/json',
+        'Authorization': 'Bearer {}'.format(pooltoken),
+    }
+    response = requests.post( apiurl + 'chat/completions', headers=headers, json=json_data )
+    res = json.loads(response.text)['choices'][0]['message']['content']
+    #print(res)
+    return res
+
+def chat(prompt, model:str, stream=True, debug=False):
+    openai.base_url = os.environ.get('OPENAI_BASE_URL')
+    openai.api_key = os.environ.get('OPENAI_API_KEY')
+    total_content = ""
+    for chunk in openai.chat.completions.create(
+        model=model,
+        messages=[{
+            "role": "user",
+            "content": prompt
+        }],
+        stream=True,
+        max_tokens=3072,temperature=0.2
+    ):
+        stream_resp = chunk.dict()
+        token = stream_resp["choices"][0]["delta"].get("content", "")
+        if token:
+            
+            total_content += token
+            yield token
+#    if debug:
+#        print(total_content)
+ 
+
+ 
+    
+def ask_internet(query:str, model:str, debug=False):
+  
+    content_list = search_web_ref(query,debug=debug)
+#    if debug:
+#        print(content_list)
+    prompt = gen_prompt(query,content_list,context_length_limit=6000,debug=debug)
+    total_token =  ""
+ 
+    for token in chat(prompt=prompt, model=model):
+        if token:
+            total_token += token
+            yield token
+
+    yield "\n\n"
+    # 是否返回参考资料
+    if True:
+        yield "---"
+        yield "\nSearxng"
+        yield "参考资料:\n"
+        count = 1
+        for url_content in content_list:
+           url = url_content.get('url')
+           yield "*[{}. {}]({})*".format(str(count),url,url )  
+           yield "\n"
+           count += 1

requirements.txt

@@ -0,0 +1,39 @@
+annotated-types==0.6.0
+anyio==4.3.0
+certifi==2024.2.2
+charset-normalizer==3.3.2
+click==8.1.7
+courlan==1.0.0
+dateparser==1.2.0
+distro==1.9.0
+exceptiongroup==1.2.0
+fastapi==0.110.1
+filelock==3.13.3
+h11==0.14.0
+htmldate==1.8.0
+httpcore==1.0.5
+httpx==0.27.0
+idna==3.6
+jusText==3.0.0
+langcodes==3.3.0
+lxml==5.1.1
+openai==1.16.2
+pydantic==2.6.4
+pydantic_core==2.16.3
+python-dateutil==2.9.0.post0
+pytz==2024.1
+regex==2023.12.25
+requests==2.31.0
+requests-file==2.0.0
+six==1.16.0
+sniffio==1.3.1
+sse-starlette==2.0.0
+starlette==0.37.2
+tld==0.13
+tldextract==5.1.2
+tqdm==4.66.2
+trafilatura==1.8.1
+typing_extensions==4.10.0
+tzlocal==5.2
+urllib3==2.2.1
+uvicorn==0.29.0

server.py

@@ -0,0 +1,312 @@
+# -*- coding: utf-8 -*-
+
+import time
+import uvicorn
+import sys
+import getopt
+import json
+import os 
+from pprint import pprint
+import requests
+import trafilatura
+from trafilatura import bare_extraction
+from concurrent.futures import ThreadPoolExecutor
+import concurrent
+import requests
+import openai
+import time 
+from datetime import datetime
+from urllib.parse import urlparse
+import platform
+import urllib.parse
+import free_ask_internet
+from pydantic import BaseModel, Field
+from fastapi import FastAPI, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
+from contextlib import asynccontextmanager
+from typing import Any, Dict, List, Literal, Optional, Union
+from sse_starlette.sse import ServerSentEvent, EventSourceResponse
+from fastapi.responses import StreamingResponse
+
+app = FastAPI()
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+class ModelCard(BaseModel):
+    id: str
+    object: str = "model"
+    created: int = Field(default_factory=lambda: int(time.time()))
+    owned_by: str = "owner"
+    root: Optional[str] = None
+    parent: Optional[str] = None
+    permission: Optional[list] = None
+
+
+class ModelList(BaseModel):
+    object: str = "list"
+    data: List[ModelCard] = []
+
+
+class ChatMessage(BaseModel):
+    role: Literal["user", "assistant", "system"]
+    content: str
+
+
+class DeltaMessage(BaseModel):
+    role: Optional[Literal["user", "assistant", "system"]] = None
+    content: Optional[str] = None
+
+class QueryRequest(BaseModel):
+    query:str
+    model: str
+    ask_type:  Literal["search", "llm"]
+    llm_auth_token: Optional[str] = os.environ.get('OPENAI_API_KEY')
+    llm_base_url: Optional[str] = os.environ.get('OPENAI_BASE_URL')
+    using_custom_llm:Optional[bool] = False
+    lang:Optional[str] = "zh-CN"
+
+class ChatCompletionRequest(BaseModel):
+    model: str
+    messages: List[ChatMessage]
+    temperature: Optional[float] = None
+    top_p: Optional[float] = None
+    max_length: Optional[int] = None
+    stream: Optional[bool] = False
+
+
+class ChatCompletionResponseChoice(BaseModel):
+    index: int
+    message: ChatMessage
+    finish_reason: Literal["stop", "length"]
+
+
+class ChatCompletionResponseStreamChoice(BaseModel):
+    index: int
+    delta: DeltaMessage
+    finish_reason: Optional[Literal["stop", "length"]]
+
+
+class ChatCompletionResponse(BaseModel):
+    model: str
+    object: Literal["chat.completion", "chat.completion.chunk"]
+    choices: List[Union[ChatCompletionResponseChoice,
+                        ChatCompletionResponseStreamChoice]]
+    created: Optional[int] = Field(default_factory=lambda: int(time.time()))
+
+class SearchItem(BaseModel):
+    url: str
+    icon_url: str
+    site_name:str
+    snippet:str
+    title:str 
+
+class SearchItemList(BaseModel):
+    search_items: List[SearchItem] = []
+ 
+class SearchResp(BaseModel):
+    code:int
+    msg:str
+    data: List[SearchItem] = []
+
+
+@app.get("/deem/v1/models", response_model=ModelList)
+async def list_models():
+    global model_args
+    model_card = ModelCard(id="gpt-3.5-turbo")
+    return ModelList(data=[model_card])
+
+
+@app.post("/deem/v1/chat/completions", response_model=ChatCompletionResponse)
+async def create_chat_completion(request: ChatCompletionRequest):
+    global model, tokenizer
+    if request.messages[-1].role != "user":
+        raise HTTPException(status_code=400, detail="Invalid request")
+    query = request.messages[-1].content
+    if query[0] == '@':
+        #print("当前问题:gpt ---> {}".format(query))
+        generate = askgpt(request.messages,"",request.model)
+    else:
+        query = query[1:]
+        if len(request.messages) > 2:
+            message = '\n'.join([msg.content for msg in request.messages])
+            query = free_ask_internet.summary_gpt(message + "\n请根据以上的内容总结，" + query +" 这个问题是要问什么？不要有模糊的代名词比如他/她之类的，不允许缺失上下文语境，需要明确提问的主题;最后只允许输出总结并完善语境的问题，不要有任何多余的文字！", request.model)
+        else:
+            pass
+        #print("当前问题:net ---> {}".format(query))
+        generate = predict(query, "", request.model)
+    return EventSourceResponse(generate, media_type="text/event-stream")
+
+ 
+def askgpt(query, history, model_id):
+    choice_data = ChatCompletionResponseStreamChoice(
+        index=0,
+        delta=DeltaMessage(role="assistant"),
+        finish_reason=None
+    )
+    chunk = ChatCompletionResponse(model=model_id, choices=[
+                                choice_data], object="chat.completion.chunk")
+    yield "{}".format(chunk.json(exclude_unset=True))
+    new_response = ""
+    current_length = 0
+    for token in free_ask_internet.ask_gpt(query, model_id,):
+
+        new_response += token
+        if len(new_response) == current_length:
+            continue
+
+        new_text = new_response[current_length:]
+        current_length = len(new_response)
+
+        choice_data = ChatCompletionResponseStreamChoice(
+            index=0,
+            delta=DeltaMessage(content=new_text,role="assistant"),
+            finish_reason=None
+        )
+        chunk = ChatCompletionResponse(model=model_id, choices=[
+                                       choice_data], object="chat.completion.chunk")
+        yield "{}".format(chunk.json(exclude_unset=True))
+
+    choice_data = ChatCompletionResponseStreamChoice(
+        index=0,
+        delta=DeltaMessage(),
+        finish_reason="stop"
+    )
+    chunk = ChatCompletionResponse(model=model_id, choices=[
+                                   choice_data], object="chat.completion.chunk")
+    yield "{}".format(chunk.json(exclude_unset=True))
+    yield '[DONE]'
+
+
+
+def predict(query: str, history: None, model_id: str):
+    choice_data = ChatCompletionResponseStreamChoice(
+        index=0,
+        delta=DeltaMessage(role="assistant"),
+        finish_reason=None
+    )
+    chunk = ChatCompletionResponse(model=model_id, choices=[
+                                choice_data], object="chat.completion.chunk")
+    yield "{}".format(chunk.json(exclude_unset=True))
+    new_response = ""
+    current_length = 0
+    for token in free_ask_internet.ask_internet(query=query, model=model_id):
+    
+        new_response += token
+        if len(new_response) == current_length:
+            continue
+
+        new_text = new_response[current_length:]
+        current_length = len(new_response)
+
+        choice_data = ChatCompletionResponseStreamChoice(
+            index=0,
+            delta=DeltaMessage(content=new_text,role="assistant"),
+            finish_reason=None
+        )
+        chunk = ChatCompletionResponse(model=model_id, choices=[
+                                       choice_data], object="chat.completion.chunk")
+        yield "{}".format(chunk.json(exclude_unset=True))
+
+    choice_data = ChatCompletionResponseStreamChoice(
+        index=0,
+        delta=DeltaMessage(),
+        finish_reason="stop"
+    )
+    chunk = ChatCompletionResponse(model=model_id, choices=[
+                                   choice_data], object="chat.completion.chunk")
+    yield "{}".format(chunk.json(exclude_unset=True))
+    yield '[DONE]'
+ 
+
+
+@app.post("/api/search/get_search_refs", response_model=SearchResp)
+async def get_search_refs(request: QueryRequest):
+
+    global search_results
+    search_results = []
+    search_item_list = []
+    if request.ask_type == "search":
+        search_links,search_results = free_ask_internet.search_web_ref(request.query)
+        for search_item in search_links:
+            snippet = search_item.get("snippet")
+            url = search_item.get("url")
+            icon_url = search_item.get("icon_url")
+            site_name = search_item.get("site_name")
+            title = search_item.get("title")
+    
+
+            si = SearchItem(snippet=snippet,url=url,icon_url=icon_url,site_name=site_name,title=title)
+
+            search_item_list.append(si)
+
+    resp = SearchResp(code=0,msg="success",data=search_item_list)
+   
+    return  resp
+
+def generator(prompt:str, model:str, llm_auth_token:str,llm_base_url:str, using_custom_llm=False,is_failed=False):
+    if is_failed:
+        yield "搜索失败，没有返回结果"
+    else:
+        total_token = ""
+        for token in  free_ask_internet.chat(prompt=prompt,model=model,llm_auth_token=llm_auth_token,llm_base_url=llm_base_url,using_custom_llm=using_custom_llm,stream=True):
+            total_token += token
+            yield token
+ 
+@app.post("/api/search/stream/{search_uuid}")
+async def stream(search_uuid:str,request: QueryRequest):
+    global search_results
+
+    if request.ask_type == "llm":
+            
+        answer_language = ' Simplified Chinese '
+        if request.lang == "zh-CN":
+            answer_language = ' Simplified Chinese '
+        if request.lang == "zh-TW":
+            answer_language = ' Traditional Chinese '
+        if request.lang == "en-US":
+            answer_language = ' English '
+        prompt = ' You are a large language AI assistant develop by nash_su. Answer user question in ' + answer_language + '. And here is the user question: ' + request.query
+        generate = generator(prompt,model=request.model,llm_auth_token=request.llm_auth_token, llm_base_url=request.llm_base_url, using_custom_llm=request.using_custom_llm)
+    else:
+        prompt = None
+        limit_count = 10
+
+        while limit_count > 0:
+            try:
+                if len(search_results) > 0:
+                    prompt = free_ask_internet.gen_prompt(request.query,search_results,lang=request.lang,context_length_limit=8000)
+                    break
+                else:
+                    limit_count -= 1
+                    time.sleep(1)
+            except Exception as err:
+                limit_count -= 1
+                time.sleep(1)
+        total_token =  ""
+        if prompt:   
+            generate = generator(prompt,model=request.model,llm_auth_token=request.llm_auth_token, llm_base_url=request.llm_base_url, using_custom_llm=request.using_custom_llm)
+        else:
+            generate = generator(prompt,model=request.model,llm_auth_token=request.llm_auth_token,llm_base_url=request.llm_base_url, using_custom_llm=request.using_custom_llm,is_failed=True)
+
+    # return EventSourceResponse(generate, media_type="text/event-stream")
+    return StreamingResponse(generate, media_type="text/event-stream")
+
+def main():
+
+    port = 8000
+
+    search_results = []
+ 
+   
+    uvicorn.run(app, host='0.0.0.0', port=port, workers=1)
+
+
+if __name__ == "__main__":
+    main()