Duplicate from DiamondYin/english-learning-copilot

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,6 @@
+.venv
+__pycache__
+.chroma
+initialize.sh
+conversations.log
+custom_gpt_voice assistant_demo.mp4

README.md

@@ -0,0 +1,14 @@
+---
+title: Custom Gpt Voice Assistant
+emoji: 🌖
+colorFrom: indigo
+colorTo: gray
+sdk: gradio
+sdk_version: 3.27.0
+app_file: app.py
+pinned: false
+license: other
+duplicated_from: DiamondYin/english-learning-copilot
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,130 @@
+import os
+import nltk 
+import openai
+import time
+import gradio as gr
+from threading import Thread #线程  用于定时器
+
+from assets.char_poses_base64 import ( #角色动作
+    CHAR_IDLE_HTML, CHAR_THINKING_HTML, CHAR_TALKING_HTML)
+
+from app_utils import (
+    get_chat_history, initialize_knowledge_base, 
+    text_to_speech_gen, logging, buzz_user)
+
+global FUNC_CALL #全局变量 用于判断角色动作
+FUNC_CALL = 0
+
+global BUZZ_TIMEOUT #全局变量 用于定时器
+BUZZ_TIMEOUT = 60
+
+GENERAL_RSPONSE_TRIGGERS = ["I don't understand the question.", "I don't know", "Hello, my name is", "mentioned in the context provided"]
+MESSAGES = [{"role": "system", "content": "You are a helpful assistant.You accompany me to practice English and engage in scene dialogue. As a hotel attendant, I am checking in. You introduce the hotel to me and recommend hotel services to me. After receiving my needs, arrange for the service personnel to work. Please remember, my English is not very good. Please have a conversation with me in simple English. After you ask questions in English, please give me some English prompts so that I know how to answer you. Let's start the conversation. You first say hello to me."}]
+
+LOGGER = logging.getLogger('voice_agent') #日志
+AUDIO_HTML = ''
+
+# Uncomment If this is your first Run: 
+nltk.download('averaged_perceptron_tagger')  #下载语料库
+conv_model, voice_model = initialize_knowledge_base()  #初始化知识库
+
+
+def idle_timer():
+    global BUZZ_TIMEOUT
+
+    while True:
+        time.sleep(BUZZ_TIMEOUT)
+        buzz_user()
+
+        if BUZZ_TIMEOUT == 80:
+            time.sleep(BUZZ_TIMEOUT)
+            BUZZ_TIMEOUT = 60
+
+
+def update_img():
+    global FUNC_CALL
+    FUNC_CALL += 1
+
+    if FUNC_CALL % 2== 0:
+        return CHAR_TALKING_HTML
+    else:
+        return CHAR_THINKING_HTML
+
+
+def get_response(history, audio_input):
+
+    query_type = 'text'
+    question =history[-1][0]
+
+    global BUZZ_TIMEOUT
+    BUZZ_TIMEOUT = 80
+
+    if not question:
+        if audio_input:
+            query_type = 'audio'
+            os.rename(audio_input, audio_input + '.wav')
+            audio_file = open(audio_input + '.wav', "rb")
+            transcript = openai.Audio.transcribe("whisper-1", audio_file)
+            question = transcript['text']
+        else:
+            return None, None
+
+    LOGGER.info("\nquery_type: %s", query_type)
+    LOGGER.info("query_text: %s", question)
+    print('\nquery_type:', query_type)
+    print('\nquery_text:', question)
+
+    if question.lower().strip() == 'hi':
+        question = 'hello'
+    
+    answer = conv_model.run(question)
+    LOGGER.info("\ndocument_response: %s", answer)
+    print('\ndocument_response:', answer)
+
+    for trigger in GENERAL_RSPONSE_TRIGGERS:
+        if trigger in answer:    
+            MESSAGES.append({"role": "user", "content": question})
+            chat = openai.ChatCompletion.create(
+                    model="gpt-3.5-turbo", 
+                    messages=MESSAGES,
+                    temperature=0.7,
+                    n=128,
+                    stop="\n"
+                    )
+            answer = chat.choices[0].message.content
+            MESSAGES.append({"role": "assistant", "content": answer})
+            LOGGER.info("general_response: %s", answer)
+            print('\ngeneral_response:', answer)
+
+    AUDIO_HTML = text_to_speech_gen(answer)
+    history[-1][1] = answer
+
+    return history, AUDIO_HTML
+
+# buzz_usr_proc = Thread(target=idle_timer)
+
+with gr.Blocks(css = """#col_image{width:800px; height:800px; margin-left: auto; margin-right: auto;}""") as demo: 
+    with gr.Row(scale=0.7):
+        output_html = gr.HTML(label="Felix's Voice", value=AUDIO_HTML)
+        output_html.visible = False
+        image1= gr.Image("assets/NPCtest1.png").style(height=700) #elem_id = "col_image"
+        #assistant_character = gr.HTML(label=None, value=CHAR_IDLE_HTML, show_label=False)
+        with gr.Column(scale=0.3):
+            chatbot = gr.Chatbot(label='Send a text or a voice input').style(height=285)
+            with gr.Column():
+                msg = gr.Textbox(placeholder='Write a chat & press Enter.', show_label=False).style(container=False)
+                with gr.Column(scale=0.5):
+                    audio_input = gr.Audio(source="microphone", type='filepath', show_label=False).style(container=False)
+                    button = gr.Button(value="Send")
+
+    msg.submit(get_chat_history, [msg, chatbot], [msg, chatbot]
+                ).then(get_response, [chatbot, audio_input], [chatbot, output_html]
+                )
+
+    button.click(get_chat_history, [msg, chatbot], [msg, chatbot]
+                ).then(get_response, [chatbot, audio_input], [chatbot, output_html]
+                )
+    
+    # buzz_usr_proc.start()
+    
+demo.launch(debug=False, favicon_path='assets/favicon.png', show_api=False, share=False)

app_utils.py

@@ -0,0 +1,90 @@
+import os
+import whisper
+from io import BytesIO #   BytesIO is a class in the io module that implements an in-memory file-like object.
+import base64 
+import boto3 # AWS Polly
+from pydub import AudioSegment #    AudioSegment is a class in the pydub module that can be used to manipulate audio files.
+from pydub.playback import play  #  play is a function in the pydub.playback module that can be used to play audio files.
+import logging
+
+from langchain import OpenAI
+from langchain.chains import RetrievalQA #  RetrievalQA is a class in the langchain.chains module that can be used to build a retrieval-based question answering system.
+from langchain.vectorstores import Chroma # Chroma is a class in the langchain.vectorstores module that can be used to store vectors.
+from langchain.document_loaders import DirectoryLoader #       
+from langchain.embeddings.openai import OpenAIEmbeddings # OpenAIGPTEmbeddings
+from langchain.text_splitter import CharacterTextSplitter #     CharacterTextSplitter is a class in the langchain.text_splitter module that can be used to split text into chunks.
+
+
+OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')
+AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID')
+AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY')
+AWS_REGION_NAME = 'ap-south-1'
+
+
+logging.basicConfig(level="INFO",
+                    filename='conversations.log',
+                    filemode='a',
+                    format='%(asctime)s %(message)s',
+                    datefmt='%H:%M:%S')
+
+
+def buzz_user():
+    input_prompt = AudioSegment.from_mp3('assets/timeout_audio.mp3')
+    play(input_prompt)
+    
+
+def initialize_knowledge_base():
+    
+    loader = DirectoryLoader('profiles', glob='**/*.txt') #文件夹加载器 profiles文件夹下的所有txt文件
+    docs = loader.load()
+
+    char_text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
+    doc_texts = char_text_splitter.split_documents(docs)
+
+    openAI_embeddings = OpenAIEmbeddings()
+    vStore = Chroma.from_documents(doc_texts, openAI_embeddings)
+
+    conv_model = RetrievalQA.from_chain_type(
+        llm=OpenAI(), 
+        chain_type="stuff", 
+        retriever=vStore.as_retriever(
+            search_kwargs={"k": 1}
+            )
+        )
+    voice_model = whisper.load_model("tiny") #加载模型  tiny模型    tiny模型是一个小型的语音识别模型，它的大小只有 50MB 左右，但是它的准确率却非常高，可以达到 95% 以上。
+
+    return conv_model, voice_model
+
+
+def text_to_speech_gen(answer):  #文字转语音
+
+    polly = boto3.client('polly',
+                aws_access_key_id=AWS_ACCESS_KEY_ID,
+                aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
+                region_name=AWS_REGION_NAME)
+
+    response = polly.synthesize_speech(
+        Text=answer, 
+        VoiceId='Matthew', 
+        OutputFormat='mp3', 
+        Engine = "neural")
+    
+    audio_stream = response['AudioStream'].read()
+    audio_html = audio_to_html(audio_stream)
+
+    return audio_html
+    
+
+def audio_to_html(audio_bytes):       #音频转html
+    audio_io = BytesIO(audio_bytes)
+    audio_io.seek(0)
+    audio_base64 = base64.b64encode(audio_io.read()).decode("utf-8")
+    audio_html = f'<audio src="data:audio/mpeg;base64,{audio_base64}" controls autoplay></audio>'
+
+    return audio_html
+
+
+def get_chat_history(user_message, history):
+    return "", history + [[user_message, None]]
+
+

profiles/john.txt

@@ -0,0 +1,19 @@
+Changsha Nicolo Hotel
+Located at the heart of the bustling city center of Changsha IFS National Financial Center and on the top of the 452 meter high Changsha International Financial Center, the hotel is a landmark with fashionable entertainment style and high-end business office. The hotel has 241 fine rooms and suites from the 86th to 92nd floors, all equipped with complete and advanced modern facilities, fashionable catering and social space and elegant conference and activity space to bring you unparalleled luxury experience, Including the Yinyuan Auditorium, a 788 square meter luxury banquet hall, Yuelang, Bar 93, Xinchef, spa, and fitness center. Standing in a hotel like a towering cloud, one can not only overlook the magnificent panoramic view of the city, but also appreciate the infinite scenery of the Xiangjiang River.
+1. Discovery Suite, Room size: 90 m ², 1 king bed, River view, 3600RMB/1night
+2. Deluxe Twin Room, Room size: 45 m ², 2 single beds, River view, 2800RMB/1night
+3. Premier River View Room 2 Twin beds, Room size: 45 m ², 2 single beds, River view, 2400RMB/1night
+Environment:
+Wuyi Square is located in the center of the business circle of History of Changsha and the oldest area in Changsha. The city site of ancient Changsha is today's Wuyi Square and its surrounding areas. For thousands of years, Wuyi Square has always been the official seat of ancient Changsha.
+Shopping malls around Wuyi Square:
+Pinghetang, Wangfujing Department Store, Spring Department Store, Lehecheng, Yinxiang City, Yinxiang Hui, 7mall.
+May Day Square Food:
+Authentic Hunan cuisine: Chef Fei, a lamp, the era of cooking smoke, tea with pleasant color, Hutou Restaurant, Mo Mo Dim sum Restaurant, Wu Susheng Pastry.
+Taiping Street
+The fish bone shaped block of Taiping Street has remained unchanged for 200 years, with a total length of 375 meters and a width of no more than 7 meters. It is adjacent to Wuyi Avenue to the south and Jiefang West Road to the north, and the transportation is very convenient. It is the most fashionable old street and the most popular night market on the internet. With just over 300 meters of history and culture, the old street has gradually evolved into the street with the most popular food and check-in points in Changsha.
+Introduction to Taiping Street Night Market in Changsha (Food+Scenic Spots+Transportation Guide)
+Fire Palace
+The Fire Palace is located in Pozi Street in the center of Changsha. It used to be a temple dedicated to the god of fire. It was built in the fifth year of Wanli (1577 AD) of the Ming Dynasty, and has been rebuilt many times since then. Because there have been snack stalls and temple fairs throughout history, it has later developed into a famous snack center in Changsha and a famous brand of Changsha cuisine,
+Changsha Pozi Street Food Strategy (Recommended Store+Address+Opening Hours)
+IFS National Financial Center
+As the "tallest building in Hunan" building, very representative, the seventh floor of the sky sculpture garden is not to be missed, the sky sculpture garden in the two huge sculpture artworks KAWS, two dolls one gray and one blue, sitting back to back in the corner of the square, has become the preferred check-in place for many people

requirements.txt

@@ -0,0 +1,147 @@
+aiofiles==23.1.0
+aiohttp==3.8.4
+aiosignal==1.3.1
+altair==4.2.2
+anyio==3.6.2
+argilla==1.6.0
+async-timeout==4.0.2
+attrs==23.1.0
+backoff==2.2.1
+boto3==1.26.115
+botocore==1.29.115
+certifi==2022.12.7
+charset-normalizer==3.1.0
+chromadb==0.3.21
+click==8.1.3
+clickhouse-connect==0.5.20
+cmake==3.26.3
+commonmark==0.9.1
+contourpy==1.0.7
+cycler==0.11.0
+dataclasses-json==0.5.7
+Deprecated==1.2.13
+duckdb==0.7.1
+entrypoints==0.4
+et-xmlfile==1.1.0
+fastapi==0.95.1
+ffmpeg-python==0.2.0
+ffmpy==0.3.0
+filelock==3.12.0
+fonttools==4.39.3
+frozenlist==1.3.3
+fsspec==2023.4.0
+future==0.18.3
+gradio==3.27.0
+gradio_client==0.1.3
+greenlet==2.0.2
+h11==0.14.0
+hnswlib==0.7.0
+httpcore==0.16.3
+httptools==0.5.0
+httpx==0.23.3
+huggingface-hub==0.13.4
+idna==3.4
+Jinja2==3.1.2
+jmespath==1.0.1
+joblib==1.2.0
+jsonschema==4.17.3
+kiwisolver==1.4.4
+langchain==0.0.144
+linkify-it-py==2.0.0
+lit==16.0.1
+llvmlite==0.39.1
+lxml==4.9.2
+lz4==4.3.2
+Markdown==3.4.3
+markdown-it-py==2.2.0
+MarkupSafe==2.1.2
+marshmallow==3.19.0
+marshmallow-enum==1.5.1
+matplotlib==3.7.1
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+monotonic==1.6
+more-itertools==9.1.0
+mpmath==1.3.0
+msg-parser==1.2.0
+multidict==6.0.4
+mypy-extensions==1.0.0
+networkx==3.1
+nltk==3.8.1
+numba==0.56.4
+numexpr==2.8.4
+numpy==1.23.5
+nvidia-cublas-cu11==11.10.3.66
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+nvidia-cuda-runtime-cu11==11.7.99
+nvidia-cudnn-cu11==8.5.0.96
+nvidia-cufft-cu11==10.9.0.58
+nvidia-curand-cu11==10.2.10.91
+nvidia-cusolver-cu11==11.4.0.1
+nvidia-cusparse-cu11==11.7.4.91
+nvidia-nccl-cu11==2.14.3
+nvidia-nvtx-cu11==11.7.91
+olefile==0.46
+openai==0.27.4
+openai-whisper @ git+https://github.com/openai/whisper.git@c09a7ae299c4c34c5839a76380ae407e7d785914
+openapi-schema-pydantic==1.2.4
+openpyxl==3.1.2
+orjson==3.8.10
+packaging==23.1
+pandas==1.5.3
+Pillow==9.5.0
+posthog==3.0.0
+pydantic==1.10.7
+pydub==0.25.1
+Pygments==2.15.1
+pypandoc==1.11
+pyparsing==3.0.9
+pyrsistent==0.19.3
+python-dateutil==2.8.2
+python-docx==0.8.11
+python-dotenv==1.0.0
+python-magic==0.4.27
+python-multipart==0.0.6
+python-pptx==0.6.21
+pytz==2023.3
+PyYAML==6.0
+regex==2023.3.23
+requests==2.28.2
+rfc3986==1.5.0
+rich==13.0.1
+s3transfer==0.6.0
+scikit-learn==1.2.2
+scipy==1.10.1
+semantic-version==2.10.0
+sentence-transformers==2.2.2
+sentencepiece==0.1.98
+six==1.16.0
+sniffio==1.3.0
+SQLAlchemy==1.4.47
+starlette==0.26.1
+sympy==1.11.1
+tenacity==8.2.2
+threadpoolctl==3.1.0
+tiktoken==0.3.1
+tokenizers==0.13.3
+toolz==0.12.0
+torch==2.0.0
+torchvision==0.15.1
+tqdm==4.65.0
+transformers==4.28.1
+triton==2.0.0
+typing-inspect==0.8.0
+typing_extensions==4.5.0
+tzdata==2023.3
+uc-micro-py==1.0.1
+unstructured==0.5.13
+urllib3==1.26.15
+uvicorn==0.21.1
+uvloop==0.17.0
+watchfiles==0.19.0
+websockets==11.0.2
+wrapt==1.14.1
+XlsxWriter==3.1.0
+yarl==1.8.2
+zstandard==0.21.0