feat: initial commit

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.json filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,5 @@
+.venv
+data
+.DS_Store
+__pycache__
+.env

app.py

@@ -1,64 +1,58 @@
+import os
 import gradio as gr
-from huggingface_hub import InferenceClient
-
-"""
-For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
-"""
-client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
-
-
-def respond(
-    message,
-    history: list[tuple[str, str]],
-    system_message,
-    max_tokens,
-    temperature,
-    top_p,
-):
-    messages = [{"role": "system", "content": system_message}]
+from llama_index.core import StorageContext, load_index_from_storage, Settings
+from llama_index.llms.azure_openai import AzureOpenAI
+from llama_index.embeddings.azure_openai import AzureOpenAIEmbedding
+from dotenv import load_dotenv
+
+load_dotenv(override=True)
+
+api_key = os.getenv("AZURE_OPENAI_API_KEY")
+api_version = "2024-05-01-preview"
+azure_endpoint = os.getenv("AZURE_OPENAI_ENDPOINT")
+
+llm = AzureOpenAI(
+    model="gpt-4o",
+    deployment_name="gpt-4o",
+    api_key=api_key,
+    azure_endpoint=azure_endpoint,
+    api_version=api_version,
+)
 
-    for val in history:
-        if val[0]:
-            messages.append({"role": "user", "content": val[0]})
-        if val[1]:
-            messages.append({"role": "assistant", "content": val[1]})
+# You need to deploy your own embedding model as well as your own chat completion model
+embed_model = AzureOpenAIEmbedding(
+    model="text-embedding-3-small",
+    deployment_name="text-embedding-3-small",
+    api_key=api_key,
+    azure_endpoint=azure_endpoint,
+    api_version=api_version,
+)
 
-    messages.append({"role": "user", "content": message})
+Settings.llm = llm
+Settings.embed_model = embed_model
 
-    response = ""
+# rebuild storage context
+storage_context = StorageContext.from_defaults(persist_dir="./index")
 
-    for message in client.chat_completion(
-        messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
-    ):
-        token = message.choices[0].delta.content
+# load index
+index = load_index_from_storage(storage_context)
 
-        response += token
-        yield response
+query_engine = index.as_query_engine(similarity_top_k=10)
 
+# Function to handle chat messages with history
+def echo(message, history):
+    context = "\n".join([f"User: {user_msg}\nBot: {bot_msg}" for user_msg, bot_msg in history])
+    full_context = f"{context}\nUser: {message}"
+    response = query_engine.query(full_context).response
+    history.append((message, response))
+    return response # history
 
-"""
-For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
-"""
 demo = gr.ChatInterface(
-    respond,
-    additional_inputs=[
-        gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
-        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
-        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
-        gr.Slider(
-            minimum=0.1,
-            maximum=1.0,
-            value=0.95,
-            step=0.05,
-            label="Top-p (nucleus sampling)",
-        ),
-    ],
+    fn=echo, 
+    examples=[
+        "光源氏はどのような人物ですか？",
+        "夕顔はどのような人物ですか？"
+    ], 
+    title="Llama Index Chatbot",
 )
-
-
-if __name__ == "__main__":
-    demo.launch()
+demo.launch()

index/default__vector_store.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7fe224b5643b9f72240d2512a9b9d8a2f667c604a76a515e43990bd6ac89881
+size 56982497

index/docstore.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff15643f2195ab3def2669052b4e7b2a1f94907658cab0dc0a6d27860439da0d
+size 11660400

index/graph_store.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e0a77744010862225c69da83c585f4f8a42fd551b044ce530dbb1eb6e16742c
+size 18

index/image__vector_store.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d17ed74c1649a438e518a8dc56a7772913dfe1ea7a7605bce069c63872431455
+size 72

index/index_store.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f99ab7691679df00c79231c7620b7b41e2b0eb374425547075a236beb35cd9d
+size 137839

requirements.txt

@@ -1 +1,7 @@
-huggingface_hub==0.25.2
+gradio
+requests
+bs4
+llama_index
+llama-index-embeddings-azure-openai
+llama-index-llms-azure-openai
+python-dotenv

src/01_dwn.ipynb

@@ -0,0 +1,107 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "ModuleNotFoundError",
+     "evalue": "No module named 'bs4'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[1], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mrequests\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mbs4\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m BeautifulSoup\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mos\u001b[39;00m\n",
+      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'bs4'"
+     ]
+    }
+   ],
+   "source": [
+    "import requests\n",
+    "from bs4 import BeautifulSoup\n",
+    "import os"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "url = \"https://genji.dl.itc.u-tokyo.ac.jp/data/info.json\"\n",
+    "\n",
+    "response = requests.get(url).json()\n",
+    "\n",
+    "selections = response[\"selections\"]\n",
+    "\n",
+    "for selection in selections:\n",
+    "\n",
+    "    members = selection[\"members\"]\n",
+    "\n",
+    "    for member in members:\n",
+    "\n",
+    "        aozora_urls = []\n",
+    "\n",
+    "        for metadata in member[\"metadata\"]:\n",
+    "\n",
+    "            if metadata[\"label\"] == \"aozora\":\n",
+    "\n",
+    "                aozora_urls = metadata[\"value\"].split(\", \")\n",
+    "\n",
+    "        for aozora_url in aozora_urls:\n",
+    "\n",
+    "            filename = aozora_url.split(\"/\")[-1].split(\".\")[0]\n",
+    "\n",
+    "            opath = f\"./data/text/{filename}.txt\"\n",
+    "\n",
+    "            if os.path.exists(opath):\n",
+    "                continue\n",
+    "                # pass\n",
+    "\n",
+    "            response = requests.get(aozora_url)\n",
+    "\n",
+    "            response.encoding = response.apparent_encoding\n",
+    "\n",
+    "            soup = BeautifulSoup(response.text, \"html.parser\")\n",
+    "\n",
+    "            div = soup.find(\"div\", class_=\"main_text\")        \n",
+    "\n",
+    "            txt = div.get_text().strip()\n",
+    "\n",
+    "            os.makedirs(os.path.dirname(opath), exist_ok=True)\n",
+    "\n",
+    "            with open(opath, \"w\") as f:\n",
+    "                f.write(txt)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

src/02_llm.ipynb

@@ -0,0 +1,119 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "from llama_index.llms.azure_openai import AzureOpenAI\n",
+    "from llama_index.embeddings.azure_openai import AzureOpenAIEmbedding\n",
+    "from llama_index.core import SimpleDirectoryReader, Settings, VectorStoreIndex"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "api_key = os.getenv(\"AZURE_OPENAI_API_KEY\")\n",
+    "api_version = \"2024-05-01-preview\"\n",
+    "azure_endpoint = os.getenv(\"AZURE_OPENAI_ENDPOINT\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "llm = AzureOpenAI(\n",
+    "    model=\"gpt-4o\",\n",
+    "    deployment_name=\"gpt-4o\",\n",
+    "    api_key=api_key,\n",
+    "    azure_endpoint=azure_endpoint,\n",
+    "    api_version=api_version,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# You need to deploy your own embedding model as well as your own chat completion model\n",
+    "embed_model = AzureOpenAIEmbedding(\n",
+    "    model=\"text-embedding-3-small\",\n",
+    "    deployment_name=\"text-embedding-3-small\",\n",
+    "    api_key=api_key,\n",
+    "    azure_endpoint=azure_endpoint,\n",
+    "    api_version=api_version,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "Settings.llm = llm\n",
+    "Settings.embed_model = embed_model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Data Source -> Documents化を行うStep\n",
+    "documents = SimpleDirectoryReader(\n",
+    "    input_dir=\"./data/text\"\n",
+    ").load_data()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "index = VectorStoreIndex.from_documents(documents)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "index.storage_context.persist(persist_dir=\"../index\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}