Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
@@ -1,21 +1,22 @@
|
|
1 |
-
import gradio as gr
|
2 |
-
import os
|
3 |
-
from typing import List, Dict
|
4 |
-
from langchain.text_splitter import (
|
5 |
-
RecursiveCharacterTextSplitter
|
6 |
-
CharacterTextSplitter
|
7 |
-
TokenTextSplitter
|
8 |
)
|
9 |
-
from langchain_community.vectorstores import FAISS, Chroma, Qdrant
|
10 |
-
from langchain_community.document_loaders import PyPDFLoader
|
11 |
-
from langchain.chains import ConversationalRetrievalChain
|
12 |
-
from langchain_community.embeddings import HuggingFaceEmbeddings
|
13 |
-
from langchain_huggingface import HuggingFaceEndpoint
|
14 |
-
from langchain.memory import ConversationBufferMemory
|
15 |
|
16 |
-
|
17 |
-
|
18 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
19 |
|
20 |
CHUNK_SIZES = {
|
21 |
"small": {"recursive": 512, "fixed": 512, "token": 256},
|
|
|
1 |
+
import gradio as gr # used for UI dev
|
2 |
+
import os # Built-in model to get/use the token for running huggingface source model which requires token to run
|
3 |
+
from typing import List, Dict # Built-in model
|
4 |
+
from langchain.text_splitter import ( # Text splitting strategies
|
5 |
+
RecursiveCharacterTextSplitter,#Text splitting strategies
|
6 |
+
CharacterTextSplitter,#Text splitting strategies
|
7 |
+
TokenTextSplitter#Text splitting strategies
|
8 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
9 |
|
10 |
+
from langchain_community.vectorstores import FAISS, Chroma, Qdrant # Vector database
|
11 |
+
from langchain_community.document_loaders import PyPDFLoader # Convert PDF to TEXT
|
12 |
+
from langchain.chains import ConversationalRetrievalChain # Entire retrival chain for conversation
|
13 |
+
from langchain_community.embeddings import HuggingFaceEmbeddings # Words to no
|
14 |
+
from langchain_huggingface import HuggingFaceEndpoint # API for generative model
|
15 |
+
from langchain.memory import ConversationBufferMemory # Chat History
|
16 |
+
|
17 |
+
list_llm = ["meta-llama/Meta-Llama-3-8B-Instruct", "mistralai/Mistral-7B-Instruct-v0.2"] # list of model
|
18 |
+
list_llm_simple = [os.path.basename(llm) for llm in list_llm] # display purpose
|
19 |
+
api_token = os.getenv("HF_TOKEN") # getting token
|
20 |
|
21 |
CHUNK_SIZES = {
|
22 |
"small": {"recursive": 512, "fixed": 512, "token": 256},
|