Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
@@ -17,12 +17,12 @@ from langchain.memory import ConversationBufferMemory # Chat History
|
|
17 |
list_llm = ["meta-llama/Meta-Llama-3-8B-Instruct", "mistralai/Mistral-7B-Instruct-v0.2"] # list of model
|
18 |
list_llm_simple = [os.path.basename(llm) for llm in list_llm] # display purpose
|
19 |
api_token = os.getenv("HF_TOKEN") # getting token
|
20 |
-
|
21 |
CHUNK_SIZES = {
|
22 |
"small": {"recursive": 512, "fixed": 512, "token": 256},
|
23 |
"medium": {"recursive": 1024, "fixed": 1024, "token": 512}
|
24 |
}
|
25 |
-
|
26 |
def get_text_splitter(strategy: str, chunk_size: int = 1024, chunk_overlap: int = 64):
|
27 |
splitters = {
|
28 |
"recursive": RecursiveCharacterTextSplitter(
|
|
|
17 |
list_llm = ["meta-llama/Meta-Llama-3-8B-Instruct", "mistralai/Mistral-7B-Instruct-v0.2"] # list of model
|
18 |
list_llm_simple = [os.path.basename(llm) for llm in list_llm] # display purpose
|
19 |
api_token = os.getenv("HF_TOKEN") # getting token
|
20 |
+
# Defining Chunk sizes
|
21 |
CHUNK_SIZES = {
|
22 |
"small": {"recursive": 512, "fixed": 512, "token": 256},
|
23 |
"medium": {"recursive": 1024, "fixed": 1024, "token": 512}
|
24 |
}
|
25 |
+
# passing Strategy , Chunk size , overlap
|
26 |
def get_text_splitter(strategy: str, chunk_size: int = 1024, chunk_overlap: int = 64):
|
27 |
splitters = {
|
28 |
"recursive": RecursiveCharacterTextSplitter(
|