sudip1987 commited on
Commit
46d0a50
·
verified ·
1 Parent(s): f0df96e

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +2 -2
app.py CHANGED
@@ -17,12 +17,12 @@ from langchain.memory import ConversationBufferMemory # Chat History
17
  list_llm = ["meta-llama/Meta-Llama-3-8B-Instruct", "mistralai/Mistral-7B-Instruct-v0.2"] # list of model
18
  list_llm_simple = [os.path.basename(llm) for llm in list_llm] # display purpose
19
  api_token = os.getenv("HF_TOKEN") # getting token
20
-
21
  CHUNK_SIZES = {
22
  "small": {"recursive": 512, "fixed": 512, "token": 256},
23
  "medium": {"recursive": 1024, "fixed": 1024, "token": 512}
24
  }
25
-
26
  def get_text_splitter(strategy: str, chunk_size: int = 1024, chunk_overlap: int = 64):
27
  splitters = {
28
  "recursive": RecursiveCharacterTextSplitter(
 
17
  list_llm = ["meta-llama/Meta-Llama-3-8B-Instruct", "mistralai/Mistral-7B-Instruct-v0.2"] # list of model
18
  list_llm_simple = [os.path.basename(llm) for llm in list_llm] # display purpose
19
  api_token = os.getenv("HF_TOKEN") # getting token
20
+ # Defining Chunk sizes
21
  CHUNK_SIZES = {
22
  "small": {"recursive": 512, "fixed": 512, "token": 256},
23
  "medium": {"recursive": 1024, "fixed": 1024, "token": 512}
24
  }
25
+ # passing Strategy , Chunk size , overlap
26
  def get_text_splitter(strategy: str, chunk_size: int = 1024, chunk_overlap: int = 64):
27
  splitters = {
28
  "recursive": RecursiveCharacterTextSplitter(