bupa1018 commited on
Commit
d0c3226
·
1 Parent(s): b700892

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +9 -12
app.py CHANGED
@@ -498,30 +498,27 @@ def initialize():
498
 
499
  download_gitlab_project_by_version()
500
  #download_gitlab_repo()
501
- #code_partial_paths = ['kadi_apy/lib/']
502
- #code_file_paths = []
503
- #doc_partial_paths = []
504
  doc_partial_paths = ['docs/source/setup/']
505
  doc_file_paths = ['docs/source/usage/lib.rst']
506
 
507
- #code_files, code_file_references = process_directory(DATA_DIR,code_partial_paths, code_file_paths)
508
- #print("LEEEEEEEEEEEENGTH of code_files: ", len(code_files))
509
 
510
 
511
  doc_files, doc_file_references = process_directory(DATA_DIR,doc_partial_paths, doc_file_paths)
512
  print("LEEEEEEEEEEEENGTH of doc_files: ", len(doc_files))
513
- #code_files, code_file_references = process_directory5(DATA_DIR, code_partial_paths, code_file_path)
514
 
515
- doc_files, doc_file_references = process_directory5(DATA_DIR, doc_partial_paths, doc_file_paths)
516
-
517
- #code_chunks = split_pythoncode_into_chunks(code_files, code_file_references, 1500, 0)
518
  doc_chunks = split_into_chunks(doc_files, doc_file_references, CHUNK_SIZE, CHUNK_OVERLAP)
519
 
520
- #print(f"Total number of code_chunks: {len(code_chunks)}")
521
- #print(f"Total number of doc_chunks: {len(doc_chunks)}")
522
 
523
  docstore = setup_vectorstore(doc_chunks, EMBEDDING_MODEL_NAME, PERSIST_DOC_DIRECTORY)
524
- #codestore = setup_vectorstore(code_chunks, EMBEDDING_MODEL_NAME, PERSIST_CODE_DIRECTORY)
525
 
526
  llm = setup_llm(LLM_MODEL_NAME, LLM_TEMPERATURE, GROQ_API_KEY)
527
 
 
498
 
499
  download_gitlab_project_by_version()
500
  #download_gitlab_repo()
501
+ code_partial_paths = ['kadi_apy/lib/']
502
+ code_file_paths = []
503
+ doc_partial_paths = []
504
  doc_partial_paths = ['docs/source/setup/']
505
  doc_file_paths = ['docs/source/usage/lib.rst']
506
 
507
+ code_files, code_file_references = process_directory(DATA_DIR,code_partial_paths, code_file_paths)
508
+ print("LEEEEEEEEEEEENGTH of code_files: ", len(code_files))
509
 
510
 
511
  doc_files, doc_file_references = process_directory(DATA_DIR,doc_partial_paths, doc_file_paths)
512
  print("LEEEEEEEEEEEENGTH of doc_files: ", len(doc_files))
 
513
 
514
+ code_chunks = split_pythoncode_into_chunks(code_files, code_file_references, 1500, 0)
 
 
515
  doc_chunks = split_into_chunks(doc_files, doc_file_references, CHUNK_SIZE, CHUNK_OVERLAP)
516
 
517
+ print(f"Total number of code_chunks: {len(code_chunks)}")
518
+ print(f"Total number of doc_chunks: {len(doc_chunks)}")
519
 
520
  docstore = setup_vectorstore(doc_chunks, EMBEDDING_MODEL_NAME, PERSIST_DOC_DIRECTORY)
521
+ codestore = setup_vectorstore(code_chunks, EMBEDDING_MODEL_NAME, PERSIST_CODE_DIRECTORY)
522
 
523
  llm = setup_llm(LLM_MODEL_NAME, LLM_TEMPERATURE, GROQ_API_KEY)
524