bupa1018 commited on
Commit
807a415
·
1 Parent(s): 15a869d

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +13 -0
app.py CHANGED
@@ -52,6 +52,19 @@ api = HfApi()
52
  def split_python_code_into_chunks(texts, file_paths):
53
  chunks = []
54
  for text, file_path in zip(texts, file_paths):
 
 
 
 
 
 
 
 
 
 
 
 
 
55
  document_chunks = chunk_python_code_with_metadata(text, file_path)
56
  chunks.extend(document_chunks)
57
  return chunks
 
52
  def split_python_code_into_chunks(texts, file_paths):
53
  chunks = []
54
  for text, file_path in zip(texts, file_paths):
55
+ """
56
+ Custom made python code splitter, algorithm iterates through child nodes of ast-tree(max child depth = 2)
57
+ aims to have full body of methods along signature (+ can handle decorators) in a chunk and adds method specific metadata
58
+ e.g visbility: public, _internal
59
+ type: "class", "methods", "command"(CLI commands)
60
+ source:
61
+
62
+
63
+ with the intend to use a filter when retrieving potentaion useful snippets.
64
+
65
+
66
+
67
+ """
68
  document_chunks = chunk_python_code_with_metadata(text, file_path)
69
  chunks.extend(document_chunks)
70
  return chunks