Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
@@ -52,6 +52,19 @@ api = HfApi()
|
|
52 |
def split_python_code_into_chunks(texts, file_paths):
|
53 |
chunks = []
|
54 |
for text, file_path in zip(texts, file_paths):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
55 |
document_chunks = chunk_python_code_with_metadata(text, file_path)
|
56 |
chunks.extend(document_chunks)
|
57 |
return chunks
|
|
|
52 |
def split_python_code_into_chunks(texts, file_paths):
|
53 |
chunks = []
|
54 |
for text, file_path in zip(texts, file_paths):
|
55 |
+
"""
|
56 |
+
Custom made python code splitter, algorithm iterates through child nodes of ast-tree(max child depth = 2)
|
57 |
+
aims to have full body of methods along signature (+ can handle decorators) in a chunk and adds method specific metadata
|
58 |
+
e.g visbility: public, _internal
|
59 |
+
type: "class", "methods", "command"(CLI commands)
|
60 |
+
source:
|
61 |
+
|
62 |
+
|
63 |
+
with the intend to use a filter when retrieving potentaion useful snippets.
|
64 |
+
|
65 |
+
|
66 |
+
|
67 |
+
"""
|
68 |
document_chunks = chunk_python_code_with_metadata(text, file_path)
|
69 |
chunks.extend(document_chunks)
|
70 |
return chunks
|