Spaces:
Sleeping
Sleeping
Update chunk_python_code.py
Browse files- chunk_python_code.py +9 -4
chunk_python_code.py
CHANGED
@@ -5,7 +5,12 @@ from langchain.schema import Document
|
|
5 |
|
6 |
def chunk_python_code_with_metadata(python_code, file_path):
|
7 |
"""
|
8 |
-
Custom Python Code Splitter
|
|
|
|
|
|
|
|
|
|
|
9 |
Entry point method to process the Python file.
|
10 |
It invokes the iterate_ast function.
|
11 |
"""
|
@@ -61,9 +66,9 @@ def _iterate_ast(python_code, documents, file_path):
|
|
61 |
elif isinstance(first_level_node, ast.Assign):
|
62 |
documents.extend(
|
63 |
_chunk_first_level_assign_node(first_level_node, python_code))
|
64 |
-
|
65 |
-
|
66 |
-
|
67 |
|
68 |
|
69 |
|
|
|
5 |
|
6 |
def chunk_python_code_with_metadata(python_code, file_path):
|
7 |
"""
|
8 |
+
Custom Python Code Splitter
|
9 |
+
chunks python file by length of func/method body
|
10 |
+
aims to have one full method/function in a chunk and full body of a class, but cutting of when first method declaration is met
|
11 |
+
|
12 |
+
able to handles decorators on methods
|
13 |
+
|
14 |
Entry point method to process the Python file.
|
15 |
It invokes the iterate_ast function.
|
16 |
"""
|
|
|
66 |
elif isinstance(first_level_node, ast.Assign):
|
67 |
documents.extend(
|
68 |
_chunk_first_level_assign_node(first_level_node, python_code))
|
69 |
+
# else:
|
70 |
+
# documents.extend(
|
71 |
+
# _handle_not_defined_case(python_code))
|
72 |
|
73 |
|
74 |
|