Spaces:
Sleeping
Sleeping
Update chunk_python_code.py
Browse files- chunk_python_code.py +6 -6
chunk_python_code.py
CHANGED
@@ -51,7 +51,7 @@ def _iterate_ast(python_code, documents, file_path):
|
|
51 |
|
52 |
if isinstance(first_level_node, ast.ClassDef):
|
53 |
documents.extend(
|
54 |
-
|
55 |
elif isinstance(first_level_node, ast.FunctionDef):
|
56 |
documents.extend(
|
57 |
_chunk_first_level_func_node(first_level_node, documents, python_code))
|
@@ -60,7 +60,7 @@ def _iterate_ast(python_code, documents, file_path):
|
|
60 |
_chunk_first_level_assign_node(first_level_node, documents, python_code))
|
61 |
else:
|
62 |
documents.extend(
|
63 |
-
|
64 |
|
65 |
|
66 |
def _chunk_import_only_python_code(python_code, file_path):
|
@@ -84,18 +84,18 @@ def _chunk_import_only_python_code(python_code, file_path):
|
|
84 |
documents.append(doc)
|
85 |
return documents
|
86 |
|
87 |
-
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
88 |
|
89 |
|
90 |
-
def
|
91 |
documents = []
|
92 |
documents.extend(
|
93 |
chunk_python_code_by_character)
|
94 |
|
95 |
return documents
|
96 |
|
|
|
97 |
|
98 |
-
def
|
99 |
documents = []
|
100 |
text_splitter = RecursiveCharacterTextSplitter(
|
101 |
chunk_size=512,
|
@@ -156,7 +156,7 @@ def _chunk_first_level_assign_node(ast_node, python_code):
|
|
156 |
|
157 |
return documents
|
158 |
|
159 |
-
def
|
160 |
"""
|
161 |
Handles classes at the first level of the AST.
|
162 |
"""
|
|
|
51 |
|
52 |
if isinstance(first_level_node, ast.ClassDef):
|
53 |
documents.extend(
|
54 |
+
_handle_first_level_class(first_level_node, documents, python_code))
|
55 |
elif isinstance(first_level_node, ast.FunctionDef):
|
56 |
documents.extend(
|
57 |
_chunk_first_level_func_node(first_level_node, documents, python_code))
|
|
|
60 |
_chunk_first_level_assign_node(first_level_node, documents, python_code))
|
61 |
else:
|
62 |
documents.extend(
|
63 |
+
_handle_notdefined_case(python_code))
|
64 |
|
65 |
|
66 |
def _chunk_import_only_python_code(python_code, file_path):
|
|
|
84 |
documents.append(doc)
|
85 |
return documents
|
86 |
|
|
|
87 |
|
88 |
|
89 |
+
def _handle_notdefined_case(python_code):
|
90 |
documents = []
|
91 |
documents.extend(
|
92 |
chunk_python_code_by_character)
|
93 |
|
94 |
return documents
|
95 |
|
96 |
+
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
97 |
|
98 |
+
def _chunk_python_code_by_character(python_code):
|
99 |
documents = []
|
100 |
text_splitter = RecursiveCharacterTextSplitter(
|
101 |
chunk_size=512,
|
|
|
156 |
|
157 |
return documents
|
158 |
|
159 |
+
def _handle_first_level_class(ast_node , python_code):
|
160 |
"""
|
161 |
Handles classes at the first level of the AST.
|
162 |
"""
|