bupa1018 commited on
Commit
4a229af
·
1 Parent(s): 9936614

Update chunk_python_code.py

Browse files
Files changed (1) hide show
  1. chunk_python_code.py +6 -6
chunk_python_code.py CHANGED
@@ -51,7 +51,7 @@ def _iterate_ast(python_code, documents, file_path):
51
 
52
  if isinstance(first_level_node, ast.ClassDef):
53
  documents.extend(
54
- handle_first_level_class(first_level_node, documents, python_code))
55
  elif isinstance(first_level_node, ast.FunctionDef):
56
  documents.extend(
57
  _chunk_first_level_func_node(first_level_node, documents, python_code))
@@ -60,7 +60,7 @@ def _iterate_ast(python_code, documents, file_path):
60
  _chunk_first_level_assign_node(first_level_node, documents, python_code))
61
  else:
62
  documents.extend(
63
- handle_notdefined_case(python_code))
64
 
65
 
66
  def _chunk_import_only_python_code(python_code, file_path):
@@ -84,18 +84,18 @@ def _chunk_import_only_python_code(python_code, file_path):
84
  documents.append(doc)
85
  return documents
86
 
87
- from langchain.text_splitter import RecursiveCharacterTextSplitter
88
 
89
 
90
- def handle_notdefined_case(python_code):
91
  documents = []
92
  documents.extend(
93
  chunk_python_code_by_character)
94
 
95
  return documents
96
 
 
97
 
98
- def chunk_python_code_by_character(python_code):
99
  documents = []
100
  text_splitter = RecursiveCharacterTextSplitter(
101
  chunk_size=512,
@@ -156,7 +156,7 @@ def _chunk_first_level_assign_node(ast_node, python_code):
156
 
157
  return documents
158
 
159
- def handle_first_level_class(ast_node , python_code):
160
  """
161
  Handles classes at the first level of the AST.
162
  """
 
51
 
52
  if isinstance(first_level_node, ast.ClassDef):
53
  documents.extend(
54
+ _handle_first_level_class(first_level_node, documents, python_code))
55
  elif isinstance(first_level_node, ast.FunctionDef):
56
  documents.extend(
57
  _chunk_first_level_func_node(first_level_node, documents, python_code))
 
60
  _chunk_first_level_assign_node(first_level_node, documents, python_code))
61
  else:
62
  documents.extend(
63
+ _handle_notdefined_case(python_code))
64
 
65
 
66
  def _chunk_import_only_python_code(python_code, file_path):
 
84
  documents.append(doc)
85
  return documents
86
 
 
87
 
88
 
89
+ def _handle_notdefined_case(python_code):
90
  documents = []
91
  documents.extend(
92
  chunk_python_code_by_character)
93
 
94
  return documents
95
 
96
+ from langchain.text_splitter import RecursiveCharacterTextSplitter
97
 
98
+ def _chunk_python_code_by_character(python_code):
99
  documents = []
100
  text_splitter = RecursiveCharacterTextSplitter(
101
  chunk_size=512,
 
156
 
157
  return documents
158
 
159
+ def _handle_first_level_class(ast_node , python_code):
160
  """
161
  Handles classes at the first level of the AST.
162
  """