Spaces:
Running
Running
Update parser.py
Browse files
parser.py
CHANGED
@@ -1,6 +1,4 @@
|
|
1 |
# parser.py
|
2 |
-
import os
|
3 |
-
os.system("pip install --upgrade ast")
|
4 |
import ast
|
5 |
|
6 |
def get_category(node):
|
@@ -25,7 +23,7 @@ def get_category(node):
|
|
25 |
return 'expression'
|
26 |
elif isinstance(node, ast.ExceptHandler):
|
27 |
return 'except'
|
28 |
-
elif isinstance(node, ast.Assign
|
29 |
return 'assigned_variable'
|
30 |
else:
|
31 |
return 'other'
|
@@ -38,8 +36,16 @@ def get_variable_role(node, parent):
|
|
38 |
elif isinstance(parent, ast.Return) and isinstance(node, ast.Name):
|
39 |
return 'returned_variable'
|
40 |
elif isinstance(parent, (ast.Assign, ast.AnnAssign, ast.AugAssign)):
|
41 |
-
if isinstance(node, ast.Name)
|
42 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
43 |
return None
|
44 |
|
45 |
def create_vector(category, level, location, total_lines, parent_path, variable_info=None):
|
@@ -130,25 +136,23 @@ def parse_node(node, lines, prev_end, level=0, total_lines=None, parent_path=Non
|
|
130 |
processed_lines.add(start_line)
|
131 |
|
132 |
# Handle variables in function definitions (input variables)
|
133 |
-
if isinstance(
|
134 |
-
|
135 |
-
|
136 |
-
if
|
137 |
-
|
138 |
-
|
139 |
-
|
140 |
-
|
141 |
-
|
142 |
-
|
143 |
-
|
144 |
-
|
145 |
-
|
146 |
-
|
147 |
-
|
148 |
-
|
149 |
-
|
150 |
-
})
|
151 |
-
processed_lines.add(var_start)
|
152 |
|
153 |
# Process nested bodies
|
154 |
nested_prev_end = start_line
|
@@ -212,11 +216,8 @@ def parse_node(node, lines, prev_end, level=0, total_lines=None, parent_path=Non
|
|
212 |
nested_prev_end = max(nested_prev_end, child_parts[-1]['location'][1] if child_parts else child_start)
|
213 |
else:
|
214 |
# Handle assignments and returns for variable detection
|
215 |
-
if isinstance(child, ast.Assign)
|
216 |
-
|
217 |
-
if isinstance(child, ast.Assign):
|
218 |
-
targets = child.targets
|
219 |
-
for target in targets:
|
220 |
if isinstance(target, ast.Name):
|
221 |
var_start = child.lineno
|
222 |
if var_start not in processed_lines and not is_blank_or_comment(lines[var_start - 1]):
|
@@ -233,6 +234,24 @@ def parse_node(node, lines, prev_end, level=0, total_lines=None, parent_path=Non
|
|
233 |
'node_id': var_node_id
|
234 |
})
|
235 |
processed_lines.add(var_start)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
236 |
elif isinstance(child, ast.Return):
|
237 |
for value in ast.walk(child):
|
238 |
if isinstance(value, ast.Name):
|
@@ -272,7 +291,7 @@ def parse_python_code(code):
|
|
272 |
try:
|
273 |
tree = ast.parse(code)
|
274 |
except SyntaxError:
|
275 |
-
return [{'category': 'error', 'source': 'Invalid Python code', 'location': (1, 1), 'level': 0, 'vector': [0, 0, 1.0, 0.0, 0, 0], 'parent_path': 'Top-Level', 'node_id': 'Error[1]'}]
|
276 |
|
277 |
parts = []
|
278 |
prev_end = 0
|
|
|
1 |
# parser.py
|
|
|
|
|
2 |
import ast
|
3 |
|
4 |
def get_category(node):
|
|
|
23 |
return 'expression'
|
24 |
elif isinstance(node, ast.ExceptHandler):
|
25 |
return 'except'
|
26 |
+
elif isinstance(node, (ast.Assign, ast.AnnAssign, ast.AugAssign)):
|
27 |
return 'assigned_variable'
|
28 |
else:
|
29 |
return 'other'
|
|
|
36 |
elif isinstance(parent, ast.Return) and isinstance(node, ast.Name):
|
37 |
return 'returned_variable'
|
38 |
elif isinstance(parent, (ast.Assign, ast.AnnAssign, ast.AugAssign)):
|
39 |
+
if isinstance(node, ast.Name):
|
40 |
+
# Handle different target structures
|
41 |
+
if isinstance(parent, ast.Assign):
|
42 |
+
for target in parent.targets:
|
43 |
+
if isinstance(target, ast.Name) and target.id == node.id:
|
44 |
+
return 'assigned_variable'
|
45 |
+
elif isinstance(parent, (ast.AnnAssign, ast.AugAssign)):
|
46 |
+
target = parent.target
|
47 |
+
if isinstance(target, ast.Name) and target.id == node.id:
|
48 |
+
return 'assigned_variable'
|
49 |
return None
|
50 |
|
51 |
def create_vector(category, level, location, total_lines, parent_path, variable_info=None):
|
|
|
136 |
processed_lines.add(start_line)
|
137 |
|
138 |
# Handle variables in function definitions (input variables)
|
139 |
+
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.args.args:
|
140 |
+
for arg in node.args.args:
|
141 |
+
var_start = start_line # Assume args are on the same line as function def for simplicity
|
142 |
+
if var_start not in processed_lines:
|
143 |
+
counters['input_variable'] += 1
|
144 |
+
var_node_id = f"InputVariable[{counters['input_variable']}]"
|
145 |
+
var_info = {'role': 'input_variable', 'name': arg.arg}
|
146 |
+
parts.append({
|
147 |
+
'category': 'input_variable',
|
148 |
+
'source': f" {arg.arg},", # Indented as part of function
|
149 |
+
'location': (var_start, var_start),
|
150 |
+
'level': level + 1,
|
151 |
+
'vector': create_vector('input_variable', level + 1, (var_start, var_start), total_lines, current_path, var_info),
|
152 |
+
'parent_path': f"{current_path[0]} -> {var_node_id}",
|
153 |
+
'node_id': var_node_id
|
154 |
+
})
|
155 |
+
processed_lines.add(var_start)
|
|
|
|
|
156 |
|
157 |
# Process nested bodies
|
158 |
nested_prev_end = start_line
|
|
|
216 |
nested_prev_end = max(nested_prev_end, child_parts[-1]['location'][1] if child_parts else child_start)
|
217 |
else:
|
218 |
# Handle assignments and returns for variable detection
|
219 |
+
if isinstance(child, ast.Assign):
|
220 |
+
for target in child.targets:
|
|
|
|
|
|
|
221 |
if isinstance(target, ast.Name):
|
222 |
var_start = child.lineno
|
223 |
if var_start not in processed_lines and not is_blank_or_comment(lines[var_start - 1]):
|
|
|
234 |
'node_id': var_node_id
|
235 |
})
|
236 |
processed_lines.add(var_start)
|
237 |
+
elif isinstance(child, ast.AnnAssign) or isinstance(child, ast.AugAssign):
|
238 |
+
target = child.target
|
239 |
+
if isinstance(target, ast.Name):
|
240 |
+
var_start = child.lineno
|
241 |
+
if var_start not in processed_lines and not is_blank_or_comment(lines[var_start - 1]):
|
242 |
+
counters['assigned_variable'] += 1
|
243 |
+
var_node_id = f"AssignedVariable[{counters['assigned_variable']}]"
|
244 |
+
var_info = {'role': 'assigned_variable', 'name': target.id}
|
245 |
+
parts.append({
|
246 |
+
'category': 'assigned_variable',
|
247 |
+
'source': lines[var_start - 1],
|
248 |
+
'location': (var_start, var_start),
|
249 |
+
'level': level + 1,
|
250 |
+
'vector': create_vector('assigned_variable', level + 1, (var_start, var_start), total_lines, current_path, var_info),
|
251 |
+
'parent_path': f"{current_path[0]} -> {var_node_id}",
|
252 |
+
'node_id': var_node_id
|
253 |
+
})
|
254 |
+
processed_lines.add(var_start)
|
255 |
elif isinstance(child, ast.Return):
|
256 |
for value in ast.walk(child):
|
257 |
if isinstance(value, ast.Name):
|
|
|
291 |
try:
|
292 |
tree = ast.parse(code)
|
293 |
except SyntaxError:
|
294 |
+
return [{'category': 'error', 'source': 'Invalid Python code', 'location': (1, 1), 'level': 0, 'vector': [0, 0, 1.0, 0.0, 0, 0, 0, 0], 'parent_path': 'Top-Level', 'node_id': 'Error[1]'}]
|
295 |
|
296 |
parts = []
|
297 |
prev_end = 0
|