Update parser.py

parser.py

@@ -25,6 +25,8 @@ def get_category(node):
         return 'return'
     elif isinstance(node, ast.Expr):
         return 'expression'
+    elif isinstance(node, ast.ExceptHandler):
+        return 'except'
     else:
         return 'other'
 
@@ -44,6 +46,11 @@ def create_vector(category, level, location, total_lines, parent_path):
                         for i, parent in enumerate(parent_path)) / max(1, len(category_map))
     return [category_id, level, center_pos, span, parent_depth, parent_weight]
 
+def is_blank_or_comment(line):
+    """Check if a line is blank or a comment."""
+    stripped = line.strip()
+    return not stripped or stripped.startswith('#')
+
 def parse_node(node, lines, prev_end, level=0, total_lines=None, parent_path=None, counters=None, processed_lines=None):
     if total_lines is None:
         total_lines = len(lines)
@@ -66,27 +73,29 @@ def parse_node(node, lines, prev_end, level=0, total_lines=None, parent_path=Non
     counters[category] += 1
     node_id = f"{category.capitalize()}[{counters[category]}]"
 
-    # Spacer before node
+    # Spacer before node (only for blank lines or comments)
     if start_line > prev_end + 1:
         spacer_lines = lines[prev_end:start_line - 1]
         spacer_lines_set = set(range(prev_end + 1, start_line))
         if not spacer_lines_set.issubset(processed_lines):
-            counters['spacer'] += 1
-            spacer_node_id = f"Spacer[{counters['spacer']}]"
-            parts.append({
-                'category': 'spacer',
-                'source': ''.join(spacer_lines),
-                'location': (prev_end + 1, start_line - 1),
-                'level': level,
-                'vector': create_vector('spacer', level, (prev_end + 1, start_line - 1), total_lines, parent_path),
-                'parent_path': ' -> '.join(parent_path) if parent_path else 'Top-Level',
-                'node_id': spacer_node_id
-            })
-            processed_lines.update(spacer_lines_set)
-
-    # Current node's header (e.g., 'def', 'if')
+            for i, line in enumerate(spacer_lines, prev_end + 1):
+                if i not in processed_lines and is_blank_or_comment(line):
+                    counters['spacer'] += 1
+                    spacer_node_id = f"Spacer[{counters['spacer']}]"
+                    parts.append({
+                        'category': 'spacer',
+                        'source': line,
+                        'location': (i, i),
+                        'level': level,
+                        'vector': create_vector('spacer', level, (i, i), total_lines, parent_path),
+                        'parent_path': ' -> '.join(parent_path) if parent_path else 'Top-Level',
+                        'node_id': spacer_node_id
+                    })
+                    processed_lines.add(i)
+
+    # Current node's header (e.g., 'def', 'if', 'try')
     current_path = parent_path + [node_id]
-    if start_line not in processed_lines:
+    if start_line not in processed_lines and not is_blank_or_comment(lines[start_line - 1]):
         parts.append({
             'category': category,
             'source': lines[start_line - 1],
@@ -103,76 +112,74 @@ def parse_node(node, lines, prev_end, level=0, total_lines=None, parent_path=Non
     for attr in ('body', 'orelse', 'handlers', 'finalbody'):
         if hasattr(node, attr) and getattr(node, attr):
             for child in getattr(node, attr):
-                if attr == 'orelse' and isinstance(node, ast.If) and child.lineno != start_line:
-                    sub_category = 'elif' if 'elif' in lines[child.lineno - 1] else 'else'
-                    sub_start = child.lineno
-                    sub_end = getattr(child, 'end_lineno', sub_start)
-                    if not any(line in processed_lines for line in range(sub_start, sub_end + 1)):
-                        counters[sub_category] += 1
-                        sub_node_id = f"{sub_category.capitalize()}[{counters[sub_category]}]"
-                        parts.append({
-                            'category': sub_category,
-                            'source': lines[sub_start - 1],
-                            'location': (sub_start, sub_start),
-                            'level': level,
-                            'vector': create_vector(sub_category, level, (sub_start, sub_start), total_lines, current_path),
-                            'parent_path': ' -> '.join(parent_path) if parent_path else 'Top-Level',
-                            'node_id': sub_node_id
-                        })
-                        processed_lines.add(sub_start)
-                        child_parts = parse_node(child, lines, sub_start, level + 1, total_lines, current_path, counters, processed_lines)
+                child_start = getattr(child, 'lineno', nested_prev_end + 1)
+                child_end = getattr(child, 'end_lineno', child_start)
+                if not any(line in processed_lines for line in range(child_start, child_end + 1)):
+                    if attr == 'orelse' and isinstance(node, ast.If) and child_start != start_line:
+                        sub_category = 'elif' if 'elif' in lines[child_start - 1] else 'else'
+                        if child_start not in processed_lines and not is_blank_or_comment(lines[child_start - 1]):
+                            counters[sub_category] += 1
+                            sub_node_id = f"{sub_category.capitalize()}[{counters[sub_category]}]"
+                            parts.append({
+                                'category': sub_category,
+                                'source': lines[child_start - 1],
+                                'location': (child_start, child_start),
+                                'level': level,
+                                'vector': create_vector(sub_category, level, (child_start, child_start), total_lines, current_path),
+                                'parent_path': ' -> '.join(parent_path) if parent_path else 'Top-Level',
+                                'node_id': sub_node_id
+                            })
+                            processed_lines.add(child_start)
+                        child_parts = parse_node(child, lines, child_start, level + 1, total_lines, current_path, counters, processed_lines)
                         parts.extend(child_parts)
-                        nested_prev_end = max(nested_prev_end, child_parts[-1]['location'][1] if child_parts else sub_start)
-                elif attr == 'handlers' and isinstance(child, ast.ExceptHandler):
-                    sub_start = child.lineno
-                    sub_end = getattr(child, 'end_lineno', sub_start)
-                    if not any(line in processed_lines for line in range(sub_start, sub_end + 1)):
-                        counters['except'] += 1
-                        sub_node_id = f"Except[{counters['except']}]"
-                        parts.append({
-                            'category': 'except',
-                            'source': lines[sub_start - 1],
-                            'location': (sub_start, sub_start),
-                            'level': level,
-                            'vector': create_vector('except', level, (sub_start, sub_start), total_lines, current_path),
-                            'parent_path': ' -> '.join(parent_path) if parent_path else 'Top-Level',
-                            'node_id': sub_node_id
-                        })
-                        processed_lines.add(sub_start)
-                        child_parts = parse_node(child, lines, sub_start, level + 1, total_lines, current_path, counters, processed_lines)
+                        nested_prev_end = max(nested_prev_end, child_parts[-1]['location'][1] if child_parts else child_start)
+                    elif attr == 'handlers' and isinstance(child, ast.ExceptHandler):
+                        if child_start not in processed_lines and not is_blank_or_comment(lines[child_start - 1]):
+                            counters['except'] += 1
+                            sub_node_id = f"Except[{counters['except']}]"
+                            parts.append({
+                                'category': 'except',
+                                'source': lines[child_start - 1],
+                                'location': (child_start, child_start),
+                                'level': level,
+                                'vector': create_vector('except', level, (child_start, child_start), total_lines, current_path),
+                                'parent_path': ' -> '.join(parent_path) if parent_path else 'Top-Level',
+                                'node_id': sub_node_id
+                            })
+                            processed_lines.add(child_start)
+                        child_parts = parse_node(child, lines, child_start, level + 1, total_lines, current_path, counters, processed_lines)
                         parts.extend(child_parts)
-                        nested_prev_end = max(nested_prev_end, child_parts[-1]['location'][1] if child_parts else sub_start)
-                elif attr == 'finalbody':
-                    sub_start = child.lineno
-                    sub_end = getattr(child, 'end_lineno', sub_start)
-                    if not any(line in processed_lines for line in range(sub_start, sub_end + 1)):
-                        counters['finally'] += 1
-                        sub_node_id = f"Finally[{counters['finally']}]"
-                        parts.append({
-                            'category': 'finally',
-                            'source': lines[sub_start - 1],
-                            'location': (sub_start, sub_start),
-                            'level': level,
-                            'vector': create_vector('finally', level, (sub_start, sub_start), total_lines, current_path),
-                            'parent_path': ' -> '.join(parent_path) if parent_path else 'Top-Level',
-                            'node_id': sub_node_id
-                        })
-                        processed_lines.add(sub_start)
-                        child_parts = parse_node(child, lines, sub_start, level + 1, total_lines, current_path, counters, processed_lines)
+                        nested_prev_end = max(nested_prev_end, child_parts[-1]['location'][1] if child_parts else child_start)
+                    elif attr == 'finalbody':
+                        if child_start not in processed_lines and not is_blank_or_comment(lines[child_start - 1]):
+                            counters['finally'] += 1
+                            sub_node_id = f"Finally[{counters['finally']}]"
+                            parts.append({
+                                'category': 'finally',
+                                'source': lines[child_start - 1],
+                                'location': (child_start, child_start),
+                                'level': level,
+                                'vector': create_vector('finally', level, (child_start, child_start), total_lines, current_path),
+                                'parent_path': ' -> '.join(parent_path) if parent_path else 'Top-Level',
+                                'node_id': sub_node_id
+                            })
+                            processed_lines.add(child_start)
+                        child_parts = parse_node(child, lines, child_start, level + 1, total_lines, current_path, counters, processed_lines)
                         parts.extend(child_parts)
-                        nested_prev_end = max(nested_prev_end, child_parts[-1]['location'][1] if child_parts else sub_start)
-                else:
-                    child_parts = parse_node(child, lines, nested_prev_end, level + 1, total_lines, current_path, counters, processed_lines)
-                    parts.extend(child_parts)
-                    nested_prev_end = child_parts[-1]['location'][1] if child_parts else nested_prev_end
+                        nested_prev_end = max(nested_prev_end, child_parts[-1]['location'][1] if child_parts else child_start)
+                    else:
+                        child_parts = parse_node(child, lines, nested_prev_end, level + 1, total_lines, current_path, counters, processed_lines)
+                        parts.extend(child_parts)
+                        nested_prev_end = child_parts[-1]['location'][1] if child_parts else nested_prev_end
 
     # Update end_line and source of the parent node if its body extends it
     if nested_prev_end > start_line and start_line not in processed_lines:
         final_end = nested_prev_end
-        parts[-1]['location'] = (start_line, final_end)
-        parts[-1]['source'] = ''.join(lines[start_line - 1:final_end])
-        parts[-1]['vector'] = create_vector(category, level, (start_line, final_end), total_lines, current_path)
-        processed_lines.update(range(start_line, final_end + 1))
+        if start_line not in processed_lines:
+            parts[-1]['location'] = (start_line, final_end)
+            parts[-1]['source'] = ''.join(lines[start_line - 1:final_end])
+            parts[-1]['vector'] = create_vector(category, level, (start_line, final_end), total_lines, current_path)
+            processed_lines.update(range(start_line, final_end + 1))
 
     return parts
 
@@ -197,18 +204,26 @@ def parse_python_code(code):
         remaining_lines = lines[prev_end:]
         remaining_lines_set = set(range(prev_end + 1, total_lines + 1))
         if not remaining_lines_set.issubset(processed_lines):
-            counters = {'spacer': 0}
-            counters['spacer'] += 1
-            spacer_node_id = f"Spacer[{counters['spacer']}]"
-            parts.append({
-                'category': 'spacer',
-                'source': ''.join(remaining_lines),
-                'location': (prev_end + 1, total_lines + 1),
-                'level': 0,
-                'vector': create_vector('spacer', 0, (prev_end + 1, total_lines + 1), total_lines, []),
-                'parent_path': 'Top-Level',
-                'node_id': spacer_node_id
-            })
-            processed_lines.update(remaining_lines_set)
-
-    return parts
+            for i, line in enumerate(remaining_lines, prev_end + 1):
+                if i not in processed_lines:
+                    if is_blank_or_comment(line):
+                        counters = {'spacer': 0}
+                        counters['spacer'] += 1
+                        spacer_node_id = f"Spacer[{counters['spacer']}]"
+                        parts.append({
+                            'category': 'spacer',
+                            'source': line,
+                            'location': (i, i),
+                            'level': 0,
+                            'vector': create_vector('spacer', 0, (i, i), total_lines, []),
+                            'parent_path': 'Top-Level',
+                            'node_id': spacer_node_id
+                        })
+                        processed_lines.add(i)
+
+    return parts
+
+def is_blank_or_comment(line):
+    """Check if a line is blank or a comment."""
+    stripped = line.strip()
+    return not stripped or stripped.startswith('#')