Update parser.py

parser.py

@@ -24,69 +24,82 @@ def get_category(node):
     else:
         return 'other'
 
-def create_vector(category, level, location, total_lines):
-    """Create a vector representation for a code part."""
-    # Vector: [category_id, level, start_line_normalized, end_line_normalized]
+def create_vector(category, level, location, total_lines, parent_path):
+    """Create a vector representation including hierarchy info."""
     category_map = {
         'import': 1, 'assignment': 2, 'function': 3, 'async_function': 4, 'class': 5,
         'if': 6, 'while': 7, 'for': 8, 'expression': 9, 'spacer': 10, 'other': 11
     }
     category_id = category_map.get(category, 0)
     start_line, end_line = location
+    # Extend vector with parent path length and a hash of the path for uniqueness
+    parent_depth = len(parent_path)
+    parent_hash = hash(' -> '.join(parent_path)) % 1000  # Simple hash for vector
     return [
         category_id,
         level,
-        start_line / total_lines,  # Normalized start position
-        end_line / total_lines     # Normalized end position
+        start_line / total_lines,
+        end_line / total_lines,
+        parent_depth,
+        parent_hash / 1000  # Normalized hash (0.0 to 1.0)
     ]
 
-def parse_node(node, lines, prev_end, level=0, total_lines=None):
-    """Recursively parse an AST node and its children, assigning hierarchy levels."""
+def parse_node(node, lines, prev_end, level=0, total_lines=None, parent_path=None, counters=None):
+    """Recursively parse an AST node with full hierarchy tracking."""
     if total_lines is None:
         total_lines = len(lines)
+    if parent_path is None:
+        parent_path = []
+    if counters is None:
+        counters = {'if': 0, 'while': 0, 'for': 0, 'function': 0, 'class': 0}  # Track counts per type
 
     parts = []
     start_line = getattr(node, 'lineno', prev_end + 1)
     end_line = getattr(node, 'end_lineno', start_line)
+    category = get_category(node)
+
+    # Assign a node ID for relevant categories
+    node_id = ''
+    if category in counters:
+        counters[category] += 1
+        node_id = f"{category.capitalize()}[{counters[category]}]"
 
     # Handle spacers before the node
     if start_line > prev_end + 1:
         spacer_lines = lines[prev_end:start_line - 1]
-        spacer_vector = create_vector('spacer', level, (prev_end + 1, start_line - 1), total_lines)
+        spacer_vector = create_vector('spacer', level, (prev_end + 1, start_line - 1), total_lines, parent_path)
         parts.append({
             'category': 'spacer',
             'source': ''.join(spacer_lines),
             'location': (prev_end + 1, start_line - 1),
             'level': level,
-            'vector': spacer_vector
+            'vector': spacer_vector,
+            'parent_path': ' -> '.join(parent_path) if parent_path else 'Top-Level',
+            'node_id': ''
         })
 
     # Capture the node's source
     stmt_lines = lines[start_line - 1:end_line]
-    node_vector = create_vector(get_category(node), level, (start_line, end_line), total_lines)
+    current_path = parent_path + ([node_id] if node_id else [])
+    node_vector = create_vector(category, level, (start_line, end_line), total_lines, current_path)
     parts.append({
-        'category': get_category(node),
+        'category': category,
         'source': ''.join(stmt_lines),
         'location': (start_line, end_line),
         'level': level,
-        'vector': node_vector
+        'vector': node_vector,
+        'parent_path': ' -> '.join(parent_path) if parent_path else 'Top-Level',
+        'node_id': node_id
     })
 
-    # Process nested nodes (e.g., class/function bodies, control structures)
-    if hasattr(node, 'body'):
-        nested_prev_end = end_line - 1
-        for child in node.body:
-            child_parts = parse_node(child, lines, nested_prev_end, level + 1, total_lines)
-            parts.extend(child_parts)
-            nested_prev_end = child_parts[-1]['location'][1]
-
-    # Handle additional bodies (e.g., else, elif, orelse for loops)
-    if hasattr(node, 'orelse') and node.orelse:
-        orelse_prev_end = parts[-1]['location'][1]
-        for child in node.orelse:
-            child_parts = parse_node(child, lines, orelse_prev_end, level + 1, total_lines)
-            parts.extend(child_parts)
-            orelse_prev_end = child_parts[-1]['location'][1]
+    # Process nested nodes (body and orelse)
+    nested_prev_end = end_line - 1
+    for attr in ('body', 'orelse'):
+        if hasattr(node, attr) and getattr(node, attr):
+            for child in getattr(node, attr):
+                child_parts = parse_node(child, lines, nested_prev_end, level + 1, total_lines, current_path, counters)
+                parts.extend(child_parts)
+                nested_prev_end = child_parts[-1]['location'][1]
 
     return parts
 
@@ -97,7 +110,7 @@ def parse_python_code(code):
     try:
         tree = ast.parse(code)
     except SyntaxError:
-        return [{'category': 'error', 'source': 'Invalid Python code', 'location': (1, 1), 'level': 0, 'vector': [0, 0, 1.0, 1.0]}]
+        return [{'category': 'error', 'source': 'Invalid Python code', 'location': (1, 1), 'level': 0, 'vector': [0, 0, 1.0, 1.0, 0, 0], 'parent_path': 'Top-Level', 'node_id': ''}]
 
     parts = []
     prev_end = 0
@@ -107,16 +120,17 @@ def parse_python_code(code):
         parts.extend(stmt_parts)
         prev_end = stmt_parts[-1]['location'][1]
 
-    # Capture trailing spacers
     if prev_end < total_lines:
         remaining_lines = lines[prev_end:]
-        spacer_vector = create_vector('spacer', 0, (prev_end + 1, total_lines + 1), total_lines)
+        spacer_vector = create_vector('spacer', 0, (prev_end + 1, total_lines + 1), total_lines, [])
         parts.append({
             'category': 'spacer',
             'source': ''.join(remaining_lines),
             'location': (prev_end + 1, total_lines + 1),
             'level': 0,
-            'vector': spacer_vector
+            'vector': spacer_vector,
+            'parent_path': 'Top-Level',
+            'node_id': ''
         })
 
     return parts