Initial commit: 2125_GCE project

gx_compiler/segmenter.py

@@ -10,6 +10,7 @@ class Segment:
     start_byte: int
     end_byte: int
     content: str
+    symbolic_lane: int = 1  # Default lane
 
 
 class SourceSegmenter:
@@ -31,17 +32,99 @@ class SourceSegmenter:
             start_byte = self._line_offset(start)
             end_byte = self._line_offset(end)
             content = '\n'.join(self.lines[start:end])
-
+            
+            # Analyze content to determine lane
+            symbolic_lane = self._infer_lane_from_content(content, i)
+            
             segments.append(Segment(
                 segment_id=segment_id,
                 start_line=start,
                 end_line=end,
                 start_byte=start_byte,
                 end_byte=end_byte,
-                content=content
+                content=content,
+                symbolic_lane=symbolic_lane
             ))
 
         return segments
+    
+    def _infer_lane_from_content(self, content: str, index: int) -> int:
+        """Infer lane from content analysis (not just ID patterns).
+        
+        Returns lane 0-7 based on content characteristics.
+        """
+        lines = content.split('\n')
+        stripped_lines = [l.strip() for l in lines if l.strip()]
+        
+        if not stripped_lines:
+            return 1  # Default semantic flow
+        
+        # Check for structural markers (lane 0)
+        has_control_flow = any(
+            any(x in l for x in ['if ', 'for ', 'while ', 'return ', 'try:', 'except', 'with '])
+            for l in stripped_lines
+        )
+        if has_control_flow:
+            return 0
+        
+        # Check for comments/annotations (lane 3)
+        has_comments = any(
+            l.startswith('#') or l.startswith('//') or '/*' in l or '*/' in l
+            for l in lines
+        )
+        if has_comments:
+            return 3
+        
+        # Check for execution hints (lane 4)
+        has_hints = any(
+            any(x in l.lower() for x in ['hint', 'note', 'todo', 'fixme', 'warning', 'danger'])
+            for l in lines
+        )
+        if has_hints:
+            return 4
+        
+        # Check for metadata/annotations (lane 3)
+        has_metadata = any(
+            any(x in l.lower() for x in ['<glyph:', 'metadata', 'tag', 'annotation', 'symbolic'])
+            for l in lines
+        )
+        if has_metadata:
+            return 3
+        
+        # Check for execution hints (dangerous commands) (lane 4)
+        has_execution_hints = any(
+            any(x in l for x in ['rm -rf', 'del ', 'os.system', 'subprocess', 'exec(', 'eval('])
+            for l in lines
+        )
+        if has_execution_hints:
+            return 4
+        
+        # Check for outlines/templates (lane 5)
+        has_template = any(
+            any(x in l.lower() for x in ['step ', 'todo:', 'placeholder', 'fill-in', 'template', 'outline'])
+            for l in lines
+        )
+        if has_template:
+            return 5
+        
+        # Check for author/contributor markers (lane 6)
+        has_contributor = any(
+            any(x in l.lower() for x in ['author', 'contributor', 'copyright', '@', 'by '])
+            for l in lines
+        )
+        if has_contributor:
+            return 6
+        
+        # Check for epoch/time markers (lane 7)
+        has_epoch = any(
+            any(x in l.lower() for x in ['epoch', 'timestamp', 'date', 'time', 'version', 'build'])
+            for l in lines
+        )
+        if has_epoch:
+            return 7
+        
+        # Default: semantic flow (lane 1)
+        return 1
 
     def _line_offset(self, line_num: int) -> int:
         total = 0
@@ -52,16 +135,65 @@ class SourceSegmenter:
     def _find_boundaries(self) -> List[tuple]:
         boundaries = []
         start = 0
-
+        
+        # First pass: find def/class boundaries (code structure)
+        code_boundaries = []
         for i, line in enumerate(self.lines):
             stripped = line.lstrip()
-
             if stripped.startswith('def ') or stripped.startswith('class '):
                 if start < i and any(self.lines[start:i]):
-                    boundaries.append((start, i))
+                    code_boundaries.append((start, i))
                 start = i
-
         if start < len(self.lines):
-            boundaries.append((start, len(self.lines)))
-
+            code_boundaries.append((start, len(self.lines)))
+        
+        # Use code boundaries if present
+        if code_boundaries:
+            return code_boundaries
+        
+        # For non-code content, split by content type changes OR blank lines
+        current_start = 0
+        current_type = None
+        blank_count = 0
+        
+        for i, line in enumerate(self.lines):
+            stripped = line.lstrip()
+            
+            # Track blank lines
+            if not stripped:
+                blank_count += 1
+                continue
+            
+            # Detect content type
+            if stripped.startswith('#') or stripped.startswith('//'):
+                new_type = 'comment'
+            elif any(x in stripped.lower() for x in ['step ', 'todo:', 'placeholder', 'outline']):
+                new_type = 'template'
+            elif any(x in stripped.lower() for x in ['author', 'contributor', 'copyright', '@']):
+                new_type = 'author'
+            elif any(x in stripped.lower() for x in ['epoch', 'timestamp', 'date', 'version']):
+                new_type = 'epoch'
+            elif any(x in stripped for x in ['if ', 'for ', 'while ', 'return ', 'try:', 'except', 'with ']):
+                new_type = 'control'
+            else:
+                new_type = 'content'
+            
+            # Split on blank line (2+ consecutive) or type change
+            if blank_count >= 2 or (current_type is not None and new_type != current_type):
+                if current_start < i - blank_count:
+                    boundaries.append((current_start, i - blank_count))
+                current_start = i - blank_count if blank_count > 0 else i
+                current_type = new_type
+                blank_count = 0
+            
+            # If no blank line but type changed, split here
+            if blank_count == 0 and current_type is not None and new_type != current_type:
+                boundaries.append((current_start, i))
+                current_start = i
+                current_type = new_type
+        
+        # Final boundary
+        if current_start < len(self.lines):
+            boundaries.append((current_start, len(self.lines)))
+        
         return boundaries if boundaries else [(0, len(self.lines))]