from typing import List from dataclasses import dataclass @dataclass class Segment: segment_id: str start_line: int end_line: int start_byte: int end_byte: int content: str symbolic_lane: int = 1 # Default lane class SourceSegmenter: def __init__(self, code: str): self.code = code self.lines = code.split('\n') @staticmethod def segment(code: str) -> List[Segment]: segmenter = SourceSegmenter(code) return segmenter._find_segments() def _find_segments(self) -> List[Segment]: boundaries = self._find_boundaries() segments = [] for i, (start, end) in enumerate(boundaries): segment_id = f"seg_{i}" start_byte = self._line_offset(start) end_byte = self._line_offset(end) content = '\n'.join(self.lines[start:end]) # Analyze content to determine lane symbolic_lane = self._infer_lane_from_content(content, i) segments.append(Segment( segment_id=segment_id, start_line=start, end_line=end, start_byte=start_byte, end_byte=end_byte, content=content, symbolic_lane=symbolic_lane )) return segments def _infer_lane_from_content(self, content: str, index: int) -> int: """Infer lane from content analysis (not just ID patterns). Returns lane 0-7 based on content characteristics. """ lines = content.split('\n') stripped_lines = [l.strip() for l in lines if l.strip()] if not stripped_lines: return 1 # Default semantic flow # Check for structural markers (lane 0) has_control_flow = any( any(x in l for x in ['if ', 'for ', 'while ', 'return ', 'try:', 'except', 'with ']) for l in stripped_lines ) if has_control_flow: return 0 # Check for comments/annotations (lane 3) has_comments = any( l.startswith('#') or l.startswith('//') or '/*' in l or '*/' in l for l in lines ) if has_comments: return 3 # Check for execution hints (lane 4) has_hints = any( any(x in l.lower() for x in ['hint', 'note', 'todo', 'fixme', 'warning', 'danger']) for l in lines ) if has_hints: return 4 # Check for metadata/annotations (lane 3) has_metadata = any( any(x in l.lower() for x in [' int: total = 0 for i in range(min(line_num, len(self.lines))): total += len(self.lines[i]) + 1 return total def _find_boundaries(self) -> List[tuple]: boundaries = [] start = 0 # First pass: find def/class boundaries (code structure) code_boundaries = [] for i, line in enumerate(self.lines): stripped = line.lstrip() if stripped.startswith('def ') or stripped.startswith('class '): if start < i and any(self.lines[start:i]): code_boundaries.append((start, i)) start = i if start < len(self.lines): code_boundaries.append((start, len(self.lines))) # Use code boundaries if present if code_boundaries: return code_boundaries # For non-code content, split by content type changes OR blank lines current_start = 0 current_type = None blank_count = 0 for i, line in enumerate(self.lines): stripped = line.lstrip() # Track blank lines if not stripped: blank_count += 1 continue # Detect content type if stripped.startswith('#') or stripped.startswith('//'): new_type = 'comment' elif any(x in stripped.lower() for x in ['step ', 'todo:', 'placeholder', 'outline']): new_type = 'template' elif any(x in stripped.lower() for x in ['author', 'contributor', 'copyright', '@']): new_type = 'author' elif any(x in stripped.lower() for x in ['epoch', 'timestamp', 'date', 'version']): new_type = 'epoch' elif any(x in stripped for x in ['if ', 'for ', 'while ', 'return ', 'try:', 'except', 'with ']): new_type = 'control' else: new_type = 'content' # Split on blank line (2+ consecutive) or type change if blank_count >= 2 or (current_type is not None and new_type != current_type): if current_start < i - blank_count: boundaries.append((current_start, i - blank_count)) current_start = i - blank_count if blank_count > 0 else i current_type = new_type blank_count = 0 # If no blank line but type changed, split here if blank_count == 0 and current_type is not None and new_type != current_type: boundaries.append((current_start, i)) current_start = i current_type = new_type # Final boundary if current_start < len(self.lines): boundaries.append((current_start, len(self.lines))) return boundaries if boundaries else [(0, len(self.lines))]