#!/usr/bin/env python3 """Glyphrunner Benchmark: XIC Symbolic Execution Executes symbolic workload via XIC. Measures throughput of symbolic execution with control flow. """ import json import time import random import threading import queue import sys import os from pathlib import Path from datetime import datetime # Ensure we're in the right directory os.chdir(Path(__file__).parent.parent) SUPERDAVE_ROOT = Path.cwd() PROGRAMS_DIR = SUPERDAVE_ROOT / "programs" # Configuration WORKER_THREADS = 500 QUEUE_SIZE = 5000 metrics = { "total_executions": 0, "successful_executions": 0, "failed_executions": 0, "start_time": time.time(), "end_time": None, } metrics_lock = threading.Lock() def generate_benchmark_variants(count: int = 50) -> list: """Generate XIC programs that implement the symbolic workload.""" variants = [] for i in range(count): # Symbolic execution variant with control flow prog = { "magic": "GXIC1", "version": 1, "model": "", "entrypoint": "main", "symbols": {"main": 0, "end": 5}, "instructions": [ {"op": "SET_MODE", "args": ["symbolic"]}, {"op": "SET_CONTEXT", "args": ["variant", f"bench_{i}"]}, {"op": "PUSH_GLYPH_CONTEXT", "args": ["glyph://benchmark"]}, {"op": "RUN_PROMPT", "args": ["Execute symbolic workload"]}, {"op": "CHAIN", "args": ["end"]}, {"op": "LOG", "args": ["Done"]}, ], } path = PROGRAMS_DIR / f"bench_glyph_v{i}.gx.json" path.write_text(json.dumps(prog, indent=2)) variants.append(("benchmark", str(path))) return variants def execute_instance(program_path: str, instance_id: int) -> dict: """Execute a single XIC program.""" global metrics try: from xic_executor import run_xic start_time = time.time() try: ctx = run_xic(program_path, debug=False) elapsed = time.time() - start_time with metrics_lock: metrics["total_executions"] += 1 metrics["successful_executions"] += 1 return {"status": "success", "elapsed": elapsed} except Exception as e: elapsed = time.time() - start_time with metrics_lock: metrics["total_executions"] += 1 metrics["failed_executions"] += 1 return {"status": "error", "error": str(e)[:50], "elapsed": elapsed} except Exception as e: with metrics_lock: metrics["failed_executions"] += 1 return {"status": "fatal", "error": str(e)[:30]} def worker_thread(work_queue: queue.Queue, variants: list): """Worker thread that processes items from the work queue.""" while True: try: item = work_queue.get(timeout=1) if item is None: break _, program_path = random.choice(variants) execute_instance(program_path, 0) work_queue.task_done() except queue.Empty: continue except Exception as e: with metrics_lock: metrics["error_count"] = metrics.get("error_count", 0) + 1 if len(metrics.get("error_log", [])) < 100: if "error_log" not in metrics: metrics["error_log"] = [] metrics["error_log"].append(f"worker: {e}") def main(): """Run Glyphrunner benchmark.""" duration = int(sys.argv[1]) if len(sys.argv) > 1 else 60 instances = int(sys.argv[2]) if len(sys.argv) > 2 else 5000 print("\n" + "="*60) print("GLYPHRUNNER BENCHMARK: XIC Symbolic Execution") print("="*60) print(f"Start Time: {datetime.now().isoformat()}") print(f"Duration: {duration} seconds") print(f"Target Instances: {instances}") print(f"Worker Threads: {WORKER_THREADS}") print() # Generate variants print("[1/3] Generating benchmark variants...") variants = generate_benchmark_variants(50) print(f"✓ Generated {len(variants)} program variants") print() # Create work queue print("[2/3] Initializing work queue...") work_queue = queue.Queue(maxsize=QUEUE_SIZE) print(f"✓ Queue created (max size: {QUEUE_SIZE})") print() # Start worker threads print(f"[3/3] Starting {WORKER_THREADS} worker threads...") workers = [] for i in range(WORKER_THREADS): w = threading.Thread(target=worker_thread, args=(work_queue, variants), daemon=True) w.start() workers.append(w) print(f"✓ All {WORKER_THREADS} workers started") print() print("Submitting work items...") print() # Submit work items start_time = time.time() last_report = start_time submitted = 0 while time.time() - start_time < duration: # Fill the queue while not work_queue.full() and time.time() - start_time < duration: work_queue.put(submitted) submitted += 1 # Report progress now = time.time() if now - last_report > 10: elapsed = now - start_time with metrics_lock: rate = metrics["total_executions"] / elapsed if elapsed > 0 else 0 print(f"⚡ {metrics['total_executions']} executions | " f"{rate:.1f} exec/sec | " f"{metrics['successful_executions']} success | " f"{metrics['failed_executions']} failed") last_report = now time.sleep(0.1) # Drain queue print("\nDraining work queue...") work_queue.join() # Stop workers for _ in range(WORKER_THREADS): work_queue.put(None) for w in workers: w.join(timeout=2) metrics["end_time"] = time.time() total_elapsed = metrics["end_time"] - metrics["start_time"] # Final report print() print("="*60) print("GLYPHRUNNER BENCHMARK RESULTS") print("="*60) print() print(f"Duration: {total_elapsed:.1f} seconds") print(f"Total Executions: {metrics['total_executions']}") print(f"Successful: {metrics['successful_executions']}") print(f"Failed: {metrics['failed_executions']}") success_rate = 100 * metrics['successful_executions'] / max(1, metrics['total_executions']) print(f"Success Rate: {success_rate:.1f}%") print() throughput = metrics['total_executions'] / total_elapsed if total_elapsed > 0 else 0 print(f"Throughput: {throughput:.1f} executions/second") print() print("="*60) if __name__ == "__main__": main()