2125_GCE/benchmark/glyphrunner_bench.py

#!/usr/bin/env python3
"""Glyphrunner Benchmark: XIC Symbolic Execution

Executes symbolic workload via XIC.
Measures throughput of symbolic execution with control flow.
"""

import json
import time
import random
import threading
import queue
import sys
import os
from pathlib import Path
from datetime import datetime

# Ensure we're in the right directory
os.chdir(Path(__file__).parent.parent)

SUPERDAVE_ROOT = Path.cwd()
PROGRAMS_DIR = SUPERDAVE_ROOT / "programs"

# Configuration
WORKER_THREADS = 500
QUEUE_SIZE = 5000

metrics = {
    "total_executions": 0,
    "successful_executions": 0,
    "failed_executions": 0,
    "start_time": time.time(),
    "end_time": None,
}
metrics_lock = threading.Lock()


def generate_benchmark_variants(count: int = 50) -> list:
    """Generate XIC programs that implement the symbolic workload."""
    variants = []

    for i in range(count):
        # Symbolic execution variant with control flow
        prog = {
            "magic": "GXIC1",
            "version": 1,
            "model": "",
            "entrypoint": "main",
            "symbols": {"main": 0, "end": 5},
            "instructions": [
                {"op": "SET_MODE", "args": ["symbolic"]},
                {"op": "SET_CONTEXT", "args": ["variant", f"bench_{i}"]},
                {"op": "PUSH_GLYPH_CONTEXT", "args": ["glyph://benchmark"]},
                {"op": "RUN_PROMPT", "args": ["Execute symbolic workload"]},
                {"op": "CHAIN", "args": ["end"]},
                {"op": "LOG", "args": ["Done"]},
            ],
        }
        path = PROGRAMS_DIR / f"bench_glyph_v{i}.gx.json"
        path.write_text(json.dumps(prog, indent=2))
        variants.append(("benchmark", str(path)))

    return variants


def execute_instance(program_path: str, instance_id: int) -> dict:
    """Execute a single XIC program."""
    global metrics

    try:
        from xic_executor import run_xic

        start_time = time.time()

        try:
            ctx = run_xic(program_path, debug=False)
            elapsed = time.time() - start_time

            with metrics_lock:
                metrics["total_executions"] += 1
                metrics["successful_executions"] += 1

            return {"status": "success", "elapsed": elapsed}

        except Exception as e:
            elapsed = time.time() - start_time
            with metrics_lock:
                metrics["total_executions"] += 1
                metrics["failed_executions"] += 1

            return {"status": "error", "error": str(e)[:50], "elapsed": elapsed}

    except Exception as e:
        with metrics_lock:
            metrics["failed_executions"] += 1
        return {"status": "fatal", "error": str(e)[:30]}


def worker_thread(work_queue: queue.Queue, variants: list):
    """Worker thread that processes items from the work queue."""
    while True:
        try:
            item = work_queue.get(timeout=1)
            if item is None:
                break

            _, program_path = random.choice(variants)
            execute_instance(program_path, 0)
            work_queue.task_done()

        except queue.Empty:
            continue
        except Exception as e:
            with metrics_lock:
                metrics["error_count"] = metrics.get("error_count", 0) + 1
                if len(metrics.get("error_log", [])) < 100:
                    if "error_log" not in metrics:
                        metrics["error_log"] = []
                    metrics["error_log"].append(f"worker: {e}")


def main():
    """Run Glyphrunner benchmark."""
    duration = int(sys.argv[1]) if len(sys.argv) > 1 else 60
    instances = int(sys.argv[2]) if len(sys.argv) > 2 else 5000

    print("\n" + "="*60)
    print("GLYPHRUNNER BENCHMARK: XIC Symbolic Execution")
    print("="*60)
    print(f"Start Time: {datetime.now().isoformat()}")
    print(f"Duration: {duration} seconds")
    print(f"Target Instances: {instances}")
    print(f"Worker Threads: {WORKER_THREADS}")
    print()

    # Generate variants
    print("[1/3] Generating benchmark variants...")
    variants = generate_benchmark_variants(50)
    print(f"✓ Generated {len(variants)} program variants")
    print()

    # Create work queue
    print("[2/3] Initializing work queue...")
    work_queue = queue.Queue(maxsize=QUEUE_SIZE)
    print(f"✓ Queue created (max size: {QUEUE_SIZE})")
    print()

    # Start worker threads
    print(f"[3/3] Starting {WORKER_THREADS} worker threads...")
    workers = []
    for i in range(WORKER_THREADS):
        w = threading.Thread(target=worker_thread, args=(work_queue, variants), daemon=True)
        w.start()
        workers.append(w)
    print(f"✓ All {WORKER_THREADS} workers started")
    print()
    print("Submitting work items...")
    print()

    # Submit work items
    start_time = time.time()
    last_report = start_time
    submitted = 0

    while time.time() - start_time < duration:
        # Fill the queue
        while not work_queue.full() and time.time() - start_time < duration:
            work_queue.put(submitted)
            submitted += 1

        # Report progress
        now = time.time()
        if now - last_report > 10:
            elapsed = now - start_time
            with metrics_lock:
                rate = metrics["total_executions"] / elapsed if elapsed > 0 else 0
                print(f"⚡ {metrics['total_executions']} executions | "
                      f"{rate:.1f} exec/sec | "
                      f"{metrics['successful_executions']} success | "
                      f"{metrics['failed_executions']} failed")
            last_report = now

        time.sleep(0.1)

    # Drain queue
    print("\nDraining work queue...")
    work_queue.join()

    # Stop workers
    for _ in range(WORKER_THREADS):
        work_queue.put(None)

    for w in workers:
        w.join(timeout=2)

    metrics["end_time"] = time.time()
    total_elapsed = metrics["end_time"] - metrics["start_time"]

    # Final report
    print()
    print("="*60)
    print("GLYPHRUNNER BENCHMARK RESULTS")
    print("="*60)
    print()
    print(f"Duration: {total_elapsed:.1f} seconds")
    print(f"Total Executions: {metrics['total_executions']}")
    print(f"Successful: {metrics['successful_executions']}")
    print(f"Failed: {metrics['failed_executions']}")
    success_rate = 100 * metrics['successful_executions'] / max(1, metrics['total_executions'])
    print(f"Success Rate: {success_rate:.1f}%")
    print()
    throughput = metrics['total_executions'] / total_elapsed if total_elapsed > 0 else 0
    print(f"Throughput: {throughput:.1f} executions/second")
    print()
    print("="*60)


if __name__ == "__main__":
    main()