Initial commit: 2125_GCE project
This commit is contained in:
GlyphRunner System
Executable
+219
@@ -0,0 +1,219 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Glyphrunner Benchmark: XIC Symbolic Execution
|
||||
|
||||
Executes symbolic workload via XIC.
|
||||
Measures throughput of symbolic execution with control flow.
|
||||
"""
|
||||
|
||||
import json
|
||||
import time
|
||||
import random
|
||||
import threading
|
||||
import queue
|
||||
import sys
|
||||
import os
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
|
||||
# Ensure we're in the right directory
|
||||
os.chdir(Path(__file__).parent.parent)
|
||||
|
||||
SUPERDAVE_ROOT = Path.cwd()
|
||||
PROGRAMS_DIR = SUPERDAVE_ROOT / "programs"
|
||||
|
||||
# Configuration
|
||||
WORKER_THREADS = 500
|
||||
QUEUE_SIZE = 5000
|
||||
|
||||
metrics = {
|
||||
"total_executions": 0,
|
||||
"successful_executions": 0,
|
||||
"failed_executions": 0,
|
||||
"start_time": time.time(),
|
||||
"end_time": None,
|
||||
}
|
||||
metrics_lock = threading.Lock()
|
||||
|
||||
|
||||
def generate_benchmark_variants(count: int = 50) -> list:
|
||||
"""Generate XIC programs that implement the symbolic workload."""
|
||||
variants = []
|
||||
|
||||
for i in range(count):
|
||||
# Symbolic execution variant with control flow
|
||||
prog = {
|
||||
"magic": "GXIC1",
|
||||
"version": 1,
|
||||
"model": "",
|
||||
"entrypoint": "main",
|
||||
"symbols": {"main": 0, "end": 5},
|
||||
"instructions": [
|
||||
{"op": "SET_MODE", "args": ["symbolic"]},
|
||||
{"op": "SET_CONTEXT", "args": ["variant", f"bench_{i}"]},
|
||||
{"op": "PUSH_GLYPH_CONTEXT", "args": ["glyph://benchmark"]},
|
||||
{"op": "RUN_PROMPT", "args": ["Execute symbolic workload"]},
|
||||
{"op": "CHAIN", "args": ["end"]},
|
||||
{"op": "LOG", "args": ["Done"]},
|
||||
],
|
||||
}
|
||||
path = PROGRAMS_DIR / f"bench_glyph_v{i}.gx.json"
|
||||
path.write_text(json.dumps(prog, indent=2))
|
||||
variants.append(("benchmark", str(path)))
|
||||
|
||||
return variants
|
||||
|
||||
|
||||
def execute_instance(program_path: str, instance_id: int) -> dict:
|
||||
"""Execute a single XIC program."""
|
||||
global metrics
|
||||
|
||||
try:
|
||||
from xic_executor import run_xic
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
ctx = run_xic(program_path, debug=False)
|
||||
elapsed = time.time() - start_time
|
||||
|
||||
with metrics_lock:
|
||||
metrics["total_executions"] += 1
|
||||
metrics["successful_executions"] += 1
|
||||
|
||||
return {"status": "success", "elapsed": elapsed}
|
||||
|
||||
except Exception as e:
|
||||
elapsed = time.time() - start_time
|
||||
with metrics_lock:
|
||||
metrics["total_executions"] += 1
|
||||
metrics["failed_executions"] += 1
|
||||
|
||||
return {"status": "error", "error": str(e)[:50], "elapsed": elapsed}
|
||||
|
||||
except Exception as e:
|
||||
with metrics_lock:
|
||||
metrics["failed_executions"] += 1
|
||||
return {"status": "fatal", "error": str(e)[:30]}
|
||||
|
||||
|
||||
def worker_thread(work_queue: queue.Queue, variants: list):
|
||||
"""Worker thread that processes items from the work queue."""
|
||||
while True:
|
||||
try:
|
||||
item = work_queue.get(timeout=1)
|
||||
if item is None:
|
||||
break
|
||||
|
||||
_, program_path = random.choice(variants)
|
||||
execute_instance(program_path, 0)
|
||||
work_queue.task_done()
|
||||
|
||||
except queue.Empty:
|
||||
continue
|
||||
except Exception as e:
|
||||
with metrics_lock:
|
||||
metrics["error_count"] = metrics.get("error_count", 0) + 1
|
||||
if len(metrics.get("error_log", [])) < 100:
|
||||
if "error_log" not in metrics:
|
||||
metrics["error_log"] = []
|
||||
metrics["error_log"].append(f"worker: {e}")
|
||||
|
||||
|
||||
def main():
|
||||
"""Run Glyphrunner benchmark."""
|
||||
duration = int(sys.argv[1]) if len(sys.argv) > 1 else 60
|
||||
instances = int(sys.argv[2]) if len(sys.argv) > 2 else 5000
|
||||
|
||||
print("\n" + "="*60)
|
||||
print("GLYPHRUNNER BENCHMARK: XIC Symbolic Execution")
|
||||
print("="*60)
|
||||
print(f"Start Time: {datetime.now().isoformat()}")
|
||||
print(f"Duration: {duration} seconds")
|
||||
print(f"Target Instances: {instances}")
|
||||
print(f"Worker Threads: {WORKER_THREADS}")
|
||||
print()
|
||||
|
||||
# Generate variants
|
||||
print("[1/3] Generating benchmark variants...")
|
||||
variants = generate_benchmark_variants(50)
|
||||
print(f"✓ Generated {len(variants)} program variants")
|
||||
print()
|
||||
|
||||
# Create work queue
|
||||
print("[2/3] Initializing work queue...")
|
||||
work_queue = queue.Queue(maxsize=QUEUE_SIZE)
|
||||
print(f"✓ Queue created (max size: {QUEUE_SIZE})")
|
||||
print()
|
||||
|
||||
# Start worker threads
|
||||
print(f"[3/3] Starting {WORKER_THREADS} worker threads...")
|
||||
workers = []
|
||||
for i in range(WORKER_THREADS):
|
||||
w = threading.Thread(target=worker_thread, args=(work_queue, variants), daemon=True)
|
||||
w.start()
|
||||
workers.append(w)
|
||||
print(f"✓ All {WORKER_THREADS} workers started")
|
||||
print()
|
||||
print("Submitting work items...")
|
||||
print()
|
||||
|
||||
# Submit work items
|
||||
start_time = time.time()
|
||||
last_report = start_time
|
||||
submitted = 0
|
||||
|
||||
while time.time() - start_time < duration:
|
||||
# Fill the queue
|
||||
while not work_queue.full() and time.time() - start_time < duration:
|
||||
work_queue.put(submitted)
|
||||
submitted += 1
|
||||
|
||||
# Report progress
|
||||
now = time.time()
|
||||
if now - last_report > 10:
|
||||
elapsed = now - start_time
|
||||
with metrics_lock:
|
||||
rate = metrics["total_executions"] / elapsed if elapsed > 0 else 0
|
||||
print(f"⚡ {metrics['total_executions']} executions | "
|
||||
f"{rate:.1f} exec/sec | "
|
||||
f"{metrics['successful_executions']} success | "
|
||||
f"{metrics['failed_executions']} failed")
|
||||
last_report = now
|
||||
|
||||
time.sleep(0.1)
|
||||
|
||||
# Drain queue
|
||||
print("\nDraining work queue...")
|
||||
work_queue.join()
|
||||
|
||||
# Stop workers
|
||||
for _ in range(WORKER_THREADS):
|
||||
work_queue.put(None)
|
||||
|
||||
for w in workers:
|
||||
w.join(timeout=2)
|
||||
|
||||
metrics["end_time"] = time.time()
|
||||
total_elapsed = metrics["end_time"] - metrics["start_time"]
|
||||
|
||||
# Final report
|
||||
print()
|
||||
print("="*60)
|
||||
print("GLYPHRUNNER BENCHMARK RESULTS")
|
||||
print("="*60)
|
||||
print()
|
||||
print(f"Duration: {total_elapsed:.1f} seconds")
|
||||
print(f"Total Executions: {metrics['total_executions']}")
|
||||
print(f"Successful: {metrics['successful_executions']}")
|
||||
print(f"Failed: {metrics['failed_executions']}")
|
||||
success_rate = 100 * metrics['successful_executions'] / max(1, metrics['total_executions'])
|
||||
print(f"Success Rate: {success_rate:.1f}%")
|
||||
print()
|
||||
throughput = metrics['total_executions'] / total_elapsed if total_elapsed > 0 else 0
|
||||
print(f"Throughput: {throughput:.1f} executions/second")
|
||||
print()
|
||||
print("="*60)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user