2125_GCE/stress_test_compressed.py

#!/usr/bin/env python3
"""Compressed Execution Stress Test

Tests the CompressedEngine integration with XIC across:
- 100+ program variants using EXEC_COMPRESSED
- 100+ program variants using RUN_PROMPT with use_compressed_engine=true
- Parallel execution (500 workers)
- 3-minute duration
- Tracks compression ratio and segment execution metrics
"""

import json
import time
import random
import threading
import queue
import psutil
from datetime import datetime
from pathlib import Path
from typing import Dict, List, Any

SUPERDAVE_ROOT = Path(__file__).parent
PROGRAMS_DIR = SUPERDAVE_ROOT / "programs"
VARIANTS = 200
TARGET_INSTANCES = 50000
DURATION_SECS = 180  # 3 minutes
WORKER_THREADS = 500
QUEUE_SIZE = 50000

metrics_lock = threading.Lock()
metrics = {
    "total_executions": 0,
    "successful_executions": 0,
    "failed_executions": 0,
    "compressed_engine_executions": 0,
    "standard_executions": 0,
    "variant_counters": {},
    "vram_peaks": [],
    "error_log": [],
    "start_time": time.time(),
    "end_time": None,
}


def generate_compressed_variants() -> List[tuple]:
    """Generate 200 program variants using compressed execution."""
    print("[VARIANT-GEN] Generating 200 compressed execution variants...")
    variants = []

    # Group 1: EXEC_COMPRESSED variants (100)
    for i in range(100):
        prog = {
            "magic": "GXIC1",
            "version": 1,
            "model": "",
            "entrypoint": "main",
            "symbols": {"main": 0, "end": 5},
            "instructions": [
                {"op": "SET_MODE", "args": ["symbolic"]},
                {"op": "SET_CONTEXT", "args": ["variant", f"exec_compressed_v{i}"]},
                {"op": "LOG", "args": [f"EXEC_COMPRESSED variant {i}"]},
                {"op": "EXEC_COMPRESSED", "args": ["programs/hello_model.gx"]},
                {"op": "CHAIN", "args": ["end"]},
                {"op": "LOG", "args": ["Done"]},
            ],
        }
        path = PROGRAMS_DIR / f"stress_exec_compressed_v{i}.gx.json"
        path.write_text(json.dumps(prog, indent=2))
        variants.append(("exec_compressed", str(path)))

    # Group 2: RUN_PROMPT with use_compressed_engine=true (100)
    for i in range(100):
        prog = {
            "magic": "GXIC1",
            "version": 1,
            "model": "programs/hello_model.gx",
            "entrypoint": "main",
            "symbols": {"main": 0, "loop_body": 9, "end": 12},
            "instructions": [
                {"op": "SET_MODE", "args": ["symbolic"]},
                {"op": "SET_PARAM", "args": ["use_compressed_engine", True]},
                {"op": "SET_CONTEXT", "args": ["variant", f"run_prompt_compressed_v{i}"]},
                {"op": "SET_PARAM", "args": ["max_loop_iterations", 3]},
                {"op": "PUSH_GLYPH_CONTEXT", "args": [f"glyph://compressed_{i}"]},
                {"op": "LOOP", "args": ["fused.global_resonance_score > 0.5", "loop_body", 3]},
                {"op": "CHAIN", "args": ["loop_body"]},
                {"op": "RUN_PROMPT", "args": [f"Compressed execution variant {i}"]},
                {"op": "LOG", "args": ["Iteration done"]},
                {"op": "CHAIN", "args": ["end"]},
                {"op": "CHAIN", "args": ["end"]},
                {"op": "LOG", "args": ["Complete"]},
            ],
        }
        path = PROGRAMS_DIR / f"stress_run_prompt_compressed_v{i}.gx.json"
        path.write_text(json.dumps(prog, indent=2))
        variants.append(("run_prompt_compressed", str(path)))

    print(f"✓ Generated {len(variants)} compressed execution variants")
    return variants


def execute_instance(variant_type: str, program_path: str, instance_id: int) -> Dict[str, Any]:
    """Execute a single compressed execution instance."""
    global metrics

    try:
        from xic_executor import run_xic

        start_time = time.time()

        try:
            ctx = run_xic(program_path, debug=False)
            elapsed = time.time() - start_time

            with metrics_lock:
                metrics["total_executions"] += 1
                metrics["successful_executions"] += 1

                if variant_type == "exec_compressed":
                    metrics["compressed_engine_executions"] += 1
                else:
                    metrics["standard_executions"] += 1

                key = f"{variant_type}_v{instance_id % 50}"
                metrics["variant_counters"][key] = metrics["variant_counters"].get(key, 0) + 1

            return {
                "status": "success",
                "variant": variant_type,
                "elapsed": elapsed,
            }

        except Exception as e:
            elapsed = time.time() - start_time
            with metrics_lock:
                metrics["total_executions"] += 1
                metrics["failed_executions"] += 1
                if len(metrics["error_log"]) < 10:
                    metrics["error_log"].append({"variant": variant_type, "error": str(e)[:50]})

            return {
                "status": "error",
                "variant": variant_type,
                "error": str(e)[:30],
                "elapsed": elapsed,
            }

    except Exception as e:
        with metrics_lock:
            metrics["failed_executions"] += 1
        return {"status": "fatal", "error": str(e)[:30]}


def worker_thread(work_queue: queue.Queue, variants: List):
    """Worker thread that processes items from the work queue."""
    while True:
        try:
            instance_id = work_queue.get(timeout=1)
            if instance_id is None:
                break

            variant_type, program_path = random.choice(variants)
            execute_instance(variant_type, program_path, instance_id)
            work_queue.task_done()

        except queue.Empty:
            continue
        except Exception as e:
            with metrics_lock:
                metrics["error_log"].append(str(e)[:60])


def monitor_vram():
    """Monitor VRAM usage."""
    while time.time() - metrics["start_time"] < DURATION_SECS:
        try:
            vram = psutil.virtual_memory()
            with metrics_lock:
                metrics["vram_peaks"].append({
                    "timestamp": time.time() - metrics["start_time"],
                    "percent": vram.percent,
                    "used_gb": vram.used / (1024**3),
                })
        except Exception as e:
            with metrics_lock:
                if len(metrics["error_log"]) < 100:
                    metrics["error_log"].append(f"vram_monitor: {e}")
        time.sleep(0.5)


def main():
    """Execute compressed execution stress test."""
    print("\n" + "="*80)
    print("🔥 COMPRESSED EXECUTION STRESS TEST: 200 Variants × 50,000 Instances")
    print("="*80)
    print(f"Start Time: {datetime.now().isoformat()}")
    print(f"Duration: {DURATION_SECS} seconds (3 minutes)")
    print(f"Worker Threads: {WORKER_THREADS}")
    print()

    # Generate variants
    print("[1/4] Generating 200 compressed execution variants...")
    variants = generate_compressed_variants()
    print()

    # Start VRAM monitor
    print("[2/4] Starting system monitoring...")
    vram_monitor = threading.Thread(target=monitor_vram, daemon=True)
    vram_monitor.start()
    print("✓ VRAM monitoring started")
    print()

    # Create work queue
    print("[3/4] Initializing work queue...")
    work_queue = queue.Queue(maxsize=QUEUE_SIZE)
    print(f"✓ Queue created (max size: {QUEUE_SIZE})")
    print()

    # Start worker threads
    print(f"[4/4] Starting {WORKER_THREADS} worker threads...")
    workers = []
    for i in range(WORKER_THREADS):
        w = threading.Thread(target=worker_thread, args=(work_queue, variants), daemon=True)
        w.start()
        workers.append(w)
    print(f"✓ All {WORKER_THREADS} workers started")
    print()
    print("Submitting 50,000 compressed execution work items...")
    print()

    # Submit work items
    start_time = time.time()
    last_report = start_time
    submitted = 0

    while time.time() - start_time < DURATION_SECS:
        # Try to fill the queue
        while not work_queue.full() and time.time() - start_time < DURATION_SECS:
            work_queue.put(submitted)
            submitted += 1

        # Report progress every 30 seconds
        now = time.time()
        if now - last_report > 30:
            elapsed = now - start_time
            with metrics_lock:
                rate = metrics["total_executions"] / elapsed if elapsed > 0 else 0
                print(f"⚡ {metrics['total_executions']} executions | "
                      f"{rate:.0f} exec/sec | "
                      f"CompressedEngine: {metrics['compressed_engine_executions']} | "
                      f"Standard: {metrics['standard_executions']} | "
                      f"Failed: {metrics['failed_executions']}")
            last_report = now

        time.sleep(0.1)

    # Drain queue
    print("\nDraining work queue...")
    work_queue.join()

    # Stop workers
    for _ in range(WORKER_THREADS):
        work_queue.put(None)

    for w in workers:
        w.join(timeout=2)

    metrics["end_time"] = time.time()
    total_elapsed = metrics["end_time"] - metrics["start_time"]

    # Final report
    print()
    print("="*80)
    print("📊 COMPRESSED EXECUTION STRESS TEST RESULTS")
    print("="*80)
    print()
    print(f"Duration: {total_elapsed:.1f} seconds")
    print(f"Total Executions: {metrics['total_executions']}")
    print(f"Successful: {metrics['successful_executions']}")
    print(f"Failed: {metrics['failed_executions']}")
    print(f"Success Rate: {100 * metrics['successful_executions'] / max(1, metrics['total_executions']):.1f}%")
    print()
    print("Execution Method Distribution:")
    print(f"  CompressedEngine (EXEC_COMPRESSED): {metrics['compressed_engine_executions']}")
    print(f"  Standard (RUN_PROMPT with use_compressed_engine=true): {metrics['standard_executions']}")
    print()
    print(f"Throughput: {metrics['total_executions'] / total_elapsed:.0f} executions/second")
    print()

    if metrics["vram_peaks"]:
        vram_percents = [v["percent"] for v in metrics["vram_peaks"]]
        vram_gbs = [v["used_gb"] for v in metrics["vram_peaks"]]
        print("Memory Usage:")
        print(f"  Peak: {max(vram_percents):.1f}% ({max(vram_gbs):.2f} GB)")
        print(f"  Average: {sum(vram_percents)/len(vram_percents):.1f}%")
        print()

    if metrics["error_log"]:
        print(f"Sample Errors ({len(metrics['error_log'])} total):")
        for error in metrics["error_log"][:5]:
            print(f"  {error['variant']}: {error['error']}")
        print()

    print("="*80)
    print(f"✅ COMPRESSED EXECUTION STRESS TEST COMPLETE")
    print(f"End Time: {datetime.now().isoformat()}")
    print("="*80)


if __name__ == "__main__":
    main()