#!/usr/bin/env python3 """1 Million Parallel Instance MEGA Stress Test with 10,000 Program Variants Generates 10,000 unique XIC program variants and executes 1,000,000 instances over 5 minutes using a queue-based worker pool approach to avoid OS thread limits. Key optimizations: - Programs generated in-memory (no file I/O overhead) - Queue-based execution (avoids 1M thread creation limit) - Minimal worker threads (500) with high throughput - Real-time metrics collection and display """ import json import time import random import threading import queue import psutil from datetime import datetime from pathlib import Path from typing import Dict, List, Any, Callable # Configuration SUPERDAVE_ROOT = Path(__file__).parent PROGRAM_VARIANTS = 10000 TARGET_INSTANCES = 1000000 DURATION_SECS = 300 # 5 minutes WORKER_THREADS = 500 QUEUE_SIZE = 50000 # Metrics collection metrics_lock = threading.Lock() metrics = { "total_executions": 0, "successful_executions": 0, "failed_executions": 0, "guardrail_triggers": 0, "variant_counters": {}, "vram_peaks": [], "throughput_samples": [], "error_log": [], "start_time": time.time(), "end_time": None, } def generate_program_variants() -> List[Callable]: """Generate 10,000 unique XIC program variants as callable functions.""" print("[VARIANT-GEN] Generating 10,000 program variants in-memory...") variants = [] # Group 1: IF variations (1,250 variants) for i in range(1250): threshold = 0.2 + ((i % 100) * 0.008) # 0.2 to 0.99 branch_count = 2 + (i % 3) # 2-4 branches def make_if_prog(idx=i, thresh=threshold, branches=branch_count): instructions = [ {"op": "SET_MODE", "args": ["symbolic"]}, {"op": "SET_CONTEXT", "args": ["variant", f"if_v{idx}_{thresh:.2f}"]}, ] for j in range(2): instructions.append({"op": "PUSH_GLYPH_CONTEXT", "args": [f"glyph://if_{idx}_{j}"]}) instructions.append({"op": "RUN_PROMPT", "args": [f"IF variant {idx}"]}) instructions.append({"op": "IF", "args": [f"fused.global_resonance_score > {thresh}", f"br_a_{idx}", f"br_b_{idx}"]}) for b in range(branches): instructions.extend([ {"op": "CHAIN", "args": [f"br_a_{idx}"] if b == 0 else {"op": "CHAIN", "args": [f"end_{idx}"]}}, {"op": "LOG", "args": [f"Branch {b}"]}, {"op": "RUN_PROMPT", "args": ["Execute"]}, ]) instructions.extend([ {"op": "CHAIN", "args": [f"br_b_{idx}"]}, {"op": "LOG", "args": ["Alt branch"]}, {"op": "RUN_PROMPT", "args": ["Alt path"]}, {"op": "CHAIN", "args": [f"end_{idx}"]}, {"op": "CHAIN", "args": [f"end_{idx}"]}, {"op": "LOG", "args": ["Complete"]}, ]) return { "magic": "GXIC1", "version": 1, "model": "", "entrypoint": "main", "symbols": {"main": 0, f"br_a_{idx}": 6, f"br_b_{idx}": 12, f"end_{idx}": 16}, "instructions": instructions, } variants.append(("if", make_if_prog)) # Group 2: LOOP variations (1,250 variants) for i in range(1250): max_iter = 2 + (i % 15) # 2-16 iterations loop_id = 1250 + i def make_loop_prog(idx=loop_id, iter_max=max_iter): return { "magic": "GXIC1", "version": 1, "model": "", "entrypoint": "main", "symbols": {"main": 0, "loop_body": 7, "end": 12}, "instructions": [ {"op": "SET_MODE", "args": ["symbolic"]}, {"op": "SET_PARAM", "args": ["max_loop_iterations", iter_max]}, {"op": "SET_CONTEXT", "args": ["variant", f"loop_v{idx}"]}, {"op": "PUSH_GLYPH_CONTEXT", "args": [f"glyph://loop_{idx}"]}, {"op": "LOOP", "args": [f"fused.global_resonance_score > 0.{5 + (idx%4)}", "loop_body", iter_max]}, {"op": "CHAIN", "args": ["loop_body"]}, {"op": "RUN_PROMPT", "args": [f"Loop {idx}"]}, {"op": "GET_GLYPH_RESONANCE", "args": [f"glyph://loop_{idx}", "global"]}, {"op": "LOG", "args": ["Iteration"]}, {"op": "CHAIN", "args": ["end"]}, {"op": "CHAIN", "args": ["end"]}, {"op": "LOG", "args": ["Done"]}, ], } variants.append(("loop", make_loop_prog)) # Group 3: MATCH variations (1,250 variants) for i in range(1250): match_id = 2500 + i pattern_count = 1 + (i % 5) def make_match_prog(idx=match_id, pat_cnt=pattern_count): patterns = [f"glyph://pat_{j}" for j in range(pat_cnt)] return { "magic": "GXIC1", "version": 1, "model": "", "entrypoint": "main", "symbols": {"main": 0, "match_true": 6, "end": 10}, "instructions": [ {"op": "SET_MODE", "args": ["symbolic"]}, {"op": "SET_CONTEXT", "args": ["variant", f"match_v{idx}"]}, {"op": "PUSH_GLYPH_CONTEXT", "args": [f"glyph://match_{idx}"]}, {"op": "RUN_PROMPT", "args": [f"Match {idx}"]}, {"op": "MATCH", "args": ["fused.glyph_ids", patterns[0], "match_true"]}, {"op": "CHAIN", "args": ["match_true"]}, {"op": "LOG", "args": ["Matched"]}, {"op": "RUN_PROMPT", "args": ["Post-match"]}, {"op": "CHAIN", "args": ["end"]}, {"op": "CHAIN", "args": ["end"]}, {"op": "LOG", "args": ["Done"]}, ], } variants.append(("match", make_match_prog)) # Group 4: Nested control flow (1,250 variants) for i in range(1250): nested_id = 3750 + i depth = 2 + (i % 3) # 2-4 nesting depth def make_nested_prog(idx=nested_id, d=depth): return { "magic": "GXIC1", "version": 1, "model": "", "entrypoint": "main", "symbols": { "main": 0, "loop_body": 7, "if_true": 11, "if_false": 14, "end": 17 }, "instructions": [ {"op": "SET_MODE", "args": ["symbolic"]}, {"op": "SET_PARAM", "args": ["max_loop_iterations", 3]}, {"op": "SET_CONTEXT", "args": ["variant", f"nested_v{idx}"]}, {"op": "PUSH_GLYPH_CONTEXT", "args": [f"glyph://nested_{idx}"]}, {"op": "LOOP", "args": ["fused.global_resonance_score > 0.5", "loop_body", 3]}, {"op": "CHAIN", "args": ["loop_body"]}, {"op": "RUN_PROMPT", "args": [f"Nested {idx}"]}, {"op": "IF", "args": ["fused.glyph_count > 0", "if_true", "if_false"]}, {"op": "CHAIN", "args": ["if_true"]}, {"op": "LOG", "args": ["True"]}, {"op": "RUN_PROMPT", "args": ["True path"]}, {"op": "CHAIN", "args": ["end"]}, {"op": "CHAIN", "args": ["if_false"]}, {"op": "LOG", "args": ["False"]}, {"op": "RUN_PROMPT", "args": ["False path"]}, {"op": "CHAIN", "args": ["end"]}, {"op": "CHAIN", "args": ["end"]}, {"op": "CHAIN", "args": ["end"]}, {"op": "LOG", "args": ["Done"]}, ], } variants.append(("nested", make_nested_prog)) # Group 5: Multi-chain (1,250 variants) for i in range(1250): chain_id = 5000 + i chain_count = 3 + (i % 6) def make_multichain_prog(idx=chain_id, ch_cnt=chain_count): instructions = [ {"op": "SET_MODE", "args": ["symbolic"]}, {"op": "SET_CONTEXT", "args": ["variant", f"multichain_v{idx}"]}, ] for j in range(ch_cnt): instructions.append({"op": "PUSH_GLYPH_CONTEXT", "args": [f"glyph://ch_{j}"]}) instructions.append({"op": "RUN_PROMPT", "args": [f"Multichain {idx}"]}) for j in range(ch_cnt): instructions.extend([ {"op": "CHAIN", "args": [f"ch_{j}"]}, {"op": "RUN_PROMPT", "args": [f"Chain {j}"]}, ]) instructions.extend([ {"op": "CHAIN", "args": ["end"]}, {"op": "LOG", "args": ["Done"]}, ]) return { "magic": "GXIC1", "version": 1, "model": "", "entrypoint": "main", "symbols": {**{f"ch_{j}": 6 + j*2 for j in range(ch_cnt)}, "end": 100}, "instructions": instructions, } variants.append(("multichain", make_multichain_prog)) # Group 6: Predicate complexity (1,250 variants) for i in range(1250): pred_id = 6250 + i predicates = [ "fused.global_resonance_score > 0.7", "fused.glyph_count >= 1", "fused.global_resonance_score > 0.5 and fused.glyph_count > 0", "dominant_contains('glyph://test')", "not (fused.global_resonance_score < 0.3)", ] pred = predicates[i % len(predicates)] def make_predicate_prog(idx=pred_id, p=pred): return { "magic": "GXIC1", "version": 1, "model": "", "entrypoint": "main", "symbols": {"main": 0, "true_b": 5, "false_b": 9, "end": 12}, "instructions": [ {"op": "SET_MODE", "args": ["symbolic"]}, {"op": "SET_CONTEXT", "args": ["variant", f"pred_v{idx}"]}, {"op": "RUN_PROMPT", "args": [f"Predicate {idx}"]}, {"op": "IF", "args": [p, "true_b", "false_b"]}, {"op": "CHAIN", "args": ["true_b"]}, {"op": "LOG", "args": ["True"]}, {"op": "RUN_PROMPT", "args": ["T"]}, {"op": "CHAIN", "args": ["end"]}, {"op": "CHAIN", "args": ["false_b"]}, {"op": "LOG", "args": ["False"]}, {"op": "RUN_PROMPT", "args": ["F"]}, {"op": "CHAIN", "args": ["end"]}, {"op": "CHAIN", "args": ["end"]}, {"op": "LOG", "args": ["Done"]}, ], } variants.append(("predicate", make_predicate_prog)) # Group 7: Glyph stacking (1,250 variants) for i in range(1250): glyph_id = 7500 + i glyph_count = 5 + (i % 20) def make_glyph_prog(idx=glyph_id, glyph_cnt=glyph_count): instructions = [ {"op": "SET_MODE", "args": ["symbolic"]}, {"op": "SET_CONTEXT", "args": ["variant", f"glyph_v{idx}_{glyph_cnt}"]}, ] for j in range(glyph_cnt): instructions.append({"op": "PUSH_GLYPH_CONTEXT", "args": [f"glyph://s_{j}"]}) instructions.extend([ {"op": "RUN_PROMPT", "args": [f"Glyph stack {idx}"]}, {"op": "LOG", "args": ["Done"]}, ]) return { "magic": "GXIC1", "version": 1, "model": "", "entrypoint": "main", "symbols": {"main": 0}, "instructions": instructions, } variants.append(("glyph_stack", make_glyph_prog)) # Group 8: Guardrail stress (1,000 variants) for i in range(1000): guardrail_id = 8750 + i def make_guardrail_prog(idx=guardrail_id): return { "magic": "GXIC1", "version": 1, "model": "", "entrypoint": "main", "symbols": {"main": 0, "loop_a": 7, "end": 13}, "instructions": [ {"op": "SET_MODE", "args": ["symbolic"]}, {"op": "SET_PARAM", "args": ["max_loop_iterations", 2]}, {"op": "SET_PARAM", "args": ["max_total_steps", 50 + (idx % 50)]}, {"op": "SET_CONTEXT", "args": ["variant", f"guardrail_v{idx}"]}, {"op": "PUSH_GLYPH_CONTEXT", "args": [f"glyph://g_{idx}"]}, {"op": "LOOP", "args": ["fused.global_resonance_score > 0.4", "loop_a", 10]}, {"op": "CHAIN", "args": ["loop_a"]}, {"op": "RUN_PROMPT", "args": ["Heavy"]}, {"op": "RUN_PROMPT", "args": ["Secondary"]}, {"op": "GET_GLYPH_RESONANCE", "args": [f"glyph://g_{idx}", "global"]}, {"op": "CHAIN", "args": ["end"]}, {"op": "CHAIN", "args": ["end"]}, {"op": "LOG", "args": ["Done"]}, ], } variants.append(("guardrail", make_guardrail_prog)) print(f"✓ Generated {len(variants)} program variants in memory") return variants def execute_instance(variant_factory: tuple, instance_id: int) -> Dict[str, Any]: """Execute a single XIC program instance.""" global metrics try: from xic_executor import run_xic from tempfile import NamedTemporaryFile variant_type, factory = variant_factory prog = factory() # Write to temp file for execution with NamedTemporaryFile(mode='w', suffix='.gx.json', delete=False) as f: json.dump(prog, f) temp_path = f.name start_time = time.time() try: ctx = run_xic(temp_path, debug=False) elapsed = time.time() - start_time with metrics_lock: metrics["total_executions"] += 1 metrics["successful_executions"] += 1 variant_key = f"{variant_type}_v{instance_id % 100}" metrics["variant_counters"][variant_key] = metrics["variant_counters"].get(variant_key, 0) + 1 if ctx._state.get("guardrails"): metrics["guardrail_triggers"] += len(ctx._state["guardrails"]) return {"status": "success", "variant": variant_type, "elapsed": elapsed} except Exception as e: elapsed = time.time() - start_time with metrics_lock: metrics["total_executions"] += 1 metrics["failed_executions"] += 1 if len(metrics["error_log"]) < 10: metrics["error_log"].append(str(e)[:50]) return {"status": "error", "variant": variant_type, "error": str(e)[:30], "elapsed": elapsed} finally: import os try: os.unlink(temp_path) except Exception as e: with metrics_lock: if len(metrics.get("error_log", [])) < 10: if "error_log" not in metrics: metrics["error_log"] = [] metrics["error_log"].append(f"cleanup: {e}") except Exception as e: with metrics_lock: metrics["failed_executions"] += 1 return {"status": "fatal", "error": str(e)[:30]} def worker_thread(work_queue: queue.Queue, variants: List): """Worker thread that processes items from the work queue.""" while True: try: instance_id = work_queue.get(timeout=1) if instance_id is None: break variant = random.choice(variants) execute_instance(variant, instance_id) work_queue.task_done() except queue.Empty: continue except Exception as e: with metrics_lock: if "error_log" not in metrics: metrics["error_log"] = [] if len(metrics.get("error_log", [])) < 100: metrics["error_log"].append(f"worker: {e}") def monitor_vram(): """Monitor VRAM usage.""" while time.time() - metrics["start_time"] < DURATION_SECS: try: vram = psutil.virtual_memory() with metrics_lock: metrics["vram_peaks"].append({ "timestamp": time.time() - metrics["start_time"], "percent": vram.percent, "used_gb": vram.used / (1024**3), }) except Exception as e: with metrics_lock: if "error_log" not in metrics: metrics["error_log"] = [] if len(metrics.get("error_log", [])) < 100: metrics["error_log"].append(f"vram_monitor: {e}") time.sleep(0.2) def main(): """Execute 1 million parallel instances × 10,000 program variants.""" print("\n" + "="*80) print("🔥 MEGA STRESS TEST: 1,000,000 Parallel Instances × 10,000 Program Variants") print("="*80) print(f"Start Time: {datetime.now().isoformat()}") print(f"Duration: {DURATION_SECS} seconds (5 minutes)") print(f"Worker Threads: {WORKER_THREADS}") print() # Generate variants print("[1/4] Generating 10,000 program variants...") variants = generate_program_variants() print() # Start VRAM monitor print("[2/4] Starting system monitoring...") vram_monitor = threading.Thread(target=monitor_vram, daemon=True) vram_monitor.start() print("✓ VRAM monitoring started") print() # Create work queue print("[3/4] Initializing work queue...") work_queue = queue.Queue(maxsize=QUEUE_SIZE) print(f"✓ Queue created (max size: {QUEUE_SIZE})") print() # Start worker threads print(f"[4/4] Starting {WORKER_THREADS} worker threads...") workers = [] for i in range(WORKER_THREADS): w = threading.Thread(target=worker_thread, args=(work_queue, variants), daemon=True) w.start() workers.append(w) print(f"✓ All {WORKER_THREADS} workers started") print() print("Submitting 1,000,000 work items...") print() # Submit work items start_time = time.time() last_report = start_time submitted = 0 while time.time() - start_time < DURATION_SECS: # Try to fill the queue while not work_queue.full() and time.time() - start_time < DURATION_SECS: work_queue.put(submitted) submitted += 1 # Report progress every 30 seconds now = time.time() if now - last_report > 30: elapsed = now - start_time with metrics_lock: rate = metrics["total_executions"] / elapsed if elapsed > 0 else 0 print(f"⚡ {metrics['total_executions']:,} executions | " f"{rate:,.0f} exec/sec | " f"{elapsed:.0f}s elapsed | " f"Success: {metrics['successful_executions']:,} | " f"Failed: {metrics['failed_executions']:,} | " f"Guardrails: {metrics['guardrail_triggers']}") last_report = now time.sleep(0.1) # Drain queue print("\nDraining work queue...") work_queue.join() # Stop workers for _ in range(WORKER_THREADS): work_queue.put(None) for w in workers: w.join(timeout=2) metrics["end_time"] = time.time() total_elapsed = metrics["end_time"] - metrics["start_time"] # Final report print() print("="*80) print("📊 MEGA STRESS TEST RESULTS") print("="*80) print() print(f"Duration: {total_elapsed:.1f} seconds") print(f"Total Executions: {metrics['total_executions']:,}") print(f"Successful: {metrics['successful_executions']:,}") print(f"Failed: {metrics['failed_executions']:,}") print(f"Success Rate: {100 * metrics['successful_executions'] / max(1, metrics['total_executions']):.1f}%") print() print(f"Throughput: {metrics['total_executions'] / total_elapsed:,.0f} executions/second") print(f"⚠️ Guardrail Triggers: {metrics['guardrail_triggers']}") print() if metrics["vram_peaks"]: vram_percents = [v["percent"] for v in metrics["vram_peaks"]] vram_gbs = [v["used_gb"] for v in metrics["vram_peaks"]] print("Memory Usage:") print(f" Peak: {max(vram_percents):.1f}% ({max(vram_gbs):.2f} GB)") print(f" Average: {sum(vram_percents)/len(vram_percents):.1f}%") print() if metrics["error_log"]: print(f"Sample Errors ({len(metrics['error_log'])} total):") for error in metrics["error_log"][:3]: print(f" {error}") print() print("="*80) print(f"✅ MEGA STRESS TEST COMPLETE") print(f"End Time: {datetime.now().isoformat()}") print("="*80) if __name__ == "__main__": main()