Files
2125_GCE/stress_test_1m_nuclear.py
T
2026-07-09 12:54:44 -04:00

560 lines
21 KiB
Python
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""1 Million Parallel Instance MEGA Stress Test with 10,000 Program Variants
Generates 10,000 unique XIC program variants and executes 1,000,000 instances
over 5 minutes using a queue-based worker pool approach to avoid OS thread limits.
Key optimizations:
- Programs generated in-memory (no file I/O overhead)
- Queue-based execution (avoids 1M thread creation limit)
- Minimal worker threads (500) with high throughput
- Real-time metrics collection and display
"""
import json
import time
import random
import threading
import queue
import psutil
from datetime import datetime
from pathlib import Path
from typing import Dict, List, Any, Callable
# Configuration
SUPERDAVE_ROOT = Path(__file__).parent
PROGRAM_VARIANTS = 10000
TARGET_INSTANCES = 1000000
DURATION_SECS = 300 # 5 minutes
WORKER_THREADS = 500
QUEUE_SIZE = 50000
# Metrics collection
metrics_lock = threading.Lock()
metrics = {
"total_executions": 0,
"successful_executions": 0,
"failed_executions": 0,
"guardrail_triggers": 0,
"variant_counters": {},
"vram_peaks": [],
"throughput_samples": [],
"error_log": [],
"start_time": time.time(),
"end_time": None,
}
def generate_program_variants() -> List[Callable]:
"""Generate 10,000 unique XIC program variants as callable functions."""
print("[VARIANT-GEN] Generating 10,000 program variants in-memory...")
variants = []
# Group 1: IF variations (1,250 variants)
for i in range(1250):
threshold = 0.2 + ((i % 100) * 0.008) # 0.2 to 0.99
branch_count = 2 + (i % 3) # 2-4 branches
def make_if_prog(idx=i, thresh=threshold, branches=branch_count):
instructions = [
{"op": "SET_MODE", "args": ["symbolic"]},
{"op": "SET_CONTEXT", "args": ["variant", f"if_v{idx}_{thresh:.2f}"]},
]
for j in range(2):
instructions.append({"op": "PUSH_GLYPH_CONTEXT", "args": [f"glyph://if_{idx}_{j}"]})
instructions.append({"op": "RUN_PROMPT", "args": [f"IF variant {idx}"]})
instructions.append({"op": "IF", "args": [f"fused.global_resonance_score > {thresh}", f"br_a_{idx}", f"br_b_{idx}"]})
for b in range(branches):
instructions.extend([
{"op": "CHAIN", "args": [f"br_a_{idx}"] if b == 0 else {"op": "CHAIN", "args": [f"end_{idx}"]}},
{"op": "LOG", "args": [f"Branch {b}"]},
{"op": "RUN_PROMPT", "args": ["Execute"]},
])
instructions.extend([
{"op": "CHAIN", "args": [f"br_b_{idx}"]},
{"op": "LOG", "args": ["Alt branch"]},
{"op": "RUN_PROMPT", "args": ["Alt path"]},
{"op": "CHAIN", "args": [f"end_{idx}"]},
{"op": "CHAIN", "args": [f"end_{idx}"]},
{"op": "LOG", "args": ["Complete"]},
])
return {
"magic": "GXIC1",
"version": 1,
"model": "",
"entrypoint": "main",
"symbols": {"main": 0, f"br_a_{idx}": 6, f"br_b_{idx}": 12, f"end_{idx}": 16},
"instructions": instructions,
}
variants.append(("if", make_if_prog))
# Group 2: LOOP variations (1,250 variants)
for i in range(1250):
max_iter = 2 + (i % 15) # 2-16 iterations
loop_id = 1250 + i
def make_loop_prog(idx=loop_id, iter_max=max_iter):
return {
"magic": "GXIC1",
"version": 1,
"model": "",
"entrypoint": "main",
"symbols": {"main": 0, "loop_body": 7, "end": 12},
"instructions": [
{"op": "SET_MODE", "args": ["symbolic"]},
{"op": "SET_PARAM", "args": ["max_loop_iterations", iter_max]},
{"op": "SET_CONTEXT", "args": ["variant", f"loop_v{idx}"]},
{"op": "PUSH_GLYPH_CONTEXT", "args": [f"glyph://loop_{idx}"]},
{"op": "LOOP", "args": [f"fused.global_resonance_score > 0.{5 + (idx%4)}", "loop_body", iter_max]},
{"op": "CHAIN", "args": ["loop_body"]},
{"op": "RUN_PROMPT", "args": [f"Loop {idx}"]},
{"op": "GET_GLYPH_RESONANCE", "args": [f"glyph://loop_{idx}", "global"]},
{"op": "LOG", "args": ["Iteration"]},
{"op": "CHAIN", "args": ["end"]},
{"op": "CHAIN", "args": ["end"]},
{"op": "LOG", "args": ["Done"]},
],
}
variants.append(("loop", make_loop_prog))
# Group 3: MATCH variations (1,250 variants)
for i in range(1250):
match_id = 2500 + i
pattern_count = 1 + (i % 5)
def make_match_prog(idx=match_id, pat_cnt=pattern_count):
patterns = [f"glyph://pat_{j}" for j in range(pat_cnt)]
return {
"magic": "GXIC1",
"version": 1,
"model": "",
"entrypoint": "main",
"symbols": {"main": 0, "match_true": 6, "end": 10},
"instructions": [
{"op": "SET_MODE", "args": ["symbolic"]},
{"op": "SET_CONTEXT", "args": ["variant", f"match_v{idx}"]},
{"op": "PUSH_GLYPH_CONTEXT", "args": [f"glyph://match_{idx}"]},
{"op": "RUN_PROMPT", "args": [f"Match {idx}"]},
{"op": "MATCH", "args": ["fused.glyph_ids", patterns[0], "match_true"]},
{"op": "CHAIN", "args": ["match_true"]},
{"op": "LOG", "args": ["Matched"]},
{"op": "RUN_PROMPT", "args": ["Post-match"]},
{"op": "CHAIN", "args": ["end"]},
{"op": "CHAIN", "args": ["end"]},
{"op": "LOG", "args": ["Done"]},
],
}
variants.append(("match", make_match_prog))
# Group 4: Nested control flow (1,250 variants)
for i in range(1250):
nested_id = 3750 + i
depth = 2 + (i % 3) # 2-4 nesting depth
def make_nested_prog(idx=nested_id, d=depth):
return {
"magic": "GXIC1",
"version": 1,
"model": "",
"entrypoint": "main",
"symbols": {
"main": 0,
"loop_body": 7,
"if_true": 11,
"if_false": 14,
"end": 17
},
"instructions": [
{"op": "SET_MODE", "args": ["symbolic"]},
{"op": "SET_PARAM", "args": ["max_loop_iterations", 3]},
{"op": "SET_CONTEXT", "args": ["variant", f"nested_v{idx}"]},
{"op": "PUSH_GLYPH_CONTEXT", "args": [f"glyph://nested_{idx}"]},
{"op": "LOOP", "args": ["fused.global_resonance_score > 0.5", "loop_body", 3]},
{"op": "CHAIN", "args": ["loop_body"]},
{"op": "RUN_PROMPT", "args": [f"Nested {idx}"]},
{"op": "IF", "args": ["fused.glyph_count > 0", "if_true", "if_false"]},
{"op": "CHAIN", "args": ["if_true"]},
{"op": "LOG", "args": ["True"]},
{"op": "RUN_PROMPT", "args": ["True path"]},
{"op": "CHAIN", "args": ["end"]},
{"op": "CHAIN", "args": ["if_false"]},
{"op": "LOG", "args": ["False"]},
{"op": "RUN_PROMPT", "args": ["False path"]},
{"op": "CHAIN", "args": ["end"]},
{"op": "CHAIN", "args": ["end"]},
{"op": "CHAIN", "args": ["end"]},
{"op": "LOG", "args": ["Done"]},
],
}
variants.append(("nested", make_nested_prog))
# Group 5: Multi-chain (1,250 variants)
for i in range(1250):
chain_id = 5000 + i
chain_count = 3 + (i % 6)
def make_multichain_prog(idx=chain_id, ch_cnt=chain_count):
instructions = [
{"op": "SET_MODE", "args": ["symbolic"]},
{"op": "SET_CONTEXT", "args": ["variant", f"multichain_v{idx}"]},
]
for j in range(ch_cnt):
instructions.append({"op": "PUSH_GLYPH_CONTEXT", "args": [f"glyph://ch_{j}"]})
instructions.append({"op": "RUN_PROMPT", "args": [f"Multichain {idx}"]})
for j in range(ch_cnt):
instructions.extend([
{"op": "CHAIN", "args": [f"ch_{j}"]},
{"op": "RUN_PROMPT", "args": [f"Chain {j}"]},
])
instructions.extend([
{"op": "CHAIN", "args": ["end"]},
{"op": "LOG", "args": ["Done"]},
])
return {
"magic": "GXIC1",
"version": 1,
"model": "",
"entrypoint": "main",
"symbols": {**{f"ch_{j}": 6 + j*2 for j in range(ch_cnt)}, "end": 100},
"instructions": instructions,
}
variants.append(("multichain", make_multichain_prog))
# Group 6: Predicate complexity (1,250 variants)
for i in range(1250):
pred_id = 6250 + i
predicates = [
"fused.global_resonance_score > 0.7",
"fused.glyph_count >= 1",
"fused.global_resonance_score > 0.5 and fused.glyph_count > 0",
"dominant_contains('glyph://test')",
"not (fused.global_resonance_score < 0.3)",
]
pred = predicates[i % len(predicates)]
def make_predicate_prog(idx=pred_id, p=pred):
return {
"magic": "GXIC1",
"version": 1,
"model": "",
"entrypoint": "main",
"symbols": {"main": 0, "true_b": 5, "false_b": 9, "end": 12},
"instructions": [
{"op": "SET_MODE", "args": ["symbolic"]},
{"op": "SET_CONTEXT", "args": ["variant", f"pred_v{idx}"]},
{"op": "RUN_PROMPT", "args": [f"Predicate {idx}"]},
{"op": "IF", "args": [p, "true_b", "false_b"]},
{"op": "CHAIN", "args": ["true_b"]},
{"op": "LOG", "args": ["True"]},
{"op": "RUN_PROMPT", "args": ["T"]},
{"op": "CHAIN", "args": ["end"]},
{"op": "CHAIN", "args": ["false_b"]},
{"op": "LOG", "args": ["False"]},
{"op": "RUN_PROMPT", "args": ["F"]},
{"op": "CHAIN", "args": ["end"]},
{"op": "CHAIN", "args": ["end"]},
{"op": "LOG", "args": ["Done"]},
],
}
variants.append(("predicate", make_predicate_prog))
# Group 7: Glyph stacking (1,250 variants)
for i in range(1250):
glyph_id = 7500 + i
glyph_count = 5 + (i % 20)
def make_glyph_prog(idx=glyph_id, glyph_cnt=glyph_count):
instructions = [
{"op": "SET_MODE", "args": ["symbolic"]},
{"op": "SET_CONTEXT", "args": ["variant", f"glyph_v{idx}_{glyph_cnt}"]},
]
for j in range(glyph_cnt):
instructions.append({"op": "PUSH_GLYPH_CONTEXT", "args": [f"glyph://s_{j}"]})
instructions.extend([
{"op": "RUN_PROMPT", "args": [f"Glyph stack {idx}"]},
{"op": "LOG", "args": ["Done"]},
])
return {
"magic": "GXIC1",
"version": 1,
"model": "",
"entrypoint": "main",
"symbols": {"main": 0},
"instructions": instructions,
}
variants.append(("glyph_stack", make_glyph_prog))
# Group 8: Guardrail stress (1,000 variants)
for i in range(1000):
guardrail_id = 8750 + i
def make_guardrail_prog(idx=guardrail_id):
return {
"magic": "GXIC1",
"version": 1,
"model": "",
"entrypoint": "main",
"symbols": {"main": 0, "loop_a": 7, "end": 13},
"instructions": [
{"op": "SET_MODE", "args": ["symbolic"]},
{"op": "SET_PARAM", "args": ["max_loop_iterations", 2]},
{"op": "SET_PARAM", "args": ["max_total_steps", 50 + (idx % 50)]},
{"op": "SET_CONTEXT", "args": ["variant", f"guardrail_v{idx}"]},
{"op": "PUSH_GLYPH_CONTEXT", "args": [f"glyph://g_{idx}"]},
{"op": "LOOP", "args": ["fused.global_resonance_score > 0.4", "loop_a", 10]},
{"op": "CHAIN", "args": ["loop_a"]},
{"op": "RUN_PROMPT", "args": ["Heavy"]},
{"op": "RUN_PROMPT", "args": ["Secondary"]},
{"op": "GET_GLYPH_RESONANCE", "args": [f"glyph://g_{idx}", "global"]},
{"op": "CHAIN", "args": ["end"]},
{"op": "CHAIN", "args": ["end"]},
{"op": "LOG", "args": ["Done"]},
],
}
variants.append(("guardrail", make_guardrail_prog))
print(f"✓ Generated {len(variants)} program variants in memory")
return variants
def execute_instance(variant_factory: tuple, instance_id: int) -> Dict[str, Any]:
"""Execute a single XIC program instance."""
global metrics
try:
from xic_executor import run_xic
from tempfile import NamedTemporaryFile
variant_type, factory = variant_factory
prog = factory()
# Write to temp file for execution
with NamedTemporaryFile(mode='w', suffix='.gx.json', delete=False) as f:
json.dump(prog, f)
temp_path = f.name
start_time = time.time()
try:
ctx = run_xic(temp_path, debug=False)
elapsed = time.time() - start_time
with metrics_lock:
metrics["total_executions"] += 1
metrics["successful_executions"] += 1
variant_key = f"{variant_type}_v{instance_id % 100}"
metrics["variant_counters"][variant_key] = metrics["variant_counters"].get(variant_key, 0) + 1
if ctx._state.get("guardrails"):
metrics["guardrail_triggers"] += len(ctx._state["guardrails"])
return {"status": "success", "variant": variant_type, "elapsed": elapsed}
except Exception as e:
elapsed = time.time() - start_time
with metrics_lock:
metrics["total_executions"] += 1
metrics["failed_executions"] += 1
if len(metrics["error_log"]) < 10:
metrics["error_log"].append(str(e)[:50])
return {"status": "error", "variant": variant_type, "error": str(e)[:30], "elapsed": elapsed}
finally:
import os
try:
os.unlink(temp_path)
except Exception as e:
with metrics_lock:
if len(metrics.get("error_log", [])) < 10:
if "error_log" not in metrics:
metrics["error_log"] = []
metrics["error_log"].append(f"cleanup: {e}")
except Exception as e:
with metrics_lock:
metrics["failed_executions"] += 1
return {"status": "fatal", "error": str(e)[:30]}
def worker_thread(work_queue: queue.Queue, variants: List):
"""Worker thread that processes items from the work queue."""
while True:
try:
instance_id = work_queue.get(timeout=1)
if instance_id is None:
break
variant = random.choice(variants)
execute_instance(variant, instance_id)
work_queue.task_done()
except queue.Empty:
continue
except Exception as e:
with metrics_lock:
if "error_log" not in metrics:
metrics["error_log"] = []
if len(metrics.get("error_log", [])) < 100:
metrics["error_log"].append(f"worker: {e}")
def monitor_vram():
"""Monitor VRAM usage."""
while time.time() - metrics["start_time"] < DURATION_SECS:
try:
vram = psutil.virtual_memory()
with metrics_lock:
metrics["vram_peaks"].append({
"timestamp": time.time() - metrics["start_time"],
"percent": vram.percent,
"used_gb": vram.used / (1024**3),
})
except Exception as e:
with metrics_lock:
if "error_log" not in metrics:
metrics["error_log"] = []
if len(metrics.get("error_log", [])) < 100:
metrics["error_log"].append(f"vram_monitor: {e}")
time.sleep(0.2)
def main():
"""Execute 1 million parallel instances × 10,000 program variants."""
print("\n" + "="*80)
print("🔥 MEGA STRESS TEST: 1,000,000 Parallel Instances × 10,000 Program Variants")
print("="*80)
print(f"Start Time: {datetime.now().isoformat()}")
print(f"Duration: {DURATION_SECS} seconds (5 minutes)")
print(f"Worker Threads: {WORKER_THREADS}")
print()
# Generate variants
print("[1/4] Generating 10,000 program variants...")
variants = generate_program_variants()
print()
# Start VRAM monitor
print("[2/4] Starting system monitoring...")
vram_monitor = threading.Thread(target=monitor_vram, daemon=True)
vram_monitor.start()
print("✓ VRAM monitoring started")
print()
# Create work queue
print("[3/4] Initializing work queue...")
work_queue = queue.Queue(maxsize=QUEUE_SIZE)
print(f"✓ Queue created (max size: {QUEUE_SIZE})")
print()
# Start worker threads
print(f"[4/4] Starting {WORKER_THREADS} worker threads...")
workers = []
for i in range(WORKER_THREADS):
w = threading.Thread(target=worker_thread, args=(work_queue, variants), daemon=True)
w.start()
workers.append(w)
print(f"✓ All {WORKER_THREADS} workers started")
print()
print("Submitting 1,000,000 work items...")
print()
# Submit work items
start_time = time.time()
last_report = start_time
submitted = 0
while time.time() - start_time < DURATION_SECS:
# Try to fill the queue
while not work_queue.full() and time.time() - start_time < DURATION_SECS:
work_queue.put(submitted)
submitted += 1
# Report progress every 30 seconds
now = time.time()
if now - last_report > 30:
elapsed = now - start_time
with metrics_lock:
rate = metrics["total_executions"] / elapsed if elapsed > 0 else 0
print(f"⚡ {metrics['total_executions']:,} executions | "
f"{rate:,.0f} exec/sec | "
f"{elapsed:.0f}s elapsed | "
f"Success: {metrics['successful_executions']:,} | "
f"Failed: {metrics['failed_executions']:,} | "
f"Guardrails: {metrics['guardrail_triggers']}")
last_report = now
time.sleep(0.1)
# Drain queue
print("\nDraining work queue...")
work_queue.join()
# Stop workers
for _ in range(WORKER_THREADS):
work_queue.put(None)
for w in workers:
w.join(timeout=2)
metrics["end_time"] = time.time()
total_elapsed = metrics["end_time"] - metrics["start_time"]
# Final report
print()
print("="*80)
print("📊 MEGA STRESS TEST RESULTS")
print("="*80)
print()
print(f"Duration: {total_elapsed:.1f} seconds")
print(f"Total Executions: {metrics['total_executions']:,}")
print(f"Successful: {metrics['successful_executions']:,}")
print(f"Failed: {metrics['failed_executions']:,}")
print(f"Success Rate: {100 * metrics['successful_executions'] / max(1, metrics['total_executions']):.1f}%")
print()
print(f"Throughput: {metrics['total_executions'] / total_elapsed:,.0f} executions/second")
print(f"⚠️ Guardrail Triggers: {metrics['guardrail_triggers']}")
print()
if metrics["vram_peaks"]:
vram_percents = [v["percent"] for v in metrics["vram_peaks"]]
vram_gbs = [v["used_gb"] for v in metrics["vram_peaks"]]
print("Memory Usage:")
print(f" Peak: {max(vram_percents):.1f}% ({max(vram_gbs):.2f} GB)")
print(f" Average: {sum(vram_percents)/len(vram_percents):.1f}%")
print()
if metrics["error_log"]:
print(f"Sample Errors ({len(metrics['error_log'])} total):")
for error in metrics["error_log"][:3]:
print(f" {error}")
print()
print("="*80)
print(f"✅ MEGA STRESS TEST COMPLETE")
print(f"End Time: {datetime.now().isoformat()}")
print("="*80)
if __name__ == "__main__":
main()