560 lines
21 KiB
Python
560 lines
21 KiB
Python
|
#!/usr/bin/env python3
|
|||
|
|
"""1 Million Parallel Instance MEGA Stress Test with 10,000 Program Variants
|
|||
|
|
|
|||
|
|
Generates 10,000 unique XIC program variants and executes 1,000,000 instances
|
|||
|
|
over 5 minutes using a queue-based worker pool approach to avoid OS thread limits.
|
|||
|
|
|
|||
|
|
Key optimizations:
|
|||
|
|
- Programs generated in-memory (no file I/O overhead)
|
|||
|
|
- Queue-based execution (avoids 1M thread creation limit)
|
|||
|
|
- Minimal worker threads (500) with high throughput
|
|||
|
|
- Real-time metrics collection and display
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
import json
|
|||
|
|
import time
|
|||
|
|
import random
|
|||
|
|
import threading
|
|||
|
|
import queue
|
|||
|
|
import psutil
|
|||
|
|
from datetime import datetime
|
|||
|
|
from pathlib import Path
|
|||
|
|
from typing import Dict, List, Any, Callable
|
|||
|
|
|
|||
|
|
# Configuration
|
|||
|
|
SUPERDAVE_ROOT = Path(__file__).parent
|
|||
|
|
PROGRAM_VARIANTS = 10000
|
|||
|
|
TARGET_INSTANCES = 1000000
|
|||
|
|
DURATION_SECS = 300 # 5 minutes
|
|||
|
|
WORKER_THREADS = 500
|
|||
|
|
QUEUE_SIZE = 50000
|
|||
|
|
|
|||
|
|
# Metrics collection
|
|||
|
|
metrics_lock = threading.Lock()
|
|||
|
|
metrics = {
|
|||
|
|
"total_executions": 0,
|
|||
|
|
"successful_executions": 0,
|
|||
|
|
"failed_executions": 0,
|
|||
|
|
"guardrail_triggers": 0,
|
|||
|
|
"variant_counters": {},
|
|||
|
|
"vram_peaks": [],
|
|||
|
|
"throughput_samples": [],
|
|||
|
|
"error_log": [],
|
|||
|
|
"start_time": time.time(),
|
|||
|
|
"end_time": None,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def generate_program_variants() -> List[Callable]:
|
|||
|
|
"""Generate 10,000 unique XIC program variants as callable functions."""
|
|||
|
|
print("[VARIANT-GEN] Generating 10,000 program variants in-memory...")
|
|||
|
|
|
|||
|
|
variants = []
|
|||
|
|
|
|||
|
|
# Group 1: IF variations (1,250 variants)
|
|||
|
|
for i in range(1250):
|
|||
|
|
threshold = 0.2 + ((i % 100) * 0.008) # 0.2 to 0.99
|
|||
|
|
branch_count = 2 + (i % 3) # 2-4 branches
|
|||
|
|
|
|||
|
|
def make_if_prog(idx=i, thresh=threshold, branches=branch_count):
|
|||
|
|
instructions = [
|
|||
|
|
{"op": "SET_MODE", "args": ["symbolic"]},
|
|||
|
|
{"op": "SET_CONTEXT", "args": ["variant", f"if_v{idx}_{thresh:.2f}"]},
|
|||
|
|
]
|
|||
|
|
for j in range(2):
|
|||
|
|
instructions.append({"op": "PUSH_GLYPH_CONTEXT", "args": [f"glyph://if_{idx}_{j}"]})
|
|||
|
|
|
|||
|
|
instructions.append({"op": "RUN_PROMPT", "args": [f"IF variant {idx}"]})
|
|||
|
|
instructions.append({"op": "IF", "args": [f"fused.global_resonance_score > {thresh}", f"br_a_{idx}", f"br_b_{idx}"]})
|
|||
|
|
|
|||
|
|
for b in range(branches):
|
|||
|
|
instructions.extend([
|
|||
|
|
{"op": "CHAIN", "args": [f"br_a_{idx}"] if b == 0 else {"op": "CHAIN", "args": [f"end_{idx}"]}},
|
|||
|
|
{"op": "LOG", "args": [f"Branch {b}"]},
|
|||
|
|
{"op": "RUN_PROMPT", "args": ["Execute"]},
|
|||
|
|
])
|
|||
|
|
|
|||
|
|
instructions.extend([
|
|||
|
|
{"op": "CHAIN", "args": [f"br_b_{idx}"]},
|
|||
|
|
{"op": "LOG", "args": ["Alt branch"]},
|
|||
|
|
{"op": "RUN_PROMPT", "args": ["Alt path"]},
|
|||
|
|
{"op": "CHAIN", "args": [f"end_{idx}"]},
|
|||
|
|
{"op": "CHAIN", "args": [f"end_{idx}"]},
|
|||
|
|
{"op": "LOG", "args": ["Complete"]},
|
|||
|
|
])
|
|||
|
|
|
|||
|
|
return {
|
|||
|
|
"magic": "GXIC1",
|
|||
|
|
"version": 1,
|
|||
|
|
"model": "",
|
|||
|
|
"entrypoint": "main",
|
|||
|
|
"symbols": {"main": 0, f"br_a_{idx}": 6, f"br_b_{idx}": 12, f"end_{idx}": 16},
|
|||
|
|
"instructions": instructions,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
variants.append(("if", make_if_prog))
|
|||
|
|
|
|||
|
|
# Group 2: LOOP variations (1,250 variants)
|
|||
|
|
for i in range(1250):
|
|||
|
|
max_iter = 2 + (i % 15) # 2-16 iterations
|
|||
|
|
loop_id = 1250 + i
|
|||
|
|
|
|||
|
|
def make_loop_prog(idx=loop_id, iter_max=max_iter):
|
|||
|
|
return {
|
|||
|
|
"magic": "GXIC1",
|
|||
|
|
"version": 1,
|
|||
|
|
"model": "",
|
|||
|
|
"entrypoint": "main",
|
|||
|
|
"symbols": {"main": 0, "loop_body": 7, "end": 12},
|
|||
|
|
"instructions": [
|
|||
|
|
{"op": "SET_MODE", "args": ["symbolic"]},
|
|||
|
|
{"op": "SET_PARAM", "args": ["max_loop_iterations", iter_max]},
|
|||
|
|
{"op": "SET_CONTEXT", "args": ["variant", f"loop_v{idx}"]},
|
|||
|
|
{"op": "PUSH_GLYPH_CONTEXT", "args": [f"glyph://loop_{idx}"]},
|
|||
|
|
{"op": "LOOP", "args": [f"fused.global_resonance_score > 0.{5 + (idx%4)}", "loop_body", iter_max]},
|
|||
|
|
{"op": "CHAIN", "args": ["loop_body"]},
|
|||
|
|
{"op": "RUN_PROMPT", "args": [f"Loop {idx}"]},
|
|||
|
|
{"op": "GET_GLYPH_RESONANCE", "args": [f"glyph://loop_{idx}", "global"]},
|
|||
|
|
{"op": "LOG", "args": ["Iteration"]},
|
|||
|
|
{"op": "CHAIN", "args": ["end"]},
|
|||
|
|
{"op": "CHAIN", "args": ["end"]},
|
|||
|
|
{"op": "LOG", "args": ["Done"]},
|
|||
|
|
],
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
variants.append(("loop", make_loop_prog))
|
|||
|
|
|
|||
|
|
# Group 3: MATCH variations (1,250 variants)
|
|||
|
|
for i in range(1250):
|
|||
|
|
match_id = 2500 + i
|
|||
|
|
pattern_count = 1 + (i % 5)
|
|||
|
|
|
|||
|
|
def make_match_prog(idx=match_id, pat_cnt=pattern_count):
|
|||
|
|
patterns = [f"glyph://pat_{j}" for j in range(pat_cnt)]
|
|||
|
|
return {
|
|||
|
|
"magic": "GXIC1",
|
|||
|
|
"version": 1,
|
|||
|
|
"model": "",
|
|||
|
|
"entrypoint": "main",
|
|||
|
|
"symbols": {"main": 0, "match_true": 6, "end": 10},
|
|||
|
|
"instructions": [
|
|||
|
|
{"op": "SET_MODE", "args": ["symbolic"]},
|
|||
|
|
{"op": "SET_CONTEXT", "args": ["variant", f"match_v{idx}"]},
|
|||
|
|
{"op": "PUSH_GLYPH_CONTEXT", "args": [f"glyph://match_{idx}"]},
|
|||
|
|
{"op": "RUN_PROMPT", "args": [f"Match {idx}"]},
|
|||
|
|
{"op": "MATCH", "args": ["fused.glyph_ids", patterns[0], "match_true"]},
|
|||
|
|
{"op": "CHAIN", "args": ["match_true"]},
|
|||
|
|
{"op": "LOG", "args": ["Matched"]},
|
|||
|
|
{"op": "RUN_PROMPT", "args": ["Post-match"]},
|
|||
|
|
{"op": "CHAIN", "args": ["end"]},
|
|||
|
|
{"op": "CHAIN", "args": ["end"]},
|
|||
|
|
{"op": "LOG", "args": ["Done"]},
|
|||
|
|
],
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
variants.append(("match", make_match_prog))
|
|||
|
|
|
|||
|
|
# Group 4: Nested control flow (1,250 variants)
|
|||
|
|
for i in range(1250):
|
|||
|
|
nested_id = 3750 + i
|
|||
|
|
depth = 2 + (i % 3) # 2-4 nesting depth
|
|||
|
|
|
|||
|
|
def make_nested_prog(idx=nested_id, d=depth):
|
|||
|
|
return {
|
|||
|
|
"magic": "GXIC1",
|
|||
|
|
"version": 1,
|
|||
|
|
"model": "",
|
|||
|
|
"entrypoint": "main",
|
|||
|
|
"symbols": {
|
|||
|
|
"main": 0,
|
|||
|
|
"loop_body": 7,
|
|||
|
|
"if_true": 11,
|
|||
|
|
"if_false": 14,
|
|||
|
|
"end": 17
|
|||
|
|
},
|
|||
|
|
"instructions": [
|
|||
|
|
{"op": "SET_MODE", "args": ["symbolic"]},
|
|||
|
|
{"op": "SET_PARAM", "args": ["max_loop_iterations", 3]},
|
|||
|
|
{"op": "SET_CONTEXT", "args": ["variant", f"nested_v{idx}"]},
|
|||
|
|
{"op": "PUSH_GLYPH_CONTEXT", "args": [f"glyph://nested_{idx}"]},
|
|||
|
|
{"op": "LOOP", "args": ["fused.global_resonance_score > 0.5", "loop_body", 3]},
|
|||
|
|
{"op": "CHAIN", "args": ["loop_body"]},
|
|||
|
|
{"op": "RUN_PROMPT", "args": [f"Nested {idx}"]},
|
|||
|
|
{"op": "IF", "args": ["fused.glyph_count > 0", "if_true", "if_false"]},
|
|||
|
|
{"op": "CHAIN", "args": ["if_true"]},
|
|||
|
|
{"op": "LOG", "args": ["True"]},
|
|||
|
|
{"op": "RUN_PROMPT", "args": ["True path"]},
|
|||
|
|
{"op": "CHAIN", "args": ["end"]},
|
|||
|
|
{"op": "CHAIN", "args": ["if_false"]},
|
|||
|
|
{"op": "LOG", "args": ["False"]},
|
|||
|
|
{"op": "RUN_PROMPT", "args": ["False path"]},
|
|||
|
|
{"op": "CHAIN", "args": ["end"]},
|
|||
|
|
{"op": "CHAIN", "args": ["end"]},
|
|||
|
|
{"op": "CHAIN", "args": ["end"]},
|
|||
|
|
{"op": "LOG", "args": ["Done"]},
|
|||
|
|
],
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
variants.append(("nested", make_nested_prog))
|
|||
|
|
|
|||
|
|
# Group 5: Multi-chain (1,250 variants)
|
|||
|
|
for i in range(1250):
|
|||
|
|
chain_id = 5000 + i
|
|||
|
|
chain_count = 3 + (i % 6)
|
|||
|
|
|
|||
|
|
def make_multichain_prog(idx=chain_id, ch_cnt=chain_count):
|
|||
|
|
instructions = [
|
|||
|
|
{"op": "SET_MODE", "args": ["symbolic"]},
|
|||
|
|
{"op": "SET_CONTEXT", "args": ["variant", f"multichain_v{idx}"]},
|
|||
|
|
]
|
|||
|
|
for j in range(ch_cnt):
|
|||
|
|
instructions.append({"op": "PUSH_GLYPH_CONTEXT", "args": [f"glyph://ch_{j}"]})
|
|||
|
|
|
|||
|
|
instructions.append({"op": "RUN_PROMPT", "args": [f"Multichain {idx}"]})
|
|||
|
|
|
|||
|
|
for j in range(ch_cnt):
|
|||
|
|
instructions.extend([
|
|||
|
|
{"op": "CHAIN", "args": [f"ch_{j}"]},
|
|||
|
|
{"op": "RUN_PROMPT", "args": [f"Chain {j}"]},
|
|||
|
|
])
|
|||
|
|
|
|||
|
|
instructions.extend([
|
|||
|
|
{"op": "CHAIN", "args": ["end"]},
|
|||
|
|
{"op": "LOG", "args": ["Done"]},
|
|||
|
|
])
|
|||
|
|
|
|||
|
|
return {
|
|||
|
|
"magic": "GXIC1",
|
|||
|
|
"version": 1,
|
|||
|
|
"model": "",
|
|||
|
|
"entrypoint": "main",
|
|||
|
|
"symbols": {**{f"ch_{j}": 6 + j*2 for j in range(ch_cnt)}, "end": 100},
|
|||
|
|
"instructions": instructions,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
variants.append(("multichain", make_multichain_prog))
|
|||
|
|
|
|||
|
|
# Group 6: Predicate complexity (1,250 variants)
|
|||
|
|
for i in range(1250):
|
|||
|
|
pred_id = 6250 + i
|
|||
|
|
predicates = [
|
|||
|
|
"fused.global_resonance_score > 0.7",
|
|||
|
|
"fused.glyph_count >= 1",
|
|||
|
|
"fused.global_resonance_score > 0.5 and fused.glyph_count > 0",
|
|||
|
|
"dominant_contains('glyph://test')",
|
|||
|
|
"not (fused.global_resonance_score < 0.3)",
|
|||
|
|
]
|
|||
|
|
pred = predicates[i % len(predicates)]
|
|||
|
|
|
|||
|
|
def make_predicate_prog(idx=pred_id, p=pred):
|
|||
|
|
return {
|
|||
|
|
"magic": "GXIC1",
|
|||
|
|
"version": 1,
|
|||
|
|
"model": "",
|
|||
|
|
"entrypoint": "main",
|
|||
|
|
"symbols": {"main": 0, "true_b": 5, "false_b": 9, "end": 12},
|
|||
|
|
"instructions": [
|
|||
|
|
{"op": "SET_MODE", "args": ["symbolic"]},
|
|||
|
|
{"op": "SET_CONTEXT", "args": ["variant", f"pred_v{idx}"]},
|
|||
|
|
{"op": "RUN_PROMPT", "args": [f"Predicate {idx}"]},
|
|||
|
|
{"op": "IF", "args": [p, "true_b", "false_b"]},
|
|||
|
|
{"op": "CHAIN", "args": ["true_b"]},
|
|||
|
|
{"op": "LOG", "args": ["True"]},
|
|||
|
|
{"op": "RUN_PROMPT", "args": ["T"]},
|
|||
|
|
{"op": "CHAIN", "args": ["end"]},
|
|||
|
|
{"op": "CHAIN", "args": ["false_b"]},
|
|||
|
|
{"op": "LOG", "args": ["False"]},
|
|||
|
|
{"op": "RUN_PROMPT", "args": ["F"]},
|
|||
|
|
{"op": "CHAIN", "args": ["end"]},
|
|||
|
|
{"op": "CHAIN", "args": ["end"]},
|
|||
|
|
{"op": "LOG", "args": ["Done"]},
|
|||
|
|
],
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
variants.append(("predicate", make_predicate_prog))
|
|||
|
|
|
|||
|
|
# Group 7: Glyph stacking (1,250 variants)
|
|||
|
|
for i in range(1250):
|
|||
|
|
glyph_id = 7500 + i
|
|||
|
|
glyph_count = 5 + (i % 20)
|
|||
|
|
|
|||
|
|
def make_glyph_prog(idx=glyph_id, glyph_cnt=glyph_count):
|
|||
|
|
instructions = [
|
|||
|
|
{"op": "SET_MODE", "args": ["symbolic"]},
|
|||
|
|
{"op": "SET_CONTEXT", "args": ["variant", f"glyph_v{idx}_{glyph_cnt}"]},
|
|||
|
|
]
|
|||
|
|
for j in range(glyph_cnt):
|
|||
|
|
instructions.append({"op": "PUSH_GLYPH_CONTEXT", "args": [f"glyph://s_{j}"]})
|
|||
|
|
|
|||
|
|
instructions.extend([
|
|||
|
|
{"op": "RUN_PROMPT", "args": [f"Glyph stack {idx}"]},
|
|||
|
|
{"op": "LOG", "args": ["Done"]},
|
|||
|
|
])
|
|||
|
|
|
|||
|
|
return {
|
|||
|
|
"magic": "GXIC1",
|
|||
|
|
"version": 1,
|
|||
|
|
"model": "",
|
|||
|
|
"entrypoint": "main",
|
|||
|
|
"symbols": {"main": 0},
|
|||
|
|
"instructions": instructions,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
variants.append(("glyph_stack", make_glyph_prog))
|
|||
|
|
|
|||
|
|
# Group 8: Guardrail stress (1,000 variants)
|
|||
|
|
for i in range(1000):
|
|||
|
|
guardrail_id = 8750 + i
|
|||
|
|
|
|||
|
|
def make_guardrail_prog(idx=guardrail_id):
|
|||
|
|
return {
|
|||
|
|
"magic": "GXIC1",
|
|||
|
|
"version": 1,
|
|||
|
|
"model": "",
|
|||
|
|
"entrypoint": "main",
|
|||
|
|
"symbols": {"main": 0, "loop_a": 7, "end": 13},
|
|||
|
|
"instructions": [
|
|||
|
|
{"op": "SET_MODE", "args": ["symbolic"]},
|
|||
|
|
{"op": "SET_PARAM", "args": ["max_loop_iterations", 2]},
|
|||
|
|
{"op": "SET_PARAM", "args": ["max_total_steps", 50 + (idx % 50)]},
|
|||
|
|
{"op": "SET_CONTEXT", "args": ["variant", f"guardrail_v{idx}"]},
|
|||
|
|
{"op": "PUSH_GLYPH_CONTEXT", "args": [f"glyph://g_{idx}"]},
|
|||
|
|
{"op": "LOOP", "args": ["fused.global_resonance_score > 0.4", "loop_a", 10]},
|
|||
|
|
{"op": "CHAIN", "args": ["loop_a"]},
|
|||
|
|
{"op": "RUN_PROMPT", "args": ["Heavy"]},
|
|||
|
|
{"op": "RUN_PROMPT", "args": ["Secondary"]},
|
|||
|
|
{"op": "GET_GLYPH_RESONANCE", "args": [f"glyph://g_{idx}", "global"]},
|
|||
|
|
{"op": "CHAIN", "args": ["end"]},
|
|||
|
|
{"op": "CHAIN", "args": ["end"]},
|
|||
|
|
{"op": "LOG", "args": ["Done"]},
|
|||
|
|
],
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
variants.append(("guardrail", make_guardrail_prog))
|
|||
|
|
|
|||
|
|
print(f"✓ Generated {len(variants)} program variants in memory")
|
|||
|
|
return variants
|
|||
|
|
|
|||
|
|
|
|||
|
|
def execute_instance(variant_factory: tuple, instance_id: int) -> Dict[str, Any]:
|
|||
|
|
"""Execute a single XIC program instance."""
|
|||
|
|
global metrics
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
from xic_executor import run_xic
|
|||
|
|
from tempfile import NamedTemporaryFile
|
|||
|
|
|
|||
|
|
variant_type, factory = variant_factory
|
|||
|
|
prog = factory()
|
|||
|
|
|
|||
|
|
# Write to temp file for execution
|
|||
|
|
with NamedTemporaryFile(mode='w', suffix='.gx.json', delete=False) as f:
|
|||
|
|
json.dump(prog, f)
|
|||
|
|
temp_path = f.name
|
|||
|
|
|
|||
|
|
start_time = time.time()
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
ctx = run_xic(temp_path, debug=False)
|
|||
|
|
elapsed = time.time() - start_time
|
|||
|
|
|
|||
|
|
with metrics_lock:
|
|||
|
|
metrics["total_executions"] += 1
|
|||
|
|
metrics["successful_executions"] += 1
|
|||
|
|
variant_key = f"{variant_type}_v{instance_id % 100}"
|
|||
|
|
metrics["variant_counters"][variant_key] = metrics["variant_counters"].get(variant_key, 0) + 1
|
|||
|
|
|
|||
|
|
if ctx._state.get("guardrails"):
|
|||
|
|
metrics["guardrail_triggers"] += len(ctx._state["guardrails"])
|
|||
|
|
|
|||
|
|
return {"status": "success", "variant": variant_type, "elapsed": elapsed}
|
|||
|
|
|
|||
|
|
except Exception as e:
|
|||
|
|
elapsed = time.time() - start_time
|
|||
|
|
with metrics_lock:
|
|||
|
|
metrics["total_executions"] += 1
|
|||
|
|
metrics["failed_executions"] += 1
|
|||
|
|
if len(metrics["error_log"]) < 10:
|
|||
|
|
metrics["error_log"].append(str(e)[:50])
|
|||
|
|
|
|||
|
|
return {"status": "error", "variant": variant_type, "error": str(e)[:30], "elapsed": elapsed}
|
|||
|
|
|
|||
|
|
finally:
|
|||
|
|
import os
|
|||
|
|
try:
|
|||
|
|
os.unlink(temp_path)
|
|||
|
|
except Exception as e:
|
|||
|
|
with metrics_lock:
|
|||
|
|
if len(metrics.get("error_log", [])) < 10:
|
|||
|
|
if "error_log" not in metrics:
|
|||
|
|
metrics["error_log"] = []
|
|||
|
|
metrics["error_log"].append(f"cleanup: {e}")
|
|||
|
|
|
|||
|
|
except Exception as e:
|
|||
|
|
with metrics_lock:
|
|||
|
|
metrics["failed_executions"] += 1
|
|||
|
|
return {"status": "fatal", "error": str(e)[:30]}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def worker_thread(work_queue: queue.Queue, variants: List):
|
|||
|
|
"""Worker thread that processes items from the work queue."""
|
|||
|
|
while True:
|
|||
|
|
try:
|
|||
|
|
instance_id = work_queue.get(timeout=1)
|
|||
|
|
if instance_id is None:
|
|||
|
|
break
|
|||
|
|
|
|||
|
|
variant = random.choice(variants)
|
|||
|
|
execute_instance(variant, instance_id)
|
|||
|
|
work_queue.task_done()
|
|||
|
|
|
|||
|
|
except queue.Empty:
|
|||
|
|
continue
|
|||
|
|
except Exception as e:
|
|||
|
|
with metrics_lock:
|
|||
|
|
if "error_log" not in metrics:
|
|||
|
|
metrics["error_log"] = []
|
|||
|
|
if len(metrics.get("error_log", [])) < 100:
|
|||
|
|
metrics["error_log"].append(f"worker: {e}")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def monitor_vram():
|
|||
|
|
"""Monitor VRAM usage."""
|
|||
|
|
while time.time() - metrics["start_time"] < DURATION_SECS:
|
|||
|
|
try:
|
|||
|
|
vram = psutil.virtual_memory()
|
|||
|
|
with metrics_lock:
|
|||
|
|
metrics["vram_peaks"].append({
|
|||
|
|
"timestamp": time.time() - metrics["start_time"],
|
|||
|
|
"percent": vram.percent,
|
|||
|
|
"used_gb": vram.used / (1024**3),
|
|||
|
|
})
|
|||
|
|
except Exception as e:
|
|||
|
|
with metrics_lock:
|
|||
|
|
if "error_log" not in metrics:
|
|||
|
|
metrics["error_log"] = []
|
|||
|
|
if len(metrics.get("error_log", [])) < 100:
|
|||
|
|
metrics["error_log"].append(f"vram_monitor: {e}")
|
|||
|
|
time.sleep(0.2)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def main():
|
|||
|
|
"""Execute 1 million parallel instances × 10,000 program variants."""
|
|||
|
|
print("\n" + "="*80)
|
|||
|
|
print("🔥 MEGA STRESS TEST: 1,000,000 Parallel Instances × 10,000 Program Variants")
|
|||
|
|
print("="*80)
|
|||
|
|
print(f"Start Time: {datetime.now().isoformat()}")
|
|||
|
|
print(f"Duration: {DURATION_SECS} seconds (5 minutes)")
|
|||
|
|
print(f"Worker Threads: {WORKER_THREADS}")
|
|||
|
|
print()
|
|||
|
|
|
|||
|
|
# Generate variants
|
|||
|
|
print("[1/4] Generating 10,000 program variants...")
|
|||
|
|
variants = generate_program_variants()
|
|||
|
|
print()
|
|||
|
|
|
|||
|
|
# Start VRAM monitor
|
|||
|
|
print("[2/4] Starting system monitoring...")
|
|||
|
|
vram_monitor = threading.Thread(target=monitor_vram, daemon=True)
|
|||
|
|
vram_monitor.start()
|
|||
|
|
print("✓ VRAM monitoring started")
|
|||
|
|
print()
|
|||
|
|
|
|||
|
|
# Create work queue
|
|||
|
|
print("[3/4] Initializing work queue...")
|
|||
|
|
work_queue = queue.Queue(maxsize=QUEUE_SIZE)
|
|||
|
|
print(f"✓ Queue created (max size: {QUEUE_SIZE})")
|
|||
|
|
print()
|
|||
|
|
|
|||
|
|
# Start worker threads
|
|||
|
|
print(f"[4/4] Starting {WORKER_THREADS} worker threads...")
|
|||
|
|
workers = []
|
|||
|
|
for i in range(WORKER_THREADS):
|
|||
|
|
w = threading.Thread(target=worker_thread, args=(work_queue, variants), daemon=True)
|
|||
|
|
w.start()
|
|||
|
|
workers.append(w)
|
|||
|
|
print(f"✓ All {WORKER_THREADS} workers started")
|
|||
|
|
print()
|
|||
|
|
print("Submitting 1,000,000 work items...")
|
|||
|
|
print()
|
|||
|
|
|
|||
|
|
# Submit work items
|
|||
|
|
start_time = time.time()
|
|||
|
|
last_report = start_time
|
|||
|
|
submitted = 0
|
|||
|
|
|
|||
|
|
while time.time() - start_time < DURATION_SECS:
|
|||
|
|
# Try to fill the queue
|
|||
|
|
while not work_queue.full() and time.time() - start_time < DURATION_SECS:
|
|||
|
|
work_queue.put(submitted)
|
|||
|
|
submitted += 1
|
|||
|
|
|
|||
|
|
# Report progress every 30 seconds
|
|||
|
|
now = time.time()
|
|||
|
|
if now - last_report > 30:
|
|||
|
|
elapsed = now - start_time
|
|||
|
|
with metrics_lock:
|
|||
|
|
rate = metrics["total_executions"] / elapsed if elapsed > 0 else 0
|
|||
|
|
print(f"⚡ {metrics['total_executions']:,} executions | "
|
|||
|
|
f"{rate:,.0f} exec/sec | "
|
|||
|
|
f"{elapsed:.0f}s elapsed | "
|
|||
|
|
f"Success: {metrics['successful_executions']:,} | "
|
|||
|
|
f"Failed: {metrics['failed_executions']:,} | "
|
|||
|
|
f"Guardrails: {metrics['guardrail_triggers']}")
|
|||
|
|
last_report = now
|
|||
|
|
|
|||
|
|
time.sleep(0.1)
|
|||
|
|
|
|||
|
|
# Drain queue
|
|||
|
|
print("\nDraining work queue...")
|
|||
|
|
work_queue.join()
|
|||
|
|
|
|||
|
|
# Stop workers
|
|||
|
|
for _ in range(WORKER_THREADS):
|
|||
|
|
work_queue.put(None)
|
|||
|
|
|
|||
|
|
for w in workers:
|
|||
|
|
w.join(timeout=2)
|
|||
|
|
|
|||
|
|
metrics["end_time"] = time.time()
|
|||
|
|
total_elapsed = metrics["end_time"] - metrics["start_time"]
|
|||
|
|
|
|||
|
|
# Final report
|
|||
|
|
print()
|
|||
|
|
print("="*80)
|
|||
|
|
print("📊 MEGA STRESS TEST RESULTS")
|
|||
|
|
print("="*80)
|
|||
|
|
print()
|
|||
|
|
print(f"Duration: {total_elapsed:.1f} seconds")
|
|||
|
|
print(f"Total Executions: {metrics['total_executions']:,}")
|
|||
|
|
print(f"Successful: {metrics['successful_executions']:,}")
|
|||
|
|
print(f"Failed: {metrics['failed_executions']:,}")
|
|||
|
|
print(f"Success Rate: {100 * metrics['successful_executions'] / max(1, metrics['total_executions']):.1f}%")
|
|||
|
|
print()
|
|||
|
|
print(f"Throughput: {metrics['total_executions'] / total_elapsed:,.0f} executions/second")
|
|||
|
|
print(f"⚠️ Guardrail Triggers: {metrics['guardrail_triggers']}")
|
|||
|
|
print()
|
|||
|
|
|
|||
|
|
if metrics["vram_peaks"]:
|
|||
|
|
vram_percents = [v["percent"] for v in metrics["vram_peaks"]]
|
|||
|
|
vram_gbs = [v["used_gb"] for v in metrics["vram_peaks"]]
|
|||
|
|
print("Memory Usage:")
|
|||
|
|
print(f" Peak: {max(vram_percents):.1f}% ({max(vram_gbs):.2f} GB)")
|
|||
|
|
print(f" Average: {sum(vram_percents)/len(vram_percents):.1f}%")
|
|||
|
|
print()
|
|||
|
|
|
|||
|
|
if metrics["error_log"]:
|
|||
|
|
print(f"Sample Errors ({len(metrics['error_log'])} total):")
|
|||
|
|
for error in metrics["error_log"][:3]:
|
|||
|
|
print(f" {error}")
|
|||
|
|
print()
|
|||
|
|
|
|||
|
|
print("="*80)
|
|||
|
|
print(f"✅ MEGA STRESS TEST COMPLETE")
|
|||
|
|
print(f"End Time: {datetime.now().isoformat()}")
|
|||
|
|
print("="*80)
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
main()
|