330 lines
12 KiB
Bash
Executable File
330 lines
12 KiB
Bash
Executable File
#!/bin/bash
|
|
#===============================================================================
|
|
# GLYPHOS VS TRANSFORMER BENCHMARK SUITE - CLEAN VERSION
|
|
#===============================================================================
|
|
|
|
# Colors
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
CYAN='\033[0;36m'
|
|
NC='\033[0m'
|
|
BOLD='\033[1m'
|
|
|
|
PROJECT_DIR="benchmark_suite"
|
|
|
|
info() { echo -e "${BLUE}[INFO]${NC} $1"; }
|
|
success() { echo -e "${GREEN}[✓]${NC} $1"; }
|
|
warn() { echo -e "${YELLOW}[!]${NC} $1"; }
|
|
error() { echo -e "${RED}[✗]${NC} $1"; }
|
|
|
|
# =============================================================================
|
|
# DEPENDENCY INSTALLATION
|
|
# =============================================================================
|
|
install_deps() {
|
|
info "Checking Python installation..."
|
|
if ! command -v python3 &> /dev/null; then
|
|
error "Python3 not found!"
|
|
exit 1
|
|
fi
|
|
info "Found: $(python3 --version)"
|
|
|
|
info "Installing PyTorch and NumPy (this may take a minute)..."
|
|
|
|
# Try multiple methods
|
|
if python3 -c "import torch" 2>/dev/null; then
|
|
success "PyTorch already installed"
|
|
elif python3 -c "import pip" >/dev/null 2>&1; then
|
|
python3 -m pip install --quiet torch numpy psutil 2>&1 || {
|
|
warn "Standard pip failed, trying system pip..."
|
|
pip install --user --quiet torch numpy 2>&1 || {
|
|
error "Auto-install failed. Please run: pip install torch numpy"
|
|
return 1
|
|
}
|
|
}
|
|
success "Dependencies installed"
|
|
else
|
|
error "pip not found. Install Python pip first."
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# =============================================================================
|
|
# PROJECT SETUP
|
|
# =============================================================================
|
|
setup_project() {
|
|
info "Creating project in ${PROJECT_DIR}/..."
|
|
|
|
# Remove old directory cleanly
|
|
rm -rf "$PROJECT_DIR"
|
|
mkdir -p "$PROJECT_DIR"
|
|
|
|
# transformer_bench.py
|
|
cat > "${PROJECT_DIR}/transformer_bench.py" << 'EOF'
|
|
#!/usr/bin/env python3
|
|
"""REAL TRANSFORMER INFERENCE BENCHMARK"""
|
|
try:
|
|
import torch
|
|
import torch.nn as nn
|
|
except ImportError:
|
|
print("\033[91m[ERROR] torch not installed. Run: pip install torch\033[0m")
|
|
exit(1)
|
|
|
|
import time
|
|
import numpy as np
|
|
|
|
class SmallTransformer(nn.Module):
|
|
def __init__(self, d_model=256, n_heads=4, n_layers=2, max_seq=1024):
|
|
super().__init__()
|
|
self.d_model = d_model
|
|
self.embedding = nn.Embedding(10000, d_model)
|
|
self.pos_embed = nn.Parameter(torch.randn(1, max_seq, d_model) * 0.02)
|
|
|
|
attn = nn.MultiheadAttention(d_model, n_heads, dropout=0, batch_first=True)
|
|
self.attention = nn.ModuleList([attn] * n_layers)
|
|
self.ffn = nn.Sequential(
|
|
nn.Linear(d_model, d_model * 4),
|
|
nn.ReLU(),
|
|
nn.Linear(d_model * 4, d_model)
|
|
)
|
|
self.norm = nn.LayerNorm(d_model)
|
|
self.lm_head = nn.Linear(d_model, 10000)
|
|
|
|
def forward(self, x):
|
|
seq_len = x.size(1)
|
|
h = self.embedding(x) + self.pos_embed[:, :seq_len, :]
|
|
mask = torch.triu(torch.ones(seq_len, seq_len), diagonal=1).bool().to(x.device)
|
|
|
|
for attn_layer in self.attention:
|
|
attn, _ = attn_layer(h, h, h, attn_mask=mask)
|
|
h = h + attn
|
|
h = h + self.ffn(h)
|
|
h = self.norm(h)
|
|
return self.lm_head(h)
|
|
|
|
def benchmark():
|
|
device = 'cuda' if torch.cuda.is_available() else 'cpu'
|
|
print(f"\033[36mDevice:\033[0m {device}")
|
|
if device == 'cuda':
|
|
print(f"\033[36mGPU:\033[0m {torch.cuda.get_device_name(0)}")
|
|
|
|
config = {'d_model': 256, 'n_heads': 4, 'n_layers': 2, 'max_seq': 1024}
|
|
model = SmallTransformer(**config).to(device).eval()
|
|
|
|
input_ids = torch.randint(0, 10000, (1, 256), device=device)
|
|
|
|
times = []
|
|
print("\033[33mRunning 5 inference cycles...\033[0m")
|
|
for i in range(5):
|
|
if device == 'cuda': torch.cuda.synchronize()
|
|
start = time.perf_counter()
|
|
with torch.inference_mode():
|
|
_ = model(input_ids)
|
|
if device == 'cuda': torch.cuda.synchronize()
|
|
times.append((time.perf_counter() - start) * 1000)
|
|
print(f" Cycle {i+1}: {times[-1]:.2f} ms")
|
|
|
|
print(f"\n\033[1m=== TRANSFORMER BASELINE RESULTS ===\033[0m")
|
|
print(f"TTFT (256 tokens): {np.mean(times):.2f} ± {np.std(times):.2f} ms")
|
|
print(f"Parameters: {sum(p.numel() for p in model.parameters()):,}")
|
|
print(f"Model: {config['n_layers']}-layer, {config['d_model']}d")
|
|
|
|
if __name__ == '__main__':
|
|
benchmark()
|
|
EOF
|
|
|
|
# glyph_os_bench.py
|
|
cat > "${PROJECT_DIR}/glyph_os_bench.py" << 'EOF'
|
|
#!/usr/bin/env python3
|
|
"""GLYPHOS SUBSTRATE BENCHMARK"""
|
|
try:
|
|
import numpy as np
|
|
except ImportError:
|
|
print("\033[91m[ERROR] numpy not installed. Run: pip install numpy\033[0m")
|
|
exit(1)
|
|
|
|
import time
|
|
|
|
def resonance(similarity):
|
|
return 1.0 / (1.0 + np.exp(-1.0 * (similarity - 4.0)))
|
|
|
|
class SubstrateGraph:
|
|
def __init__(self, node_count=4096, edges_per_node=4):
|
|
np.random.seed(42) # Reproducible
|
|
self.nodes = np.random.rand(node_count).astype(np.float32)
|
|
self.edges = [(i, (i * 7 + e * 13 + 3) % node_count)
|
|
for i in range(node_count) for e in range(edges_per_node)]
|
|
|
|
def converge(self, max_epochs=100, threshold=0.001):
|
|
for epoch in range(max_epochs):
|
|
new_nodes = self.nodes.copy()
|
|
max_delta = 0.0
|
|
for i in range(len(self.nodes)):
|
|
neighbors = [self.nodes[j] for _, j in self.edges if _ == i]
|
|
if neighbors:
|
|
sims = np.where(np.abs(self.nodes[i] - neighbors) < 0.5, 5.0, 0.0)
|
|
weights = np.array([resonance(s) for s in sims])
|
|
w_sum = np.sum(weights)
|
|
new_nodes[i] = np.sum(np.array(neighbors) * weights) / w_sum if w_sum > 0 else self.nodes[i]
|
|
delta = abs(new_nodes[i] - self.nodes[i])
|
|
max_delta = max(max_delta, delta)
|
|
self.nodes = new_nodes
|
|
if max_delta < threshold:
|
|
return epoch + 1, max_delta
|
|
return max_epochs, max_delta
|
|
|
|
def benchmark():
|
|
print("\033[35mGlyphOS Substrate Benchmark\033[0m")
|
|
|
|
# TTC test
|
|
print("\033[33mRunning convergence test (4096 nodes)...\033[0m")
|
|
start = time.perf_counter()
|
|
epochs, delta = SubstrateGraph(4096, 4).converge(100)
|
|
ttc = (time.perf_counter() - start) * 1000
|
|
print(f" Converged in {epochs} epochs, delta={delta:.4f}")
|
|
|
|
# NEPS test
|
|
print("\033[33mRunning throughput test...\033[0m")
|
|
start = time.perf_counter()
|
|
for _ in range(20):
|
|
SubstrateGraph(4096, 4).converge(5)
|
|
elapsed = time.perf_counter() - start
|
|
neps = (4096 * 20) / elapsed
|
|
|
|
print(f"\n\033[1m=== GLYPHOS BASELINE RESULTS ===\033[0m")
|
|
print(f"TTC (4096 nodes): {ttc:.2f} ms in {epochs} epochs")
|
|
print(f"NEPS: {neps:,.0f} node-epochs/sec")
|
|
print(f"\033[93mNote: Measures constraint graph relaxation, not AI inference\033[0m")
|
|
|
|
if __name__ == '__main__':
|
|
benchmark()
|
|
EOF
|
|
|
|
# compare.py
|
|
cat > "${PROJECT_DIR}/compare.py" << 'EOF'
|
|
#!/usr/bin/env python3
|
|
"""COMPARISON REPORT"""
|
|
print("""
|
|
\033[1;36m============================================================\033[0m
|
|
\033[1;36m GLYPHOS vs TRANSFORMER - COMPARATIVE ANALYSIS \033[0m
|
|
\033[1;36m============================================================\033[0m
|
|
|
|
\033[1;33m1. WHAT EACH BENCHMARK MEASURES\033[0m
|
|
------------------------------------------------------------
|
|
\033[1mTRANSFORMER:\033[0m
|
|
✓ Full vocabulary embedding (10,000+ classes)
|
|
✓ Multi-head attention O(N²) for ALL token pairs
|
|
✓ Softmax normalization (exponential operations)
|
|
✓ Residual connections + Layer Normalization
|
|
✓ Language model output head
|
|
|
|
\033[1mGLYPHOS:\033[0m
|
|
✓ Sparse graph with 4 edges per node
|
|
✓ Simple weighted averaging of neighbors
|
|
✓ Binary similarity check (fixed threshold)
|
|
✓ Sigmoid activation (cheap approximation)
|
|
✓ No vocabulary, no language modeling
|
|
|
|
\033[91mKEY POINT: These solve fundamentally DIFFERENT problems!\033[0m
|
|
|
|
\033[1;33m2. OPERATIONAL COST COMPARISON\033[0m
|
|
------------------------------------------------------------
|
|
| Component | Transformer | GlyphOS |
|
|
|--------------------|------------------|------------------|
|
|
| Attention Ops | \033[91m~500M/token\033[0m | \033[92m~16K/node\033[0m |
|
|
| Memory Pattern | \033[91mRandom/Cache miss\033[0m|\033[92m Sequential/Clean\033[0m|
|
|
| Scaling Behavior | \033[91mO(N²)\033[0m | \033[92mO(edges) ≈ O(N)\033[0m |
|
|
| Training Required | \033[91mYes (weeks)\033[0m | \033[92mNo (static)\033[0m |
|
|
| Capability | \033[93mText generation\033[0m | \033[93mGraph relaxation\033[0m |
|
|
|
|
\033[1;33m3. APPLES-TO-APPLES COMPARISON NEEDS\033[0m
|
|
------------------------------------------------------------
|
|
□ Same task (e.g., text completion)
|
|
✓ Same sequence length
|
|
✓ Same hardware
|
|
□ Same evaluation metric (perplexity, BLEU, etc.)
|
|
□ Same parameter budget
|
|
|
|
\033[91mWithout these, performance claims are misleading.\033[0m
|
|
|
|
\033[36mRun actual benchmarks to see real timings.\033[0m
|
|
""")
|
|
EOF
|
|
|
|
success "All files created in ${PROJECT_DIR}/"
|
|
}
|
|
|
|
# =============================================================================
|
|
# MENU SYSTEM
|
|
# =============================================================================
|
|
show_menu() {
|
|
clear
|
|
echo -e "${BOLD}${CYAN}"
|
|
echo "╔═══════════════════════════════════════════════════════════════╗"
|
|
echo "║ GLYPHOS vs TRANSFORMER BENCHMARK SUITE ║"
|
|
echo "╠═══════════════════════════════════════════════════════════════╣"
|
|
echo "║ [1] Run Transformer Baseline ║"
|
|
echo "║ [2] Run GlyphOS Substrate ║"
|
|
echo "║ [3] Run BOTH Benchmarks ║"
|
|
echo "║ [4] View Comparison Report ║"
|
|
echo "║ [5] Reset Project Files ║"
|
|
echo "║ [6] Exit ║"
|
|
echo "╚═══════════════════════════════════════════════════════════════╝"
|
|
echo -e "${NC}"
|
|
|
|
read -p "Choice [1-6]: " choice
|
|
|
|
case $choice in
|
|
1)
|
|
[ ! -f "${PROJECT_DIR}/transformer_bench.py" ] && {
|
|
warn "Setup needed..."; setup_project;
|
|
}
|
|
python3 "${PROJECT_DIR}/transformer_bench.py"
|
|
;;
|
|
2)
|
|
[ ! -f "${PROJECT_DIR}/glyph_os_bench.py" ] && {
|
|
warn "Setup needed..."; setup_project;
|
|
}
|
|
python3 "${PROJECT_DIR}/glyph_os_bench.py"
|
|
;;
|
|
3)
|
|
[ ! -f "${PROJECT_DIR}/transformer_bench.py" ] && setup_project
|
|
python3 "${PROJECT_DIR}/transformer_bench.py"
|
|
echo ""
|
|
echo "---"
|
|
echo ""
|
|
python3 "${PROJECT_DIR}/glyph_os_bench.py"
|
|
;;
|
|
4)
|
|
[ ! -f "${PROJECT_DIR}/compare.py" ] && setup_project
|
|
python3 "${PROJECT_DIR}/compare.py"
|
|
;;
|
|
5)
|
|
warn "Resetting project files..."
|
|
setup_project
|
|
;;
|
|
6)
|
|
echo -e "${GREEN}Goodbye!${NC}"
|
|
exit 0
|
|
;;
|
|
*)
|
|
error "Invalid choice"
|
|
sleep 1
|
|
;;
|
|
esac
|
|
|
|
echo
|
|
read -p "Press Enter to continue..."
|
|
clear
|
|
show_menu
|
|
}
|
|
|
|
# =============================================================================
|
|
# MAIN
|
|
# =============================================================================
|
|
clear
|
|
install_deps
|
|
setup_project
|
|
show_menu
|