From 695725857d73ace1aeeaf52d894ace748e04502a Mon Sep 17 00:00:00 2001 From: gyt Date: Thu, 9 Jul 2026 13:28:28 -0400 Subject: [PATCH] Initial commit: aether-tools-rs workspace --- .gitignore | 1 + Cargo.toml | 22 ++ crates/aec-atom/Cargo.toml | 18 + crates/aec-atom/src/main.rs | 250 ++++++++++++++ crates/aether-ask/Cargo.toml | 17 + crates/aether-ask/src/main.rs | 326 ++++++++++++++++++ crates/aether-ask/tests/integration.rs | 132 +++++++ crates/aether-net-mini/Cargo.toml | 19 + crates/aether-net-mini/src/main.rs | 245 +++++++++++++ crates/aether-recall-mcp/Cargo.toml | 18 + crates/aether-recall-mcp/src/main.rs | 311 +++++++++++++++++ .../aether-recall-mcp/tests/cross_binary.rs | 159 +++++++++ crates/aether-recall-mcp/tests/integration.rs | 130 +++++++ examples/library_usage/Cargo.toml | 8 + examples/library_usage/src/main.rs | 87 +++++ examples/stress_test/Cargo.toml | 9 + examples/stress_test/src/main.rs | 160 +++++++++ 17 files changed, 1912 insertions(+) create mode 100644 .gitignore create mode 100644 Cargo.toml create mode 100644 crates/aec-atom/Cargo.toml create mode 100644 crates/aec-atom/src/main.rs create mode 100644 crates/aether-ask/Cargo.toml create mode 100644 crates/aether-ask/src/main.rs create mode 100644 crates/aether-ask/tests/integration.rs create mode 100644 crates/aether-net-mini/Cargo.toml create mode 100644 crates/aether-net-mini/src/main.rs create mode 100644 crates/aether-recall-mcp/Cargo.toml create mode 100644 crates/aether-recall-mcp/src/main.rs create mode 100644 crates/aether-recall-mcp/tests/cross_binary.rs create mode 100644 crates/aether-recall-mcp/tests/integration.rs create mode 100644 examples/library_usage/Cargo.toml create mode 100644 examples/library_usage/src/main.rs create mode 100644 examples/stress_test/Cargo.toml create mode 100644 examples/stress_test/src/main.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2f7896d --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..f847c5a --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,22 @@ +[workspace] +resolver = "2" +members = ["crates/*", "examples/*"] + +[workspace.package] +version = "0.1.0" +edition = "2021" +license = "MIT OR Apache-2.0" + +[workspace.dependencies] +aether-atom = { path = "/home/cosmos/aether-atom" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha3 = "0.10" +tokio = { version = "1", features = ["rt-multi-thread", "sync", "macros", "io-util", "net"] } +clap = { version = "4", features = ["derive"] } + +[profile.release] +opt-level = 3 +lto = "thin" +codegen-units = 1 +strip = "symbols" diff --git a/crates/aec-atom/Cargo.toml b/crates/aec-atom/Cargo.toml new file mode 100644 index 0000000..33e735d --- /dev/null +++ b/crates/aec-atom/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "aec-atom" +version.workspace = true +edition.workspace = true + +[[bin]] +name = "aec-atom" +path = "src/main.rs" + +[dependencies] +aether-atom = { path = "/home/cosmos/aether-atom" } +sha3 = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +regex = "1" + +[dev-dependencies] +tempfile = "3" diff --git a/crates/aec-atom/src/main.rs b/crates/aec-atom/src/main.rs new file mode 100644 index 0000000..0cf8160 --- /dev/null +++ b/crates/aec-atom/src/main.rs @@ -0,0 +1,250 @@ +//! aec-atom — minimal REPL over aether-atom. +//! +//! Replaces the 1088-line aec interpreter with a thin shell. Six +//! builtins: lookup, remember, stats, replay, fingerprint, canonicalize. +//! The primitive is a single-token command followed by quoted args. +//! +//! Usage: +//! aec-atom # interactive REPL on stdin +//! AEC_ATOM_DATA_DIR=/path aec-atom # custom data dir + +use aether_atom::{data_dir, fingerprint, rust_canonicalize, Lattice}; +use std::env; +use std::io::{BufRead, BufReader, Write}; + +fn extract_quoted(s: &str) -> Vec { + // Split on whitespace; treat `"x y"` as a single token. + let mut out = Vec::new(); + let mut rest = s; + while !rest.is_empty() { + rest = rest.trim_start(); + if rest.is_empty() { break; } + if rest.starts_with('"') { + if let Some(end) = rest[1..].find('"') { + out.push(rest[1..end+1].to_string()); + rest = &rest[end + 2..]; + } else { + out.push(rest[1..].to_string()); + rest = ""; + } + } else { + let end = rest.find(char::is_whitespace).unwrap_or(rest.len()); + out.push(rest[..end].to_string()); + rest = &rest[end..]; + } + } + out +} + +fn handle(line: &str, lattice: &mut Lattice) -> String { + let trimmed = line.trim(); + if trimmed.is_empty() { return String::new(); } + if trimmed.starts_with('#') { return String::new(); } + let parts = extract_quoted(trimmed); + let cmd = parts.first().map(String::as_str).unwrap_or(""); + let args = &parts[1..]; + + match cmd { + "lookup" | "l" => { + let f = args.first().map(String::as_str).unwrap_or(""); + if f.is_empty() { + return "lookup: missing argument".into(); + } + let sig = fingerprint(f.as_bytes()); + match lattice.lookup(&sig, "global") { + Some(hit) => format!("[recall hit] {} -> {}", sig, String::from_utf8_lossy(hit.value.as_ref())), + None => format!("[recall miss] {} (work was never recorded)", sig), + } + } + "remember" | "r" => { + if args.len() < 2 { + return "remember: need ".into(); + } + let sig = fingerprint(args[0].as_bytes()); + let bytes = args[1].as_bytes().to_vec(); + lattice.record(sig, "global", &bytes, Some(true)); + format!("[recorded] {} -> {} bytes", sig, bytes.len()) + } + "demote" | "d" => { + if args.is_empty() { + return "demote: need ".into(); + } + let sig = fingerprint(args[0].as_bytes()); + lattice.record(sig, "global", b"[demoted]", Some(false)); + format!("[demoted] {}", sig) + } + "stats" | "s" => { + format!( + "records: {} lookups: {} hits: {} misses: {} outcomes: {} hit_rate: {:.1}%", + lattice.record_count(), + lattice.lookups, + lattice.hits, + lattice.misses, + lattice.outcomes, + lattice.hit_rate() * 100.0 + ) + } + "compact" | "c" => { + let retention_secs = env::var("AEC_COMPACT_DAYS") + .ok().and_then(|v| v.parse().ok()) + .unwrap_or(30u64) * 86_400; + match lattice.compact(retention_secs) { + Ok(dropped) => format!("compacted: dropped {} old lookups", dropped), + Err(e) => format!("compact error: {}", e), + } + } + "replay" | "R" => { + let path = env::var("AEC_REPLAY_PATH").unwrap_or_else(|_| { + format!("{}/events.jsonl", data_dir("AETHER_RECALL_DIR").display()) + }); + match std::fs::read_to_string(&path) { + Ok(text) => { + let lines: Vec<&str> = text.lines().filter(|l| !l.trim().is_empty()).collect(); + let start = lines.len().saturating_sub(20); + let mut out = format!("last 20 of {} events:\n", lines.len()); + for l in &lines[start..] { + out.push_str(l); + out.push('\n'); + } + out + } + Err(e) => format!("replay error: {}", e), + } + } + "fingerprint" | "fp" => { + let input = args.first().map(String::as_str).unwrap_or(""); + let sig = fingerprint(input.as_bytes()); + format!("{}", sig) + } + "canonicalize" | "c14" => { + let input = args.first().map(String::as_str).unwrap_or(""); + let can = rust_canonicalize(input); + format!("canonical: {}", String::from_utf8_lossy(&can)) + } + "help" | "h" | "?" => { + "commands:\n lookup / l — atom.lookup\n remember / r — atom.record (worked=true)\n demote / d — atom.record (worked=false)\n stats / s — hit_rate + counters\n compact / c — drop lookups older than AEC_COMPACT_DAYS\n replay / R — last 20 jsonl events\n fingerprint / fp — sha3 hex\n canonicalize / c14 — Rust canonical form\n help / h / ? — this\n quit / q — exit".into() + } + "quit" | "q" | "exit" => "::quit::".into(), + other => format!("unknown: {} (try `help`)", other), + } +} + +fn main() -> Result<(), Box> { + let dir = data_dir("AETHER_RECALL_DIR"); + let mut lattice = Lattice::open(&dir)?; + eprintln!("aec-atom ready (memory: {:?})", dir); + eprintln!("type 'help' for commands, 'quit' to exit"); + + let stdin = std::io::stdin(); + let mut stdout = std::io::stdout(); + let reader = BufReader::new(stdin.lock()); + + for line in reader.lines() { + let line = match line { + Ok(l) => l, + Err(_) => break, + }; + let out = handle(&line, &mut lattice); + if out == "::quit::" { break; } + if out.is_empty() { continue; } + writeln!(stdout, "{}", out)?; + stdout.flush()?; + write!(stdout, "ae> ")?; + stdout.flush()?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + #[test] + fn extract_quoted_simple() { + let p = extract_quoted(r#"hello "world foo" "#); + assert_eq!(p, vec!["hello", "world foo"]); + } + + #[test] + fn help_lists_all_commands() { + let dir = tempdir().unwrap(); + let mut lat = Lattice::open(dir.path()).unwrap(); + let resp = handle("help", &mut lat); + assert!(resp.contains("lookup")); + assert!(resp.contains("remember")); + assert!(resp.contains("canonicalize")); + } + + #[test] + fn remember_then_lookup_round_trip() { + let dir = tempdir().unwrap(); + let mut lat = Lattice::open(dir.path()).unwrap(); + let _ = handle(r#"remember "error[E0308]: foo" "fix this""#, &mut lat); + let resp = handle(r#"lookup "error[E0308]: foo""#, &mut lat); + assert!(resp.contains("[recall hit]")); + assert!(resp.contains("fix this")); + } + + #[test] + fn lookup_miss_after_no_record() { + let dir = tempdir().unwrap(); + let mut lat = Lattice::open(dir.path()).unwrap(); + let resp = handle(r#"lookup "never seen""#, &mut lat); + assert!(resp.contains("[recall miss]")); + } + + #[test] + fn stats_returns_counters() { + let dir = tempdir().unwrap(); + let mut lat = Lattice::open(dir.path()).unwrap(); + let _ = handle(r#"remember "x" "y""#, &mut lat); + let _ = handle(r#"lookup "x""#, &mut lat); + let resp = handle("stats", &mut lat); + assert!(resp.contains("records:")); + assert!(resp.contains("outcomes: 1")); + assert!(resp.contains("lookups: 1")); + } + + #[test] + fn fingerprint_command_matches_atom() { + let dir = tempdir().unwrap(); + let mut lat = Lattice::open(dir.path()).unwrap(); + let a = handle(r#"fingerprint "hello""#, &mut lat); + let b = format!("{}", fingerprint(b"hello")); + assert_eq!(a, b); + } + + #[test] + fn canonicalize_command_produces_canonical() { + let dir = tempdir().unwrap(); + let mut lat = Lattice::open(dir.path()).unwrap(); + let a = handle(r#"canonicalize "error[E0308]: expected i32, found &str""#, &mut lat); + assert!(a.contains("error[E0308]")); + } + + #[test] + fn unknown_command_handled() { + let dir = tempdir().unwrap(); + let mut lat = Lattice::open(dir.path()).unwrap(); + let resp = handle("nonsense", &mut lat); + assert!(resp.starts_with("unknown:")); + } + + #[test] + fn empty_line_no_output() { + let dir = tempdir().unwrap(); + let mut lat = Lattice::open(dir.path()).unwrap(); + assert_eq!(handle("", &mut lat), ""); + assert_eq!(handle(" ", &mut lat), ""); + assert_eq!(handle("# comment", &mut lat), ""); + } + + #[test] + fn quit_signal() { + let dir = tempdir().unwrap(); + let mut lat = Lattice::open(dir.path()).unwrap(); + assert_eq!(handle("quit", &mut lat), "::quit::"); + assert_eq!(handle("q", &mut lat), "::quit::"); + } +} diff --git a/crates/aether-ask/Cargo.toml b/crates/aether-ask/Cargo.toml new file mode 100644 index 0000000..ee00fa3 --- /dev/null +++ b/crates/aether-ask/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "aether-ask" +version.workspace = true +edition.workspace = true + +[[bin]] +name = "aether-ask" +path = "src/main.rs" + +[dependencies] +aether-atom = { path = "/home/cosmos/aether-atom" } +serde = { workspace = true } +serde_json = { workspace = true } +tokio = { workspace = true, features = ["rt-multi-thread", "io-util", "net", "process", "time", "macros"] } + +[dev-dependencies] +serde_json = { workspace = true } diff --git a/crates/aether-ask/src/main.rs b/crates/aether-ask/src/main.rs new file mode 100644 index 0000000..b6d109e --- /dev/null +++ b/crates/aether-ask/src/main.rs @@ -0,0 +1,326 @@ +//! aether-ask — HTTP front for the atom with ollama fallback. +//! +//! The original aether-ask was 386 LOC across stdin-JSON-RPC spawn. +//! This rewrite uses the atom for storage and tokio for the HTTP +//! loop. The same 3 verbs (cache, fallback, remember) over a +//! hyper-thin HTTP surface. +//! +//! Routes: +//! GET / — dark-mode single-page UI +//! GET /q/ — cached-or-fallback response as
+//!   POST /ask           — JSON `{question, answer}`, also JSON in/JSON out
+//!   GET  /api/stats     — atom stats
+//!
+//! Tunables via env:
+//!   AETHER_ASK_PORT         (default 8082)
+//!   OLLAMA_HOST            (default 127.0.0.1)
+//!   OLLAMA_PORT            (default 11434)
+//!   OLLAMA_MODEL           (default dolphin-llama3:8b)
+//!   AETHER_ASK_DATA_DIR     (default $HOME/.aether-recall)
+
+use aether_atom::{data_dir, fingerprint, Lattice};
+use serde_json::{json, Value};
+use std::env;
+use std::io::Write;
+use std::net::TcpStream;
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt};
+use tokio::net::TcpListener;
+use tokio::sync::Mutex;
+
+fn ask_port() -> u16 {
+    env::var("AETHER_ASK_PORT")
+        .ok()
+        .and_then(|s| s.parse().ok())
+        .unwrap_or(8082)
+}
+fn ollama_host() -> String {
+    env::var("OLLAMA_HOST").unwrap_or_else(|_| "127.0.0.1".into())
+}
+fn ollama_port() -> u16 {
+    env::var("OLLAMA_PORT").ok().and_then(|s| s.parse().ok()).unwrap_or(11434)
+}
+fn ollama_model() -> String {
+    env::var("OLLAMA_MODEL").unwrap_or_else(|_| "dolphin-llama3:8b".into())
+}
+
+fn http_response(code: u16, ct: &str, body: &str) -> Vec {
+    format!(
+        "HTTP/1.1 {code} {}\r\nContent-Type: {ct}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
+        reason(code),
+        body.len()
+    )
+    .into_bytes()
+    .iter()
+    .chain(body.as_bytes().iter())
+    .copied()
+    .collect()
+}
+
+fn reason(c: u16) -> &'static str {
+    match c {
+        200 => "OK",
+        400 => "Bad Request",
+        404 => "Not Found",
+        500 => "Internal Server Error",
+        _ => "Status",
+    }
+}
+
+const HTML: &str = r#"Aether Ask
+
+

AetherAsk

+

canonicalize → remember → ask. cached first, ollama fallback.

+ +
+"#; + +#[tokio::main(flavor = "current_thread")] +async fn main() -> Result<(), Box> { + let port = ask_port(); + let addr = format!("0.0.0.0:{}", port); + let listener = TcpListener::bind(&addr).await?; + let dir = data_dir("AETHER_RECALL_DIR"); + let lattice = Arc::new(Mutex::new(Lattice::open(&dir)?)); + eprintln!("aether-ask listening on http://{} (memory: {:?})", addr, dir); + eprintln!(" ollama fallback at {}:{} model={}", ollama_host(), ollama_port(), ollama_model()); + + loop { + let (stream, _) = listener.accept().await?; + let lat = lattice.clone(); + tokio::spawn(async move { + if let Err(e) = handle_http(stream, lat).await { + eprintln!("http conn error: {}", e); + } + }); + } +} + +async fn handle_http( + stream: tokio::net::TcpStream, + lattice: Arc>, +) -> Result<(), Box> { + use tokio::io::BufReader; + let (read_half, mut write_half) = stream.into_split(); + let mut reader = BufReader::new(read_half); + let mut head = Vec::new(); + let mut total: usize = 0; + + // Read headers + loop { + let mut line = String::new(); + let n = reader.read_line(&mut line).await?; + if n == 0 { break; } + total += n; + if line == "\r\n" || line == "\n" { break; } + head.push(line); + if total > 16384 { break; } + } + if head.is_empty() { return Ok(()); } + + let head_str = head.concat(); + let mut parts = head_str.split_whitespace(); + let method = parts.next().unwrap_or("").to_string(); + let path = parts.next().unwrap_or("").to_string(); + let path_no_q = path.split('?').next().unwrap_or(&path).to_string(); + + let body_bytes = { + let cl = head_str + .lines() + .find_map(|l| l.to_ascii_lowercase().strip_prefix("content-length: ") + .and_then(|n| n.trim().parse::().ok())) + .unwrap_or(0); + if cl > 0 && cl < 65536 { + let mut b = vec![0u8; cl]; + let mut read = 0; + while read < cl { + let n = reader.read(&mut b[read..]).await?; + if n == 0 { break; } + read += n; + } + b.truncate(read); + Some(b) + } else { + None + } + }; + + let q = match (method.as_str(), path_no_q.as_str()) { + ("GET", p) if p.starts_with("/q/") => percent_decode(&p[3..]), + ("GET", "/api/stats") => { + let lat = lattice.lock().await; + let body = json!({ + "records": lat.record_count(), + "lookups": lat.lookups, + "hits": lat.hits, + "misses": lat.misses, + "hit_rate": lat.hit_rate(), + "outcomes": lat.outcomes, + }).to_string(); + let resp = http_response(200, "application/json", &body); + write_half.write_all(&resp).await?; + return Ok(()); + } + ("GET", "/") | ("GET", "/index.html") => { + let resp = http_response(200, "text/html; charset=utf-8", HTML); + write_half.write_all(&resp).await?; + return Ok(()); + } + ("POST", "/ask") => { + let b = body_bytes.unwrap_or_default(); + // Parse `{"question":"..."}` out of the POST body. Fall back + // to the whole body if not JSON. + match serde_json::from_slice::(&b) { + Ok(v) => v.get("question").and_then(|q| q.as_str()).unwrap_or("").to_string(), + Err(_) => String::from_utf8_lossy(&b).trim().to_string(), + } + } + _ => { + let resp = http_response(404, "text/plain", "not found"); + write_half.write_all(&resp).await?; + return Ok(()); + } + }; + + if q.is_empty() { + let resp = http_response(400, "application/json", + "{\"error\":\"missing question\"}"); + write_half.write_all(&resp).await?; + return Ok(()); + } + + // Cache-or-fallback: check atom first, ollama on miss, then remember. + let cached: Option> = { + let mut lat = lattice.lock().await; + lat.lookup(&fingerprint(q.as_bytes()), "global") + .map(|h| h.value.as_ref().clone()) + }; + + let (body_bytes, source) = match cached { + Some(c) => (c, "deterministic"), + None => { + let ans = call_ollama_blocking(&q); + { + let mut lat = lattice.lock().await; + lat.record(fingerprint(q.as_bytes()), "global", &ans, Some(true)); + } + (ans, "neural") + } + }; + + let answer_str = String::from_utf8_lossy(&body_bytes); + let body = match method.as_str() { + "POST" => json!({ "question": q, "answer": answer_str, "source": source }).to_string(), + _ => format!("[{}] {}\n", source, answer_str), + }; + let ct = if method == "POST" { "application/json" } else { "text/html; charset=utf-8" }; + let resp = http_response(200, ct, &body); + write_half.write_all(&resp).await?; + Ok(()) +} + +fn percent_decode(s: &str) -> String { + let mut out: Vec = Vec::with_capacity(s.len()); + let bytes = s.as_bytes(); + let mut i = 0; + while i < bytes.len() { + match bytes[i] { + b'+' => { out.push(b' '); i += 1; } + b'%' if i + 2 < bytes.len() => { + let h = (bytes[i+1] as char).to_digit(16); + let l = (bytes[i+2] as char).to_digit(16); + if let (Some(h), Some(l)) = (h, l) { + out.push((h*16 + l) as u8); + i += 3; + } else { + out.push(b'%'); i += 1; + } + } + b => { out.push(b); i += 1; } + } + } + String::from_utf8_lossy(&out).into_owned() +} + +fn call_ollama_blocking(question: &str) -> Vec { + let host = ollama_host(); + let port = ollama_port(); + let body = json!({ + "model": ollama_model(), + "prompt": question, + "stream": false, + "options": { "temperature": 0.1, "num_predict": 512 } + }).to_string(); + + let mut stream = match TcpStream::connect_timeout( + &format!("{}:{}", host, port).parse().unwrap(), + Duration::from_secs(5), + ) { + Ok(s) => s, + Err(_) => return b"[error: ollama unavailable]".to_vec(), + }; + let req = format!( + "POST /api/generate HTTP/1.1\r\nHost: {}:{}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + host, port, body.len(), body + ); + if stream.write_all(req.as_bytes()).is_err() { + return b"[error: ollama write failed]".to_vec(); + } + let mut buf = String::new(); + if std::io::Read::read_to_string(&mut std::io::BufReader::new(&mut stream), &mut buf).is_err() { + return b"[error: ollama read failed]".to_vec(); + } + if let Some(start) = buf.find('{') { + if let Some(end) = buf[start..].rfind('}') { + if let Ok(v) = serde_json::from_str::(&buf[start..=start+end]) { + if let Some(s) = v.get("response").and_then(|r| r.as_str()) { + return s.as_bytes().to_vec(); + } + } + } + } + b"[error: malformed ollama response]".to_vec() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn percent_decode_basic() { + assert_eq!(percent_decode("hello"), "hello"); + assert_eq!(percent_decode("a+b"), "a b"); + assert_eq!(percent_decode("a%20b"), "a b"); + assert_eq!(percent_decode("a%2Bb"), "a+b"); + assert_eq!(percent_decode("%E2%9C%93"), "\u{2713}"); + } + + #[tokio::test] + async fn http_routes_dispatch_by_method_and_path() { + // Just exercise the routing logic in isolation; we don't bind the + // listener here. The end-to-end test uses a real socket. + let cases = vec![ + ("GET", "/", true), + ("GET", "/api/stats", true), + ("GET", "/q/hello", false), // asks the lattice; might miss in fresh dir + ("POST", "/ask", true), + ("GET", "/garbage", false), + ]; + for (m, p, _) in cases { + // trivial smoke that the matcher doesn't panic + let _ = (m.to_string(), p.to_string()); + } + } +} diff --git a/crates/aether-ask/tests/integration.rs b/crates/aether-ask/tests/integration.rs new file mode 100644 index 0000000..beb81d8 --- /dev/null +++ b/crates/aether-ask/tests/integration.rs @@ -0,0 +1,132 @@ +//! End-to-end integration tests for aether-ask. +//! +//! Spawns the actual HTTP server on a real port and verifies the +//! /api/stats, /q/, and POST /ask flows. + +use serde_json::json; +use std::io::{Read, Write}; +use std::net::TcpStream; +use std::path::PathBuf; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +fn binary_path() -> PathBuf { + PathBuf::from(env!("CARGO_BIN_EXE_aether-ask")) +} + +struct Ask { + child: Child, + port: u16, +} + +impl Ask { + fn spawn(data_dir: &PathBuf) -> Self { + let stamp = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); + let port = 31000 + ((stamp % 1000) as u16); + let mut cmd = Command::new(binary_path()); + cmd.env("AETHER_RECALL_DIR", data_dir) + .env("AETHER_ASK_PORT", port.to_string()) + .env("OLLAMA_HOST", "127.0.0.1") + .env("OLLAMA_PORT", "1") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + let mut child = cmd.spawn().expect("spawn aether-ask"); + + // Wait for it to bind + let addr = format!("127.0.0.1:{}", port); + for _ in 0..50 { + if TcpStream::connect_timeout(&addr.parse().unwrap(), Duration::from_millis(50)).is_ok() { + return Self { child, port }; + } + std::thread::sleep(Duration::from_millis(50)); + } + let _ = child.kill(); + let _ = child.wait(); + panic!("aether-ask did not bind to {}", addr); + } + + fn http(&self, req: &str) -> String { + let mut s = TcpStream::connect(("127.0.0.1", self.port)).unwrap(); + s.set_read_timeout(Some(Duration::from_secs(3))).ok(); + s.write_all(req.as_bytes()).unwrap(); + let mut buf = Vec::new(); + s.read_to_end(&mut buf).ok(); + String::from_utf8_lossy(&buf).to_string() + } +} + +impl Drop for Ask { + fn drop(&mut self) { + let _ = self.child.kill(); + } +} + +#[test] +fn stats_endpoint_returns_json() { + let dir = tempdir_in("/tmp"); + let ask = Ask::spawn(&dir); + let r = ask.http("GET /api/stats HTTP/1.1\r\nHost: localhost\r\n\r\n"); + assert!(r.contains("\"records\":"), "got: {r}"); + assert!(r.contains("\"hit_rate\":")); +} + +#[test] +fn get_root_serves_html() { + let dir = tempdir_in("/tmp"); + let ask = Ask::spawn(&dir); + let r = ask.http("GET / HTTP/1.1\r\nHost: localhost\r\n\r\n"); + assert!(r.starts_with("HTTP/1.1 200")); + assert!(r.contains("AetherAsk")); +} + +#[test] +fn post_ask_returns_json() { + let dir = tempdir_in("/tmp"); + let ask = Ask::spawn(&dir); + let body = json!({"question":"hello-world-test-q"}).to_string(); + let r = ask.http(&format!( + "POST /ask HTTP/1.1\r\nHost: localhost\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{}", + body.len(), body + )); + assert!(r.starts_with("HTTP/1.1 200")); + let body = r.split("\r\n\r\n").nth(1).unwrap_or(""); + let json: serde_json::Value = serde_json::from_str(body).expect("json body"); + // No ollama: should report the source as neural with the error message + // or, if cached from a prior ask, maybe deterministic. ollama is + // unreachable so we expect the [error] fallback body path. + let answer = json["answer"].as_str().unwrap(); + // Either successfully recorded with deterministic fallback, or stored + // the ollama error. We don't enforce the exact text, only that answer field exists. + assert!(!answer.is_empty()); + // The next call must hit the cache (deterministic) + let body2 = json!({"question":"hello-world-test-q"}).to_string(); + let r2 = ask.http(&format!( + "POST /ask HTTP/1.1\r\nHost: localhost\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{}", + body2.len(), body2 + )); + let body2 = r2.split("\r\n\r\n").nth(1).unwrap_or(""); + let json2: serde_json::Value = serde_json::from_str(body2).unwrap(); + assert_eq!(json2["source"], "deterministic", + "second identical ask must be a cache hit, got: {json2:?}"); + assert_eq!(json2["answer"], json["answer"]); +} + +#[test] +fn get_q_route_responds_with_deterministic_after_seeding() { + let dir = tempdir_in("/tmp"); + let ask = Ask::spawn(&dir); + // First, record a value directly via /api/... no — we have only /q/ and /ask. + // So we issue the same query twice. First ollama fails -> [error …]; second is cache hit. + let q = "second-call-determ-test"; + let _ = ask.http(&format!("GET /q/{q} HTTP/1.1\r\nHost: localhost\r\n\r\n")); + let r2 = ask.http(&format!("GET /q/{q} HTTP/1.1\r\nHost: localhost\r\n\r\n")); + assert!(r2.contains("[deterministic]"), "got: {r2}"); +} + +fn tempdir_in(parent: &str) -> PathBuf { + let stamp = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); + let p = PathBuf::from(parent).join(format!("aether-ask-it-{}", stamp)); + std::fs::create_dir_all(&p).unwrap(); + p +} diff --git a/crates/aether-net-mini/Cargo.toml b/crates/aether-net-mini/Cargo.toml new file mode 100644 index 0000000..81fa36f --- /dev/null +++ b/crates/aether-net-mini/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "aether-net-mini" +version.workspace = true +edition.workspace = true + +[[bin]] +name = "aether-net-mini" +path = "src/main.rs" + +[dependencies] +aether-atom = { path = "/home/cosmos/aether-atom" } +serde = { workspace = true } +serde_json = { workspace = true } +sha3 = { workspace = true } +tokio = { workspace = true } +clap = { workspace = true } + +[dev-dependencies] +tempfile = "3" diff --git a/crates/aether-net-mini/src/main.rs b/crates/aether-net-mini/src/main.rs new file mode 100644 index 0000000..ca897ba --- /dev/null +++ b/crates/aether-net-mini/src/main.rs @@ -0,0 +1,245 @@ +//! aether-net-mini — DHT abstraction over aether-atom. +//! +//! The original aether-net was 1173 LOC of single-threaded TCP JSON-RPC +//! with a custom in-memory hashmap. This rewrite uses the atom for +//! storage and tokio for the network loop. Every byte of public API +//! (`put`, `get`) is a one-liner over the atom. +//! +//! Reduced surface: 2 methods instead of 15. Persistence is implicit +//! (atom ledger survives restart). Scopes work the same way (atom +//! supports `global` and arbitrary per-repo overrides). + +use aether_atom::{fingerprint, Lattice}; +use clap::{Parser, Subcommand}; +use serde_json::{json, Value}; +use std::io::{BufRead, BufReader, Write}; +use std::net::TcpStream; +use std::sync::Arc; +use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader as ABufReader}; +use tokio::net::TcpListener as ATcpListener; +use tokio::sync::Mutex; + +#[derive(Parser, Debug)] +#[command(name = "aether-net-mini", about = "DHT over aether-atom")] +struct Cli { + #[command(subcommand)] + cmd: Cmd, +} + +#[derive(Subcommand, Debug)] +enum Cmd { + /// Long-running daemon. Stores all content in aether-atom. + Daemon { + #[arg(long, default_value = "127.0.0.1:0")] + listen: String, + #[arg(long)] + data_dir: Option, + }, + /// One-shot client: connect, send a single JSON-RPC request, print + /// response. + Client { + #[arg(long)] + server: String, + #[arg(long)] + method: String, + #[arg(long, default_value = "{}", value_parser = parse_json_arg)] + params: Value, + }, +} + +fn parse_json_arg(s: &str) -> Result { + serde_json::from_str(s).map_err(|e| e.to_string()) +} + +fn err_response(id: Value, code: i64, msg: &str) -> Value { + json!({ "jsonrpc": "2.0", "id": id, "error": { "code": code, "message": msg } }) +} +fn ok_response(id: Value, result: Value) -> Value { + json!({ "jsonrpc": "2.0", "id": id, "result": result }) +} + +/// The two real verbs over aether-atom's lattice: +/// - `put(data)` → hash it, store +/// - `get(hash)` → retrieve +/// +/// Subscribe / replicate are future features; not in this surface. +async fn dispatch( + method: &str, + params: Value, + lattice: &mut Lattice, +) -> Result { + match method { + "put" => { + let data = params + .get("data") + .and_then(|v| v.as_str()) + .ok_or_else(|| "missing 'data'".to_string())?; + let bytes = data.as_bytes(); + let sig = fingerprint(bytes); + lattice.record(sig, "global", bytes, Some(true)); + Ok(json!({ "hash": sig.as_bytes().iter().map(|b| format!("{:02x}", b)).collect::() })) + } + "get" => { + let hash_hex = params + .get("hash") + .and_then(|v| v.as_str()) + .ok_or_else(|| "missing 'hash'".to_string())?; + let sig_bytes = aether_atom::hex_decode(hash_hex) + .ok_or_else(|| "invalid hex hash".to_string())?; + let sig = aether_atom::Fingerprint(sig_bytes); + match lattice.lookup(&sig, "global") { + Some(hit) => Ok(json!({ + "hash": hash_hex, + "found": true, + "content": String::from_utf8_lossy(hit.value.as_ref()).to_string(), + })), + None => Ok(json!({ "hash": hash_hex, "found": false })), + } + } + other => Err(format!("unknown method: {}", other)), + } +} + +#[tokio::main(flavor = "current_thread")] +async fn main() -> Result<(), Box> { + let cli = Cli::parse(); + match cli.cmd { + Cmd::Daemon { listen, data_dir } => { + let dir = match data_dir { + Some(d) => std::path::PathBuf::from(d), + None => aether_atom::data_dir("AETHER_NET_DATA_DIR"), + }; + let lattice = Arc::new(Mutex::new(Lattice::open(&dir)?)); + eprintln!("aether-net-mini daemon listening on {} (data: {:?})", listen, dir); + let listener = ATcpListener::bind(&listen).await?; + + loop { + let (stream, addr) = listener.accept().await?; + let lat = lattice.clone(); + tokio::spawn(async move { + if let Err(e) = handle_conn(stream, lat).await { + eprintln!(" conn {} error: {}", addr, e); + } + }); + } + } + Cmd::Client { server, method, params } => { + let mut s = TcpStream::connect(&server)?; + let req = json!({ "jsonrpc": "2.0", "id": 1, "method": method, "params": params }); + let req_str = serde_json::to_string(&req)? + "\n"; + s.write_all(req_str.as_bytes())?; + let mut reader = BufReader::new(s); + let mut line = String::new(); + reader.read_line(&mut line)?; + println!("{}", line.trim()); + } + } + Ok(()) +} + +async fn handle_conn( + stream: tokio::net::TcpStream, + lattice: Arc>, +) -> Result<(), Box> { + let (read_half, mut write_half) = stream.into_split(); + let mut reader = ABufReader::new(read_half); + let mut line = String::new(); + + loop { + line.clear(); + let n = reader.read_line(&mut line).await?; + if n == 0 { + break; // EOF + } + if line.trim().is_empty() { + continue; + } + let resp = match serde_json::from_str::(&line) { + Ok(msg) => { + let id = msg.get("id").cloned().unwrap_or(Value::Null); + let method = msg.get("method").and_then(|m| m.as_str()).unwrap_or(""); + let params = msg.get("params").cloned().unwrap_or_else(|| json!({})); + let mut lat = lattice.lock().await; + match dispatch(method, params, &mut lat).await { + Ok(result) => ok_response(id, result), + Err(msg) => err_response(id, -32000, &msg), + } + } + Err(_) => continue, // skip malformed lines + }; + let s = serde_json::to_string(&resp)? + "\n"; + write_half.write_all(s.as_bytes()).await?; + write_half.flush().await?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + #[tokio::test] + async fn put_then_get_round_trip() { + let dir = tempdir().unwrap(); + let mut lat = Lattice::open(dir.path()).unwrap(); + + let v = dispatch( + "put", + json!({ "data": "hello world" }), + &mut lat, + ) + .await + .unwrap(); + let hash = v["hash"].as_str().unwrap().to_string(); + + let v = dispatch( + "get", + json!({ "hash": hash }), + &mut lat, + ) + .await + .unwrap(); + assert_eq!(v["found"], true); + assert_eq!(v["content"], "hello world"); + } + + #[tokio::test] + async fn get_unknown_returns_not_found() { + let dir = tempdir().unwrap(); + let mut lat = Lattice::open(dir.path()).unwrap(); + let v = dispatch( + "get", + json!({ "hash": "0000000000000000000000000000000000000000000000000000000000000000" }), + &mut lat, + ) + .await + .unwrap(); + assert_eq!(v["found"], false); + } + + #[tokio::test] + async fn put_missing_data_errors() { + let dir = tempdir().unwrap(); + let mut lat = Lattice::open(dir.path()).unwrap(); + let r = dispatch("put", json!({}), &mut lat).await; + assert!(r.is_err()); + assert!(r.unwrap_err().contains("data")); + } + + #[tokio::test] + async fn get_invalid_hex_errors() { + let dir = tempdir().unwrap(); + let mut lat = Lattice::open(dir.path()).unwrap(); + let r = dispatch("get", json!({ "hash": "Z" }), &mut lat).await; + assert!(r.is_err()); + } + + #[tokio::test] + async fn unknown_method_errors() { + let dir = tempdir().unwrap(); + let mut lat = Lattice::open(dir.path()).unwrap(); + let r = dispatch("foo", json!({}), &mut lat).await; + assert!(r.is_err()); + } +} diff --git a/crates/aether-recall-mcp/Cargo.toml b/crates/aether-recall-mcp/Cargo.toml new file mode 100644 index 0000000..5a410b5 --- /dev/null +++ b/crates/aether-recall-mcp/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "aether-recall-mcp" +version.workspace = true +edition.workspace = true + +[[bin]] +name = "aether-recall-mcp" +path = "src/main.rs" + +[dependencies] +aether-atom = { path = "/home/cosmos/aether-atom" } +serde = { workspace = true } +serde_json = { workspace = true } +sha3 = { workspace = true } + +[dev-dependencies] +serde_json = { workspace = true } +tempfile = "3" diff --git a/crates/aether-recall-mcp/src/main.rs b/crates/aether-recall-mcp/src/main.rs new file mode 100644 index 0000000..9e69835 --- /dev/null +++ b/crates/aether-recall-mcp/src/main.rs @@ -0,0 +1,311 @@ +//! aether-recall-mcp — the entire aether-recall tool, refactored as a +//! thin MCP transport over the atom. +//! +//! What used to be 830 LOC across 5 modules is now this: 4 tool handlers +//! that call `aether_atom::{ask, remember, demote}`. The canonicalization, +//! fingerprinting, ledger, scope fallback, persistence, and TTL compaction +//! all come from the atom. + +use aether_atom::{data_dir, fingerprint, rust_canonicalize, Lattice}; +use serde_json::{json, Value}; +use std::env; +use std::io::{BufRead, BufReader, Write}; +use std::time::SystemTime; + +fn now_ts() -> u64 { + SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) +} + +fn ok(id: Value, result: Value) -> Value { + json!({ "jsonrpc": "2.0", "id": id, "result": result }) +} +fn err(id: Value, code: i64, message: &str) -> Value { + json!({ "jsonrpc": "2.0", "id": id, "error": { "code": code, "message": message } }) +} + +fn tool_result(content: Value, is_error: bool) -> Value { + json!({ "content": [{ "type": "text", "text": content.to_string() }], "isError": is_error }) +} + +struct Server { + lattice: Lattice, +} + +impl Server { + fn new(lattice: Lattice) -> Self { + Self { lattice } + } + + fn dispatch(&mut self, msg: Value) -> Option { + let id = msg.get("id").cloned().unwrap_or(Value::Null); + let is_request = !id.is_null(); + let method = msg.get("method").and_then(|m| m.as_str()).unwrap_or(""); + let params = msg.get("params").cloned().unwrap_or_else(|| json!({})); + + let respond = |result: Value| -> Option { + if is_request { + Some(ok(id.clone(), result)) + } else { + None + } + }; + + match method { + "initialize" => respond(json!({ + "protocolVersion": "2025-06-18", + "serverInfo": { "name": "aether-recall", "version": "0.1.0" }, + "capabilities": { "tools": {} } + })), + "tools/list" => respond(tools_list()), + "tools/call" => respond(self.tool_call(¶ms)), + "ping" => respond(json!({})), + m if m.starts_with("notifications/") => None, + _ => { + if is_request { + Some(err(id, -32601, "method not found")) + } else { + None + } + } + } + } + + fn tool_call(&mut self, params: &Value) -> Value { + let name = params.get("name").and_then(|n| n.as_str()).unwrap_or(""); + let args = params.get("arguments").cloned().unwrap_or_else(|| json!({})); + match name { + "lookup_resolution" => self.tool_lookup(&args), + "record_outcome" => self.tool_record(&args), + "memory_stats" => self.tool_stats(), + "compact" => self.tool_compact(), + other => tool_result( + json!({ "error": format!("unknown tool: {}", other) }), + true, + ), + } + } + + /// `lookup_resolution(canonical_text)` — content-only lookup; returns + /// cached resolution if present. + fn tool_lookup(&mut self, args: &Value) -> Value { + let failure = match args.get("failure").and_then(|v| v.as_str()) { + Some(f) if !f.trim().is_empty() => f, + _ => { + return tool_result( + json!({ + "error": "lookup_resolution: parameter 'failure' is required and must be a non-empty string", + "tool": "lookup_resolution", + "parameter": "failure", + "expected": "non-empty string" + }), + true, + ); + } + }; + + let canon = rust_canonicalize(failure); + let sig = fingerprint(&canon); + match self.lattice.lookup(&sig, "global") { + Some(hit) => tool_result( + json!({ + "hit": true, + "signature": sig.as_bytes().iter() + .map(|b| format!("{:02x}", b)) + .collect::(), + "value": String::from_utf8_lossy(hit.value.as_ref()).to_string(), + "value_hex": hit.value.iter() + .map(|b| format!("{:02x}", b)) + .collect::(), + "worked": hit.worked, + "scope": hit.scope, + }), + false, + ), + None => tool_result( + json!({ + "hit": false, + "signature": sig.as_bytes().iter() + .map(|b| format!("{:02x}", b)) + .collect::(), + "note": "no verified resolution yet; call record_outcome after fixing" + }), + false, + ), + } + } + + /// `record_outcome(canonical_text, resolution, worked)` — direct + /// seeding. Use after a fresh fix has been verified. + fn tool_record(&mut self, args: &Value) -> Value { + let failure = match args.get("failure").and_then(|v| v.as_str()) { + Some(f) if !f.trim().is_empty() => f, + _ => { + return tool_result( + json!({ + "error": "record_outcome: parameter 'failure' is required and must be a non-empty string", + "tool": "record_outcome", + "parameter": "failure", + "expected": "non-empty string" + }), + true, + ); + } + }; + let resolution = match args.get("resolution").and_then(|v| v.as_str()) { + Some(r) if !r.trim().is_empty() => r, + _ => { + return tool_result( + json!({ + "error": "record_outcome: parameter 'resolution' is required and must be a non-empty string", + "tool": "record_outcome", + "parameter": "resolution", + "expected": "non-empty string" + }), + true, + ); + } + }; + let worked = args.get("worked").and_then(|v| v.as_bool()).unwrap_or(true); + let scope = args.get("scope").and_then(|v| v.as_str()).unwrap_or("global"); + + let canon = rust_canonicalize(failure); + let sig = fingerprint(&canon); + self.lattice.record(sig, scope, resolution.as_bytes(), Some(worked)); + + tool_result( + json!({ + "recorded": true, + "signature": sig.as_bytes().iter() + .map(|b| format!("{:02x}", b)) + .collect::(), + "scope": scope, + "worked": worked, + "ts": now_ts(), + }), + false, + ) + } + + /// `memory_stats()` — recall ledger health. + fn tool_stats(&mut self) -> Value { + tool_result( + json!({ + "records": self.lattice.record_count(), + "lookups": self.lattice.lookups, + "hits": self.lattice.hits, + "misses": self.lattice.misses, + "hit_rate": self.lattice.hit_rate(), + "outcomes": self.lattice.outcomes, + }), + false, + ) + } + + /// `compact(retention_days)` — drop lookup events older than N + /// days; default 30. Outcomes (durable facts) are preserved. + fn tool_compact(&mut self) -> Value { + let retention_secs = match env::var("AETHER_RECALL_LOOKUP_RETENTION_DAYS") { + Ok(s) => s.parse::().unwrap_or(30).saturating_mul(86_400), + Err(_) => 30 * 86_400, + }; + match self.lattice.compact(retention_secs) { + Ok(dropped) => tool_result( + json!({ + "compacted": true, + "dropped_lookups": dropped, + "retention_secs": retention_secs, + "retention_days": retention_secs / 86_400, + }), + false, + ), + Err(e) => tool_result( + json!({ "error": format!("compaction failed: {}", e) }), + true, + ), + } + } +} + +fn tools_list() -> Value { + json!({ + "tools": [ + { + "name": "lookup_resolution", + "description": "Look up a previously-recorded resolution for a canonical failure text", + "inputSchema": { + "type": "object", + "properties": { + "failure": { "type": "string", "description": "Canonical form of the failure to look up" } + }, + "required": ["failure"] + } + }, + { + "name": "record_outcome", + "description": "Record an outcome: this canonical failure has this verified resolution", + "inputSchema": { + "type": "object", + "properties": { + "failure": { "type": "string" }, + "resolution": { "type": "string" }, + "worked": { "type": "boolean" }, + "scope": { "type": "string", "default": "global" } + }, + "required": ["failure", "resolution"] + } + }, + { + "name": "memory_stats", + "description": "Return recall ledger health: hit_rate, lookups, outcomes, records", + "inputSchema": { "type": "object", "properties": {} } + }, + { + "name": "compact", + "description": "Drop lookup events older than AETHER_RECALL_LOOKUP_RETENTION_DAYS", + "inputSchema": { "type": "object", "properties": {} } + } + ] + }) +} + +fn main() { + let dir = data_dir("AETHER_RECALL_DIR"); + let lattice = match Lattice::open(&dir) { + Ok(l) => l, + Err(e) => { + eprintln!("aether-recall-mcp: failed to open lattice at {:?}: {}", dir, e); + std::process::exit(1); + } + }; + eprintln!("aether-recall-mcp ready (memory: {:?})", dir); + + let mut server = Server::new(lattice); + let stdin = std::io::stdin(); + let mut stdout = std::io::stdout(); + + for line in BufReader::new(stdin.lock()).lines() { + let line = match line { + Ok(l) => l, + Err(_) => break, + }; + if line.trim().is_empty() { + continue; + } + let msg: Value = match serde_json::from_str(&line) { + Ok(v) => v, + Err(e) => { + eprintln!("aether-recall-mcp: parse error: {}", e); + continue; + } + }; + if let Some(resp) = server.dispatch(msg) { + if let Ok(s) = serde_json::to_string(&resp) { + let _ = writeln!(stdout, "{}", s); + let _ = stdout.flush(); + } + } + } +} diff --git a/crates/aether-recall-mcp/tests/cross_binary.rs b/crates/aether-recall-mcp/tests/cross_binary.rs new file mode 100644 index 0000000..f026308 --- /dev/null +++ b/crates/aether-recall-mcp/tests/cross_binary.rs @@ -0,0 +1,159 @@ +//! Cross-binary integration test. +//! +//! Spawns `aether-recall-mcp` and `aether-ask` against a shared data +//! directory. Issues a record via the MCP transport, then a query via +//! the HTTP transport. Asserts the second process sees what the +//! first wrote. If this fails, the system is not an integrated whole. + +use serde_json::json; +use std::io::{Read, Write}; +use std::net::TcpStream; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +/// Find the per-target binaries. We resolve via the per-target +/// `CARGO_BIN_EXE_` for our own crate's bin, and via the +/// workspace target/release path for sibling crates' bins. +fn bin_mcp() -> PathBuf { + let env = format!("CARGO_BIN_EXE_{}", "aether-recall-mcp"); + PathBuf::from(std::env::var(&env).expect("CARGO_BIN_EXE_aether-recall-mcp")) +} +fn bin_ask() -> PathBuf { + // Sibling-crate binaries are not exposed via CARGO_BIN_EXE_ in this + // crate's tests; they sit in workspace target/{debug,release}. + // We use the absolute workspace path because the test's CWD isn't + // guaranteed to be the workspace root. + // ../target/debug/aether-ask from this crate's tests/ dir at + // crates/aether-recall-mcp/tests/ is too fragile; instead use CARGO_MANIFEST_DIR. + // But we don't know which profile. Probe both. + let manifest = std::env::var("CARGO_MANIFEST_DIR") + .expect("CARGO_MANIFEST_DIR"); + let crate_dir = PathBuf::from(manifest); // crates/aether-recall-mcp + let workspace_dir = crate_dir + .parent().and_then(|p| p.parent()).expect("workspace dir") + .to_path_buf(); + for profile in ["release", "debug"] { + let p = workspace_dir.join("target").join(profile).join("aether-ask"); + if p.exists() { return p; } + } + panic!("aether-ask binary not found under {workspace_dir:?}/target/{{release,debug}}"); +} + +struct McpClient { + child: Child, +} +impl McpClient { + fn spawn(data_dir: &Path) -> Self { + let mut cmd = Command::new(bin_mcp()); + cmd.env("AETHER_RECALL_DIR", data_dir) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()); + Self { child: cmd.spawn().expect("spawn mcp") } + } + fn send(&mut self, req: serde_json::Value) -> serde_json::Value { + let stdin = self.child.stdin.as_mut().unwrap(); + writeln!(stdin, "{}", serde_json::to_string(&req).unwrap()).unwrap(); + stdin.flush().unwrap(); + let stdout = self.child.stdout.as_mut().unwrap(); + let mut buf = Vec::new(); + let mut byte = [0u8; 1]; + while stdout.read(&mut byte).unwrap() > 0 { + buf.push(byte[0]); + if byte[0] == b'\n' { break; } + } + serde_json::from_slice(&buf).unwrap() + } +} +impl Drop for McpClient { + fn drop(&mut self) { let _ = self.child.kill(); } +} + +#[test] +fn mcp_record_then_mcp_lookup_returns_hit() { + let dir = tmp(); + let mut m = McpClient::spawn(&dir); + m.send(json!({ + "jsonrpc":"2.0","id":1,"method":"tools/call", + "params":{"name":"record_outcome","arguments":{ + "failure":"E0308 canonical test","resolution":"use .to_string()","worked":true + }} + })); + let r = m.send(json!({ + "jsonrpc":"2.0","id":2,"method":"tools/call", + "params":{"name":"lookup_resolution","arguments":{"failure":"E0308 canonical test"}} + })); + let content = r["result"]["content"][0]["text"].as_str().unwrap(); + assert!(content.contains("\"hit\":true"), "got: {content}"); + assert!(content.contains("use .to_string()")); +} + +#[test] +fn ask_returns_value_after_mcp_recorded() { + let dir = tmp(); + + // 1. MCP records a fact in process A + { + let mut m = McpClient::spawn(&dir); + m.send(json!({ + "jsonrpc":"2.0","id":1,"method":"tools/call", + "params":{"name":"record_outcome","arguments":{ + "failure":"cross-binary integration", + "resolution":"verified by separate processes", + "worked":true + }} + })); + } // process A drops → killed + + // 2. ASK spins up in process B, hits the SAME data dir, asserts hit + let port = ephemeral_port(); + let mut ask_child = Command::new(bin_ask()) + .env("AETHER_RECALL_DIR", &dir) + .env("AETHER_ASK_PORT", port.to_string()) + .env("OLLAMA_PORT", "1") // unreachable → fast neural-fail path + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn ask"); + + let addr = format!("127.0.0.1:{}", port); + for _ in 0..60 { + if TcpStream::connect_timeout(&addr.parse().unwrap(), Duration::from_millis(100)).is_ok() { + break; + } + std::thread::sleep(Duration::from_millis(50)); + } + + let mut s = TcpStream::connect(("127.0.0.1", port)).unwrap(); + s.set_read_timeout(Some(Duration::from_secs(3))).ok(); + let body = json!({"question":"cross-binary integration"}).to_string(); + let req = format!( + "POST /ask HTTP/1.1\r\nHost: localhost\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), body + ); + s.write_all(req.as_bytes()).unwrap(); + let mut buf = Vec::new(); + s.read_to_end(&mut buf).ok(); + let text = String::from_utf8_lossy(&buf).to_string(); + let (_head, body) = text.split_once("\r\n\r\n").unwrap_or(("", &text)); + let v: serde_json::Value = serde_json::from_str(body).expect("json"); + let _ = ask_child.kill(); + + assert_eq!(v["source"].as_str(), Some("deterministic"), + "ASK must hit the cached MCP record across processes; got: {v:?}"); + assert_eq!(v["answer"].as_str(), Some("verified by separate processes")); +} + +fn tmp() -> PathBuf { + let stamp = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); + let p = std::env::temp_dir().join(format!("cross-bin-it-{stamp}")); + std::fs::create_dir_all(&p).unwrap(); + p +} + +fn ephemeral_port() -> u16 { + let stamp = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); + 33_000 + ((stamp % 1000) as u16) +} diff --git a/crates/aether-recall-mcp/tests/integration.rs b/crates/aether-recall-mcp/tests/integration.rs new file mode 100644 index 0000000..a6ab789 --- /dev/null +++ b/crates/aether-recall-mcp/tests/integration.rs @@ -0,0 +1,130 @@ +//! End-to-end integration tests for aether-recall-mcp. +//! +//! Spawns the actual binary, talks JSON-RPC over stdio, verifies +//! the round-trip and the canonical-form abstraction. + +use serde_json::json; +use std::io::{Read, Write}; +use std::path::PathBuf; +use std::process::{Child, Command, Stdio}; + +fn binary_path() -> PathBuf { + PathBuf::from(env!("CARGO_BIN_EXE_aether-recall-mcp")) +} + +struct Mcp { + child: Child, +} + +impl Mcp { + fn spawn(data_dir: &PathBuf) -> Self { + let mut cmd = Command::new(binary_path()); + cmd.env("AETHER_RECALL_DIR", data_dir) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let child = cmd.spawn().expect("spawn aether-recall-mcp"); + Self { child } + } + + fn send(&mut self, req: serde_json::Value) -> serde_json::Value { + let stdin = self.child.stdin.as_mut().expect("stdin"); + let s = serde_json::to_string(&req).unwrap(); + writeln!(stdin, "{}", s).unwrap(); + stdin.flush().unwrap(); + + let stdout = self.child.stdout.as_mut().expect("stdout"); + let mut buf = Vec::new(); + let mut byte = [0u8; 1]; + while stdout.read(&mut byte).unwrap() > 0 { + buf.push(byte[0]); + if byte[0] == b'\n' { break; } + } + serde_json::from_slice(&buf).unwrap() + } +} + +impl Drop for Mcp { + fn drop(&mut self) { + let _ = self.child.kill(); + } +} + +#[test] +fn full_lifecycle() { + let dir = tempdir_in("/tmp"); + let mut m = Mcp::spawn(&dir); + + // 1. initialize + let r = m.send(json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}})); + assert_eq!(r["result"]["serverInfo"]["name"], "aether-recall"); + + // 2. cold lookup → miss + let r = m.send(json!({ + "jsonrpc":"2.0","id":2,"method":"tools/call", + "params":{"name":"lookup_resolution","arguments":{"failure":"error[E0308]: expected i32, found &str"}} + })); + let content = r["result"]["content"][0]["text"].as_str().unwrap(); + assert!(content.contains("\"hit\":false"), "got: {content}"); + + // 3. record_outcome + let r = m.send(json!({ + "jsonrpc":"2.0","id":3,"method":"tools/call", + "params":{"name":"record_outcome","arguments":{"failure":"error[E0308]: expected i32, found &str","resolution":"use .to_string()","worked":true}} + })); + let content = r["result"]["content"][0]["text"].as_str().unwrap(); + assert!(content.contains("\"recorded\":true")); + + // 4. warm lookup → hit + let r = m.send(json!({ + "jsonrpc":"2.0","id":4,"method":"tools/call", + "params":{"name":"lookup_resolution","arguments":{"failure":"error[E0308]: expected i32, found &str"}} + })); + let content = r["result"]["content"][0]["text"].as_str().unwrap(); + assert!(content.contains("\"hit\":true"), "got: {content}"); + assert!(content.contains("use .to_string()"), "got: {content}"); + + // 5. path-equivalent surface form → hit (canonical abstraction) + let r = m.send(json!({ + "jsonrpc":"2.0","id":5,"method":"tools/call", + "params":{"name":"lookup_resolution","arguments":{"failure":"error[E0308]: expected i32, found &str\n --> src/foo.rs:42:5"}} + })); + let content = r["result"]["content"][0]["text"].as_str().unwrap(); + assert!(content.contains("\"hit\":true"), "got: {content}"); + + // 6. memory_stats: 3 lookups, 2 hits, 1 miss + let r = m.send(json!({ + "jsonrpc":"2.0","id":6,"method":"tools/call", + "params":{"name":"memory_stats","arguments":{}} + })); + let content = r["result"]["content"][0]["text"].as_str().unwrap(); + assert!(content.contains("\"lookups\":3")); + assert!(content.contains("\"hits\":2")); + assert!(content.contains("\"misses\":1")); + assert!(content.contains("\"hit_rate\":0.6")); + + // 7. compact succeeds + let r = m.send(json!({ + "jsonrpc":"2.0","id":7,"method":"tools/call", + "params":{"name":"compact","arguments":{}} + })); + let content = r["result"]["content"][0]["text"].as_str().unwrap(); + assert!(content.contains("\"compacted\":true")); + + // 8. error: empty failure + let r = m.send(json!({ + "jsonrpc":"2.0","id":8,"method":"tools/call", + "params":{"name":"lookup_resolution","arguments":{"failure":""}} + })); + assert_eq!(r["result"]["isError"], true); + let content = r["result"]["content"][0]["text"].as_str().unwrap(); + assert!(content.contains("\"tool\":\"lookup_resolution\"")); +} + +fn tempdir_in(parent: &str) -> PathBuf { + use std::time::{SystemTime, UNIX_EPOCH}; + let stamp = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); + let p = PathBuf::from(parent).join(format!("aether-recall-mcp-it-{}", stamp)); + std::fs::create_dir_all(&p).unwrap(); + p +} diff --git a/examples/library_usage/Cargo.toml b/examples/library_usage/Cargo.toml new file mode 100644 index 0000000..9fc7ed3 --- /dev/null +++ b/examples/library_usage/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "library_usage" +version = "0.1.0" +edition = "2021" +publish = false + +[dependencies] +aether-atom = { path = "/home/cosmos/aether-atom" } diff --git a/examples/library_usage/src/main.rs b/examples/library_usage/src/main.rs new file mode 100644 index 0000000..942bf71 --- /dev/null +++ b/examples/library_usage/src/main.rs @@ -0,0 +1,87 @@ +//! Library usage — third-party code consuming aether-atom. +//! +//! This is what every binary in `crates/` does, distilled to ~50 LOC. +//! Use the atom directly (via `Lattice`); there is no helper layer to +//! hide behind — the primitives *are* the API. +//! +//! Runs in an ephemeral temp directory by default so this example +//! cannot pollute `$HOME/.aether-recall` unless `AETHER_RECALL_DIR` +//! is set explicitly. + +use aether_atom::{fingerprint, rust_canonicalize, Lattice}; + +fn open_lattice() -> Result { + // Use the standard path; override via env if desired. + Lattice::open(&aether_atom::data_dir("AETHER_RECALL_DIR")) +} + +fn main() { + // The example is read-only against the user's real ledger, but to + // demonstrate without state there, point it at a clean tmp dir. + let dir = match std::env::var("AETHER_RECALL_DIR") { + Ok(d) => std::path::PathBuf::from(d), + Err(_) => { + let stamp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos(); + let p = std::env::temp_dir().join(format!("library-usage-demo-{stamp}")); + std::fs::create_dir_all(&p).unwrap(); + p + } + }; + let mut lat = Lattice::open(&dir).expect("open atom"); + println!("memory: dir={:?}", dir); + + let question = "error[E0308]: expected i32, found &str"; + let sig = fingerprint(&rust_canonicalize(question)); + + match lat.lookup(&sig, "global") { + Some(hit) => println!("hit: {}", String::from_utf8_lossy(hit.value.as_ref())), + None => { + println!("miss — recording placeholder answer"); + lat.record(sig, "global", b"use .to_string()", Some(true)); + } + } + + // Path-equivalent lookup: the rust_canonicalize step collapses + // `path:line:col` back to identical bytes, so this hits too. + let equiv_sig = fingerprint(&rust_canonicalize( + "error[E0308]: expected i32, found &str\n --> src/foo.rs:42:5", + )); + assert_eq!(equiv_sig, sig, "path-line-col must not change the canonical form"); + + let stats = lat; + println!("stats: records={} lookups={} hits={} misses={} hit_rate={:.2}", + stats.record_count(), + stats.lookups, + stats.hits, + stats.misses, + stats.hit_rate() + ); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tmp_dir() -> std::path::PathBuf { + let stamp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos(); + let p = std::env::temp_dir().join(format!("library-usage-test-{stamp}")); + std::fs::create_dir_all(&p).unwrap(); + p + } + + #[test] + fn open_lattice_creates_dir_if_missing() { + let dir = tmp_dir(); + let mut lat = Lattice::open(&dir).unwrap(); + lat.record( + fingerprint(b"hello"), + "global", + b"world", + Some(true), + ); + let hit = lat.lookup(&fingerprint(b"hello"), "global"); + assert_eq!(hit.unwrap().value.as_ref(), b"world"); + } +} diff --git a/examples/stress_test/Cargo.toml b/examples/stress_test/Cargo.toml new file mode 100644 index 0000000..fc0db36 --- /dev/null +++ b/examples/stress_test/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "stress_test" +version = "0.1.0" +edition = "2021" +publish = false + +[dependencies] +serde = { workspace = true } +serde_json = { workspace = true } diff --git a/examples/stress_test/src/main.rs b/examples/stress_test/src/main.rs new file mode 100644 index 0000000..0b33081 --- /dev/null +++ b/examples/stress_test/src/main.rs @@ -0,0 +1,160 @@ +//! Stress test for aether-recall-mcp. +//! +//! Spawns one MCP server, keeps stdin/stdout open, sends many requests +//! over the lifetime of the binary. Each request uses a UNIQUE failure +//! text so the canonicalizer can't collapse records to duplicates. +//! +//! Usage: `cargo run --release -p stress_test -- ` +//! where N is the number of records (default 50_000). Each invocation +//! prints throughput stats on exit. + +use serde_json::{json, Value}; +use std::env; +use std::io::{BufRead, BufReader, Read, Write}; +use std::path::PathBuf; +use std::process::{Command, Stdio}; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; + +fn main() { + let n: usize = env::args().nth(1) + .and_then(|s| s.parse().ok()) + .unwrap_or(50_000); + + // Use an isolated tmp dir so the stress run never collides with the + // user's real ledger. + let stamp = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); + let data_dir = std::env::temp_dir().join(format!("stress-{stamp}")); + std::fs::create_dir_all(&data_dir).unwrap(); + + // Resolve the binary at runtime. env!("CARGO_BIN_EXE_") only + // works when this crate *owns* the binary; here we depend on it + // from a sibling crate, so we probe the workspace target/ dir. + let manifest = env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR"); + let examples_dir = PathBuf::from(manifest); + let workspace_dir = examples_dir.parent().and_then(|p| p.parent()).unwrap().to_path_buf(); + let bin = ["release", "debug"].iter() + .map(|p| workspace_dir.join("target").join(p).join("aether-recall-mcp")) + .find(|p| p.exists()) + .expect("aether-recall-mcp binary missing — run `cargo build --release --workspace` first"); + println!("spawning {:?} for {n} ops", bin); + let mut child = Command::new(bin) + .env("AETHER_RECALL_DIR", &data_dir) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn mcp"); + // Take stdin/stdout once each. stdin becomes our writer; stdout + // becomes a buffered reader for incoming responses. + let stdout = child.stdout.take().unwrap(); + let mut writer = child.stdin.take().unwrap(); + let mut out = BufReader::new(stdout); + + // PHASE 1: seed N unique failure records + let t = Instant::now(); + for i in 0..n { + let req = json!({ + "jsonrpc":"2.0","id": i, + "method":"tools/call", + "params":{"name":"record_outcome","arguments":{ + "failure": format!("UNIQ_X{i:08}_error_{i}: cannot resolve type 'Z{i}' to inferred type 'W{i}'"), + "resolution": format!("fix {}: type annotation `W`", i), + "worked": true + }} + }); + write_line(&mut writer, &req.to_string()); + drain_response(&mut out); + } + let seed_elapsed = t.elapsed(); + println!( + "seed: {n} records in {:.2?} ({:.1} records/s)", + seed_elapsed, + n as f64 / seed_elapsed.as_secs_f64() + ); + + // PHASE 2: look each one up via MCP + let t = Instant::now(); + let mut hits = 0usize; + for i in 0..n { + let req = json!({ + "jsonrpc":"2.0","id": i + n, + "method":"tools/call", + "params":{"name":"lookup_resolution","arguments":{ + "failure": format!("UNIQ_X{i:08}_error_{i}: cannot resolve type 'Z{i}' to inferred type 'W{i}'") + }} + }); + write_line(&mut writer, &req.to_string()); + let resp = drain_response(&mut out); + if resp_hit(&resp) { + hits += 1; + } + } + let lookup_elapsed = t.elapsed(); + println!( + "lookups: {n} in {:.2?} ({:.1}/s) — hits={hits} ({:.1}%)", + lookup_elapsed, + n as f64 / lookup_elapsed.as_secs_f64(), + hits as f64 * 100.0 / n as f64 + ); + + // PHASE 3: stats + write_line(&mut writer, &json!({ + "jsonrpc":"2.0","id": n*2+1, + "method":"tools/call", + "params":{"name":"memory_stats","arguments":{}} + }).to_string()); + let stats_resp = drain_response(&mut out); + println!("stats: {}", stats_resp.get("result").and_then(|r| r.get("content")).and_then(|c| c.as_array()).and_then(|a| a.first()).and_then(|t| t.get("text")).and_then(|s| s.as_str()).unwrap_or("?")); + + let _ = child.kill(); + let _ = child.wait(); +} + +fn write_line(w: &mut std::process::ChildStdin, line: &str) { + w.write_all(line.as_bytes()).unwrap(); + w.write_all(b"\n").unwrap(); + w.flush().unwrap(); +} + +fn drain_response(r: &mut BufReader) -> Value { + let mut line = String::new(); + r.read_line(&mut line).unwrap(); + let mut s = &line[..]; + while !s.trim().is_empty() && !s.starts_with('{') && !s.starts_with('[') { + line.clear(); + r.read_line(&mut line).unwrap(); + s = &line[..]; + } + serde_json::from_str(s.trim()).unwrap_or_else(|_| { + let mut all = String::new(); + r.read_to_string(&mut all).ok(); + panic!("non-JSON response from MCP: {:?}\nnext bytes: {:?}", s.trim(), &all[..all.len().min(2048)]); + }) +} + +fn resp_hit(v: &Value) -> bool { + v.get("result") + .and_then(|r| r.get("content")) + .and_then(|c| c.as_array()) + .and_then(|a| a.first()) + .and_then(|t| t.get("text")) + .and_then(|s| s.as_str()) + .map(|s| s.contains("\"hit\":true")) + .unwrap_or(false) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn resp_hit_recognises_true_and_false() { + let yes: Value = serde_json::from_str( + r#"{"result":{"content":[{"text":"{\"hit\":true, \"signature\":\"abc\"}"}]}}"# + ).unwrap(); + let no: Value = serde_json::from_str( + r#"{"result":{"content":[{"text":"{\"hit\":false}"}]}}"# + ).unwrap(); + assert!(resp_hit(&yes)); + assert!(!resp_hit(&no)); + } +}