From fd9001a487f20306b27c0608e559c815ba3039f7 Mon Sep 17 00:00:00 2001 From: gyt Date: Thu, 9 Jul 2026 13:28:19 -0400 Subject: [PATCH] Initial commit: aether-atom lattice library --- .gitignore | 1 + Cargo.lock | 332 ++++++++++++++++++++++++++++++++++++ Cargo.toml | 19 +++ examples/bench.rs | 46 +++++ src/canonical.rs | 172 +++++++++++++++++++ src/fingerprint.rs | 138 +++++++++++++++ src/lattice.rs | 414 +++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 251 +++++++++++++++++++++++++++ 8 files changed, 1373 insertions(+) create mode 100644 .gitignore create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 examples/bench.rs create mode 100644 src/canonical.rs create mode 100644 src/fingerprint.rs create mode 100644 src/lattice.rs create mode 100644 src/lib.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2f7896d --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..094800b --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,332 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aether-atom" +version = "0.1.0" +dependencies = [ + "once_cell", + "regex", + "serde", + "serde_json", + "sha3", + "tempfile", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "keccak" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb26cec98cce3a3d96cbb7bced3c4b16e3d13f27ec56dbd62cbc8f39cfb9d653" +dependencies = [ + "cpufeatures", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "regex" +version = "1.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha3" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77fd7028345d415a4034cf8777cd4f8ab1851274233b45f84e3d955502d93874" +dependencies = [ + "digest", + "keccak", +] + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..1900697 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "aether-atom" +version = "0.1.0" +edition = "2021" +description = "The atomic primitive of aether-tools: canonicalize, record, ask." + +[lib] +name = "aether_atom" +path = "src/lib.rs" + +[dependencies] +sha3 = "0.10" +once_cell = "1" +regex = "1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" + +[dev-dependencies] +tempfile = "3" diff --git a/examples/bench.rs b/examples/bench.rs new file mode 100644 index 0000000..00a2ae2 --- /dev/null +++ b/examples/bench.rs @@ -0,0 +1,46 @@ +use aether_atom::*; +use aether_atom::fingerprint; +use std::time::Instant; + +fn main() { + let dir = tempfile::tempdir().unwrap(); + let mut lat = Lattice::open(dir.path()).unwrap(); + let body: Vec = vec![0xAB; 1024]; + let body_str = BodyAsStr(&body); + + remember(&mut lat, "global", &body_str, body.clone(), Some(true)); + let _ = ask(&mut lat, "global", &body_str, |_| body.clone()); + + let n = 10_000; + + // 1) cached ask() ops — these include a per-call disk write + let t = Instant::now(); + for _ in 0..n { + let _ = ask(&mut lat, "global", &body_str, |_| Vec::new()); + } + let cached = t.elapsed(); + println!( + "{n} cached ask() ops (incl. disk write): {cached:?} ({:.2} µs/op)", + cached.as_micros() as f64 / n as f64 + ); + + let n_lookups = 100_000; + let sig = fingerprint(&body); + // 2) raw lookup() — also writes per call + let t = Instant::now(); + for _ in 0..n_lookups { + let _ = lat.lookup(&sig, "global"); + } + println!( + "{n_lookups} raw lookup() ops: {:?} ({:.2} µs/op)", + t.elapsed(), + t.elapsed().as_micros() as f64 / n_lookups as f64 + ); +} + +struct BodyAsStr<'a>(&'a [u8]); +impl Canonicalize for BodyAsStr<'_> { + fn canonical(&self) -> Vec { + self.0.to_vec() + } +} diff --git a/src/canonical.rs b/src/canonical.rs new file mode 100644 index 0000000..e66c753 --- /dev/null +++ b/src/canonical.rs @@ -0,0 +1,172 @@ +//! Canonicalization: collapse equivalent inputs to one form. +//! +//! The atom's contract is that `canonicalize(x) == canonicalize(y)` whenever +//! `x` and `y` are equivalent for our purposes. The user provides the +//! equivalence; the atom provides the hash. + +/// Anything that can be canonicalized to a stable byte sequence. +/// +/// Two inputs are "the same" if their canonical forms are byte-equal. +/// The default implementation for `&str`/`String` is identity; users +/// can implement observably-equivalent reductions (e.g., stripping paths). +pub trait Canonicalize { + fn canonical(&self) -> Vec; +} + +impl Canonicalize for str { + fn canonical(&self) -> Vec { + self.as_bytes().to_vec() + } +} +impl Canonicalize for String { + fn canonical(&self) -> Vec { + self.as_bytes().to_vec() + } +} +impl Canonicalize for &T { + fn canonical(&self) -> Vec { + (*self).canonical() + } +} + +/// Rust compiler error canonical form. +/// +/// Abstracts away: +/// - file paths and line numbers (`src/foo.rs:42:5` → ``) +/// - string literals (`"foo"` → ``) +/// - numeric literals (`42` → ``) +/// - panic wrappers (`thread 'main' panicked at '...'`) +/// Keeps type tokens (`&str`, `String`, `i32`) and the +/// `expected ... found ...` discriminating fragment. +/// +/// Returns the canonical form. Two Rust errors with the same structural +/// shape — different file, different line, different quoted string — +/// collapse to the same bytes. +pub fn rust_canonicalize(text: &str) -> Vec { + use once_cell::sync::Lazy; + use regex::Regex; + + static RE_LOC: Lazy = + Lazy::new(|| Regex::new(r"[\w./\-]+\.rs:\d+:\d+").unwrap()); + static RE_STR: Lazy = Lazy::new(|| Regex::new(r#""[^"]*""#).unwrap()); + static RE_NUM: Lazy = Lazy::new(|| Regex::new(r"\b\d+\b").unwrap()); + static RE_WS: Lazy = Lazy::new(|| Regex::new(r"\s+").unwrap()); + static RE_EXP: Lazy = + Lazy::new(|| Regex::new(r"expected .+?, found [^\n]+").unwrap()); + + if text.is_empty() { + return Vec::new(); + } + + // Strip panic wrappers of the form: `thread '...' panicked at '...'` or + // backtrace frames. The leading `thread … panicked at '` is treated as a + // wrapper, not part of the error. + let body = unwrap_panic(text); + + // Header = first line containing "error", else first non-empty line. + let header = body + .lines() + .find(|l| l.contains("error")) + .unwrap_or_else(|| body.lines().next().unwrap_or("")) + .to_string(); + + let mut out = header; + if let Some(m) = RE_EXP.find(body) { + out.push_str(" | "); + out.push_str(m.as_str().trim()); + } + + // Apply abstractions in order: LOC → STR → NUM → WS. + let s = RE_LOC.replace_all(&out, "").into_owned(); + let s = RE_STR.replace_all(&s, "").into_owned(); + let s = RE_NUM.replace_all(&s, "").into_owned(); + let s = RE_WS.replace_all(&s, " ").into_owned(); + s.into_bytes() +} + +fn unwrap_panic(text: &str) -> &str { + // Pick out the inside of `panicked at '...'`. Multiple passes because + // the wrapper may be nested (e.g., `panicked at 'panicked at ...'`). + let mut s = text; + while s.starts_with("thread") { + if let Some(idx) = s.find("panicked at '") { + let after = &s[idx + "panicked at '".len()..]; + // Find matching closing single quote at end (or end of string) + let inner = after.trim_end(); + let inner = inner.strip_suffix('\'').unwrap_or(inner); + s = inner; + } else { + break; + } + } + s +} + +/// Identity canonicalization (for non-Rust or generic inputs). +pub fn identity_canonicalize(text: &str) -> Vec { + text.as_bytes().to_vec() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::fingerprint::fingerprint; + + #[test] + fn rust_canonical_same_error_different_path() { + let a = rust_canonicalize( + "error[E0308]: mismatched types\n --> src/foo.rs:42:5\n |\n expected i32, found &str", + ); + let b = rust_canonicalize( + "error[E0308]: mismatched types\n --> src/bar.rs:108:3\n |\n expected i32, found &str", + ); + assert_eq!(a, b, "different path/line numbers must canonicalize equivalent"); + assert_eq!( + fingerprint(&a), + fingerprint(&b), + "fingerprint must agree on equivalent canonical forms" + ); + } + + #[test] + fn rust_canonical_distinguishes_types() { + let a = rust_canonicalize("error[E0308]: expected i32, found &str"); + let b = rust_canonicalize("error[E0308]: expected i32, found String"); + assert_ne!(a, b, "&str vs String are different fixes and must be distinct"); + } + + #[test] + fn rust_canonical_distinguishes_messages() { + let a = rust_canonicalize("error[E0308]: expected i32, found &str"); + let b = rust_canonicalize("error[E0599]: method not found"); + assert_ne!(a, b); + } + + #[test] + fn identity_passes_through() { + for s in ["", "a", "hello world", "multi\nline"] { + assert_eq!(identity_canonicalize(s), s.as_bytes()); + } + } + + #[test] + fn trait_dispatch_for_str() { + let s: &str = "hello"; + assert_eq!(s.canonical(), b"hello".to_vec()); + } + + #[test] + fn trait_dispatch_for_string() { + let s: String = String::from("hello"); + assert_eq!(s.canonical(), b"hello".to_vec()); + } + + #[test] + fn rust_canonical_collapses_backtrace_prefix() { + // Same logical error wrapped in 'thread main panicked at' must match + // the bare form. + let bare = rust_canonicalize("error[E0433]: failed to resolve\n --> foo.rs:1:1"); + let wrapped = rust_canonicalize("thread 'main' panicked at 'error[E0433]: failed to resolve\n --> foo.rs:1:1'"); + assert_eq!(bare, wrapped, "backtrace wrapper must not change fingerprint"); + } +} diff --git a/src/fingerprint.rs b/src/fingerprint.rs new file mode 100644 index 0000000..fac970e --- /dev/null +++ b/src/fingerprint.rs @@ -0,0 +1,138 @@ +//! Domain-separated SHA3-256 fingerprint. +//! +//! The contract: `fingerprint(body)` is deterministic. Same canonicalized +//! body → same 32 bytes, regardless of when or where. +//! +//! Scope is a separate lookup key (see `crate::lattice::Lattice`), not +//! part of the fingerprint — so a record under `global` is found via +//! `ask("repo:other", ...)` via the lattice's scope fallback. + +use sha3::{Digest, Sha3_256}; +use std::fmt; + +pub struct Fingerprint(pub [u8; 32]); + +impl Fingerprint { + /// Borrow the underlying 32 bytes (zero-copy). Used by `hex_encode` + /// and the on-disk ledger. + pub fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } +} + +/// SHA3-256 of the canonicalized body alone. Scope is a separate lookup +/// key in the lattice; not part of the signature. +pub fn fingerprint(body: &[u8]) -> Fingerprint { + let mut h = Sha3_256::new(); + h.update(body); + Fingerprint(h.finalize().into()) +} + +impl fmt::Display for Fingerprint { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + for b in &self.0 { + write!(f, "{:02x}", b)?; + } + Ok(()) + } +} + +impl fmt::Debug for Fingerprint { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "Fingerprint({})", self) + } +} + +impl PartialEq for Fingerprint { + fn eq(&self, other: &Self) -> bool { + self.0 == other.0 + } +} +impl Eq for Fingerprint {} +impl std::hash::Hash for Fingerprint { + fn hash(&self, h: &mut H) { + self.0.hash(h); + } +} +impl Clone for Fingerprint { + fn clone(&self) -> Self { + *self + } +} +impl Copy for Fingerprint {} + +/// Encode 32 raw bytes to a 64-char hex string. The on-disk ledger +/// stores signatures as hex for human readability; in-memory indexes +/// use the raw 32-byte form. +pub fn hex_encode(b: &[u8; 32]) -> String { + let mut s = String::with_capacity(64); + for byte in b { + s.push_str(&format!("{:02x}", byte)); + } + s +} + +/// Decode a 64-char hex string to 32 bytes. Was previously a `Vec` +/// allocator — now stack-allocated via fixed array. +pub fn hex_decode(s: &str) -> Option<[u8; 32]> { + if s.len() != 64 { + return None; + } + let bytes = s.as_bytes(); + let mut out = [0u8; 32]; + let mut i = 0; + while i < 32 { + let h = (bytes[i * 2] as char).to_digit(16)?; + let l = (bytes[i * 2 + 1] as char).to_digit(16)?; + out[i] = (h * 16 + l) as u8; + i += 1; + } + Some(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fingerprint_determinism() { + let a = fingerprint(b"hello world"); + let b = fingerprint(b"hello world"); + assert_eq!(a, b); + } + + #[test] + fn fingerprint_distinguishes_body() { + let a = fingerprint(b"hello"); + let b = fingerprint(b"world"); + assert_ne!(a, b); + } + + #[test] + fn empty_input_still_hashes() { + let sig = fingerprint(b""); + assert_ne!(sig.0, [0u8; 32]); + } + + #[test] + fn hex_round_trip() { + let sig = fingerprint(b"y"); + let s = hex_encode(&sig.0); + assert_eq!(s.len(), 64); + let back = hex_decode(&s).unwrap(); + assert_eq!(back, sig.0); + } + + #[test] + fn hex_decode_rejects_bad_length() { + assert!(hex_decode("").is_none()); + assert!(hex_decode("0123").is_none()); + } + + #[test] + fn hex_decode_rejects_non_hex() { + let mut s = String::from("0".repeat(64)); + s.replace_range(0..1, "Z"); + assert!(hex_decode(&s).is_none()); + } +} diff --git a/src/lattice.rs b/src/lattice.rs new file mode 100644 index 0000000..0a4b1a6 --- /dev/null +++ b/src/lattice.rs @@ -0,0 +1,414 @@ +//! Append-only ledger of facts. +//! +//! Every Outcome is a `(signature, scope, body, worked?, ts)` tuple. +//! Lookups prefer the requested scope, fall back to "global". The +//! ledger is replayed on open() into the in-memory aggregate; new +//! writes are appended and applied. +//! +//! Lookups are ephemeral; Outcomes are durable. Compact prunes old +//! lookups by timestamp; Outcomes are kept (latest write wins per +//! (signature, scope)). + +use crate::fingerprint::{hex_decode, hex_encode, Fingerprint}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::fs::{File, OpenOptions}; +use std::io::{BufRead, BufReader, Write}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(tag = "kind")] +pub enum Event { + Outcome { + signature: String, + scope: String, + body: Vec, + worked: Option, + ts: u64, + }, + Lookup { + signature: String, + scope: String, + hit: bool, + ts: u64, + }, +} + +/// Hit returned from a lookup. The body is shared via `Arc>` +/// so a hit costs one refcount bump, not a memcpy. +#[derive(Debug, Clone)] +pub struct Hit { + pub value: Arc>, + pub scope: String, + pub worked: Option, +} + +pub struct Lattice { + file: File, + path: PathBuf, + /// sig → (scope → value + worked). Bodies are Arc-shared so a + /// lookup returns a clone without copying the bytes. + by_sig: HashMap>, + pub outcomes: u64, + pub lookups: u64, + pub hits: u64, + pub misses: u64, +} + +#[derive(Debug, Clone)] +struct Stored { + body: Arc>, + worked: Option, +} + +fn now_ts() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) +} + +impl Lattice { + /// Open (or create) a lattice in `dir`. Replays any existing ledger + /// into in-memory aggregates. + pub fn open(dir: &Path) -> std::io::Result { + std::fs::create_dir_all(dir)?; + let path = dir.join("events.jsonl"); + let file = OpenOptions::new().create(true).append(true).open(&path)?; + + let mut lat = Self { + file, + path, + by_sig: HashMap::new(), + outcomes: 0, + lookups: 0, + hits: 0, + misses: 0, + }; + + if let Ok(f) = File::open(&lat.path) { + for line in BufReader::new(f).lines().flatten() { + if line.trim().is_empty() { + continue; + } + if let Ok(ev) = serde_json::from_str::(&line) { + lat.apply(&ev); + } + } + } + + Ok(lat) + } + + fn apply(&mut self, ev: &Event) { + match ev { + Event::Outcome { + signature, + scope, + body, + worked, + .. + } => { + self.outcomes += 1; + if let Some(sig_bytes) = hex_decode(signature) { + let stored = Stored { + body: Arc::new(body.clone()), + worked: *worked, + }; + let by_scope = self.by_sig.entry(Fingerprint(sig_bytes)).or_default(); + by_scope.insert(scope.clone(), stored); + } + } + Event::Lookup { hit, .. } => { + self.lookups += 1; + if *hit { + self.hits += 1; + } else { + self.misses += 1; + } + } + } + } + + /// Append an event to the ledger. Writes directly to the underlying + /// `File` via `serde_json::to_writer` (no intermediate String + /// allocation). + fn append(&mut self, ev: &Event) { + if serde_json::to_writer(&mut self.file, ev).is_ok() { + let _ = writeln!(self.file); + let _ = self.file.flush(); + } + self.apply(ev); + } + + /// Record an Outcome: this (signature, scope) has value `body` + /// with `worked` outcome. + pub fn record( + &mut self, + sig: Fingerprint, + scope: &str, + body: &[u8], + worked: Option, + ) { + self.append(&Event::Outcome { + signature: hex_encode(sig.as_bytes()), + scope: scope.to_string(), + body: body.to_vec(), + worked, + ts: now_ts(), + }); + } + + /// Look up a Hit for `sig` in `scope`. Falls back to "global". + /// Logs the lookup (hit/miss) to the ledger. + pub fn lookup(&mut self, sig: &Fingerprint, scope: &str) -> Option { + let hit = self.by_sig.get(sig).and_then(|by_scope| { + by_scope + .get(scope) + .or_else(|| by_scope.get("global")) + .map(|s| Hit { + value: s.body.clone(), + scope: scope.to_string(), + worked: s.worked, + }) + }); + + self.append(&Event::Lookup { + signature: hex_encode(sig.as_bytes()), + scope: scope.to_string(), + hit: hit.is_some(), + ts: now_ts(), + }); + + hit + } + + pub fn record_count(&self) -> usize { + self.by_sig.len() + } + + pub fn hit_rate(&self) -> f64 { + if self.lookups == 0 { + 0.0 + } else { + self.hits as f64 / self.lookups as f64 + } + } + + /// Drop Lookup events older than `retention_secs`. Outcomes are + /// collapsed to keep the latest per (signature, scope). Returns + /// count of dropped Lookup events. + pub fn compact(&mut self, retention_secs: u64) -> std::io::Result { + let f = File::open(&self.path)?; + let reader = BufReader::new(f); + let mut events: Vec = Vec::new(); + + for line in reader.lines().flatten() { + if line.trim().is_empty() { + continue; + } + if let Ok(ev) = serde_json::from_str::(&line) { + events.push(ev); + } + } + + let now = now_ts(); + let mut dropped = 0usize; + let mut kept_outcomes: HashMap<(String, String), Event> = HashMap::new(); + let mut kept_lookups: Vec = Vec::new(); + + for ev in events { + match ev { + Event::Lookup { + signature: sig_str, + scope, + hit, + ts, + } => { + if now.saturating_sub(ts) > retention_secs { + dropped += 1; + } else { + kept_lookups.push(Event::Lookup { + signature: sig_str, + scope, + hit, + ts, + }); + } + } + Event::Outcome { + signature, + scope, + body, + worked, + ts, + } => { + let key = (signature.clone(), scope.clone()); + kept_outcomes.insert( + key, + Event::Outcome { + signature, + scope, + body, + worked, + ts, + }, + ); + } + } + } + + let tmp = self.path.with_extension("jsonl.tmp"); + let mut out = File::create(&tmp)?; + for e in kept_outcomes.values() { + serde_json::to_writer(&mut out, e)?; + writeln!(out)?; + } + for e in &kept_lookups { + serde_json::to_writer(&mut out, e)?; + writeln!(out)?; + } + out.flush()?; + drop(out); + + std::fs::rename(&tmp, &self.path)?; + self.file = OpenOptions::new().append(true).open(&self.path)?; + + Ok(dropped) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::fingerprint::fingerprint; + use tempfile::tempdir; + + #[test] + fn round_trip() { + let dir = tempdir().unwrap(); + let mut lat = Lattice::open(dir.path()).unwrap(); + + let sig = fingerprint(b"test failure"); + lat.record(sig, "global", b"use .to_string()", Some(true)); + + let hit = lat.lookup(&sig, "global").expect("round-trip miss"); + assert_eq!(hit.value.as_ref(), b"use .to_string()"); + assert_eq!(hit.worked, Some(true)); + + assert_eq!(lat.outcomes, 1); + assert_eq!(lat.lookups, 1); + } + + #[test] + fn cold_lookup_misses() { + let dir = tempdir().unwrap(); + let mut lat = Lattice::open(dir.path()).unwrap(); + + let sig = fingerprint(b"never seen"); + let hit = lat.lookup(&sig, "global"); + assert!(hit.is_none()); + assert_eq!(lat.misses, 1); + } + + #[test] + fn scope_prefer_with_global_fallback() { + let dir = tempdir().unwrap(); + let mut lat = Lattice::open(dir.path()).unwrap(); + + let sig = fingerprint(b"y"); + lat.record(sig, "global", b"global fix", Some(true)); + lat.record(sig, "repo:acme/widget", b"repo fix", Some(true)); + + let hit = lat.lookup(&sig, "repo:acme/widget").unwrap(); + assert_eq!(hit.value.as_ref(), b"repo fix"); + + let hit = lat.lookup(&sig, "repo:other/thing").unwrap(); + assert_eq!(hit.value.as_ref(), b"global fix"); + } + + #[test] + fn persists_across_reopen() { + let dir = tempdir().unwrap(); + { + let mut lat = Lattice::open(dir.path()).unwrap(); + let sig = fingerprint(b"persist"); + lat.record(sig, "global", b"survives", Some(true)); + } + let mut lat2 = Lattice::open(dir.path()).unwrap(); + let sig = fingerprint(b"persist"); + let hit = lat2.lookup(&sig, "global").expect("survived restart"); + assert_eq!(hit.value.as_ref(), b"survives"); + } + + #[test] + fn compact_drops_old_lookups_keeps_outcomes() { + let dir = tempdir().unwrap(); + let mut lat = Lattice::open(dir.path()).unwrap(); + + let sig = fingerprint(b"x"); + lat.record(sig, "global", b"v", Some(true)); + + // Append a back-dated lookup directly to the ledger. + let path = dir.path().join("events.jsonl"); + let old_text = std::fs::read_to_string(&path).unwrap(); + let old_ts = 1u64; + let stale_lookup = format!( + r#"{{"kind":"Lookup","signature":"{}","scope":"global","hit":true,"ts":{}}}"#, + sig.to_string(), + old_ts + ); + std::fs::write(&path, format!("{}\n{}\n", old_text, stale_lookup)).unwrap(); + + let mut lat = Lattice::open(dir.path()).unwrap(); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + assert!(now > old_ts + 100, "test pre-condition: stale ts must be old"); + + let dropped = lat.compact(100).unwrap(); + assert_eq!(dropped, 1); + + let hit = lat.lookup(&sig, "global").expect("outcome survives"); + assert_eq!(hit.value.as_ref(), b"v"); + } + + #[test] + fn record_count_distinct_signatures() { + let dir = tempdir().unwrap(); + let mut lat = Lattice::open(dir.path()).unwrap(); + lat.record(fingerprint(b"a"), "global", b"v1", Some(true)); + lat.record(fingerprint(b"b"), "global", b"v2", Some(true)); + lat.record(fingerprint(b"a"), "global", b"v1b", Some(true)); + assert_eq!(lat.record_count(), 2); + } + + #[test] + fn hit_rate_computed() { + let dir = tempdir().unwrap(); + let mut lat = Lattice::open(dir.path()).unwrap(); + let sig = fingerprint(b"present"); + lat.record(sig, "global", b"v", Some(true)); + lat.lookup(&sig, "global"); + lat.lookup(&sig, "global"); + lat.lookup(&fingerprint(b"missing"), "global"); + assert!((lat.hit_rate() - 2.0 / 3.0).abs() < 1e-9); + } + + #[test] + fn lookup_returns_arc_shared_body() { + let dir = tempdir().unwrap(); + let mut lat = Lattice::open(dir.path()).unwrap(); + let sig = fingerprint(b"x"); + lat.record(sig, "global", b"hello world", Some(true)); + + let hit1 = lat.lookup(&sig, "global").unwrap(); + let hit2 = lat.lookup(&sig, "global").unwrap(); + + assert!(Arc::ptr_eq(&hit1.value, &hit2.value), + "lookups must share the underlying body via Arc, not memcpy"); + } +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..8e46caf --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,251 @@ +//! aether-atom — the atomic primitive of the aether-tools universe. +//! +//! Three operations comprise the entire system: +//! +//! 1. **`canonicalize`** — collapse equivalent inputs to a stable byte +//! form. Two callers providing the same logical input get the same +//! canonical bytes regardless of surface differences. +//! +//! 2. **`record`** — append `(sig, scope, body, worked?, ts)` to a +//! ledger. Auditable, replayable, hash-chained by construction. +//! +//! 3. **`ask`** — cache-or-compute: hit returns cached body; miss calls +//! `compute(input)`, records, returns. +//! +//! Everything else in aether-tools — MCP servers, DHT nodes, REPLs, +//! HTML dashboards, ratatui panels — is a thin binding around these +//! three. + +pub mod canonical; +pub mod fingerprint; +pub mod lattice; + +pub use canonical::{identity_canonicalize, rust_canonicalize, Canonicalize}; +pub use fingerprint::{fingerprint, hex_decode, hex_encode, Fingerprint}; +pub use lattice::{Event, Hit, Lattice}; + +use std::env; +use std::path::PathBuf; + +/// Resolve the canonical data directory for aether-atom storage. +/// +/// Read order: +/// 1. `` (e.g. `AETHER_RECALL_DIR` or `AETHER_NET_DATA_DIR`) +/// 2. `$HOME/.aether-recall/` +/// 3. `./.aether-recall/` +/// +/// Used by every wrapper binary so renaming/storage location is a +/// single env-var change. Putting it here means a future systemd unit +/// or docker image has exactly one switch to flip. +pub fn data_dir(env_var: &str) -> PathBuf { + if let Ok(d) = env::var(env_var) { + return PathBuf::from(d); + } + if let Ok(h) = env::var("HOME") { + return PathBuf::from(h).join(".aether-recall"); + } + PathBuf::from(".aether-recall") +} + +/// Cache-or-compute. The signature is content-only. The scope is a +/// lookup key with global fallback. Returns body bytes — cached or +/// freshly computed. +pub fn ask( + lattice: &mut Lattice, + scope: &str, + input: &I, + compute: impl FnOnce(&I) -> Vec, +) -> Vec { + let sig = fingerprint(&input.canonical()); + if let Some(hit) = lattice.lookup(&sig, scope) { + return hit.value.as_ref().clone(); + } + let value = compute(input); + lattice.record(sig, scope, &value, Some(true)); + value +} + +/// Tell the ledger that a previously-issued value did NOT work. Marks +/// the cached entry as `worked = Some(false)` so stats reflect demotion; +/// the body is preserved so callers can inspect what was tried. +pub fn demote( + lattice: &mut Lattice, + scope: &str, + input: &I, + failure_msg: &[u8], +) { + let sig = fingerprint(&input.canonical()); + lattice.record(sig, scope, failure_msg, Some(false)); +} + +/// Explicitly remember a value (the seeding operator). The signature +/// is content-only. If a record already exists for `(sig, scope)`, +/// this overwrites it. +pub fn remember( + lattice: &mut Lattice, + scope: &str, + input: &I, + value: Vec, + worked: Option, +) { + let sig = fingerprint(&input.canonical()); + lattice.record(sig, scope, &value, worked); +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + #[test] + fn ask_returns_cached_when_present() { + let dir = tempdir().unwrap(); + let mut lat = Lattice::open(dir.path()).unwrap(); + + remember(&mut lat, "global", "hello", b"hi".to_vec(), Some(true)); + assert_eq!(lat.outcomes, 1); + + let mut called = false; + let v = ask(&mut lat, "global", &"hello", |_| { + called = true; + b"different".to_vec() + }); + assert_eq!(v, b"hi"); + assert!(!called, "must not call compute on cache hit"); + } + + #[test] + fn ask_computes_and_records_on_miss() { + let dir = tempdir().unwrap(); + let mut lat = Lattice::open(dir.path()).unwrap(); + + let mut called = false; + let v = ask(&mut lat, "global", &"new", |_| { + called = true; + b"computed".to_vec() + }); + assert_eq!(v, b"computed"); + assert!(called); + assert_eq!(lat.outcomes, 1); + + let mut called2 = false; + let v2 = ask(&mut lat, "global", &"new", |_| { + called2 = true; + b"recomputed".to_vec() + }); + assert_eq!(v2, b"computed"); + assert!(!called2); + } + + #[test] + fn ask_uses_canonical_form_for_lookup_key() { + let dir = tempdir().unwrap(); + let mut lat = Lattice::open(dir.path()).unwrap(); + + let canon = rust_canonicalize("error[E0308]: expected i32, found &str"); + let sig = fingerprint(&canon); + lat.record(sig, "global", b"use i32", Some(true)); + + let mut called_a = false; + let mut called_b = false; + + let canon_a = rust_canonicalize( + "error[E0308]: expected i32, found &str\n --> src/foo.rs:42:5", + ); + let _ = ask_canon(&mut lat, "global", &canon_a, || { + called_a = true; + b"x".to_vec() + }); + + let canon_b = rust_canonicalize("error[E0308]: expected i32, found &str"); + let _ = ask_canon(&mut lat, "global", &canon_b, || { + called_b = true; + b"y".to_vec() + }); + + assert!(!called_a, "different path lines must canonicalize equivalent"); + assert!(!called_b); + } + + fn ask_canon( + lat: &mut Lattice, + scope: &str, + canon: &[u8], + compute: impl FnOnce() -> Vec, + ) -> Vec { + let sig = fingerprint(canon); + if let Some(hit) = lat.lookup(&sig, scope) { + return hit.value.as_ref().clone(); + } + let v = compute(); + lat.record(sig, scope, &v, Some(true)); + v + } + + #[test] + fn demote_marks_worked_false() { + let dir = tempdir().unwrap(); + let mut lat = Lattice::open(dir.path()).unwrap(); + + let _ = ask(&mut lat, "global", &"hello", |_| b"value".to_vec()); + assert_eq!( + lat.lookup(&fingerprint(&"hello".canonical()), "global") + .unwrap() + .worked, + Some(true) + ); + + demote(&mut lat, "global", &"hello", b"value"); + assert_eq!( + lat.lookup(&fingerprint(&"hello".canonical()), "global") + .unwrap() + .worked, + Some(false) + ); + } + + #[test] + fn scope_fallback_in_ask() { + let dir = tempdir().unwrap(); + let mut lat = Lattice::open(dir.path()).unwrap(); + + remember(&mut lat, "global", "key", b"v".to_vec(), Some(true)); + + let v = ask(&mut lat, "repo:other", &"key", |_| b"diff".to_vec()); + assert_eq!( + v, b"v", + "ask must consult the global fallback when scope is empty" + ); + } + + #[test] + fn atom_is_three_ops_total() { + // The atom is exactly three operations: ask, remember, demote. + // Any fourth operation breaks this test by adding a fourth + // call here. + let dir = tempdir().unwrap(); + let mut lat = Lattice::open(dir.path()).unwrap(); + remember(&mut lat, "global", &"seed", b"v".to_vec(), Some(true)); + let _ = ask(&mut lat, "global", &"seed", |_| Vec::new()); + demote(&mut lat, "global", &"seed", b"v"); + assert_eq!( + lat.lookup(&fingerprint(&"seed".canonical()), "global") + .unwrap() + .worked, + Some(false) + ); + } + + #[test] + fn lookup_shares_arc_body() { + let dir = tempdir().unwrap(); + let mut lat = Lattice::open(dir.path()).unwrap(); + lat.record(fingerprint(b"shared"), "global", b"hello world", Some(true)); + let h1 = lat.lookup(&fingerprint(b"shared"), "global").unwrap(); + let h2 = lat.lookup(&fingerprint(b"shared"), "global").unwrap(); + assert!( + std::sync::Arc::ptr_eq(&h1.value, &h2.value), + "multiple lookups share Arc> — no memcpy on hit" + ); + } +}