use std::path::PathBuf;
use std::time::Instant;
use mnemo_engine::{
Embedder, EmbedderIdentity, KdfParams, Memory, MemoryType, Mnemo, MnemoConfig, ThreadTurn,
};
use serde_json::json;
const DIMS: usize = 384;
const TURNS_PER_THREAD: usize = 500;
const PASSPHRASE: &str = "bench-pw";
struct HashEmbedder;
impl Embedder for HashEmbedder {
fn identity(&self) -> EmbedderIdentity {
EmbedderIdentity {
name: "hash-384".into(),
dimensions: DIMS,
normalize: false,
..Default::default()
}
}
fn embed(&self, texts: &[&str]) -> mnemo_engine::Result<Vec<Vec<f32>>> {
Ok(texts.iter().map(|t| hash_vector(t)).collect())
}
}
fn hash_vector(t: &str) -> Vec<f32> {
let mut h: u64 = 0xcbf2_9ce4_8422_2325;
for b in t.bytes() {
h ^= b as u64;
h = h.wrapping_mul(0x0100_0000_01b3);
}
(0..DIMS)
.map(|_| {
h = h
.wrapping_mul(6_364_136_223_846_793_005)
.wrapping_add(1_442_695_040_888_963_407);
((h >> 40) as f32 / (1u64 << 24) as f32) * 2.0 - 1.0 + 1e-3
})
.collect()
}
fn repo_text() -> String {
let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..");
let mut s = String::new();
for f in [
"PLAN.md",
"CHANGELOG.md",
"README.md",
"reviewer_notes.md",
"mnemo/README.md",
"mnemo-python/README.md",
"AGENTS.md",
] {
if let Ok(t) = std::fs::read_to_string(root.join(f)) {
s.push_str(&t);
s.push(' ');
}
}
s.split_whitespace().collect::<Vec<_>>().join(" ")
}
fn turn_lengths(n: usize) -> Vec<usize> {
let mut x: u64 = 0x7E5D_0001;
let mut next = move || {
x ^= x << 13;
x ^= x >> 7;
x ^= x << 17;
x
};
(0..n)
.map(|_| {
let r = next() % 100;
let u = (next() % 10_000) as usize;
if r < 60 {
100 + u * 500 / 10_000
} else if r < 90 {
1_000 + u * 3_000 / 10_000
} else {
8_000 + u * 32_000 / 10_000
}
})
.collect()
}
struct TextCursor {
text: String,
off: usize,
}
impl TextCursor {
fn take(&mut self, len: usize) -> String {
let t = &self.text;
if self.off + len >= t.len() {
self.off = 0;
}
let mut lo = self.off;
while !t.is_char_boundary(lo) {
lo += 1;
}
let mut hi = (lo + len).min(t.len());
while !t.is_char_boundary(hi) {
hi -= 1;
}
self.off = hi;
t[lo..hi].to_string()
}
}
fn measure_reads(path: &std::path::Path, n_threads: usize, n_turns: usize) -> serde_json::Value {
let mut db = Mnemo::open(path, PASSPHRASE).expect("open");
let t = Instant::now();
let first = db.thread_export_turns("thread-0").expect("export");
let cold_ms = ms(t);
assert_eq!(first.len(), TURNS_PER_THREAD.min(n_turns));
let picks: Vec<usize> = (0..20.min(n_threads.saturating_sub(1)))
.map(|k| 1 + (k * 7 + 3) % (n_threads - 1))
.collect();
let time_all = |db: &mut Mnemo| {
let mut xs: Vec<f64> = picks
.iter()
.map(|th| {
let t = Instant::now();
let turns = db
.thread_export_turns(&format!("thread-{th}"))
.expect("export");
assert!(!turns.is_empty());
ms(t)
})
.collect();
xs.sort_by(|a, b| a.partial_cmp(b).unwrap());
xs.get(xs.len() / 2).copied().unwrap_or(0.0)
};
let first_touch = time_all(&mut db);
let warm = time_all(&mut db);
json!({ "cold_ms": cold_ms, "first_touch_median_ms": first_touch, "warm_median_ms": warm })
}
fn ms(t: Instant) -> f64 {
t.elapsed().as_secs_f64() * 1_000.0
}
fn main() {
let args: Vec<String> = std::env::args().skip(1).collect();
let arg = |name: &str, default: usize| {
args.iter()
.position(|a| a == name)
.and_then(|i| args.get(i + 1))
.and_then(|v| v.parse().ok())
.unwrap_or(default)
};
let n_facts = arg("--facts", 10_000);
let n_turns = arg("--turns", 50_000);
let compress_after: Option<u64> = args
.iter()
.position(|a| a == "--compress-after")
.and_then(|i| args.get(i + 1))
.and_then(|v| v.parse().ok());
let tag = match (
std::env::var("MNEMO_DEV_TURN_ZSTD").as_deref(),
compress_after,
) {
(Ok("0"), _) => "plain",
(_, Some(0)) => "eager",
_ => "cold",
};
let base = std::env::var("CARGO_TARGET_DIR").unwrap_or_else(|_| "target".to_string());
let dir = PathBuf::from(base).join("thread_profile");
std::fs::create_dir_all(&dir).expect("create target/thread_profile");
let path = dir.join(format!("thread_profile_{tag}.mnemo"));
let _ = std::fs::remove_file(&path);
let cfg = MnemoConfig {
dimensions: DIMS,
kdf: KdfParams::fast(),
format_version: 9,
..Default::default()
};
let mut cur = TextCursor {
text: repo_text(),
off: 0,
};
let mut db = Mnemo::create(&path, PASSPHRASE, cfg).expect("create");
db.set_embedder(HashEmbedder).expect("embedder");
if let Some(secs) = compress_after {
db.set_turn_compress_after(secs);
}
let t = Instant::now();
for i in 0..n_facts {
let text = cur.take(300);
let v = hash_vector(&text);
db.remember(Memory::new(text, MemoryType::Semantic, v))
.expect("remember");
if (i + 1) % 1_000 == 0 {
db.flush().expect("flush");
}
}
db.flush().expect("flush");
let facts_ms = ms(t);
let lens = turn_lengths(n_turns);
let text_bytes: usize = lens.iter().sum();
let n_threads = n_turns.div_ceil(TURNS_PER_THREAD);
let t = Instant::now();
for (th, group) in lens.chunks(TURNS_PER_THREAD).enumerate() {
let turns: Vec<ThreadTurn> = group
.iter()
.enumerate()
.map(|(j, &len)| ThreadTurn {
turn_id: format!("th{th}-t{j}"),
role: if j % 2 == 0 { "user" } else { "assistant" }.into(),
content: cur.take(len),
timestamp: None,
})
.collect();
db.ingest_thread(&format!("thread-{th}"), &turns)
.expect("ingest");
db.flush().expect("flush");
}
let ingest_ms = ms(t);
let records = db.len();
db.close().expect("close");
let file_bytes = std::fs::metadata(&path).expect("stat").len();
let before = measure_reads(&path, n_threads, n_turns);
Mnemo::compact_file(path.to_str().expect("utf-8 path"), PASSPHRASE).expect("compact");
let live_bytes = std::fs::metadata(&path).expect("stat").len();
let after = measure_reads(&path, n_threads, n_turns);
let out = json!({
"turn_records": tag,
"facts": n_facts,
"turns": n_turns,
"threads": n_threads,
"records": records,
"turn_text_bytes": text_bytes,
"file_bytes": file_bytes,
"live_bytes_after_compact": live_bytes,
"live_bytes_per_turn": live_bytes as f64 / n_turns as f64,
"facts_ms": facts_ms,
"ingest_ms": ingest_ms,
"ingest_ms_per_thread": ingest_ms / n_threads as f64,
"reads_before_compact": before,
"reads_after_compact": after,
});
println!("{}", serde_json::to_string_pretty(&out).unwrap());
let json_path = dir.join(format!("thread_profile_{tag}.json"));
std::fs::write(&json_path, serde_json::to_string_pretty(&out).unwrap()).expect("write json");
let _ = std::fs::remove_file(&path);
}