use std::path::{Path, PathBuf};
use std::time::Instant;
use anyhow::{Context, Result, bail};
use chrono::{DateTime, Duration as ChronoDuration, TimeZone, Utc};
use clap::Parser;
use rand::rngs::StdRng;
use rand::{Rng, SeedableRng};
use uuid::Uuid;
use second_brain_core::index::{EMBEDDING_DIM, Indexes};
use second_brain_core::schema::{Memory, MemoryType};
use second_brain_core::sqlite_store::SqliteStore;
use second_brain_core::store::Store;
const MEMORY_TYPES: [MemoryType; 8] = [
MemoryType::Episodic,
MemoryType::Semantic,
MemoryType::Procedural,
MemoryType::Decision,
MemoryType::Architecture,
MemoryType::Debugging,
MemoryType::Task,
MemoryType::Question,
];
const MIN_BODY_CHARS: usize = 200;
const MAX_BODY_CHARS: usize = 4000;
const WORD_POOL: &[&str] = &[
"the",
"a",
"of",
"and",
"with",
"for",
"to",
"in",
"that",
"this",
"on",
"as",
"by",
"from",
"is",
"are",
"was",
"were",
"will",
"can",
"should",
"must",
"when",
"where",
"which",
"because",
"however",
"therefore",
"then",
"also",
"more",
"most",
"some",
"many",
"few",
"several",
"each",
"every",
"both",
"either",
"neither",
"not",
"only",
"just",
"still",
"yet",
"already",
"now",
"soon",
"later",
"before",
"after",
"during",
"while",
"since",
"until",
"though",
"although",
"so",
"thus",
"hence",
"memory",
"retrieval",
"embedding",
"vector",
"index",
"cache",
"database",
"latency",
"throughput",
"concurrency",
"daemon",
"snapshot",
"tenant",
"cosine",
"similarity",
"ranking",
"tokenizer",
"query",
"corpus",
"workspace",
"sqlite",
"tantivy",
"usearch",
"outbox",
"watermark",
"schema",
"migration",
"cluster",
"replica",
"shard",
"partition",
"checkpoint",
"transaction",
"mutex",
"thread",
"runtime",
"serialize",
"deserialize",
"protocol",
"endpoint",
"gateway",
"eviction",
"compaction",
"benchmark",
"percentile",
"jitter",
"backoff",
"retry",
"timeout",
"heartbeat",
"consensus",
"quorum",
"leader",
"follower",
"journal",
"commit",
"rollback",
"isolation",
"durability",
"consistency",
"availability",
"tolerance",
"balancer",
"proxy",
"router",
"session",
"token",
"credential",
"encryption",
"checksum",
"signature",
"certificate",
"socket",
"packet",
"header",
"payload",
"compress",
"decompress",
"model",
"encoder",
"decoder",
"reranker",
"gate",
"threshold",
"blend",
"recall",
"precision",
"score",
"rank",
"candidate",
"fusion",
"lexical",
"dense",
"hybrid",
"warmup",
"iteration",
"worker",
"pool",
"scope",
"lock",
"writer",
"reader",
"segment",
"tier",
"archive",
"supersede",
"distill",
"ingest",
"sync",
"conflict",
"resolve",
"apply",
"batch",
"record",
"sequence",
"log",
"event",
"machine",
"identity",
"config",
"directory",
"path",
"file",
"disk",
"usage",
"byte",
"size",
"capacity",
"growth",
"reserve",
"save",
"load",
"open",
"close",
"drain",
"rebuild",
"consistent",
"drift",
];
#[derive(Parser)]
#[command(
name = "sb-synth-store",
about = "Generates reproducible synthetic second-brain stores for capacity testing"
)]
struct Cli {
#[arg(
long,
help = "Output directory; one tenant-<i>/ subdirectory per store"
)]
out: PathBuf,
#[arg(long, help = "Number of stores to generate")]
stores: usize,
#[arg(long, help = "Number of memories per store")]
memories: usize,
#[arg(
long,
help = "Seed controlling every random choice; the same seed always produces byte-identical stores"
)]
seed: u64,
}
fn main() -> Result<()> {
let cli = Cli::parse();
if cli.stores == 0 {
bail!("--stores must be at least 1, got 0");
}
if cli.memories == 0 {
bail!("--memories must be at least 1, got 0");
}
if cli.out.exists() {
let mut entries = std::fs::read_dir(&cli.out)
.with_context(|| format!("reading --out directory {}", cli.out.display()))?;
if entries.next().is_some() {
bail!(
"--out {} already exists and is not empty; choose an empty or new directory",
cli.out.display()
);
}
}
std::fs::create_dir_all(&cli.out)
.with_context(|| format!("creating --out directory {}", cli.out.display()))?;
let started = Instant::now();
for i in 0..cli.stores {
generate_store(&cli.out, i, cli.memories, cli.seed)
.with_context(|| format!("generating store tenant-{i}"))?;
println!("tenant-{i}: wrote {} memories", cli.memories);
}
let elapsed = started.elapsed();
let total_bytes =
dir_size(&cli.out).with_context(|| format!("measuring size of {}", cli.out.display()))?;
println!(
"synth_store: {} stores x {} memories, {:.2} MiB on disk, {:.2}s wall clock",
cli.stores,
cli.memories,
total_bytes as f64 / (1024.0 * 1024.0),
elapsed.as_secs_f64()
);
Ok(())
}
fn generate_store(out: &Path, store_idx: usize, memory_count: usize, seed: u64) -> Result<()> {
let store_dir = out.join(format!("tenant-{store_idx}"));
std::fs::create_dir_all(&store_dir)?;
let db_path = store_dir.join("brain.db");
let store = SqliteStore::open(&db_path, format!("synth-{store_idx}"))
.with_context(|| format!("opening {}", db_path.display()))?;
let mut rng =
StdRng::seed_from_u64(seed.wrapping_add(store_idx as u64 * 0x9E37_79B9_7F4A_7C15));
for mem_idx in 0..memory_count {
let memory = synth_memory(&mut rng, store_idx, mem_idx);
store.store_memory(&memory)?;
}
let indexes_dir = db_path.with_extension("indexes");
let mut indexes = Indexes::open(&indexes_dir)
.with_context(|| format!("opening indexes at {}", indexes_dir.display()))?;
let applied = indexes.drain(&store)?;
if applied != memory_count {
bail!(
"index drain applied {applied} rows but {memory_count} memories were written to {}",
db_path.display()
);
}
if !indexes.consistent_with(&store)? {
bail!(
"indexes at {} are inconsistent with {} after drain",
indexes_dir.display(),
db_path.display()
);
}
Ok(())
}
fn synth_memory(rng: &mut StdRng, store_idx: usize, mem_idx: usize) -> Memory {
let content = generate_body(rng, MIN_BODY_CHARS, MAX_BODY_CHARS);
let memory_type = MEMORY_TYPES[rng.random_range(0..MEMORY_TYPES.len())];
let mut memory = Memory::new(
content,
memory_type,
"synth".to_string(),
format!("tenant-{store_idx}-mem-{mem_idx}"),
);
memory.id = deterministic_uuid(rng);
memory.embedding = random_unit_embedding(rng);
memory.machine_id = format!("synth-{store_idx}");
let created = synth_epoch()
+ ChronoDuration::days(rng.random_range(0..400))
+ ChronoDuration::seconds(rng.random_range(0..86_400));
memory.created_at = created;
memory.updated_at = created;
memory.last_accessed = created;
memory.access_count = rng.random_range(0..50);
memory.confidence = rng.random_range(0.5..1.0);
memory
}
fn synth_epoch() -> DateTime<Utc> {
Utc.with_ymd_and_hms(2025, 1, 1, 0, 0, 0).unwrap()
}
fn deterministic_uuid(rng: &mut StdRng) -> Uuid {
let mut bytes = [0u8; 16];
for b in bytes.iter_mut() {
*b = rng.random();
}
Uuid::from_bytes(bytes)
}
fn random_unit_embedding(rng: &mut StdRng) -> Vec<f32> {
let mut v: Vec<f32> = (0..EMBEDDING_DIM)
.map(|_| rng.random::<f32>() * 2.0 - 1.0)
.collect();
let norm: f32 = v.iter().map(|x| x * x).sum::<f32>().sqrt();
if norm > f32::EPSILON {
for x in v.iter_mut() {
*x /= norm;
}
}
v
}
fn generate_body(rng: &mut StdRng, min_len: usize, max_len: usize) -> String {
let target = rng.random_range(min_len..=max_len);
let mut body = String::with_capacity(target + 32);
let mut sentence_words = 0usize;
let mut sentence_limit = rng.random_range(8..16);
while body.len() < target {
let word = WORD_POOL[rng.random_range(0..WORD_POOL.len())];
if !body.is_empty() {
body.push(' ');
}
if sentence_words == 0 {
let mut chars = word.chars();
if let Some(first) = chars.next() {
body.push(first.to_ascii_uppercase());
body.push_str(chars.as_str());
}
} else {
body.push_str(word);
}
sentence_words += 1;
if sentence_words >= sentence_limit {
body.push('.');
sentence_words = 0;
sentence_limit = rng.random_range(8..16);
}
}
if !body.ends_with('.') {
body.push('.');
}
if body.len() > max_len {
let mut end = max_len;
while end > 0 && !body.is_char_boundary(end) {
end -= 1;
}
body.truncate(end);
if !body.ends_with('.') {
body.push('.');
}
}
body
}
fn dir_size(path: &Path) -> Result<u64> {
let mut total = 0u64;
for entry in std::fs::read_dir(path)? {
let entry = entry?;
let meta = entry.metadata()?;
if meta.is_dir() {
total += dir_size(&entry.path())?;
} else {
total += meta.len();
}
}
Ok(total)
}