use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
use spool::app;
use spool::config::{
AppConfig, DeveloperConfig, EmbeddingConfig, ModuleConfig, OutputConfig, ProjectConfig,
SceneConfig, VaultConfig, VaultLimits,
};
use spool::domain::{
DebugTrace, MemoryLifecycleState, MemoryOrigin, MemoryRecord, MemoryScope, MemorySourceKind,
Note, OutputFormat, RouteInput, TargetTool,
};
use spool::{engine, vault};
use std::collections::HashSet;
use std::fs;
use tempfile::TempDir;
struct Fixture {
_temp: TempDir,
config: AppConfig,
input: RouteInput,
notes: Vec<Note>,
debug: DebugTrace,
}
fn make_fixture(note_count: usize) -> Fixture {
let temp = tempfile::tempdir().unwrap();
let vault_root = temp.path().join("vault");
let repo_root = temp.path().join("repo");
fs::create_dir_all(vault_root.join("10-Projects")).unwrap();
fs::create_dir_all(vault_root.join("20-Areas")).unwrap();
fs::create_dir_all(vault_root.join("40-Permanent")).unwrap();
fs::create_dir_all(&repo_root).unwrap();
let vault_root = vault_root.canonicalize().unwrap();
let repo_root = repo_root.canonicalize().unwrap();
for index in 0..note_count {
let root = match index % 3 {
0 => "10-Projects",
1 => "20-Areas",
_ => "40-Permanent",
};
let name = format!("note-{index:04}.md");
let content = if index % 8 == 0 {
format!(
"# Repo Path Routing {index}\n\n## Project Matcher\n\nThis benchmark note mentions repo_path, routing, and [[Project Matcher]].\n\n### Scanner\n\nThe markdown scanner and wikilink parser are part of the retrieval path.\n"
)
} else if index % 11 == 0 {
format!(
"# Vault Scanner {index}\n\n## Frontmatter\n\nThis note discusses markdown headings, frontmatter, and scanner limits.\n"
)
} else {
format!(
"# General Note {index}\n\nBackground content for note {index}. This text is intentionally repetitive for stable benchmark input.\n"
)
};
fs::write(vault_root.join(root).join(name), content).unwrap();
}
let note_roots = vec![
"10-Projects".to_string(),
"20-Areas".to_string(),
"40-Permanent".to_string(),
];
let config = AppConfig {
vault: VaultConfig {
root: vault_root.clone(),
limits: VaultLimits {
max_files: note_count + 100,
max_file_bytes: 512 * 1024,
max_total_bytes: 64 * 1024 * 1024,
max_depth: 12,
},
},
output: OutputConfig {
default_format: OutputFormat::Prompt,
max_chars: 20_000,
max_notes: 8,
max_lifecycle: 8,
},
developer: DeveloperConfig::default(),
projects: vec![ProjectConfig {
id: "spool".to_string(),
name: "spool".to_string(),
repo_paths: vec![repo_root.clone()],
note_roots: note_roots.clone(),
default_tags: vec!["rust".to_string(), "cli".to_string()],
modules: vec![
ModuleConfig {
id: "routing".to_string(),
path_prefixes: vec!["src/engine".to_string()],
keywords: vec!["routing".to_string(), "repo_path".to_string()],
},
ModuleConfig {
id: "vault".to_string(),
path_prefixes: vec!["src/vault".to_string()],
keywords: vec!["scanner".to_string(), "wikilink".to_string()],
},
],
}],
scenes: vec![SceneConfig {
id: "planning".to_string(),
keywords: vec!["refine".to_string(), "benchmark".to_string()],
preferred_notes: vec!["20-Areas/note-0001.md".to_string()],
}],
embedding: EmbeddingConfig::default(),
};
let input = RouteInput {
task: "Refine Repo-Path routing benchmark".to_string(),
cwd: repo_root,
files: vec![
"src/engine/project_matcher.rs".to_string(),
"src/vault/scanner.rs".to_string(),
],
target: TargetTool::Codex,
format: OutputFormat::Prompt,
};
let (notes, scan_roots) =
vault::scan_notes_with_debug(&config.vault.root, ¬e_roots, &config.vault.limits)
.unwrap();
let debug = DebugTrace {
matched_project_id: Some("spool".to_string()),
note_roots,
scan_roots,
limits: config.vault.limits.clone(),
note_count: notes.len(),
};
Fixture {
_temp: temp,
config,
input,
notes,
debug,
}
}
fn bench_scan_notes(c: &mut Criterion) {
let mut group = c.benchmark_group("scan_notes_with_debug");
group.sample_size(10);
for note_count in [250usize, 1_000usize, 5_000usize] {
let fixture = make_fixture(note_count);
group.bench_with_input(
BenchmarkId::from_parameter(note_count),
&fixture,
|b, fixture| {
b.iter(|| {
let _ = vault::scan_notes_with_debug(
&fixture.config.vault.root,
&fixture.config.projects[0].note_roots,
&fixture.config.vault.limits,
)
.unwrap();
});
},
);
}
group.finish();
}
fn bench_build_context(c: &mut Criterion) {
let mut group = c.benchmark_group("build_context");
group.sample_size(10);
for note_count in [250usize, 1_000usize, 5_000usize] {
let fixture = make_fixture(note_count);
group.bench_with_input(
BenchmarkId::from_parameter(note_count),
&fixture,
|b, fixture| {
b.iter(|| {
let _ = engine::build_context(
&fixture.config,
&fixture.notes,
fixture.input.clone(),
fixture.debug.clone(),
);
});
},
);
}
group.finish();
}
fn bench_build_bundle(c: &mut Criterion) {
let mut group = c.benchmark_group("build_bundle");
group.sample_size(10);
for note_count in [250usize, 1_000usize, 5_000usize] {
let fixture = make_fixture(note_count);
group.bench_with_input(
BenchmarkId::from_parameter(note_count),
&fixture,
|b, fixture| {
b.iter(|| {
let _ = app::build_bundle(&fixture.config, fixture.input.clone()).unwrap();
});
},
);
}
group.finish();
}
const MEMORY_TYPES: &[&str] = &[
"constraint",
"decision",
"project",
"preference",
"incident",
"workflow",
"pattern",
"person",
"session",
];
const SAMPLE_ENTITIES: &[&str] = &[
"Rust",
"TypeScript",
"React",
"SQLite",
"Docker",
"tokio",
"serde",
"MCP",
"OAuth",
"CI",
"tantivy",
"Obsidian",
"Tauri",
"clap",
];
const SAMPLE_TAGS: &[&str] = &[
"database",
"auth",
"performance",
"testing",
"deployment",
"refactoring",
"security",
"api",
"frontend",
"backend",
];
const SAMPLE_TRIGGERS: &[&str] = &[
"deploy",
"migrate",
"refactor",
"review",
"test",
"benchmark",
"optimize",
"debug",
"release",
"rollback",
];
fn make_lifecycle_records(count: usize) -> Vec<(String, MemoryRecord)> {
(0..count)
.map(|i| {
let memory_type = MEMORY_TYPES[i % MEMORY_TYPES.len()];
let entities: Vec<String> = (0..3)
.map(|j| SAMPLE_ENTITIES[(i + j) % SAMPLE_ENTITIES.len()].to_string())
.collect();
let tags: Vec<String> = (0..2)
.map(|j| SAMPLE_TAGS[(i + j) % SAMPLE_TAGS.len()].to_string())
.collect();
let triggers: Vec<String> = (0..2)
.map(|j| SAMPLE_TRIGGERS[(i + j) % SAMPLE_TRIGGERS.len()].to_string())
.collect();
let record = MemoryRecord {
title: format!("Memory record {i}: {memory_type} about {}", entities[0]),
summary: format!(
"This is a {} memory about {} and {}. It covers important aspects of the system.",
memory_type, entities[0], tags[0]
),
memory_type: memory_type.to_string(),
scope: if i % 3 == 0 {
MemoryScope::Project
} else {
MemoryScope::User
},
state: if i % 5 == 0 {
MemoryLifecycleState::Canonical
} else {
MemoryLifecycleState::Accepted
},
origin: MemoryOrigin {
source_kind: MemorySourceKind::Manual,
source_ref: format!("bench:{i}"),
},
project_id: if i % 3 == 0 {
Some("spool".to_string())
} else {
None
},
user_id: None,
sensitivity: None,
entities,
tags,
triggers,
related_files: vec![format!("src/module_{}.rs", i % 20)],
related_records: if i > 0 {
vec![format!("rec-{:05}", i - 1)]
} else {
Vec::new()
},
supersedes: None,
applies_to: if i % 4 == 0 {
vec!["spool".to_string()]
} else {
Vec::new()
},
valid_until: None,
};
(format!("rec-{i:05}"), record)
})
.collect()
}
fn bench_score_lifecycle_candidate(c: &mut Criterion) {
let mut group = c.benchmark_group("score_lifecycle_candidate");
group.sample_size(100);
let input = RouteInput {
task: "refactor retrieval scoring pipeline for performance".to_string(),
cwd: std::path::PathBuf::from("/tmp/repo"),
files: vec![
"src/engine/scorer.rs".to_string(),
"src/engine/selector.rs".to_string(),
],
target: TargetTool::Codex,
format: OutputFormat::Prompt,
};
let project = spool::domain::MatchedProject {
id: "spool".to_string(),
name: "spool".to_string(),
reason: "bench".to_string(),
};
let records = make_lifecycle_records(100);
group.bench_function("single_record", |b| {
let (ref record_id, ref record) = records[0];
b.iter(|| {
engine::scorer::score_lifecycle_candidate(
Some(&project),
record_id,
record,
&input,
None,
Some(&records),
)
});
});
group.finish();
}
fn bench_select_lifecycle_candidates(c: &mut Criterion) {
let mut group = c.benchmark_group("select_lifecycle_candidates");
group.sample_size(10);
let input = RouteInput {
task: "refactor retrieval scoring pipeline for performance".to_string(),
cwd: std::path::PathBuf::from("/tmp/repo"),
files: vec![
"src/engine/scorer.rs".to_string(),
"src/engine/selector.rs".to_string(),
],
target: TargetTool::Codex,
format: OutputFormat::Prompt,
};
let project = spool::domain::MatchedProject {
id: "spool".to_string(),
name: "spool".to_string(),
reason: "bench".to_string(),
};
let excluded = HashSet::new();
for count in [1_000usize, 10_000usize, 50_000usize] {
let records = make_lifecycle_records(count);
group.bench_with_input(
BenchmarkId::from_parameter(count),
&records,
|b, records| {
b.iter(|| {
engine::selector::select_lifecycle_candidates(
Some(&project),
records,
&input,
8,
&excluded,
None,
)
});
},
);
}
group.finish();
}
criterion_group!(
retrieval,
bench_scan_notes,
bench_build_context,
bench_build_bundle,
bench_score_lifecycle_candidate,
bench_select_lifecycle_candidates,
);
criterion_main!(retrieval);