use crate::{
config::{McPaths, read_settings},
instructions::discover_agents_with_additional_markdown,
model_catalog::{ModelCatalogEntry, cached_model_context_window, write_catalog_cache},
output::{
ActivityEvent, ActivityId, ActivityKind, ActivityMetadata, ActivityStatus, OutputEvent,
},
providers::{ProviderEvent, StreamParser},
sessions::SessionManager,
skills::{SkillRoots, discover_skills},
tools::{ToolRuntime, process::terminate_child_tree_and_wait},
tui::state::MissionControlState,
};
use ratatui::{
Terminal, TerminalOptions, Viewport,
backend::{Backend, CrosstermBackend, TestBackend},
buffer::Buffer,
layout::{Position, Rect, Size},
};
use serde::Serialize;
use serde_json::{Value, json};
use std::{
cell::{Cell, RefCell},
collections::{BTreeMap, HashMap},
fs,
hint::black_box,
io::{self, Write},
path::{Path, PathBuf},
process::{Command, Stdio},
rc::Rc,
time::{Duration, Instant},
};
use tempfile::TempDir;
const RESPONSES_SSE: &str = include_str!("../../scripts/fixtures/cpu_profiling/responses.sse");
const CHAT_COMPLETIONS_SSE: &str =
include_str!("../../scripts/fixtures/cpu_profiling/chat_completions.sse");
const RENDERING_TRANSCRIPT: &str =
include_str!("../../scripts/fixtures/cpu_profiling/rendering_transcript.md");
const DEFAULT_ITERATIONS: u64 = 25;
thread_local! {
static EMIT_RENDER_SUMMARIES: Cell<bool> = const { Cell::new(true) };
}
#[derive(Debug)]
struct ProfileResult {
scenario: &'static str,
iterations: u64,
elapsed: Duration,
primary_metric: &'static str,
metric_value: u64,
artifact: &'static str,
counters: Vec<(&'static str, u64)>,
}
impl ProfileResult {
fn print(&self) {
let counters = self
.counters
.iter()
.map(|(key, value)| format!("{key}:{value}"))
.collect::<Vec<_>>()
.join(",");
println!(
"profile_cpu scenario={} iterations={} elapsed_ms={} primary_metric={} metric_value={} artifact={} counters={}",
self.scenario,
self.iterations,
self.elapsed.as_millis(),
self.primary_metric,
self.metric_value,
self.artifact,
counters
);
}
}
fn profile_iterations() -> u64 {
std::env::var("PROFILE_CPU_ITERATIONS")
.ok()
.and_then(|value| value.parse::<u64>().ok())
.filter(|value| *value > 0)
.unwrap_or(DEFAULT_ITERATIONS)
}
fn profile_min_duration() -> Duration {
std::env::var("PROFILE_CPU_MIN_SECONDS")
.ok()
.and_then(|value| value.parse::<u64>().ok())
.map(Duration::from_secs)
.unwrap_or_default()
}
fn run_profile(
scenario: &'static str,
primary_metric: &'static str,
artifact: &'static str,
mut run_once: impl FnMut() -> u64,
) -> ProfileResult {
let min_iterations = profile_iterations();
let min_duration = profile_min_duration();
let started = Instant::now();
let mut iterations = 0;
let mut metric_value = 0u64;
while iterations < min_iterations || started.elapsed() < min_duration {
metric_value = metric_value.saturating_add(run_once());
iterations += 1;
}
let elapsed = started.elapsed();
ProfileResult {
scenario,
iterations,
elapsed,
primary_metric,
metric_value,
artifact,
counters: vec![("per_iteration", metric_value / iterations.max(1))],
}
}
const FIND_BENCH_PREFIX: &str = "fffind_benchmark ";
const FIND_BENCH_DEFAULT_REPETITIONS: usize = 25;
const FIND_BENCH_DEFAULT_LIMIT: usize = 50;
const FIND_BENCH_MAX_LIMIT: usize = 200;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum FindBenchBaseline {
IgnoreWalk,
None,
}
impl FindBenchBaseline {
fn from_env() -> Self {
match std::env::var("FFFIND_BENCH_BASELINE")
.unwrap_or_else(|_| "ignore_walk".to_string())
.trim()
{
"none" => Self::None,
"ignore_walk" | "" => Self::IgnoreWalk,
value => {
eprintln!(
"fffind_benchmark warning=unsupported_baseline value={value:?} using=ignore_walk"
);
Self::IgnoreWalk
}
}
}
}
#[derive(Debug)]
struct FindBenchConfig {
root: PathBuf,
root_label: String,
root_hash: String,
queries: Vec<String>,
paths: Vec<String>,
repetitions: usize,
limit: usize,
cwd_churn: usize,
baseline: FindBenchBaseline,
show_results: bool,
}
impl FindBenchConfig {
fn from_env() -> anyhow::Result<Self> {
let root = std::env::var("FFFIND_BENCH_ROOT")
.map(PathBuf::from)
.unwrap_or_else(|_| PathBuf::from(env!("CARGO_MANIFEST_DIR")))
.canonicalize()?;
let root_label = root
.file_name()
.and_then(|name| name.to_str())
.filter(|name| !name.is_empty())
.unwrap_or("root")
.to_string();
let root_hash = hash_root_label(&root);
let queries = comma_env("FFFIND_BENCH_QUERIES").unwrap_or_else(|| {
["lib", "tools", "profile", "readme", "fff"]
.into_iter()
.map(ToOwned::to_owned)
.collect()
});
let paths = comma_env("FFFIND_BENCH_PATHS")
.unwrap_or_else(|| ["src", "docs"].into_iter().map(ToOwned::to_owned).collect());
let repetitions = parse_usize_env(
"FFFIND_BENCH_REPETITIONS",
FIND_BENCH_DEFAULT_REPETITIONS,
1,
usize::MAX,
);
let limit = parse_usize_env(
"FFFIND_BENCH_LIMIT",
FIND_BENCH_DEFAULT_LIMIT,
1,
FIND_BENCH_MAX_LIMIT,
);
let cwd_churn = parse_usize_env("FFFIND_BENCH_CWD_CHURN", 0, 0, usize::MAX);
let show_results = std::env::var("FFFIND_BENCH_SHOW_RESULTS")
.ok()
.is_some_and(|value| value == "1");
Ok(Self {
root,
root_label,
root_hash,
queries,
paths,
repetitions,
limit,
cwd_churn,
baseline: FindBenchBaseline::from_env(),
show_results,
})
}
fn existing_filter_paths(&self) -> Vec<String> {
self.paths
.iter()
.filter_map(|path| {
let full_path = self.root.join(path);
if full_path.is_dir() {
Some(path.clone())
} else {
eprintln!(
"fffind_benchmark warning=missing_filter_path path={path:?} action=skipped"
);
None
}
})
.collect()
}
}
fn comma_env(name: &str) -> Option<Vec<String>> {
std::env::var(name).ok().and_then(|value| {
let values = value
.split(',')
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
.collect::<Vec<_>>();
(!values.is_empty()).then_some(values)
})
}
fn parse_usize_env(name: &str, default: usize, min: usize, max: usize) -> usize {
std::env::var(name)
.ok()
.and_then(|value| value.parse::<usize>().ok())
.map(|value| value.clamp(min, max))
.unwrap_or(default)
}
fn hash_root_label(root: &Path) -> String {
use sha2::{Digest, Sha256};
let mut hasher = Sha256::new();
hasher.update(root.to_string_lossy().as_bytes());
crate::hex::lower_hex(hasher.finalize())[..16].to_string()
}
#[derive(Debug, Clone, Copy, Serialize)]
struct FindDurationStats {
min: f64,
p50: f64,
p95: f64,
max: f64,
}
fn duration_stats(durations: &[Duration]) -> FindDurationStats {
let mut millis = durations
.iter()
.map(|duration| duration.as_secs_f64() * 1000.0)
.collect::<Vec<_>>();
millis.sort_by(f64::total_cmp);
FindDurationStats {
min: round_millis(millis.first().copied().unwrap_or_default()),
p50: round_millis(percentile(&millis, 0.50)),
p95: round_millis(percentile(&millis, 0.95)),
max: round_millis(millis.last().copied().unwrap_or_default()),
}
}
fn percentile(sorted: &[f64], percentile: f64) -> f64 {
if sorted.is_empty() {
return 0.0;
}
let rank = (percentile * sorted.len() as f64).ceil() as usize;
sorted[rank.saturating_sub(1).min(sorted.len() - 1)]
}
fn round_millis(value: f64) -> f64 {
(value * 1000.0).round() / 1000.0
}
#[derive(Debug, Clone, Copy, Default)]
struct ResourceSnapshot {
rss_kb: Option<i64>,
fd_count: Option<i64>,
thread_count: Option<i64>,
}
impl ResourceSnapshot {
fn capture() -> Self {
Self {
rss_kb: current_rss_kb(),
fd_count: current_fd_count(),
thread_count: current_thread_count(),
}
}
fn delta(self, after: Self) -> ResourceDelta {
ResourceDelta {
rss_kb_delta: option_delta(self.rss_kb, after.rss_kb),
fd_delta: option_delta(self.fd_count, after.fd_count),
thread_delta: option_delta(self.thread_count, after.thread_count),
}
}
}
#[derive(Debug, Clone, Copy, Default, Serialize)]
struct ResourceDelta {
rss_kb_delta: Option<i64>,
fd_delta: Option<i64>,
thread_delta: Option<i64>,
}
fn option_delta(before: Option<i64>, after: Option<i64>) -> Option<i64> {
Some(after? - before?)
}
fn current_rss_kb() -> Option<i64> {
let output = Command::new("ps")
.args(["-o", "rss=", "-p", &std::process::id().to_string()])
.output()
.ok()?;
if !output.status.success() {
return None;
}
String::from_utf8_lossy(&output.stdout)
.trim()
.parse::<i64>()
.ok()
}
fn current_fd_count() -> Option<i64> {
fs::read_dir("/dev/fd").ok()?.count().try_into().ok()
}
#[cfg(target_os = "linux")]
fn current_thread_count() -> Option<i64> {
fs::read_dir("/proc/self/task")
.ok()?
.count()
.try_into()
.ok()
}
#[cfg(all(unix, not(target_os = "linux")))]
fn current_thread_count() -> Option<i64> {
let output = Command::new("ps")
.args(["-M", "-p", &std::process::id().to_string()])
.output()
.ok()?;
if !output.status.success() {
return None;
}
let lines = String::from_utf8_lossy(&output.stdout)
.lines()
.filter(|line| !line.trim().is_empty())
.count();
lines.checked_sub(1)?.try_into().ok()
}
#[cfg(not(unix))]
fn current_thread_count() -> Option<i64> {
None
}
#[derive(Debug)]
struct FindBenchObservation {
elapsed: Vec<Duration>,
matches_returned: u64,
total_matched: u64,
truncated: bool,
index_ready: bool,
top_results: Option<Vec<String>>,
}
fn run_find_observation(
runtime: &ToolRuntime,
query: &str,
kind: &str,
path: Option<&str>,
limit: usize,
iterations: usize,
include_top_results: bool,
) -> anyhow::Result<FindBenchObservation> {
let mut elapsed = Vec::with_capacity(iterations);
let mut matches_returned = 0;
let mut total_matched = 0;
let mut truncated = false;
let mut index_ready = false;
let mut top_results = None;
for index in 0..iterations {
let arguments = match path {
Some(path) => json!({"query": query, "kind": kind, "path": path, "limit": limit}),
None => json!({"query": query, "kind": kind, "limit": limit}),
};
let started = Instant::now();
let result = runtime.dispatch("find", arguments);
elapsed.push(started.elapsed());
if !result.success {
anyhow::bail!("find benchmark dispatch failed: {}", result.content);
}
matches_returned = metadata_u64(&result.metadata, "matches_returned");
total_matched = metadata_u64(&result.metadata, "total_matched");
truncated = result
.metadata
.get("truncated")
.and_then(Value::as_bool)
.unwrap_or(false);
index_ready = result
.metadata
.get("index_ready")
.and_then(Value::as_bool)
.unwrap_or(false);
if include_top_results && index == 0 {
top_results = Some(
result
.content
.lines()
.take(10)
.map(ToOwned::to_owned)
.collect(),
);
}
}
Ok(FindBenchObservation {
elapsed,
matches_returned,
total_matched,
truncated,
index_ready,
top_results,
})
}
fn metadata_u64(metadata: &Value, key: &str) -> u64 {
metadata
.get(key)
.and_then(Value::as_u64)
.unwrap_or_default()
}
#[derive(Debug, Serialize)]
struct IgnoreWalkBaselineSummary {
engine: &'static str,
p50_ms: f64,
p95_ms: f64,
matches_returned: usize,
total_matched: usize,
}
fn run_ignore_walk_baseline(
root: &Path,
query: &str,
path_filter: Option<&str>,
limit: usize,
iterations: usize,
) -> anyhow::Result<IgnoreWalkBaselineSummary> {
let mut elapsed = Vec::with_capacity(iterations);
let mut matches_returned = 0;
let mut total_matched = 0;
for _ in 0..iterations {
let started = Instant::now();
let matches = ignore_walk_matches(root, query, path_filter, limit)?;
elapsed.push(started.elapsed());
matches_returned = matches.0;
total_matched = matches.1;
}
let stats = duration_stats(&elapsed);
Ok(IgnoreWalkBaselineSummary {
engine: "ignore_walk",
p50_ms: stats.p50,
p95_ms: stats.p95,
matches_returned,
total_matched,
})
}
fn ignore_walk_matches(
root: &Path,
query: &str,
path_filter: Option<&str>,
limit: usize,
) -> anyhow::Result<(usize, usize)> {
let start = path_filter
.map(|path| root.join(path))
.unwrap_or_else(|| root.to_path_buf());
let query = query.to_lowercase();
let mut returned = 0;
let mut total = 0;
for entry in ignore::WalkBuilder::new(start)
.standard_filters(true)
.build()
.filter_map(Result::ok)
{
if !entry
.file_type()
.is_some_and(|file_type| file_type.is_file())
{
continue;
}
let relative = entry
.path()
.strip_prefix(root)
.unwrap_or(entry.path())
.to_string_lossy()
.replace('\\', "/");
if relative.to_lowercase().contains(&query) {
total += 1;
if returned < limit {
returned += 1;
}
}
}
Ok((returned, total))
}
fn print_find_benchmark_line(value: Value) {
println!("{FIND_BENCH_PREFIX}{value}");
}
#[allow(
clippy::too_many_arguments,
reason = "benchmark output builder mirrors JSON fields for call-site clarity"
)]
fn find_benchmark_line(
cfg: &FindBenchConfig,
scenario: &str,
query: Option<&str>,
kind: Option<&str>,
path: Option<&str>,
observation: &FindBenchObservation,
resources: ResourceDelta,
baseline: Option<IgnoreWalkBaselineSummary>,
) -> Value {
let mut value = json!({
"schema": 1,
"scenario": scenario,
"root_label": cfg.root_label,
"root_hash": cfg.root_hash,
"query": query,
"kind": kind,
"path": path,
"iterations": observation.elapsed.len(),
"elapsed_ms": duration_stats(&observation.elapsed),
"matches_returned": observation.matches_returned,
"total_matched": observation.total_matched,
"truncated": observation.truncated,
"index_ready": observation.index_ready,
"rss_kb_delta": resources.rss_kb_delta,
"fd_delta": resources.fd_delta,
"thread_delta": resources.thread_delta,
"baseline": baseline,
});
if let Some(top_results) = observation.top_results.as_ref() {
value["top_results"] = json!(top_results);
}
value
}
fn baseline_for_file_scenario(
cfg: &FindBenchConfig,
query: &str,
path: Option<&str>,
iterations: usize,
) -> anyhow::Result<Option<IgnoreWalkBaselineSummary>> {
match cfg.baseline {
FindBenchBaseline::IgnoreWalk => {
run_ignore_walk_baseline(&cfg.root, query, path, cfg.limit, iterations).map(Some)
}
FindBenchBaseline::None => Ok(None),
}
}
fn collect_churn_dirs(root: &Path, count: usize) -> Vec<PathBuf> {
if count == 0 {
return Vec::new();
}
let mut dirs = Vec::new();
for entry in ignore::WalkBuilder::new(root)
.max_depth(Some(3))
.standard_filters(true)
.build()
.filter_map(Result::ok)
{
if dirs.len() >= count {
break;
}
let path = entry.path();
if path != root
&& entry
.file_type()
.is_some_and(|file_type| file_type.is_dir())
{
dirs.push(path.to_path_buf());
}
}
dirs
}
fn run_cwd_churn_scenario(
cfg: &FindBenchConfig,
runtime: &ToolRuntime,
query: &str,
) -> anyhow::Result<Option<Value>> {
let dirs = collect_churn_dirs(&cfg.root, cfg.cwd_churn);
if cfg.cwd_churn == 0 {
return Ok(None);
}
if dirs.is_empty() {
eprintln!("fffind_benchmark warning=no_child_dirs_for_cwd_churn action=skipped");
return Ok(None);
}
let before = ResourceSnapshot::capture();
let mut elapsed = Vec::with_capacity(dirs.len());
let mut matches_returned = 0;
let mut total_matched = 0;
let mut truncated = false;
let mut index_ready = false;
for dir in &dirs {
let cwd_runtime = runtime.clone_for_cwd_with_subagent_depth(dir, 0)?;
let started = Instant::now();
let result = cwd_runtime.dispatch(
"find",
json!({"query": query, "kind": "files", "limit": cfg.limit}),
);
elapsed.push(started.elapsed());
if !result.success {
anyhow::bail!("find cwd churn dispatch failed: {}", result.content);
}
matches_returned = metadata_u64(&result.metadata, "matches_returned");
total_matched = metadata_u64(&result.metadata, "total_matched");
truncated = result
.metadata
.get("truncated")
.and_then(Value::as_bool)
.unwrap_or(false);
index_ready = result
.metadata
.get("index_ready")
.and_then(Value::as_bool)
.unwrap_or(false);
}
let resources = before.delta(ResourceSnapshot::capture());
Ok(Some(json!({
"schema": 1,
"scenario": "cwd_churn",
"root_label": cfg.root_label,
"root_hash": cfg.root_hash,
"query": query,
"kind": "files",
"path": null,
"iterations": elapsed.len(),
"configured_churn": cfg.cwd_churn,
"elapsed_ms": duration_stats(&elapsed),
"matches_returned": matches_returned,
"total_matched": total_matched,
"truncated": truncated,
"index_ready": index_ready,
"rss_kb_delta": resources.rss_kb_delta,
"fd_delta": resources.fd_delta,
"thread_delta": resources.thread_delta,
"baseline": null,
})))
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
struct ContentSearchPairKey {
scenario: String,
path_label: Option<String>,
pattern_hash: String,
mode: String,
limit: u64,
context: u64,
}
#[derive(Debug, Clone, Copy)]
struct ContentSearchP95 {
p95: f64,
}
#[derive(Debug, Clone, Copy, Serialize)]
struct ContentSearchPairedSpeedupStats {
samples: usize,
wins: usize,
median: f64,
min: f64,
max: f64,
}
fn print_content_search_benchmark_summary(
cfg: &ContentSearchBenchConfig,
scenario_count: usize,
observed_values: &[Value],
) {
print_content_search_benchmark_line(content_search_summary_value(
cfg,
scenario_count,
observed_values,
));
}
fn content_search_summary_value(
cfg: &ContentSearchBenchConfig,
scenario_count: usize,
observed_values: &[Value],
) -> Value {
let mut fastest_p50_by_mode = serde_json::Map::new();
let mut fastest_p95_by_mode = serde_json::Map::new();
for mode in [ContentSearchMode::Plain, ContentSearchMode::Regex] {
let mode_name = mode.as_str();
let warm_values = observed_values
.iter()
.filter(|value| content_search_warm_success_value(value, mode_name))
.collect::<Vec<_>>();
if let Some(value) = warm_values
.iter()
.filter_map(|value| {
Some((
value["engine"].as_str()?,
value["elapsed_ms"]["p50"].as_f64()?,
))
})
.min_by(|left, right| left.1.total_cmp(&right.1))
{
fastest_p50_by_mode.insert(mode_name.to_string(), json!(value.0));
}
if let Some(value) = warm_values
.iter()
.filter_map(|value| {
Some((
value["engine"].as_str()?,
value["elapsed_ms"]["p95"].as_f64()?,
))
})
.min_by(|left, right| left.1.total_cmp(&right.1))
{
fastest_p95_by_mode.insert(mode_name.to_string(), json!(value.0));
}
}
let mut grep_by_key: HashMap<ContentSearchPairKey, ContentSearchP95> = HashMap::new();
let mut candidate_by_key: HashMap<ContentSearchPairKey, ContentSearchP95> = HashMap::new();
for value in observed_values {
if !content_search_warm_success_value(value, value["mode"].as_str().unwrap_or_default()) {
continue;
}
let Some(key) = content_search_pair_key(value) else {
continue;
};
let Some(engine) = value["engine"].as_str() else {
continue;
};
let Some(p95) = value["elapsed_ms"]["p95"].as_f64() else {
continue;
};
let p95 = ContentSearchP95 { p95 };
if engine.starts_with("ffgrep_") {
grep_by_key
.entry(key)
.and_modify(|existing| {
if p95.p95 < existing.p95 {
*existing = p95;
}
})
.or_insert(p95);
} else if engine.starts_with("candidate_") {
candidate_by_key
.entry(key)
.and_modify(|existing| {
if p95.p95 < existing.p95 {
*existing = p95;
}
})
.or_insert(p95);
}
}
let mut plain_ratios = Vec::new();
let mut regex_ratios = Vec::new();
for (key, candidate) in &candidate_by_key {
let Some(grep) = grep_by_key.get(key) else {
continue;
};
if candidate.p95 <= 0.0 {
continue;
}
let ratio = grep.p95 / candidate.p95;
match key.mode.as_str() {
"plain" => plain_ratios.push(ratio),
"regex" => regex_ratios.push(ratio),
_ => {}
}
}
let plain_paired = paired_speedup_stats(&mut plain_ratios);
let regex_paired = paired_speedup_stats(&mut regex_ratios);
let plain_speedup = plain_paired.map(|stats| stats.median);
let regex_speedup = regex_paired.map(|stats| stats.median);
let total_failures = observed_values
.iter()
.map(|value| value["failures"].as_u64().unwrap_or_default())
.sum::<u64>();
let failure_kind_counts = content_search_failure_kind_counts(observed_values);
let diagnostic_kind_counts = content_search_diagnostic_kind_counts(observed_values);
let failure_scenarios = observed_values
.iter()
.filter(|value| value["failures"].as_u64().unwrap_or_default() > 0)
.map(content_search_failure_scenario)
.collect::<Vec<_>>();
let resource_growth_observed = observed_values.iter().any(|value| {
value["fd_delta"].as_i64().is_some_and(|delta| delta > 16)
|| value["thread_delta"]
.as_i64()
.is_some_and(|delta| delta > 16)
});
let regex_fallback_observed = observed_values.iter().any(|value| {
value["regex_fallback_error_present"]
.as_bool()
.unwrap_or(false)
});
let adapter_limit_mismatch_observed = observed_values.iter().any(|value| {
value["adapter_limit_mismatch_observed"]
.as_bool()
.unwrap_or(false)
});
let raw_limit_overrun_observed = observed_values.iter().any(|value| {
value["raw_limit_overrun_observed"]
.as_bool()
.unwrap_or(false)
});
let zero_match_compatibility_risk = observed_values.iter().any(|candidate| {
let Some(engine) = candidate["engine"].as_str() else {
return false;
};
if !engine.starts_with("candidate_")
|| candidate["matches_returned"].as_u64().unwrap_or_default() != 0
{
return false;
}
observed_values.iter().any(|grep| {
grep["engine"]
.as_str()
.is_some_and(|engine| engine.starts_with("ffgrep_"))
&& grep["scenario"] == candidate["scenario"]
&& grep["path_label"] == candidate["path_label"]
&& grep["pattern_hash"] == candidate["pattern_hash"]
&& grep["mode"] == candidate["mode"]
&& grep["limit"] == candidate["limit"]
&& grep["context"] == candidate["context"]
&& grep["matches_returned"].as_u64().unwrap_or_default() > 0
})
});
let rss_known = observed_values
.iter()
.any(|value| !value["rss_kb_delta"].is_null());
let compatibility_risk_observed = total_failures > 0
|| regex_fallback_observed
|| zero_match_compatibility_risk
|| adapter_limit_mismatch_observed;
let recommended = total_failures == 0
&& plain_speedup.is_some_and(|speedup| speedup >= 1.2)
&& regex_speedup.is_some_and(|speedup| speedup >= 1.0)
&& !compatibility_risk_observed
&& !resource_growth_observed
&& rss_known;
json!({
"schema": 1,
"scenario": "summary",
"root_label": cfg.root_label,
"root_hash": cfg.root_hash,
"paths": cfg.paths,
"repetitions": cfg.repetitions,
"limits": cfg.limits,
"contexts": cfg.contexts,
"baseline": cfg.baseline.as_str(),
"enable_content_indexing": cfg.content_indexing,
"cwd_churn": cfg.cwd_churn,
"scenario_lines": scenario_count,
"fastest_warm_p50_engine_by_mode": fastest_p50_by_mode,
"fastest_warm_p95_engine_by_mode": fastest_p95_by_mode,
"candidate_plain_speedup_vs_ffgrep_native_p95": plain_speedup,
"candidate_regex_speedup_vs_ffgrep_native_p95": regex_speedup,
"paired_p95_speedup_vs_ffgrep_by_mode": {
"plain": plain_paired,
"regex": regex_paired,
},
"total_failures": total_failures,
"failure_kind_counts": failure_kind_counts,
"diagnostic_kind_counts": diagnostic_kind_counts,
"failure_scenario_count": failure_scenarios.len(),
"failure_scenarios": failure_scenarios,
"compatibility_risk_observed": compatibility_risk_observed,
"actual_failure_observed": total_failures > 0,
"regex_fallback_observed": regex_fallback_observed,
"semantic_mismatch_observed": zero_match_compatibility_risk,
"zero_match_compatibility_risk": zero_match_compatibility_risk,
"adapter_limit_mismatch_observed": adapter_limit_mismatch_observed,
"raw_limit_overrun_observed": raw_limit_overrun_observed,
"raw_limit_overrun_diagnostic": "performance_only_visible_output_capped",
"resource_growth_observed": resource_growth_observed,
"step4_exploration": {
"recommended": recommended,
"regex_equivalent_speedup_required": true,
"caveat": "benchmark approves only separate content-search candidate exploration, not production replacement"
}
})
}
fn content_search_warm_success_value(value: &Value, mode_name: &str) -> bool {
value["mode"].as_str() == Some(mode_name)
&& value["scenario"]
.as_str()
.is_some_and(|scenario| scenario.starts_with("warm_") || scenario == "high_limit_tail")
&& value["successes"].as_u64().unwrap_or_default() > 0
&& value["failures"].as_u64().unwrap_or_default() == 0
}
fn content_search_pair_key(value: &Value) -> Option<ContentSearchPairKey> {
Some(ContentSearchPairKey {
scenario: value["scenario"].as_str()?.to_string(),
path_label: value["path_label"].as_str().map(ToOwned::to_owned),
pattern_hash: value["pattern_hash"].as_str()?.to_string(),
mode: value["mode"].as_str()?.to_string(),
limit: value["limit"].as_u64()?,
context: value["context"].as_u64()?,
})
}
fn paired_speedup_stats(ratios: &mut [f64]) -> Option<ContentSearchPairedSpeedupStats> {
if ratios.is_empty() {
return None;
}
ratios.sort_by(f64::total_cmp);
let median = if ratios.len().is_multiple_of(2) {
let upper = ratios.len() / 2;
(ratios[upper - 1] + ratios[upper]) / 2.0
} else {
ratios[ratios.len() / 2]
};
Some(ContentSearchPairedSpeedupStats {
samples: ratios.len(),
wins: ratios.iter().filter(|ratio| **ratio > 1.0).count(),
median: round_millis(median),
min: round_millis(ratios[0]),
max: round_millis(ratios[ratios.len() - 1]),
})
}
fn content_search_failure_kind_counts(observed_values: &[Value]) -> BTreeMap<String, u64> {
let mut counts = BTreeMap::new();
for value in observed_values {
let Some(kinds) = value["failure_kind_counts"].as_object() else {
continue;
};
for (kind, count) in kinds {
*counts.entry(kind.clone()).or_default() += count.as_u64().unwrap_or_default();
}
}
counts
}
fn content_search_diagnostic_kind_counts(observed_values: &[Value]) -> BTreeMap<String, u64> {
let mut counts = BTreeMap::new();
for value in observed_values {
let Some(kinds) = value["diagnostic_kind_counts"].as_object() else {
continue;
};
for (kind, count) in kinds {
*counts.entry(kind.clone()).or_default() += count.as_u64().unwrap_or_default();
}
}
counts
}
fn content_search_failure_scenario(value: &Value) -> Value {
json!({
"scenario": value["scenario"],
"path_label": value["path_label"],
"pattern_label": value["pattern_label"],
"pattern_hash": value["pattern_hash"],
"mode": value["mode"],
"engine": value["engine"],
"limit": value["limit"],
"context": value["context"],
"failures": value["failures"],
"failure_kind_counts": value["failure_kind_counts"],
})
}
const CONTENT_SEARCH_BENCH_PREFIX: &str = "content_search_benchmark ";
const CONTENT_SEARCH_BENCH_DEFAULT_REPETITIONS: usize = 25;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ContentSearchBaseline {
Grep,
Candidate,
Both,
}
impl ContentSearchBaseline {
fn from_env() -> Self {
match std::env::var("FFF_CONTENT_BENCH_BASELINE")
.unwrap_or_else(|_| "both".to_string())
.trim()
{
"grep" => Self::Grep,
"candidate" => Self::Candidate,
"both" | "" => Self::Both,
value => {
eprintln!(
"content_search_benchmark warning=unsupported_baseline value={value:?} using=both"
);
Self::Both
}
}
}
fn includes_grep(self) -> bool {
matches!(self, Self::Grep | Self::Both)
}
fn includes_candidate(self) -> bool {
matches!(self, Self::Candidate | Self::Both)
}
fn as_str(self) -> &'static str {
match self {
Self::Grep => "grep",
Self::Candidate => "candidate",
Self::Both => "both",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
enum ContentSearchMode {
Plain,
Regex,
}
impl ContentSearchMode {
fn as_str(self) -> &'static str {
match self {
Self::Plain => "plain",
Self::Regex => "regex",
}
}
fn candidate_engine(self) -> &'static str {
match self {
Self::Plain => "candidate_plain",
Self::Regex => "candidate_regex",
}
}
}
#[derive(Debug, Clone)]
struct ContentSearchPattern {
label: String,
hash: String,
text: String,
mode: ContentSearchMode,
}
impl ContentSearchPattern {
fn grep_pattern(&self) -> String {
match self.mode {
ContentSearchMode::Plain => regex::escape(&self.text),
ContentSearchMode::Regex => self.text.clone(),
}
}
}
#[derive(Debug)]
struct ContentSearchBenchConfig {
root: PathBuf,
root_label: String,
root_hash: String,
paths: Vec<String>,
plain_patterns: Vec<ContentSearchPattern>,
regex_patterns: Vec<ContentSearchPattern>,
repetitions: usize,
limits: Vec<usize>,
contexts: Vec<usize>,
baseline: ContentSearchBaseline,
content_indexing: bool,
cwd_churn: usize,
show_patterns: bool,
}
impl ContentSearchBenchConfig {
fn from_env() -> anyhow::Result<Self> {
let root = std::env::var("FFF_CONTENT_BENCH_ROOT")
.map(PathBuf::from)
.unwrap_or_else(|_| PathBuf::from(env!("CARGO_MANIFEST_DIR")))
.canonicalize()?;
let root_label = root
.file_name()
.and_then(|name| name.to_str())
.filter(|name| !name.is_empty())
.unwrap_or("root")
.to_string();
let root_hash = hash_root_label(&root);
let paths = content_benchmark_paths_from_env()?;
let plain_patterns = content_patterns_from_env(
"FFF_CONTENT_BENCH_PATTERNS_PLAIN",
ContentSearchMode::Plain,
&[
"ToolRuntime",
"grep",
"serde_json",
"ProfileResult",
"content_search",
],
);
let regex_patterns = content_patterns_from_env(
"FFF_CONTENT_BENCH_PATTERNS_REGEX",
ContentSearchMode::Regex,
&[
"fn [a-zA-Z0-9_]+",
"pub\\(crate\\) struct",
"impl ToolRuntime",
],
);
let repetitions = parse_usize_env(
"FFF_CONTENT_BENCH_REPETITIONS",
CONTENT_SEARCH_BENCH_DEFAULT_REPETITIONS,
1,
usize::MAX,
);
let limits = parse_usize_list_env("FFF_CONTENT_BENCH_LIMITS", &[20, 100], 1);
let contexts = parse_usize_list_env("FFF_CONTENT_BENCH_CONTEXTS", &[0, 2], 0);
let content_indexing = parse_bool_env("FFF_CONTENT_BENCH_CONTENT_INDEXING", true);
let cwd_churn = parse_usize_env("FFF_CONTENT_BENCH_CWD_CHURN", 0, 0, usize::MAX);
let show_patterns = parse_bool_env("FFF_CONTENT_BENCH_SHOW_PATTERNS", false);
Ok(Self {
root,
root_label,
root_hash,
paths,
plain_patterns,
regex_patterns,
repetitions,
limits,
contexts,
baseline: ContentSearchBaseline::from_env(),
content_indexing,
cwd_churn,
show_patterns,
})
}
fn existing_paths(&self) -> Vec<String> {
self.paths
.iter()
.filter_map(|path| {
let full_path = self.root.join(path);
if full_path.is_dir() {
Some(path.clone())
} else {
eprintln!(
"content_search_benchmark warning=missing_path path={path:?} action=skipped"
);
None
}
})
.collect()
}
}
fn content_benchmark_paths_from_env() -> anyhow::Result<Vec<String>> {
comma_env("FFF_CONTENT_BENCH_PATHS")
.unwrap_or_else(|| ["src", "docs"].into_iter().map(ToOwned::to_owned).collect())
.into_iter()
.enumerate()
.map(|(index, path)| {
sanitize_content_benchmark_path(&path).map_err(|error| {
anyhow::anyhow!(
"invalid FFF_CONTENT_BENCH_PATHS entry {}: {error}",
index + 1
)
})
})
.collect()
}
fn sanitize_content_benchmark_path(path: &str) -> anyhow::Result<String> {
let path = Path::new(path);
if path.is_absolute() {
anyhow::bail!("absolute paths are not allowed");
}
let mut parts = Vec::new();
for component in path.components() {
match component {
std::path::Component::Normal(part) => {
let part = part
.to_str()
.ok_or_else(|| anyhow::anyhow!("non-utf8 paths are not allowed"))?;
parts.push(part.to_string());
}
std::path::Component::CurDir => {}
std::path::Component::ParentDir => {
anyhow::bail!("parent-directory paths are not allowed");
}
std::path::Component::RootDir | std::path::Component::Prefix(_) => {
anyhow::bail!("rooted paths are not allowed");
}
}
}
if parts.is_empty() {
anyhow::bail!("empty paths are not allowed");
}
Ok(parts.join("/"))
}
fn parse_bool_env(name: &str, default: bool) -> bool {
std::env::var(name)
.ok()
.map(|value| matches!(value.trim(), "1" | "true" | "TRUE" | "yes" | "YES"))
.unwrap_or(default)
}
fn parse_usize_list_env(name: &str, default: &[usize], min: usize) -> Vec<usize> {
let mut values = std::env::var(name)
.ok()
.map(|value| {
value
.split(',')
.filter_map(|value| value.trim().parse::<usize>().ok())
.filter(|value| *value >= min)
.collect::<Vec<_>>()
})
.filter(|values| !values.is_empty())
.unwrap_or_else(|| default.to_vec());
values.sort_unstable();
values.dedup();
values
}
fn content_patterns_from_env(
env_name: &str,
mode: ContentSearchMode,
defaults: &[&str],
) -> Vec<ContentSearchPattern> {
comma_env(env_name)
.unwrap_or_else(|| defaults.iter().map(|value| (*value).to_string()).collect())
.into_iter()
.enumerate()
.map(|(index, text)| ContentSearchPattern {
label: format!("{}_{:02}", mode.as_str(), index + 1),
hash: hash_content_search_pattern(&text),
text,
mode,
})
.collect()
}
fn hash_content_search_pattern(pattern: &str) -> String {
use sha2::{Digest, Sha256};
let mut hasher = Sha256::new();
hasher.update(pattern.as_bytes());
crate::hex::lower_hex(hasher.finalize())[..16].to_string()
}
#[derive(Debug)]
struct ContentCandidatePicker {
runtime: ToolRuntime,
index_ready: bool,
cold_index_wait_ms: f64,
content_indexing: bool,
}
impl ContentCandidatePicker {
fn new(scope: &Path, content_indexing: bool) -> anyhow::Result<Self> {
let started = Instant::now();
Ok(Self {
runtime: ToolRuntime::new(scope)?,
index_ready: true,
cold_index_wait_ms: round_millis(started.elapsed().as_secs_f64() * 1000.0),
content_indexing,
})
}
}
#[derive(Debug, Clone)]
struct ContentSearchObservation {
engine: String,
elapsed: Vec<Duration>,
successes: u64,
failures: u64,
matches_returned: u64,
raw_matches_returned: Option<u64>,
output_bytes: u64,
truncated: bool,
timed_out: bool,
exit_code: Option<i64>,
index_ready: Option<bool>,
cold_index_wait_ms: Option<f64>,
enable_content_indexing: Option<bool>,
files_searched: Option<u64>,
total_files: Option<u64>,
filtered_file_count: Option<u64>,
files_with_matches: Option<u64>,
regex_fallback_error_present: Option<bool>,
next_file_offset_nonzero: Option<bool>,
limit_truncation_observed: bool,
pipe_output_truncation_observed: bool,
adapter_limit_mismatch_observed: bool,
raw_limit_overrun_observed: bool,
failure_kind_counts: BTreeMap<&'static str, u64>,
diagnostic_kind_counts: BTreeMap<&'static str, u64>,
}
fn run_grep_content_observation(
runtime: &ToolRuntime,
pattern: &str,
path: Option<&str>,
limit: usize,
context: usize,
iterations: usize,
) -> ContentSearchObservation {
let mut elapsed = Vec::with_capacity(iterations);
let mut successes = 0;
let mut failures = 0;
let mut matches_returned = 0;
let mut output_bytes = 0;
let mut truncated = false;
let mut timed_out = false;
let mut exit_code = None;
let mut engine = "ffgrep_unknown".to_string();
let mut limit_truncation_observed = false;
let mut pipe_output_truncation_observed = false;
let mut failure_kind_counts = BTreeMap::new();
let diagnostic_kind_counts = BTreeMap::new();
for _ in 0..iterations {
let arguments = match path {
Some(path) => {
json!({"pattern": pattern, "path": path, "limit": limit, "context": context})
}
None => json!({"pattern": pattern, "limit": limit, "context": context}),
};
let started = Instant::now();
let result = runtime.dispatch("grep", arguments);
elapsed.push(started.elapsed());
if result.success {
successes += 1;
} else {
failures += 1;
increment_failure_kind(
&mut failure_kind_counts,
classify_grep_failure(&result.metadata),
);
}
engine = match result.metadata.get("engine").and_then(Value::as_str) {
Some(other) => format!("ffgrep_{other}"),
None => "ffgrep_unknown".to_string(),
};
matches_returned = metadata_u64(&result.metadata, "matches_returned");
output_bytes = result.content.len() as u64;
let current_truncated = metadata_bool(&result.metadata, "truncated");
truncated |= current_truncated;
timed_out |= metadata_bool(&result.metadata, "timed_out");
exit_code = result.metadata.get("exit_code").and_then(Value::as_i64);
limit_truncation_observed |=
current_truncated && result.success && matches_returned >= limit as u64;
pipe_output_truncation_observed |=
grep_pipe_output_truncation_observed(&result.metadata, result.success);
}
ContentSearchObservation {
engine,
elapsed,
successes,
failures,
matches_returned,
raw_matches_returned: Some(matches_returned),
output_bytes,
truncated,
timed_out,
exit_code,
index_ready: None,
cold_index_wait_ms: None,
enable_content_indexing: None,
files_searched: None,
total_files: None,
filtered_file_count: None,
files_with_matches: None,
regex_fallback_error_present: None,
next_file_offset_nonzero: None,
limit_truncation_observed,
pipe_output_truncation_observed,
adapter_limit_mismatch_observed: matches_returned > limit as u64,
raw_limit_overrun_observed: false,
failure_kind_counts,
diagnostic_kind_counts,
}
}
fn increment_failure_kind(
failure_kind_counts: &mut BTreeMap<&'static str, u64>,
kind: &'static str,
) {
*failure_kind_counts.entry(kind).or_default() += 1;
}
fn classify_grep_failure(metadata: &Value) -> &'static str {
if metadata_bool(metadata, "timed_out") {
return "timed_out";
}
if metadata
.get("cleanup_warning")
.and_then(Value::as_str)
.is_some_and(|warning| !warning.is_empty())
{
return "cleanup_warning";
}
if metadata_bool(metadata, "stdout_truncated") || metadata_bool(metadata, "truncated") {
return "pipe_or_output_truncation";
}
match metadata.get("exit_code").and_then(Value::as_i64) {
Some(2) => "rg_error_exit",
Some(code) if code != 0 && code != 1 => "nonzero_exit",
None => "missing_exit_status",
_ => "unknown_failure",
}
}
fn grep_pipe_output_truncation_observed(metadata: &Value, success: bool) -> bool {
metadata_bool(metadata, "stdout_truncated")
|| metadata_bool(metadata, "stderr_truncated")
|| (!success
&& metadata_bool(metadata, "truncated")
&& !metadata_bool(metadata, "timed_out"))
}
fn run_candidate_content_observation(
picker: &ContentCandidatePicker,
pattern: &str,
mode: ContentSearchMode,
limit: usize,
context: usize,
iterations: usize,
) -> anyhow::Result<ContentSearchObservation> {
let search_pattern = match mode {
ContentSearchMode::Plain => regex::escape(pattern),
ContentSearchMode::Regex => pattern.to_string(),
};
let mut observation = run_grep_content_observation(
&picker.runtime,
&search_pattern,
None,
limit,
context,
iterations,
);
observation.engine = mode.candidate_engine().to_string();
observation.index_ready = Some(picker.index_ready);
observation.cold_index_wait_ms = Some(picker.cold_index_wait_ms);
observation.enable_content_indexing = Some(picker.content_indexing);
observation.raw_matches_returned = Some(observation.matches_returned);
observation.files_searched = None;
observation.total_files = None;
observation.filtered_file_count = None;
observation.files_with_matches = None;
observation.regex_fallback_error_present = Some(false);
observation.next_file_offset_nonzero = Some(false);
observation.raw_limit_overrun_observed = false;
Ok(observation)
}
fn metadata_bool(metadata: &Value, key: &str) -> bool {
metadata.get(key).and_then(Value::as_bool).unwrap_or(false)
}
fn print_content_search_benchmark_line(value: Value) {
println!("{CONTENT_SEARCH_BENCH_PREFIX}{value}");
}
#[allow(
clippy::too_many_arguments,
reason = "benchmark output builder mirrors JSON fields for call-site clarity"
)]
fn content_search_benchmark_line(
cfg: &ContentSearchBenchConfig,
scenario: &str,
pattern: &ContentSearchPattern,
path_label: Option<&str>,
limit: usize,
context: usize,
observation: &ContentSearchObservation,
resources: ResourceDelta,
) -> Value {
let mut value = json!({
"schema": 1,
"scenario": scenario,
"root_label": cfg.root_label,
"root_hash": cfg.root_hash,
"path_label": path_label,
"pattern_label": pattern.label,
"pattern_hash": pattern.hash,
"mode": pattern.mode.as_str(),
"engine": observation.engine,
"iterations": observation.elapsed.len(),
"limit": limit,
"context": context,
"elapsed_ms": duration_stats(&observation.elapsed),
"successes": observation.successes,
"failures": observation.failures,
"matches_returned": observation.matches_returned,
"raw_matches_returned": observation.raw_matches_returned,
"output_bytes": observation.output_bytes,
"truncated": observation.truncated,
"timed_out": observation.timed_out,
"exit_code": observation.exit_code,
"index_ready": observation.index_ready,
"cold_index_wait_ms": observation.cold_index_wait_ms,
"enable_content_indexing": observation.enable_content_indexing,
"files_searched": observation.files_searched,
"total_files": observation.total_files,
"filtered_file_count": observation.filtered_file_count,
"files_with_matches": observation.files_with_matches,
"regex_fallback_error_present": observation.regex_fallback_error_present,
"next_file_offset_nonzero": observation.next_file_offset_nonzero,
"limit_truncation_observed": observation.limit_truncation_observed,
"pipe_output_truncation_observed": observation.pipe_output_truncation_observed,
"adapter_limit_mismatch_observed": observation.adapter_limit_mismatch_observed,
"raw_limit_overrun_observed": observation.raw_limit_overrun_observed,
"failure_kind_counts": observation.failure_kind_counts,
"diagnostic_kind_counts": observation.diagnostic_kind_counts,
"rss_kb_delta": resources.rss_kb_delta,
"fd_delta": resources.fd_delta,
"thread_delta": resources.thread_delta,
});
if cfg.show_patterns {
value["pattern"] = json!(pattern.text);
}
value
}
#[allow(
clippy::too_many_arguments,
reason = "benchmark harness records each scenario dimension explicitly"
)]
fn record_content_search_observation(
cfg: &ContentSearchBenchConfig,
scenario_count: &mut usize,
observed_values: &mut Vec<Value>,
scenario: &str,
pattern: &ContentSearchPattern,
path_label: Option<&str>,
limit: usize,
context: usize,
observation: ContentSearchObservation,
resources: ResourceDelta,
) {
let value = content_search_benchmark_line(
cfg,
scenario,
pattern,
path_label,
limit,
context,
&observation,
resources,
);
print_content_search_benchmark_line(value.clone());
observed_values.push(value);
*scenario_count += 1;
}
fn content_search_scenario_name(
root_scoped: bool,
mode: ContentSearchMode,
limit: usize,
context: usize,
max_limit: usize,
) -> &'static str {
if context > 0 {
return "warm_context";
}
if limit == max_limit && max_limit > 100 {
return "high_limit_tail";
}
match (root_scoped, mode) {
(false, ContentSearchMode::Plain) => "warm_root_plain",
(false, ContentSearchMode::Regex) => "warm_root_regex",
(true, ContentSearchMode::Plain) => "warm_scoped_plain",
(true, ContentSearchMode::Regex) => "warm_scoped_regex",
}
}
fn assert_content_sentinel_observation(
label: &str,
observation: &ContentSearchObservation,
min_matches: u64,
) {
assert!(
observation.successes > 0 && observation.failures == 0,
"content benchmark sentinel {label} failed: {observation:?}"
);
assert!(
observation.matches_returned >= min_matches,
"content benchmark sentinel {label} returned too few matches: {observation:?}"
);
}
fn run_content_search_benchmark_sentinel() -> anyhow::Result<()> {
assert_eq!(sanitize_content_benchmark_path("./src")?, "src");
assert!(sanitize_content_benchmark_path("/tmp").is_err());
assert!(sanitize_content_benchmark_path("../outside").is_err());
let temp = TempDir::new()?;
fs::create_dir_all(temp.path().join("src"))?;
fs::write(
temp.path().join("src/a.rs"),
"before context\nneedle_plain alpha\nfn sentinel_func() {}\nlimit_token one\nlimit_token two\nlimit_token three\nafter context\n",
)?;
fs::write(
temp.path().join("src/b.rs"),
"needle_plain beta\nfn sentinel_other() {}\nlimit_token two\n",
)?;
fs::write(
temp.path().join("src/c.rs"),
"CASE_NEEDLE should not match lower-case benchmark query\n",
)?;
let runtime = ToolRuntime::new(temp.path())?;
let picker = ContentCandidatePicker::new(temp.path(), true)?;
let grep_plain = run_grep_content_observation(&runtime, "needle_plain", Some("src"), 10, 0, 1);
assert_content_sentinel_observation("grep_plain", &grep_plain, 2);
let candidate_plain = run_candidate_content_observation(
&picker,
"needle_plain",
ContentSearchMode::Plain,
10,
0,
1,
)?;
assert_content_sentinel_observation("candidate_plain", &candidate_plain, 2);
let grep_regex =
run_grep_content_observation(&runtime, "fn sentinel_func", Some("src"), 10, 0, 1);
assert_content_sentinel_observation("grep_regex", &grep_regex, 1);
let candidate_regex = run_candidate_content_observation(
&picker,
"fn sentinel_func",
ContentSearchMode::Regex,
10,
0,
1,
)?;
assert_content_sentinel_observation("candidate_regex", &candidate_regex, 1);
let grep_limit = run_grep_content_observation(&runtime, "limit_token", Some("src"), 1, 0, 1);
assert_eq!(
grep_limit.matches_returned, 1,
"content benchmark sentinel grep limit failed: {grep_limit:?}"
);
assert!(
grep_limit.limit_truncation_observed,
"content benchmark sentinel grep limit truncation flag failed: {grep_limit:?}"
);
let candidate_limit = run_candidate_content_observation(
&picker,
"limit_token",
ContentSearchMode::Plain,
1,
0,
1,
)?;
assert_eq!(
candidate_limit.matches_returned, 1,
"content benchmark sentinel candidate limit cap failed: {candidate_limit:?}"
);
assert!(
candidate_limit
.raw_matches_returned
.is_some_and(|raw| raw >= candidate_limit.matches_returned),
"content benchmark sentinel candidate raw limit count failed: {candidate_limit:?}"
);
assert!(
candidate_limit.limit_truncation_observed,
"content benchmark sentinel candidate limit truncation flag failed: {candidate_limit:?}"
);
assert!(
!candidate_limit.adapter_limit_mismatch_observed,
"content benchmark sentinel candidate adapter-visible limit mismatch failed: {candidate_limit:?}"
);
let grep_context =
run_grep_content_observation(&runtime, "needle_plain", Some("src"), 10, 1, 1);
assert_content_sentinel_observation("grep_context", &grep_context, 2);
let candidate_context = run_candidate_content_observation(
&picker,
"needle_plain",
ContentSearchMode::Plain,
10,
1,
1,
)?;
assert_content_sentinel_observation("candidate_context", &candidate_context, 2);
let grep_case_sensitive =
run_grep_content_observation(&runtime, "case_needle", Some("src"), 10, 0, 1);
assert_eq!(
grep_case_sensitive.matches_returned, 0,
"content benchmark sentinel grep case parity failed: {grep_case_sensitive:?}"
);
let candidate_case_sensitive = run_candidate_content_observation(
&picker,
"case_needle",
ContentSearchMode::Plain,
10,
0,
1,
)?;
assert_eq!(
candidate_case_sensitive.matches_returned, 0,
"content benchmark sentinel candidate case parity failed: {candidate_case_sensitive:?}"
);
Ok(())
}
#[allow(
clippy::too_many_arguments,
reason = "benchmark harness coordinates paired engines with explicit dimensions"
)]
fn run_content_search_engine_pair(
cfg: &ContentSearchBenchConfig,
scenario_count: &mut usize,
observed_values: &mut Vec<Value>,
scenario: &str,
pattern: &ContentSearchPattern,
path_label: Option<&str>,
grep_path: Option<&str>,
grep_runtime: &ToolRuntime,
candidate_picker: Option<&ContentCandidatePicker>,
limit: usize,
context: usize,
iterations: usize,
) -> anyhow::Result<()> {
if cfg.baseline.includes_grep() {
let before = ResourceSnapshot::capture();
let grep_pattern = pattern.grep_pattern();
let observation = run_grep_content_observation(
grep_runtime,
&grep_pattern,
grep_path,
limit,
context,
iterations,
);
let resources = before.delta(ResourceSnapshot::capture());
record_content_search_observation(
cfg,
scenario_count,
observed_values,
scenario,
pattern,
path_label,
limit,
context,
observation,
resources,
);
}
if cfg.baseline.includes_candidate()
&& let Some(candidate_picker) = candidate_picker
{
let before = ResourceSnapshot::capture();
let observation = run_candidate_content_observation(
candidate_picker,
&pattern.text,
pattern.mode,
limit,
context,
iterations,
)?;
let resources = before.delta(ResourceSnapshot::capture());
record_content_search_observation(
cfg,
scenario_count,
observed_values,
scenario,
pattern,
path_label,
limit,
context,
observation,
resources,
);
}
Ok(())
}
#[test]
#[ignore = "diagnostic-only content-search benchmark; run explicitly with --ignored --nocapture"]
fn content_search_benchmark_compares_ffgrep_native_and_candidate_engine() {
run_content_search_benchmark_sentinel().expect("content search benchmark sentinel");
let cfg = ContentSearchBenchConfig::from_env().expect("content search benchmark config");
let runtime = ToolRuntime::new(&cfg.root).expect("content search benchmark runtime");
let root_picker = cfg
.baseline
.includes_candidate()
.then(|| ContentCandidatePicker::new(&cfg.root, cfg.content_indexing))
.transpose()
.expect("content search benchmark root picker");
let existing_paths = cfg.existing_paths();
let max_limit = cfg.limits.iter().copied().max().unwrap_or(100);
let mut scenario_count = 0;
let mut observed_values = Vec::new();
if let Some(cold_pattern) = cfg.plain_patterns.first() {
for limit in &cfg.limits {
for context in &cfg.contexts {
if cfg.baseline.includes_grep() {
let cold_runtime = ToolRuntime::new(&cfg.root).expect("cold grep runtime");
let before = ResourceSnapshot::capture();
let grep_pattern = cold_pattern.grep_pattern();
let observation = run_grep_content_observation(
&cold_runtime,
&grep_pattern,
None,
*limit,
*context,
1,
);
let resources = before.delta(ResourceSnapshot::capture());
record_content_search_observation(
&cfg,
&mut scenario_count,
&mut observed_values,
"cold_root_plain",
cold_pattern,
None,
*limit,
*context,
observation,
resources,
);
}
if cfg.baseline.includes_candidate() {
let before = ResourceSnapshot::capture();
let started = Instant::now();
let cold_picker = ContentCandidatePicker::new(&cfg.root, cfg.content_indexing)
.expect("cold candidate content picker");
let mut observation = run_candidate_content_observation(
&cold_picker,
&cold_pattern.text,
ContentSearchMode::Plain,
*limit,
*context,
1,
)
.expect("cold candidate content observation");
observation.elapsed = vec![started.elapsed()];
let resources = before.delta(ResourceSnapshot::capture());
record_content_search_observation(
&cfg,
&mut scenario_count,
&mut observed_values,
"cold_root_plain",
cold_pattern,
None,
*limit,
*context,
observation,
resources,
);
}
}
}
}
for pattern in cfg.plain_patterns.iter().chain(cfg.regex_patterns.iter()) {
for limit in &cfg.limits {
for context in &cfg.contexts {
let scenario =
content_search_scenario_name(false, pattern.mode, *limit, *context, max_limit);
run_content_search_engine_pair(
&cfg,
&mut scenario_count,
&mut observed_values,
scenario,
pattern,
None,
None,
&runtime,
root_picker.as_ref(),
*limit,
*context,
cfg.repetitions,
)
.expect("warm root content scenario");
}
}
}
for path in &existing_paths {
let scoped_picker = cfg
.baseline
.includes_candidate()
.then(|| ContentCandidatePicker::new(&cfg.root.join(path), cfg.content_indexing))
.transpose()
.expect("scoped content benchmark picker");
for pattern in cfg.plain_patterns.iter().chain(cfg.regex_patterns.iter()) {
for limit in &cfg.limits {
for context in &cfg.contexts {
let scenario = content_search_scenario_name(
true,
pattern.mode,
*limit,
*context,
max_limit,
);
run_content_search_engine_pair(
&cfg,
&mut scenario_count,
&mut observed_values,
scenario,
pattern,
Some(path),
Some(path),
&runtime,
scoped_picker.as_ref(),
*limit,
*context,
cfg.repetitions,
)
.expect("warm scoped content scenario");
}
}
}
}
if cfg.cwd_churn > 0
&& let Some(pattern) = cfg.plain_patterns.first()
{
for dir in collect_churn_dirs(&cfg.root, cfg.cwd_churn) {
let path_label = dir
.strip_prefix(&cfg.root)
.ok()
.and_then(|path| path.to_str())
.map(|path| path.replace('\\', "/"))
.unwrap_or_else(|| "child".to_string());
let churn_runtime = runtime
.clone_for_cwd_with_subagent_depth(&dir, 0)
.expect("cwd churn runtime");
let churn_picker = cfg
.baseline
.includes_candidate()
.then(|| ContentCandidatePicker::new(&dir, cfg.content_indexing))
.transpose()
.expect("cwd churn picker");
run_content_search_engine_pair(
&cfg,
&mut scenario_count,
&mut observed_values,
"cwd_churn",
pattern,
Some(&path_label),
None,
&churn_runtime,
churn_picker.as_ref(),
max_limit,
0,
1,
)
.expect("cwd churn content scenario");
}
}
print_content_search_benchmark_summary(&cfg, scenario_count, &observed_values);
}
fn print_find_summary(cfg: &FindBenchConfig, scenario_count: usize, observed_values: &[Value]) {
let max_warm_p95_ms = observed_values
.iter()
.filter(|value| {
value["scenario"].as_str().is_some_and(|scenario| {
scenario.starts_with("warm_") || scenario == "repeated_same_query"
})
})
.filter_map(|value| value["elapsed_ms"]["p95"].as_f64())
.max_by(f64::total_cmp)
.map(round_millis);
let baseline_speedup_observed = observed_values.iter().any(|value| {
let Some(fff_p95) = value["elapsed_ms"]["p95"].as_f64() else {
return false;
};
let Some(baseline_p95) = value["baseline"]["p95_ms"].as_f64() else {
return false;
};
fff_p95 < baseline_p95
});
let resource_growth_observed = observed_values.iter().any(|value| {
value["fd_delta"].as_i64().is_some_and(|delta| delta > 16)
|| value["thread_delta"]
.as_i64()
.is_some_and(|delta| delta > 16)
});
print_find_benchmark_line(json!({
"schema": 1,
"scenario": "summary",
"root_label": cfg.root_label,
"root_hash": cfg.root_hash,
"queries": cfg.queries,
"paths": cfg.paths,
"repetitions": cfg.repetitions,
"limit": cfg.limit,
"baseline": match cfg.baseline {
FindBenchBaseline::IgnoreWalk => "ignore_walk",
FindBenchBaseline::None => "none",
},
"cwd_churn": cfg.cwd_churn,
"scenario_lines": scenario_count,
"max_warm_p95_ms": max_warm_p95_ms,
"baseline_speedup_observed": baseline_speedup_observed,
"resource_growth_observed": resource_growth_observed,
"step4_exploration": {
"recommended": baseline_speedup_observed && !resource_growth_observed,
"caveat": "fffind path benchmark does not prove grep content-search compatibility"
},
"step5_exploration": {
"recommended": max_warm_p95_ms.is_some_and(|p95| p95 <= 50.0) && !resource_growth_observed,
"caveat": "validate threshold on production repo before TUI autocomplete work"
}
}));
}
fn seed_startup_fixture() -> anyhow::Result<(TempDir, McPaths, PathBuf)> {
let temp = TempDir::new()?;
let mc_root = temp.path().join("mc-home");
let repo_root = temp.path().join("repo");
let paths = McPaths::from_root(mc_root.clone());
paths.ensure_runtime_dirs()?;
fs::create_dir_all(&paths.skills)?;
fs::create_dir_all(&paths.prompts)?;
fs::create_dir_all(repo_root.join(".agents/skills/repo-review"))?;
fs::create_dir_all(paths.skills.join("local-plan"))?;
fs::create_dir_all(&paths.sessions)?;
fs::write(
&paths.settings_file,
r#"{"selected_model":{"provider":"openai-codex","model":"codex-local"},"file_autocomplete_respects_gitignore":true}"#,
)?;
fs::write(&paths.user_agents, "User synthetic instruction.\n")?;
fs::write(repo_root.join("AGENTS.md"), "Repo synthetic instruction.\n")?;
fs::write(
paths.skills.join("local-plan/SKILL.md"),
"---\ndescription: Synthetic planning skill\n---\n# Local plan\n",
)?;
fs::write(
repo_root.join(".agents/skills/repo-review/SKILL.md"),
"---\ndescription: Synthetic review skill\n---\n# Repo review\n",
)?;
for index in 0..50 {
let id = format!("00000000-0000-4000-8000-{index:012}");
fs::write(
paths.sessions.join(format!("{id}.jsonl")),
format!(r#"{{"event_type":"user","text":"synthetic prompt {index}"}}\n"#),
)?;
}
write_catalog_cache(
&paths,
"openai-codex",
&[ModelCatalogEntry {
context_window: Some(128_000),
..ModelCatalogEntry::new_codex("codex-local")
}],
)?;
Ok((temp, paths, repo_root))
}
fn startup_discovery_once(paths: &McPaths, repo_root: &Path) -> anyhow::Result<u64> {
let settings = read_settings(paths)?;
let instructions = discover_agents_with_additional_markdown(
&paths.user_agents,
repo_root,
&settings.instructions.additional_markdown_paths,
)?;
let skills = discover_skills(&SkillRoots {
mc_skills: paths.skills.clone(),
repo_skills: repo_root.join(".agents/skills"),
legacy_agents_skills: repo_root.join("missing-legacy"),
additional_paths: Vec::new(),
});
let sessions = SessionManager::new(paths.sessions.clone()).list()?;
let context = cached_model_context_window(paths, "openai-codex", "codex-local").unwrap_or(0);
Ok(settings
.selected_model
.provider
.as_deref()
.unwrap_or_default()
.len() as u64
+ instructions.len() as u64
+ skills.skills.len() as u64
+ sessions.len() as u64
+ context as u64)
}
fn parse_sse_once() -> anyhow::Result<u64> {
let mut events = Vec::new();
for fixture in [RESPONSES_SSE, CHAT_COMPLETIONS_SSE] {
let mut parser = StreamParser::default();
for chunk in fixture.as_bytes().chunks(37) {
let chunk = std::str::from_utf8(chunk)?;
events.extend(parser.push_chunk(chunk)?);
}
events.extend(parser.finish()?);
}
let text_events = events
.iter()
.filter(|event| matches!(event, ProviderEvent::TextDelta(_)))
.count();
let tool_events = events
.iter()
.filter(|event| matches!(event, ProviderEvent::ToolCall(_)))
.count();
Ok((text_events + tool_events + events.len()) as u64)
}
fn heavy_transcript_once() -> u64 {
let mut state = MissionControlState::default();
for index in 0..40 {
state.apply_output_event(&OutputEvent::UserPrompt {
text: format!("inspect synthetic turn {index}"),
});
state.apply_output_event(&OutputEvent::AssistantComplete {
text: RENDERING_TRANSCRIPT.to_string(),
});
}
let lines = state.cached_transcript_lines(80);
let visual_rows = state.transcript_visual_rows(96);
black_box(lines.len() as u64 + visual_rows as u64)
}
fn streaming_simulation_once() -> u64 {
let mut state = MissionControlState::default();
state.apply_output_event(&OutputEvent::UserPrompt {
text: "stream synthetic assistant response".to_string(),
});
for index in 0..250 {
state.apply_output_event(&OutputEvent::AssistantDelta {
text: format!("delta-{index} "),
});
let id = ActivityId::new(format!("tool-{index}"));
state.apply_activity_event(ActivityEvent::Started {
id: id.clone(),
parent_id: None,
kind: ActivityKind::Tool,
status: ActivityStatus::Running,
metadata: ActivityMetadata::new(format!("synthetic tool {index}")),
});
state.apply_activity_event(ActivityEvent::Delta {
id,
preview: format!("preview line {index}"),
});
}
state.apply_output_event(&OutputEvent::AssistantComplete {
text: "final synthetic assistant response".repeat(20),
});
let transcript = state.cached_transcript_lines(80).len();
let nodes = state.visible_nodes().len();
let detail_rows = state.detail_visual_rows(80);
black_box((transcript + nodes + detail_rows) as u64)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
enum RenderBackendMode {
Test,
CrosstermSink,
CrosstermReal,
}
impl RenderBackendMode {
fn from_env() -> Self {
match std::env::var("PROFILE_TUI_RENDER_BACKEND")
.unwrap_or_else(|_| "test".to_string())
.trim()
{
"test" | "" => Self::Test,
"crossterm-sink" => Self::CrosstermSink,
"crossterm-real" => Self::CrosstermReal,
value => {
eprintln!(
"tui_render_pipeline warning=unsupported_backend value={value:?} using=test"
);
Self::Test
}
}
}
fn as_str(self) -> &'static str {
match self {
Self::Test => "test",
Self::CrosstermSink => "crossterm-sink",
Self::CrosstermReal => "crossterm-real",
}
}
}
#[derive(Debug, Default, Clone, Copy, Serialize)]
struct DurationMetricStats {
min: f64,
p50: f64,
p95: f64,
max: f64,
avg: f64,
}
#[derive(Debug, Default, Clone, Copy, Serialize)]
struct U64MetricStats {
min: u64,
p50: u64,
p95: u64,
max: u64,
avg: f64,
}
#[derive(Debug, Default, Clone, Copy, Serialize)]
struct CountingWriterStats {
stdout_bytes: u64,
write_calls: u64,
write_duration: Duration,
flush_calls: u64,
flush_duration: Duration,
}
#[derive(Debug, Clone, Copy, Default, Serialize)]
struct RenderFrameStats {
render_draw: Duration,
diff: Duration,
diff_cells: u64,
backend_draw: Duration,
stdout_bytes: u64,
write_calls: u64,
write_duration: Duration,
flush_calls: u64,
flush_duration: Duration,
terminal_draw: Option<Duration>,
controlled_draw: Duration,
}
#[derive(Debug, Serialize)]
struct RenderScenarioSummary {
schema: u64,
scenario: &'static str,
backend_mode: &'static str,
area: (u16, u16),
frames: usize,
warmup_frames: usize,
elapsed_ms: f64,
frames_per_sec: f64,
terminal_draw_ms: Option<DurationMetricStats>,
controlled_draw_ms: DurationMetricStats,
render_draw_ms: DurationMetricStats,
diff_ms: DurationMetricStats,
diff_cells: U64MetricStats,
changed_cell_ratio: f64,
backend_draw_ms: DurationMetricStats,
stdout_bytes: U64MetricStats,
write_calls: U64MetricStats,
write_ms: DurationMetricStats,
flush_calls: U64MetricStats,
flush_ms: DurationMetricStats,
}
fn render_duration_stats(values: impl Iterator<Item = Duration>) -> DurationMetricStats {
let values = values.collect::<Vec<_>>();
if values.is_empty() {
return DurationMetricStats::default();
}
let avg = values.iter().sum::<Duration>().as_secs_f64() * 1000.0 / values.len() as f64;
let mut millis = values
.iter()
.map(|duration| duration.as_secs_f64() * 1000.0)
.collect::<Vec<_>>();
millis.sort_by(f64::total_cmp);
DurationMetricStats {
min: round_millis(millis[0]),
p50: round_millis(percentile(&millis, 0.50)),
p95: round_millis(percentile(&millis, 0.95)),
max: round_millis(*millis.last().unwrap()),
avg: round_millis(avg),
}
}
fn render_u64_stats(values: impl Iterator<Item = u64>) -> U64MetricStats {
let mut values = values.collect::<Vec<_>>();
if values.is_empty() {
return U64MetricStats::default();
}
values.sort_unstable();
let avg = values.iter().sum::<u64>() as f64 / values.len() as f64;
U64MetricStats {
min: values[0],
p50: percentile_u64(&values, 0.50),
p95: percentile_u64(&values, 0.95),
max: *values.last().unwrap(),
avg: round_millis(avg),
}
}
fn percentile_u64(sorted: &[u64], percentile: f64) -> u64 {
if sorted.is_empty() {
return 0;
}
let rank = (percentile * sorted.len() as f64).ceil() as usize;
sorted[rank.saturating_sub(1).min(sorted.len() - 1)]
}
#[derive(Debug)]
struct CountingWriter<W: Write> {
inner: W,
stats: Rc<RefCell<CountingWriterStats>>,
}
impl<W: Write> CountingWriter<W> {
fn new(inner: W, stats: Rc<RefCell<CountingWriterStats>>) -> Self {
Self { inner, stats }
}
}
impl<W: Write> Write for CountingWriter<W> {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
let started = Instant::now();
let written = self.inner.write(buf)?;
let elapsed = started.elapsed();
let mut stats = self.stats.borrow_mut();
stats.stdout_bytes = stats.stdout_bytes.saturating_add(written as u64);
stats.write_calls = stats.write_calls.saturating_add(1);
stats.write_duration += elapsed;
Ok(written)
}
fn flush(&mut self) -> io::Result<()> {
let started = Instant::now();
self.inner.flush()?;
let elapsed = started.elapsed();
let mut stats = self.stats.borrow_mut();
stats.flush_calls = stats.flush_calls.saturating_add(1);
stats.flush_duration += elapsed;
Ok(())
}
}
#[derive(Debug)]
struct CountingBackend<B> {
inner: B,
writer_stats: Option<Rc<RefCell<CountingWriterStats>>>,
last_backend_draw: Duration,
last_writer_delta: CountingWriterStats,
}
impl<B> CountingBackend<B> {
fn new(inner: B, writer_stats: Option<Rc<RefCell<CountingWriterStats>>>) -> Self {
Self {
inner,
writer_stats,
last_backend_draw: Duration::ZERO,
last_writer_delta: CountingWriterStats::default(),
}
}
fn take_frame_io_stats(&mut self) -> (Duration, CountingWriterStats) {
let draw = self.last_backend_draw;
let writer = self.last_writer_delta;
self.last_backend_draw = Duration::ZERO;
self.last_writer_delta = CountingWriterStats::default();
(draw, writer)
}
}
impl<B: Backend> Backend for CountingBackend<B> {
type Error = B::Error;
fn draw<'a, I>(&mut self, content: I) -> Result<(), Self::Error>
where
I: Iterator<Item = (u16, u16, &'a ratatui::buffer::Cell)>,
{
let before = self
.writer_stats
.as_ref()
.map(|stats| *stats.borrow())
.unwrap_or_default();
let started = Instant::now();
let result = self.inner.draw(content);
self.last_backend_draw = started.elapsed();
let after = self
.writer_stats
.as_ref()
.map(|stats| *stats.borrow())
.unwrap_or_default();
self.last_writer_delta.stdout_bytes =
after.stdout_bytes.saturating_sub(before.stdout_bytes);
self.last_writer_delta.write_calls = after.write_calls.saturating_sub(before.write_calls);
self.last_writer_delta.write_duration = after
.write_duration
.checked_sub(before.write_duration)
.unwrap_or_default();
result
}
fn append_lines(&mut self, n: u16) -> Result<(), Self::Error> {
self.inner.append_lines(n)
}
fn hide_cursor(&mut self) -> Result<(), Self::Error> {
self.inner.hide_cursor()
}
fn show_cursor(&mut self) -> Result<(), Self::Error> {
self.inner.show_cursor()
}
fn get_cursor_position(&mut self) -> Result<Position, Self::Error> {
self.inner.get_cursor_position()
}
fn set_cursor_position<P: Into<Position>>(&mut self, position: P) -> Result<(), Self::Error> {
self.inner.set_cursor_position(position)
}
fn clear(&mut self) -> Result<(), Self::Error> {
self.inner.clear()
}
fn clear_region(&mut self, clear_type: ratatui::backend::ClearType) -> Result<(), Self::Error> {
self.inner.clear_region(clear_type)
}
fn size(&self) -> Result<Size, Self::Error> {
self.inner.size()
}
fn window_size(&mut self) -> Result<ratatui::backend::WindowSize, Self::Error> {
self.inner.window_size()
}
fn flush(&mut self) -> Result<(), Self::Error> {
let before = self
.writer_stats
.as_ref()
.map(|stats| *stats.borrow())
.unwrap_or_default();
let result = self.inner.flush();
let after = self
.writer_stats
.as_ref()
.map(|stats| *stats.borrow())
.unwrap_or_default();
self.last_writer_delta.flush_calls = after.flush_calls.saturating_sub(before.flush_calls);
self.last_writer_delta.flush_duration = after
.flush_duration
.checked_sub(before.flush_duration)
.unwrap_or_default();
result
}
}
fn controlled_render_frame<B: Backend>(
terminal: &mut Terminal<CountingBackend<B>>,
previous: &mut Buffer,
state: &MissionControlState,
) -> Result<RenderFrameStats, B::Error> {
let total_started = Instant::now();
terminal.current_buffer_mut().reset();
let render_started = Instant::now();
{
let mut frame = terminal.get_frame();
crate::tui::render::draw(&mut frame, state);
}
let render_draw = render_started.elapsed();
let current = terminal.current_buffer_mut().clone();
let diff_started = Instant::now();
let updates = previous.diff(¤t);
let diff = diff_started.elapsed();
let diff_cells = updates.len() as u64;
terminal.backend_mut().draw(updates.into_iter())?;
terminal.backend_mut().flush()?;
let (backend_draw, writer) = terminal.backend_mut().take_frame_io_stats();
*previous = current;
terminal.swap_buffers();
Ok(RenderFrameStats {
render_draw,
diff,
diff_cells,
backend_draw,
stdout_bytes: writer.stdout_bytes,
write_calls: writer.write_calls,
write_duration: writer.write_duration,
flush_calls: writer.flush_calls,
flush_duration: writer.flush_duration,
terminal_draw: None,
controlled_draw: total_started.elapsed(),
})
}
fn render_scenario_summary(
scenario: &'static str,
mode: RenderBackendMode,
area: Rect,
warmup_frames: usize,
frames: Vec<RenderFrameStats>,
elapsed: Duration,
) -> RenderScenarioSummary {
let measured = frames
.iter()
.skip(warmup_frames)
.copied()
.collect::<Vec<_>>();
let measured = if measured.is_empty() {
&frames
} else {
&measured
};
let terminal_values = measured
.iter()
.filter_map(|frame| frame.terminal_draw)
.collect::<Vec<_>>();
let area_cells = u64::from(area.width)
.saturating_mul(u64::from(area.height))
.max(1);
RenderScenarioSummary {
schema: 1,
scenario,
backend_mode: mode.as_str(),
area: (area.width, area.height),
frames: measured.len(),
warmup_frames,
elapsed_ms: round_millis(elapsed.as_secs_f64() * 1000.0),
frames_per_sec: round_millis(measured.len() as f64 / elapsed.as_secs_f64().max(0.001)),
terminal_draw_ms: (!terminal_values.is_empty())
.then(|| render_duration_stats(terminal_values.into_iter())),
controlled_draw_ms: render_duration_stats(
measured.iter().map(|frame| frame.controlled_draw),
),
render_draw_ms: render_duration_stats(measured.iter().map(|frame| frame.render_draw)),
diff_ms: render_duration_stats(measured.iter().map(|frame| frame.diff)),
diff_cells: render_u64_stats(measured.iter().map(|frame| frame.diff_cells)),
changed_cell_ratio: round_millis(
measured.iter().map(|frame| frame.diff_cells).sum::<u64>() as f64
/ measured.len().max(1) as f64
/ area_cells as f64,
),
backend_draw_ms: render_duration_stats(measured.iter().map(|frame| frame.backend_draw)),
stdout_bytes: render_u64_stats(measured.iter().map(|frame| frame.stdout_bytes)),
write_calls: render_u64_stats(measured.iter().map(|frame| frame.write_calls)),
write_ms: render_duration_stats(measured.iter().map(|frame| frame.write_duration)),
flush_calls: render_u64_stats(measured.iter().map(|frame| frame.flush_calls)),
flush_ms: render_duration_stats(measured.iter().map(|frame| frame.flush_duration)),
}
}
fn print_render_summary(summary: &RenderScenarioSummary) {
EMIT_RENDER_SUMMARIES.with(|emit| {
if emit.get() {
println!(
"tui_render_pipeline {}",
serde_json::to_string(summary).unwrap()
);
}
});
}
fn render_profile_line(summaries: &[RenderScenarioSummary], elapsed: Duration) -> ProfileResult {
let frames = summaries
.iter()
.map(|summary| summary.frames as u64)
.sum::<u64>();
let avg_frames_per_sec = if summaries.is_empty() {
0
} else {
(summaries
.iter()
.map(|summary| summary.frames_per_sec)
.sum::<f64>()
/ summaries.len() as f64) as u64
};
let avg_render_us = average_stat_us(summaries, |summary| summary.render_draw_ms.avg);
let avg_diff_us = average_stat_us(summaries, |summary| summary.diff_ms.avg);
let avg_diff_cells = (summaries
.iter()
.map(|summary| summary.diff_cells.avg)
.sum::<f64>()
/ summaries.len().max(1) as f64) as u64;
let avg_stdout_bytes = (summaries
.iter()
.map(|summary| summary.stdout_bytes.avg)
.sum::<f64>()
/ summaries.len().max(1) as f64) as u64;
let avg_flush_us = average_stat_us(summaries, |summary| summary.flush_ms.avg);
ProfileResult {
scenario: "tui_render_pipeline",
iterations: summaries.len() as u64,
elapsed,
primary_metric: "frames",
metric_value: frames,
artifact: "target/profiling/issue-48/tui-render-pipeline.json",
counters: vec![
("frames_per_sec", avg_frames_per_sec),
("avg_render_us", avg_render_us),
("avg_diff_us", avg_diff_us),
("avg_diff_cells", avg_diff_cells),
("avg_stdout_bytes", avg_stdout_bytes),
("avg_flush_us", avg_flush_us),
],
}
}
fn average_stat_us(
summaries: &[RenderScenarioSummary],
value: impl Fn(&RenderScenarioSummary) -> f64,
) -> u64 {
if summaries.is_empty() {
return 0;
}
(summaries.iter().map(value).sum::<f64>() * 1000.0 / summaries.len() as f64) as u64
}
fn seed_render_heavy_transcript_state(turns: usize) -> MissionControlState {
let mut state = MissionControlState::default();
for index in 0..turns {
state.apply_output_event(&OutputEvent::UserPrompt {
text: format!("inspect synthetic turn {index}"),
});
state.apply_output_event(&OutputEvent::AssistantComplete {
text: RENDERING_TRANSCRIPT.to_string(),
});
}
state
}
fn seed_render_streaming_state() -> MissionControlState {
let mut state = MissionControlState::default();
state.apply_output_event(&OutputEvent::UserPrompt {
text: "stream synthetic assistant response".to_string(),
});
state
}
fn apply_streaming_frame(state: &mut MissionControlState, frame: usize) {
for index in 0..5 {
let sequence = frame * 5 + index;
state.apply_output_event(&OutputEvent::AssistantDelta {
text: format!("delta-{sequence} "),
});
let id = ActivityId::new(format!("render-tool-{sequence}"));
state.apply_activity_event(ActivityEvent::Started {
id: id.clone(),
parent_id: None,
kind: ActivityKind::Tool,
status: ActivityStatus::Running,
metadata: ActivityMetadata::new(format!("synthetic tool {sequence}")),
});
state.apply_activity_event(ActivityEvent::Delta {
id: id.clone(),
preview: format!("preview line {sequence}"),
});
state.apply_activity_event(ActivityEvent::Finished {
id,
status: ActivityStatus::Success,
metadata: None,
});
}
}
fn seed_closed_modal_state() -> MissionControlState {
seed_render_heavy_transcript_state(8)
}
fn open_synthetic_skills_modal(state: &mut MissionControlState) {
state.open_skills_modal(
vec![
crate::tui::state::SkillToggleRow {
name: "synthetic-render-skill".to_string(),
source: "test".to_string(),
enabled: true,
},
crate::tui::state::SkillToggleRow {
name: "disabled-render-skill".to_string(),
source: "test".to_string(),
enabled: false,
},
],
8,
);
}
fn run_test_backend_render_scenario(
scenario: &'static str,
area: Rect,
frames: usize,
mut state: MissionControlState,
mut mutate: impl FnMut(&mut MissionControlState, usize, Rect),
) -> RenderScenarioSummary {
let warmup_frames = 1;
let mut terminal_draw_terminal = Terminal::with_options(
TestBackend::new(area.width, area.height),
TerminalOptions {
viewport: Viewport::Fixed(area),
},
)
.expect("test backend terminal draw terminal");
let mut controlled_terminal = Terminal::with_options(
CountingBackend::new(TestBackend::new(area.width, area.height), None),
TerminalOptions {
viewport: Viewport::Fixed(area),
},
)
.expect("test backend controlled terminal");
let mut previous = Buffer::empty(area);
let mut stats = Vec::with_capacity(frames + warmup_frames);
let started = Instant::now();
for frame_index in 0..frames + warmup_frames {
mutate(&mut state, frame_index, area);
let mut frame_stats =
controlled_render_frame(&mut controlled_terminal, &mut previous, &state)
.expect("controlled test backend frame");
let terminal_started = Instant::now();
terminal_draw_terminal
.draw(|frame| crate::tui::render::draw(frame, &state))
.expect("terminal draw");
frame_stats.terminal_draw = Some(terminal_started.elapsed());
stats.push(frame_stats);
}
let summary = render_scenario_summary(
scenario,
RenderBackendMode::Test,
area,
warmup_frames,
stats,
started.elapsed(),
);
print_render_summary(&summary);
summary
}
fn run_crossterm_render_scenario(
scenario: &'static str,
mode: RenderBackendMode,
area: Rect,
frames: usize,
mut state: MissionControlState,
mut mutate: impl FnMut(&mut MissionControlState, usize, Rect),
) -> RenderScenarioSummary {
let warmup_frames = 1;
let writer_stats = Rc::new(RefCell::new(CountingWriterStats::default()));
let writer: Box<dyn Write> = match mode {
RenderBackendMode::CrosstermSink => Box::new(io::sink()),
RenderBackendMode::CrosstermReal => Box::new(io::stdout()),
RenderBackendMode::Test => unreachable!("test mode uses TestBackend"),
};
let writer = CountingWriter::new(writer, Rc::clone(&writer_stats));
let backend = CountingBackend::new(CrosstermBackend::new(writer), Some(writer_stats));
let mut terminal = Terminal::with_options(
backend,
TerminalOptions {
viewport: Viewport::Fixed(area),
},
)
.expect("crossterm controlled terminal");
let mut previous = Buffer::empty(area);
let mut stats = Vec::with_capacity(frames + warmup_frames);
let started = Instant::now();
for frame_index in 0..frames + warmup_frames {
mutate(&mut state, frame_index, area);
stats.push(
controlled_render_frame(&mut terminal, &mut previous, &state)
.expect("controlled crossterm frame"),
);
}
let summary = render_scenario_summary(
scenario,
mode,
area,
warmup_frames,
stats,
started.elapsed(),
);
print_render_summary(&summary);
summary
}
fn run_render_scenario(
scenario: &'static str,
area: Rect,
frames: usize,
state: MissionControlState,
mutate: impl FnMut(&mut MissionControlState, usize, Rect),
) -> RenderScenarioSummary {
match RenderBackendMode::from_env() {
RenderBackendMode::Test => {
run_test_backend_render_scenario(scenario, area, frames, state, mutate)
}
RenderBackendMode::CrosstermSink => run_crossterm_render_scenario(
scenario,
RenderBackendMode::CrosstermSink,
area,
frames,
state,
mutate,
),
RenderBackendMode::CrosstermReal => run_crossterm_render_scenario(
scenario,
RenderBackendMode::CrosstermReal,
area,
frames,
state,
mutate,
),
}
}
fn run_resize_transition_scenario(
scenario: &'static str,
previous_area: Rect,
current_area: Rect,
state: MissionControlState,
) -> RenderScenarioSummary {
let mode = RenderBackendMode::from_env();
if mode != RenderBackendMode::Test {
return run_render_scenario(scenario, current_area, 1, state, |_, _, _| {});
}
let mut terminal = Terminal::with_options(
CountingBackend::new(
TestBackend::new(previous_area.width, previous_area.height),
None,
),
TerminalOptions {
viewport: Viewport::Fixed(previous_area),
},
)
.expect("resize transition terminal");
controlled_render_frame(&mut terminal, &mut Buffer::empty(previous_area), &state)
.expect("resize transition baseline frame");
let mut terminal = Terminal::with_options(
CountingBackend::new(
TestBackend::new(current_area.width, current_area.height),
None,
),
TerminalOptions {
viewport: Viewport::Fixed(current_area),
},
)
.expect("resize transition resized terminal");
let started = Instant::now();
let stats = vec![
controlled_render_frame(&mut terminal, &mut Buffer::empty(current_area), &state)
.expect("resize transition first frame"),
];
let summary = render_scenario_summary(
scenario,
RenderBackendMode::Test,
current_area,
0,
stats,
started.elapsed(),
);
print_render_summary(&summary);
summary
}
fn render_scenario_idle() -> RenderScenarioSummary {
run_render_scenario(
"idle",
Rect::new(0, 0, 120, 36),
20,
MissionControlState::default(),
|_, _, _| {},
)
}
fn render_scenario_streaming() -> RenderScenarioSummary {
run_render_scenario(
"streaming",
Rect::new(0, 0, 120, 36),
20,
seed_render_streaming_state(),
|state, frame, _| apply_streaming_frame(state, frame),
)
}
fn render_scenario_heavy_transcript() -> RenderScenarioSummary {
run_render_scenario(
"heavy_transcript",
Rect::new(0, 0, 120, 36),
12,
seed_render_heavy_transcript_state(24),
|_, _, _| {},
)
}
fn render_scenario_scroll() -> RenderScenarioSummary {
run_render_scenario(
"scroll",
Rect::new(0, 0, 120, 36),
18,
seed_render_heavy_transcript_state(24),
|state, frame, _| state.set_scroll_offset(&state.scroll_views.transcript, frame % 16),
)
}
fn render_scenario_resize() -> Vec<RenderScenarioSummary> {
let sizes = [(80, 24), (120, 40), (160, 50), (50, 12)];
let mut summaries = Vec::with_capacity(sizes.len());
for (index, (width, height)) in sizes.into_iter().enumerate() {
let previous = if index == 0 {
Rect::new(0, 0, 120, 36)
} else {
let (previous_width, previous_height) = sizes[index - 1];
Rect::new(0, 0, previous_width, previous_height)
};
let current = Rect::new(0, 0, width, height);
summaries.push(run_resize_transition_scenario(
match (width, height) {
(80, 24) => "resize_first_frame_80x24",
(120, 40) => "resize_first_frame_120x40",
(160, 50) => "resize_first_frame_160x50",
(50, 12) => "resize_first_frame_50x12",
_ => "resize_first_frame",
},
previous,
current,
seed_render_heavy_transcript_state(10),
));
}
summaries
}
fn render_scenario_modal() -> RenderScenarioSummary {
run_render_scenario(
"modal_open",
Rect::new(0, 0, 120, 36),
12,
seed_closed_modal_state(),
|state, frame, _| {
if frame == 1 {
open_synthetic_skills_modal(state);
}
},
)
}
fn run_tui_render_pipeline(filter: Option<&str>) -> Vec<RenderScenarioSummary> {
let mut summaries = Vec::new();
match filter {
Some("idle") => summaries.push(render_scenario_idle()),
Some("streaming") => summaries.push(render_scenario_streaming()),
Some("heavy_transcript") => summaries.push(render_scenario_heavy_transcript()),
Some("scroll") => summaries.push(render_scenario_scroll()),
Some("resize") => summaries.extend(render_scenario_resize()),
Some("modal") => summaries.push(render_scenario_modal()),
None => {
summaries.push(render_scenario_idle());
summaries.push(render_scenario_streaming());
summaries.push(render_scenario_heavy_transcript());
summaries.push(render_scenario_scroll());
summaries.extend(render_scenario_resize());
summaries.push(render_scenario_modal());
}
Some(other) => panic!("unknown render scenario {other}"),
}
summaries
}
fn run_tui_render_pipeline_profile(
filter: Option<&str>,
started: Instant,
) -> Vec<RenderScenarioSummary> {
let min_iterations = profile_iterations();
let min_duration = profile_min_duration();
let mut summaries = Vec::new();
let mut iterations = 0;
while iterations < min_iterations || started.elapsed() < min_duration {
EMIT_RENDER_SUMMARIES.with(|emit| emit.set(iterations + 1 >= min_iterations));
summaries.extend(run_tui_render_pipeline(filter));
iterations += 1;
}
EMIT_RENDER_SUMMARIES.with(|emit| emit.set(true));
summaries
}
fn assert_render_summaries(summaries: &[RenderScenarioSummary]) {
assert!(!summaries.is_empty());
for summary in summaries {
assert!(summary.frames > 0, "{summary:?}");
assert!(summary.frames_per_sec > 0.0, "{summary:?}");
assert!(
summary.render_draw_ms.max >= summary.render_draw_ms.min,
"{summary:?}"
);
assert!(
summary.diff_cells.max >= summary.diff_cells.min,
"{summary:?}"
);
}
}
#[cfg(unix)]
fn timeout_cleanup_once() -> anyhow::Result<u64> {
use std::os::unix::process::CommandExt;
let mut child = Command::new("/bin/bash")
.arg("-lc")
.arg("(sleep 5) & wait")
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null())
.process_group(0)
.spawn()?;
std::thread::sleep(Duration::from_millis(5));
let cleanup = terminate_child_tree_and_wait(&mut child)?;
Ok(u64::from(cleanup.cleanup_warning.is_some()) + u64::from(cleanup.status.is_some()))
}
#[cfg(not(unix))]
fn timeout_cleanup_once() -> anyhow::Result<u64> {
let mut child = Command::new("cmd")
.args(["/C", "ping", "127.0.0.1", "-n", "5"])
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn()?;
std::thread::sleep(Duration::from_millis(5));
let cleanup = terminate_child_tree_and_wait(&mut child)?;
Ok(u64::from(cleanup.cleanup_warning.is_some()) + u64::from(cleanup.status.is_some()))
}
#[test]
#[ignore = "diagnostic-only fffind benchmark; run explicitly with --ignored --nocapture"]
fn find_benchmark_measures_cold_warm_filtered_and_baseline() {
let cfg = FindBenchConfig::from_env().expect("fffind benchmark config");
let runtime = ToolRuntime::new(&cfg.root).expect("fffind benchmark runtime");
let existing_paths = cfg.existing_filter_paths();
let cold_query = cfg.queries.first().expect("at least one query").clone();
let mut scenario_count = 0;
let mut observed_values = Vec::new();
let before = ResourceSnapshot::capture();
let cold = run_find_observation(
&runtime,
&cold_query,
"files",
None,
cfg.limit,
1,
cfg.show_results,
)
.expect("cold fffind benchmark");
let resources = before.delta(ResourceSnapshot::capture());
let baseline = baseline_for_file_scenario(&cfg, &cold_query, None, 1).expect("cold baseline");
let value = find_benchmark_line(
&cfg,
"cold_unfiltered_files",
Some(&cold_query),
Some("files"),
None,
&cold,
resources,
baseline,
);
print_find_benchmark_line(value.clone());
observed_values.push(value);
scenario_count += 1;
for query in &cfg.queries {
let before = ResourceSnapshot::capture();
let observation = run_find_observation(
&runtime,
query,
"files",
None,
cfg.limit,
cfg.repetitions,
false,
)
.expect("warm unfiltered fffind benchmark");
let resources = before.delta(ResourceSnapshot::capture());
let baseline = baseline_for_file_scenario(&cfg, query, None, cfg.repetitions)
.expect("warm unfiltered baseline");
let value = find_benchmark_line(
&cfg,
"warm_unfiltered_files",
Some(query),
Some("files"),
None,
&observation,
resources,
baseline,
);
print_find_benchmark_line(value.clone());
observed_values.push(value);
scenario_count += 1;
}
for path in &existing_paths {
for query in &cfg.queries {
let before = ResourceSnapshot::capture();
let observation = run_find_observation(
&runtime,
query,
"files",
Some(path),
cfg.limit,
cfg.repetitions,
false,
)
.expect("warm filtered fffind benchmark");
let resources = before.delta(ResourceSnapshot::capture());
let baseline = baseline_for_file_scenario(&cfg, query, Some(path), cfg.repetitions)
.expect("warm filtered baseline");
let value = find_benchmark_line(
&cfg,
"warm_filtered_files",
Some(query),
Some("files"),
Some(path),
&observation,
resources,
baseline,
);
print_find_benchmark_line(value.clone());
observed_values.push(value);
scenario_count += 1;
}
}
for (scenario, kind) in [("warm_directories", "directories"), ("warm_mixed", "mixed")] {
for query in &cfg.queries {
let before = ResourceSnapshot::capture();
let observation = run_find_observation(
&runtime,
query,
kind,
None,
cfg.limit,
cfg.repetitions,
false,
)
.expect("warm kind fffind benchmark");
let resources = before.delta(ResourceSnapshot::capture());
let value = find_benchmark_line(
&cfg,
scenario,
Some(query),
Some(kind),
None,
&observation,
resources,
None,
);
print_find_benchmark_line(value.clone());
observed_values.push(value);
scenario_count += 1;
}
}
let before = ResourceSnapshot::capture();
let repeated = run_find_observation(
&runtime,
&cold_query,
"files",
None,
cfg.limit,
cfg.repetitions,
false,
)
.expect("repeated same query fffind benchmark");
let resources = before.delta(ResourceSnapshot::capture());
let baseline = baseline_for_file_scenario(&cfg, &cold_query, None, cfg.repetitions)
.expect("repeated same query baseline");
let value = find_benchmark_line(
&cfg,
"repeated_same_query",
Some(&cold_query),
Some("files"),
None,
&repeated,
resources,
baseline,
);
print_find_benchmark_line(value.clone());
observed_values.push(value);
scenario_count += 1;
if let Some(value) = run_cwd_churn_scenario(&cfg, &runtime, &cold_query).expect("cwd churn") {
print_find_benchmark_line(value.clone());
observed_values.push(value);
scenario_count += 1;
}
print_find_summary(&cfg, scenario_count, &observed_values);
}
#[test]
#[ignore = "diagnostic-only TUI render profiling harness; run explicitly with --ignored --nocapture"]
fn profile_tui_render_pipeline() {
let started = Instant::now();
let summaries = run_tui_render_pipeline_profile(None, started);
assert_render_summaries(&summaries);
render_profile_line(&summaries, started.elapsed()).print();
}
#[test]
#[ignore = "diagnostic-only TUI render profiling harness; run explicitly with --ignored --nocapture"]
fn profile_tui_render_pipeline_idle() {
let summaries = run_tui_render_pipeline(Some("idle"));
assert_render_summaries(&summaries);
}
#[test]
#[ignore = "diagnostic-only TUI render profiling harness; run explicitly with --ignored --nocapture"]
fn profile_tui_render_pipeline_streaming() {
let summaries = run_tui_render_pipeline(Some("streaming"));
assert_render_summaries(&summaries);
}
#[test]
#[ignore = "diagnostic-only TUI render profiling harness; run explicitly with --ignored --nocapture"]
fn profile_tui_render_pipeline_heavy_transcript() {
let summaries = run_tui_render_pipeline(Some("heavy_transcript"));
assert_render_summaries(&summaries);
}
#[test]
#[ignore = "diagnostic-only TUI render profiling harness; run explicitly with --ignored --nocapture"]
fn profile_tui_render_pipeline_scroll() {
let summaries = run_tui_render_pipeline(Some("scroll"));
assert_render_summaries(&summaries);
}
#[test]
#[ignore = "diagnostic-only TUI render profiling harness; run explicitly with --ignored --nocapture"]
fn profile_tui_render_pipeline_resize() {
let summaries = run_tui_render_pipeline(Some("resize"));
assert_render_summaries(&summaries);
}
#[test]
#[ignore = "diagnostic-only TUI render profiling harness; run explicitly with --ignored --nocapture"]
fn profile_tui_render_pipeline_modal() {
let summaries = run_tui_render_pipeline(Some("modal"));
assert_render_summaries(&summaries);
}
#[test]
#[ignore = "CPU profiling harness; run explicitly with --ignored --nocapture"]
fn profile_cpu_startup_discovery() {
let (_temp, paths, repo_root) = seed_startup_fixture().expect("seed startup fixture");
let result = run_profile(
"startup_discovery",
"discovered_items",
"target/profiling/issue-48/startup-discovery",
|| startup_discovery_once(&paths, &repo_root).expect("startup discovery"),
);
assert!(result.metric_value > 0);
result.print();
}
#[test]
#[ignore = "CPU profiling harness; run explicitly with --ignored --nocapture"]
fn profile_cpu_sse_parser() {
let result = run_profile(
"provider_sse_parser",
"events_parsed",
"target/profiling/issue-48/sse-parser",
|| parse_sse_once().expect("parse sse"),
);
assert!(result.metric_value > 0);
result.print();
}
#[test]
#[ignore = "CPU profiling harness; run explicitly with --ignored --nocapture"]
fn profile_cpu_rendering_heavy_transcript() {
let result = run_profile(
"rendering_heavy_transcript",
"rendered_units",
"target/profiling/issue-48/rendering-heavy-transcript",
heavy_transcript_once,
);
assert!(result.metric_value > 0);
result.print();
}
#[test]
#[ignore = "CPU profiling harness; run explicitly with --ignored --nocapture"]
fn profile_cpu_tui_streaming_simulation() {
let result = run_profile(
"tui_streaming_simulation",
"state_units",
"target/profiling/issue-48/tui-streaming-simulation",
streaming_simulation_once,
);
assert!(result.metric_value > 0);
result.print();
}
#[test]
#[ignore = "CPU profiling harness; run explicitly with --ignored --nocapture"]
fn profile_cpu_tool_timeout_cleanup() {
let result = run_profile(
"tool_timeout_cleanup",
"cleanup_units",
"target/profiling/issue-48/tool-timeout-cleanup",
|| timeout_cleanup_once().expect("timeout cleanup"),
);
result.print();
}