use crate::{
cancellation::AgentCancellation,
config::{
CliConfigOverrides, CustomProviderConfig, McPaths, Settings, ensure_settings_schema_files,
load_config_with_settings, load_effective_provider_selection,
load_startup_config_with_settings, read_settings, write_settings,
},
context::ContextBudget,
providers::{
ChatMessage, Provider, ProviderEvent, ProviderRequest, ProviderSelection,
build_chat_completion_tools_json, chat_completion_tools_json,
openai_compatible_chat_completions_body,
},
subagents::profiles::discover_subagent_profiles,
subagents::{ResolvedProviderOverride, SubagentProviderOverride},
};
use serde_json::json;
use std::{fs, hint::black_box, path::Path, sync::Arc, time::Instant};
struct BenchmarkProvider;
impl Provider for BenchmarkProvider {
fn stream_cancellable(
&self,
_request: ProviderRequest,
_cancellation: &AgentCancellation,
_on_event: &mut dyn FnMut(ProviderEvent) -> anyhow::Result<()>,
) -> anyhow::Result<()> {
unreachable!("benchmark provider is never called")
}
}
const SAMPLES: usize = 7;
#[derive(Debug)]
struct Measurement {
name: &'static str,
repetitions: usize,
samples_ns: Vec<u128>,
}
impl Measurement {
fn run(name: &'static str, repetitions: usize, mut operation: impl FnMut()) -> Self {
let mut samples_ns = Vec::with_capacity(SAMPLES);
for _ in 0..SAMPLES {
let started = Instant::now();
for _ in 0..repetitions {
operation();
}
samples_ns.push(started.elapsed().as_nanos());
}
Self {
name,
repetitions,
samples_ns,
}
}
fn print(&self) {
let mut sorted = self.samples_ns.clone();
sorted.sort_unstable();
let divisor = self.repetitions.max(1) as u128;
println!(
"issue_379_measurement {}",
json!({
"name": self.name,
"profile": if cfg!(debug_assertions) { "debug" } else { "release" },
"samples": self.samples_ns.len(),
"repetitions_per_sample": self.repetitions,
"min_ns_per_operation": sorted[0] / divisor,
"median_ns_per_operation": sorted[sorted.len() / 2] / divisor,
"max_ns_per_operation": sorted[sorted.len() - 1] / divisor,
})
);
}
}
fn repetitions() -> usize {
std::env::var("ISSUE_379_BENCH_REPETITIONS")
.ok()
.and_then(|value| value.parse().ok())
.filter(|value| *value > 0)
.unwrap_or(100)
}
#[test]
#[ignore = "diagnostic-only issue #379 benchmark; run explicitly with --ignored --nocapture"]
fn profile_issue_379_repeated_setup_paths() {
let repetitions = repetitions();
let temp = tempfile::tempdir().unwrap();
let paths = McPaths::from_root(temp.path().join("mc"));
fs::create_dir_all(&paths.root).unwrap();
fs::create_dir_all(&paths.state).unwrap();
fs::create_dir_all(&paths.subagents).unwrap();
let mut settings = Settings::default();
settings.tools.disabled.push("browser".to_string());
settings.custom_providers.insert(
"local-ai".to_string(),
CustomProviderConfig {
label: "Local AI".to_string(),
base_url: "http://localhost:8080/v1".to_string(),
fast_mode: None,
api_key_env_var: None,
models_dev_provider: None,
use_responses_endpoint: false,
supports_text_verbosity: false,
reasoning_protocol: Default::default(),
extra_models: vec!["model-a".to_string()],
request_headers: Default::default(),
},
);
write_settings(&paths, &settings).unwrap();
for index in 0..4 {
fs::write(
paths.subagents.join(format!("profile-{index}.md")),
format!(
"---\nname: Profile {index}\ndescription: Benchmark profile {index}\n---\nProfile prompt {index}\n"
),
)
.unwrap();
}
ensure_settings_schema_files(&paths).unwrap();
let startup_baseline = Measurement::run("startup_schema_then_config_load", repetitions, || {
ensure_settings_schema_files(&paths).unwrap();
black_box(load_config_with_settings(paths.clone(), CliConfigOverrides::default()).unwrap());
});
let startup_reused = Measurement::run(
"startup_combined_schema_and_config_load",
repetitions,
|| {
black_box(
load_startup_config_with_settings(paths.clone(), CliConfigOverrides::default())
.unwrap(),
);
},
);
let loaded_settings = read_settings(&paths).unwrap();
let loaded_profiles = discover_subagent_profiles(&paths.subagents);
let interactive_baseline = Measurement::run("classic_prompt_rediscovery", repetitions, || {
black_box(read_settings(&paths).unwrap());
black_box(discover_subagent_profiles(&paths.subagents));
});
let interactive_reused =
Measurement::run("classic_prompt_loaded_state_reuse", repetitions, || {
black_box(loaded_settings.clone());
black_box(loaded_profiles.clone());
});
let override_baseline = Measurement::run("ten_override_resolutions", repetitions, || {
for _ in 0..10 {
black_box(load_effective_provider_selection(&paths, "local-ai", "model-a").unwrap());
}
});
let cached_settings = loaded_settings.clone();
let override_paths = paths.clone();
let override_cache = SubagentProviderOverride::new(
paths.clone(),
Arc::new(move |selection: &ProviderSelection, _cwd: &Path| {
let active_config = load_effective_provider_selection(
&override_paths,
&selection.provider,
&selection.model,
)?;
Ok(ResolvedProviderOverride {
provider: Arc::new(BenchmarkProvider),
scope: crate::thinking::capability_scope_for_provider(
&active_config.custom_providers,
&selection.provider,
),
active_config,
context_budget: ContextBudget::default(),
settings: cached_settings.clone(),
})
}),
);
let selection = ProviderSelection {
provider: "local-ai".to_string(),
model: "model-a".to_string(),
};
black_box(override_cache.resolve(&selection, temp.path()).unwrap());
let override_reused = Measurement::run("ten_cached_override_selections", repetitions, || {
for _ in 0..10 {
black_box(override_cache.resolve(&selection, temp.path()).unwrap());
}
});
black_box(chat_completion_tools_json(true));
let tools_baseline = Measurement::run("chat_tools_rebuild_and_reshape", repetitions, || {
black_box(build_chat_completion_tools_json(true));
});
let tools_cached = Measurement::run("chat_tools_provider_ready_cache", repetitions, || {
black_box(chat_completion_tools_json(true));
});
let no_tools = ProviderRequest::new_without_tools("model", vec![ChatMessage::user("hello")]);
let static_tools = ProviderRequest::new("model", vec![ChatMessage::user("hello")]);
let disabled_tools = ProviderRequest::new("model", vec![ChatMessage::user("hello")])
.with_disabled_tool_names(vec!["browser".to_string()]);
let dynamic_tools = ProviderRequest::new("model", vec![ChatMessage::user("hello")])
.with_dynamic_tool_definitions(vec![json!({
"type": "function",
"name": "mcp__bench__echo",
"description": "Echo",
"parameters": {"type":"object","properties":{}}
})]);
for (name, request) in [
("provider_body_no_tools", no_tools),
("provider_body_static_tools", static_tools),
("provider_body_disabled_tool", disabled_tools),
("provider_body_dynamic_tool", dynamic_tools),
] {
Measurement::run(name, repetitions, || {
black_box(openai_compatible_chat_completions_body("model", &request));
})
.print();
}
for measurement in [
startup_baseline,
startup_reused,
interactive_baseline,
interactive_reused,
override_baseline,
override_reused,
tools_baseline,
tools_cached,
] {
measurement.print();
}
}