use super::*;
use std::io::{Read, Write};
use std::net::TcpListener;
use std::path::PathBuf;
use std::sync::{Arc, Mutex};
use crate::configuration::{GatewayConfig, ResolvedConfig, ResolvedDynamicPluginConfig};
use crate::server::GatewayOverrides;
use crate::test_support::{EnvScope, accept_bounded, read_headers};
fn start_doctor_http_capture_server() -> (String, Arc<Mutex<String>>, std::thread::JoinHandle<()>) {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let url = format!("http://{}", listener.local_addr().unwrap());
let body = Arc::new(Mutex::new(String::new()));
let thread_body = Arc::clone(&body);
let handle = std::thread::spawn(move || {
let mut stream = accept_bounded(&listener);
let mut data = Vec::new();
let mut buf = [0_u8; 1];
while !data.ends_with(b"\r\n\r\n") {
stream.read_exact(&mut buf).unwrap();
data.push(buf[0]);
}
let headers = String::from_utf8_lossy(&data).to_string();
let length = headers
.lines()
.find_map(|line| {
line.split_once(':').and_then(|(name, value)| {
name.eq_ignore_ascii_case("content-length")
.then_some(value.trim())
})
})
.and_then(|value| value.trim().parse::<usize>().ok())
.unwrap();
let mut request_body = vec![0_u8; length];
stream.read_exact(&mut request_body).unwrap();
*thread_body.lock().unwrap() = String::from_utf8(request_body).unwrap();
stream
.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
.unwrap();
});
(url, body, handle)
}
fn empty_report() -> DoctorReport {
DoctorReport {
schema_version: 1,
binary_version: "0.0.0-test",
target_agent: None,
environment: EnvironmentInfo {
os: "macos 25.3.0".into(),
arch: "aarch64",
shell: Some("zsh".into()),
},
configuration: ConfigurationInfo {
explicit: None,
workspace: ConfigLayer {
path: PathBuf::from("/x/.nemo-relay/config.toml"),
status: Status::Info,
active: false,
details: "not present".into(),
},
global: ConfigLayer {
path: PathBuf::from("/x/.config/nemo-relay/config.toml"),
status: Status::Info,
active: false,
details: "not present".into(),
},
system: ConfigLayer {
path: PathBuf::from("/etc/nemo-relay/config.toml"),
status: Status::Info,
active: false,
details: "not present".into(),
},
upstream_auth: UpstreamAuthInfo {
openai: SecretPresence::Unset,
anthropic: SecretPresence::Unset,
},
plugin_configs: vec![],
plugin_resolution: Check {
name: "Plugin resolution",
status: Status::Info,
details: "plugins.toml not configured".into(),
},
resolution: Check {
name: "Resolution",
status: Status::Pass,
details: "valid".into(),
},
default_agent: None,
configured_agents: vec![],
dynamic_plugins: vec![],
},
agents: vec![],
host_plugins: vec![],
observability: vec![],
completions: vec![],
}
}
async fn collect_live_observability(gateway: &GatewayConfig) -> Vec<Check> {
collect_observability(gateway, DoctorProbeMode::Live).await
}
async fn collect_offline_observability(gateway: &GatewayConfig) -> Vec<Check> {
collect_observability(gateway, DoctorProbeMode::Offline).await
}
async fn live_observability_http_exporter_checks(config: &serde_json::Value) -> Vec<Check> {
observability_http_exporter_checks(config, DoctorProbeMode::Live).await
}
async fn offline_observability_http_exporter_checks(config: &serde_json::Value) -> Vec<Check> {
observability_http_exporter_checks(config, DoctorProbeMode::Offline).await
}
#[cfg(test)]
async fn offline_atof_endpoint(index: usize, endpoint: &serde_json::Value) -> Check {
probe_atof_stream_sink(index, endpoint, DoctorProbeMode::Offline).await
}
#[test]
fn exit_code_passes_when_no_failures() {
let report = empty_report();
assert_eq!(exit_code(&report), 0);
}
#[test]
fn exit_code_fails_when_observability_check_fails() {
let mut report = empty_report();
report.observability.push(Check {
name: "ATIF dir",
status: Status::Fail,
details: "not writable".into(),
});
assert_eq!(exit_code(&report), 1);
}
#[test]
fn exit_code_passes_with_warn_only() {
let mut report = empty_report();
report.observability.push(Check {
name: "OpenInference endpoint",
status: Status::Warn,
details: "HTTP 500".into(),
});
assert_eq!(exit_code(&report), 0);
}
#[test]
fn exit_code_fails_when_workspace_config_is_invalid() {
let mut report = empty_report();
report.configuration.workspace.status = Status::Fail;
report.configuration.workspace.details = "invalid TOML".into();
assert_eq!(exit_code(&report), 1);
}
#[test]
fn exit_code_fails_when_config_resolution_fails() {
let mut report = empty_report();
report.configuration.resolution.status = Status::Fail;
report.configuration.resolution.details = "invalid gateway configuration shape".into();
assert_eq!(exit_code(&report), 1);
}
#[test]
fn exit_code_fails_when_agent_readiness_fails() {
let mut report = empty_report();
report.agents.push(AgentInfo {
name: "codex",
status: Status::Fail,
configured: true,
command: "codex".into(),
path: None,
version: None,
annotation: "configured command not found on $PATH".into(),
});
assert_eq!(exit_code(&report), 1);
}
#[test]
fn exit_code_fails_when_an_installed_host_plugin_is_unready() {
let mut report = empty_report();
report
.host_plugins
.push(crate::installation::marketplace::HostPluginReadiness {
host: "codex".into(),
remediation: "nemo-relay install codex --force".into(),
state_path: PathBuf::from("/tmp/codex.json"),
marketplace: Some(PathBuf::from("/tmp/codex-marketplace")),
plugin: Some(PathBuf::from(
"/tmp/codex-marketplace/plugins/nemo-relay-plugin",
)),
checks: vec![crate::installation::marketplace::HostPluginReadinessCheck {
name: "Host CLI".into(),
ok: false,
details: "required `codex` CLI was not found on PATH".into(),
}],
relay: None,
host_plugin_registered: None,
host_marketplace_registered: None,
plugin_setup: None,
});
assert_eq!(exit_code(&report), 1);
let rendered = format_human(&report);
assert!(rendered.contains("Persistent integrations"));
assert!(rendered.contains("repair: nemo-relay install codex --force"));
let json: serde_json::Value = serde_json::from_str(&format_json(&report).unwrap()).unwrap();
assert_eq!(json["schema_version"], 1);
assert_eq!(json["host_plugins"][0]["checks"][0]["ok"], false);
assert_eq!(
json["host_plugins"][0]["remediation"],
"nemo-relay install codex --force"
);
}
#[test]
fn format_human_emits_fixed_section_order() {
let report = empty_report();
let rendered = format_human(&report);
let env_idx = rendered.find("Environment").expect("Environment header");
let cfg_idx = rendered
.find("Configuration")
.expect("Configuration header");
let plugins_idx = rendered
.find("Plugin configuration")
.expect("Plugin configuration header");
let agents_idx = rendered.find("Agents detected").expect("Agents header");
let obs_idx = rendered
.find("Observability")
.expect("Observability header");
let comp_idx = rendered.find("Completions").expect("Completions header");
assert!(env_idx < cfg_idx);
assert!(cfg_idx < plugins_idx);
assert!(plugins_idx < agents_idx);
assert!(agents_idx < obs_idx);
assert!(obs_idx < comp_idx);
}
#[test]
fn format_human_renders_completion_details_without_check_name() {
let mut report = empty_report();
report.completions.push(Check {
name: "Completions",
status: Status::Pass,
details: "zsh: /tmp/_nemo-relay".into(),
});
let rendered = format_human(&report);
assert!(rendered.contains(" Completions\n zsh: /tmp/_nemo-relay\n"));
assert!(!rendered.contains(" Completions"));
}
#[test]
fn format_human_distinguishes_plugin_files_from_plugin_resolution() {
let mut report = empty_report();
report.configuration.plugin_configs.push(ConfigLayer {
path: PathBuf::from("/tmp/plugins.toml"),
status: Status::Pass,
active: true,
details: "discovered and contributes to plugin resolution".into(),
});
let rendered = format_human(&report);
assert!(rendered.contains("Plugin files /tmp/plugins.toml"));
assert!(rendered.contains("Plugins · plugins.toml not configured"));
}
#[test]
fn format_human_reports_effective_upstream_auth_presence() {
let mut report = empty_report();
report.configuration.upstream_auth = UpstreamAuthInfo {
openai: SecretPresence::Configured,
anthropic: SecretPresence::Unset,
};
let rendered = format_human(&report);
assert!(rendered.contains("Upstream openai=configured anthropic=unset"));
}
#[test]
fn format_human_reports_unknown_upstream_auth_presence() {
let mut report = empty_report();
report.configuration.upstream_auth = UpstreamAuthInfo::unknown();
let rendered = format_human(&report);
assert!(rendered.contains("Upstream openai=unknown anthropic=unknown"));
}
#[test]
fn format_human_reports_all_checks_passed_on_clean_report() {
let report = empty_report();
let rendered = format_human(&report);
assert!(rendered.contains("All checks passed."));
assert!(!rendered.contains("warnings"));
}
#[test]
fn format_human_uses_symbols_for_agent_statuses() {
let mut report = empty_report();
report.agents = vec![
AgentInfo {
name: "claude",
status: Status::Pass,
configured: true,
command: "claude".into(),
path: Some(PathBuf::from("/bin/claude")),
version: Some("1.0.0".into()),
annotation: "hooks: injected during run".into(),
},
AgentInfo {
name: "codex",
status: Status::Info,
configured: false,
command: "codex".into(),
path: None,
version: None,
annotation: "not configured".into(),
},
];
let rendered = format_human(&report);
assert!(rendered.contains(" ✓ claude"));
assert!(rendered.contains(" · codex"));
assert!(!rendered.contains(" pass "));
assert!(!rendered.contains(" info "));
}
#[test]
fn format_human_reports_failure_summary_when_anything_failed() {
let mut report = empty_report();
report.observability.push(Check {
name: "ATIF dir",
status: Status::Fail,
details: "not writable".into(),
});
let rendered = format_human(&report);
assert!(rendered.contains("Some checks FAILED"));
}
#[test]
fn format_human_reports_config_resolution_failure() {
let mut report = empty_report();
report.configuration.resolution.status = Status::Fail;
report.configuration.resolution.details =
"could not resolve merged config: invalid plugin TOML".into();
let rendered = format_human(&report);
assert!(rendered.contains("Resolution ✗ could not resolve merged config"));
assert!(rendered.contains("Some checks FAILED"));
}
#[tokio::test]
async fn agents_report_surfaces_merged_config_resolution_errors() {
let temp = tempfile::tempdir().unwrap();
let config_home = temp.path().join("config");
let config = config_home.join("nemo-relay").join("config.toml");
std::fs::create_dir_all(config.parent().unwrap()).unwrap();
std::fs::write(&config, "[upstream\n").unwrap();
let _env = EnvScope::set(&[
("XDG_CONFIG_HOME", Some(config_home.as_os_str())),
(
"NEMO_RELAY_CONFIG_SCOPE",
Some(std::ffi::OsStr::new("user")),
),
]);
let error = agents_report().await.unwrap_err().to_string();
assert!(error.contains("config"), "{error}");
assert!(error.contains("TOML"), "{error}");
}
#[test]
fn format_human_distinguishes_pass_with_warnings_from_clean_pass() {
let mut report = empty_report();
report.observability.push(Check {
name: "ATIF dir",
status: Status::Warn,
details: "directory missing — will be created on first write".into(),
});
let rendered = format_human(&report);
assert!(rendered.contains("All checks passed"));
assert!(
rendered.contains("warnings"),
"warn-only report should surface the word `warnings` in the footer, got:\n{rendered}"
);
}
#[test]
fn format_json_is_stable_and_versioned() {
let report = empty_report();
let json = format_json(&report).unwrap();
let parsed: serde_json::Value = serde_json::from_str(&json).unwrap();
assert_eq!(parsed["schema_version"], 1);
assert!(parsed["target_agent"].is_null());
assert!(parsed["environment"]["os"].is_string());
assert!(parsed["agents"].is_array());
assert!(parsed["configuration"]["dynamic_plugins"].is_array());
}
#[test]
fn format_json_reports_discovered_dynamic_plugin_fields() {
let mut report = empty_report();
report.configuration.dynamic_plugins = vec![DynamicPluginReferenceInfo {
plugin_id: "acme.worker".into(),
manifest_ref: "/tmp/plugins/acme/relay-plugin.toml".into(),
source: PathBuf::from("/tmp/plugins.toml"),
host_config_status: DynamicPluginHostConfigStatus::Present,
}];
let json = format_json(&report).unwrap();
let parsed: serde_json::Value = serde_json::from_str(&json).unwrap();
let plugin = &parsed["configuration"]["dynamic_plugins"][0];
assert_eq!(plugin["plugin_id"], "acme.worker");
assert_eq!(
plugin["manifest_ref"],
"/tmp/plugins/acme/relay-plugin.toml"
);
assert_eq!(plugin["source"], "/tmp/plugins.toml");
assert_eq!(plugin["host_config_status"], "present");
}
#[test]
fn check_dir_writable_does_not_create_missing_dir() {
let temp = tempfile::tempdir().unwrap();
let missing = temp.path().join("missing-atif");
assert!(check_dir_writable(&missing).is_err());
assert!(
!missing.exists(),
"doctor should not create missing ATIF directories while probing"
);
}
#[test]
fn layer_status_reports_missing_valid_invalid_and_non_directory_paths() {
let temp = tempfile::tempdir().unwrap();
let missing = temp.path().join("missing.toml");
assert_eq!(layer_status(&missing).status, Status::Info);
let valid = temp.path().join("config.toml");
std::fs::write(&valid, "[upstream]\nopenai_base_url = \"http://local\"\n").unwrap();
let valid_layer = layer_status(&valid);
assert_eq!(valid_layer.status, Status::Pass);
assert!(valid_layer.active);
let invalid_shape = temp.path().join("invalid-shape.toml");
std::fs::write(
&invalid_shape,
"[upstream.openai]\nbase_url = \"http://local\"\n",
)
.unwrap();
let invalid_shape_layer = layer_status(&invalid_shape);
assert_eq!(invalid_shape_layer.status, Status::Fail);
assert!(!invalid_shape_layer.active);
assert!(
invalid_shape_layer
.details
.contains("invalid gateway configuration shape")
);
assert!(invalid_shape_layer.details.contains("openai"));
let invalid = temp.path().join("invalid.toml");
std::fs::write(&invalid, "[upstream\n").unwrap();
let invalid_layer = layer_status(&invalid);
assert_eq!(invalid_layer.status, Status::Fail);
assert!(invalid_layer.details.contains("invalid TOML"));
let dir = temp.path().join("config-dir");
std::fs::create_dir(&dir).unwrap();
let dir_layer = layer_status(&dir);
assert_eq!(dir_layer.status, Status::Fail);
assert!(
dir_layer.details.contains("unreadable") || dir_layer.details.contains("Is a directory")
);
}
#[test]
fn plugin_layer_status_marks_a_discovered_plugins_toml_as_contributing() {
let temp = tempfile::tempdir().unwrap();
let path = temp.path().join("plugins.toml");
std::fs::write(&path, "version = 1\ncomponents = []\n").unwrap();
let layer = plugin_layer_status(&path, std::slice::from_ref(&path), None);
assert_eq!(layer.status, Status::Pass);
assert!(layer.active);
assert!(layer.details.contains("contributes to plugin resolution"));
}
#[test]
fn plugin_layer_status_marks_non_contributing_files_as_inactive() {
let temp = tempfile::tempdir().unwrap();
let path = temp.path().join("plugins.toml");
std::fs::write(&path, "version = 1\ncomponents = []\n").unwrap();
let layer = plugin_layer_status(&path, &[], None);
assert_eq!(layer.status, Status::Pass);
assert!(!layer.active);
assert!(layer.details.contains("does not contribute"));
}
#[test]
fn plugin_layer_status_surfaces_source_specific_semantic_errors() {
let temp = tempfile::tempdir().unwrap();
let path = temp.path().join("plugins.toml");
std::fs::write(&path, "[plugins]\ndynamic = 1\n").unwrap();
let error = format!("invalid dynamic plugin config in {}", path.display());
let layer = plugin_layer_status(&path, &[], Some(&error));
assert_eq!(layer.status, Status::Fail);
assert!(!layer.active);
assert!(layer.details.contains("invalid plugin configuration"));
}
#[test]
fn plugin_resolution_check_covers_all_resolution_outcomes() {
let valid = Check {
name: "Resolution",
status: Status::Pass,
details: "valid".into(),
};
let failed = Check {
name: "Resolution",
status: Status::Fail,
details: "merged configuration failed".into(),
};
let mut with_runtime_config = ResolvedConfig::default();
with_runtime_config.gateway.plugin_config = Some(serde_json::json!({"version": 1}));
let mut with_dynamic_plugin = ResolvedConfig::default();
with_dynamic_plugin
.dynamic_plugins
.push(ResolvedDynamicPluginConfig {
plugin_id: "acme.worker".into(),
manifest_ref: "/tmp/relay-plugin.toml".into(),
config: serde_json::Map::new(),
has_explicit_config: false,
source: PathBuf::from("/tmp/plugins.toml"),
});
let cases = [
(
ResolvedConfig::default(),
&valid,
Some("invalid plugin TOML"),
Status::Fail,
"could not resolve plugins.toml",
),
(
ResolvedConfig::default(),
&failed,
None,
Status::Fail,
"merged configuration failed",
),
(
with_runtime_config,
&valid,
None,
Status::Info,
"effective plugin configuration loaded",
),
(
with_dynamic_plugin,
&valid,
None,
Status::Info,
"dynamic plugin configuration loaded",
),
(
ResolvedConfig::default(),
&valid,
None,
Status::Info,
"plugins.toml not configured",
),
];
for (resolved, resolution, plugin_error, status, detail) in cases {
let check = plugin_resolution_check(&resolved, resolution, plugin_error);
assert_eq!(check.name, "Plugin resolution");
assert_eq!(check.status, status);
assert!(check.details.contains(detail));
}
}
#[test]
fn collect_configuration_uses_xdg_global_path_and_renders_resolution_branches() {
let temp = tempfile::tempdir().unwrap();
let workspace = temp.path().join("workspace");
let workspace_config = workspace.join(".nemo-relay/config.toml");
std::fs::create_dir_all(workspace_config.parent().unwrap()).unwrap();
std::fs::write(
&workspace_config,
"[upstream]\nopenai_base_url = \"http://local\"\n",
)
.unwrap();
let xdg = temp.path().join("xdg");
let global_config = xdg.join("nemo-relay/config.toml");
std::fs::create_dir_all(global_config.parent().unwrap()).unwrap();
std::fs::write(&global_config, "[upstream\n").unwrap();
let home = temp.path().join("home");
std::fs::create_dir(&home).unwrap();
let _env = EnvScope::set(&[
("XDG_CONFIG_HOME", Some(xdg.as_os_str())),
("HOME", Some(home.as_os_str())),
("USERPROFILE", None),
]);
let configuration = collect_configuration(
Some(&workspace),
Some(&home),
&ResolvedConfig::default(),
&GatewayOverrides::default(),
Check {
name: "Resolution",
status: Status::Warn,
details: "using fallback layer".into(),
},
vec!["codex".into(), "hermes".into()],
&PluginConfigurationDiagnostics {
sources: vec![],
error: None,
resolution: Check {
name: "Plugin resolution",
status: Status::Pass,
details: "valid".into(),
},
},
);
assert_eq!(configuration.workspace.status, Status::Pass);
assert!(configuration.workspace.active);
assert_eq!(configuration.global.path, global_config);
assert_eq!(configuration.global.status, Status::Fail);
assert_eq!(configuration.upstream_auth.openai, SecretPresence::Unset);
assert_eq!(configuration.upstream_auth.anthropic, SecretPresence::Unset);
assert!(configuration.global.details.contains("invalid TOML"));
let mut report = empty_report();
report.configuration = configuration;
let rendered = format_human(&report);
assert!(rendered.contains("Global"));
assert!(rendered.contains("Resolution ! using fallback layer"));
assert!(rendered.contains("Agents codex, hermes"));
}
#[test]
fn agent_helper_statuses_cover_configured_target_and_hook_paths() {
assert_eq!(
crate::process::command_argv("codex --full-auto"),
["codex", "--full-auto"]
);
assert_eq!(
agent_command(CodingAgent::ClaudeCode, &AgentConfigs::default()),
"claude"
);
assert_eq!(
agent_command_status(Some(std::path::Path::new("/bin/codex")), false, true),
Status::Warn
);
assert_eq!(agent_command_status(None, true, false), Status::Fail);
assert_eq!(
combine_status(Status::Pass, Status::Warn, true),
Status::Warn
);
assert_eq!(
combine_status(Status::Pass, Status::Warn, false),
Status::Pass
);
let mut agents = AgentConfigs::default();
agents.hermes.hooks_path = Some(PathBuf::from("/tmp/hermes.yaml"));
assert!(agent_configured(CodingAgent::Hermes, &agents));
assert_eq!(configured_agent_names(&agents), vec!["hermes".to_string()]);
assert_eq!(
hook_status(CodingAgent::ClaudeCode, &agents),
(Status::Pass, "hooks: injected during run".into())
);
assert_eq!(
hook_status(CodingAgent::Codex, &agents),
(Status::Pass, "hooks: injected during run".into())
);
assert_eq!(
hook_status(CodingAgent::Hermes, &AgentConfigs::default()),
(
Status::Pass,
"hooks: injected through an isolated HERMES_HOME during run".into()
)
);
}
#[test]
fn collect_completions_reports_shell_specific_paths() {
let temp = tempfile::tempdir().unwrap();
let zsh_completion = temp.path().join(".zfunc/_nemo-relay");
std::fs::create_dir_all(zsh_completion.parent().unwrap()).unwrap();
std::fs::write(&zsh_completion, "#compdef nemo-relay\n").unwrap();
let _env = EnvScope::set(&[("SHELL", Some(std::ffi::OsStr::new("/bin/zsh")))]);
let checks = collect_completions(Some(temp.path()));
assert_eq!(checks[0].status, Status::Pass);
assert!(checks[0].details.contains("_nemo-relay"));
drop(_env);
let _env = EnvScope::set(&[("SHELL", Some(std::ffi::OsStr::new("/bin/fish")))]);
let checks = collect_completions(Some(temp.path()));
assert_eq!(checks[0].status, Status::Info);
assert!(checks[0].details.contains("nemo-relay.fish"));
drop(_env);
let _env = EnvScope::set(&[("SHELL", None)]);
let checks = collect_completions(Some(temp.path()));
assert_eq!(checks[0].status, Status::Info);
assert!(checks[0].details.contains("no $SHELL"));
}
#[test]
fn collect_environment_and_completions_cover_missing_home_and_unknown_shell() {
let _env = EnvScope::set(&[
("SHELL", Some(std::ffi::OsStr::new("/opt/bin/elvish"))),
("COMSPEC", Some(std::ffi::OsStr::new("C:/opt/bin/elvish"))),
]);
let environment = collect_environment();
assert_eq!(environment.shell.as_deref(), Some("elvish"));
let checks = collect_completions(None);
assert_eq!(checks[0].status, Status::Info);
assert!(checks[0].details.contains("could not resolve home dir"));
drop(_env);
let _env = EnvScope::set(&[("SHELL", Some(std::ffi::OsStr::new("/bin/nu")))]);
let home = tempfile::tempdir().unwrap();
let checks = collect_completions(Some(home.path()));
assert_eq!(checks[0].status, Status::Info);
assert!(checks[0].details.contains("no known completion path"));
}
#[cfg(unix)]
#[tokio::test]
#[allow(clippy::await_holding_lock)]
async fn collect_agents_filters_target_and_records_version() {
let temp = tempfile::tempdir().unwrap();
let _cwd = crate::test_support::CwdTestScope::locked();
let codex = temp.path().join("codex");
std::fs::write(&codex, "#!/bin/sh\nprintf 'codex-cli 0.143.0\\n'\n").unwrap();
make_executable(&codex);
let mut resolved = ResolvedConfig::default();
resolved.agents.codex.command = Some(codex.to_string_lossy().into_owned());
let agents = collect_agents(Some(CodingAgent::Codex), &resolved).await;
assert_eq!(agents.len(), 1);
assert_eq!(agents[0].name, "codex");
assert_eq!(agents[0].status, Status::Pass);
assert_eq!(agents[0].path.as_deref(), Some(codex.as_path()));
assert_eq!(agents[0].version.as_deref(), Some("codex-cli 0.143.0"));
}
#[cfg(unix)]
#[tokio::test]
#[allow(clippy::await_holding_lock)]
async fn collect_agents_preserves_wrapper_argv_for_version_validation() {
let temp = tempfile::tempdir().unwrap();
let _cwd = crate::test_support::CwdTestScope::locked();
let wrapper = temp.path().join("npx");
std::fs::write(
&wrapper,
"#!/bin/sh\n[ \"$1\" = codex ] && [ \"$2\" = --version ] || exit 9\nprintf 'codex-cli 0.143.0\\n'\n",
)
.unwrap();
make_executable(&wrapper);
let mut resolved = ResolvedConfig::default();
resolved.agents.codex.command = Some(format!("{} codex", wrapper.display()));
let agents = collect_agents(Some(CodingAgent::Codex), &resolved).await;
assert_eq!(agents[0].status, Status::Pass);
assert_eq!(agents[0].path.as_deref(), Some(wrapper.as_path()));
assert_eq!(agents[0].version.as_deref(), Some("codex-cli 0.143.0"));
}
#[cfg(unix)]
#[tokio::test]
#[allow(clippy::await_holding_lock)]
async fn collect_agents_distinguishes_required_and_optional_version_failures() {
let temp = tempfile::tempdir().unwrap();
let _cwd = crate::test_support::CwdTestScope::locked();
let codex = temp.path().join("codex");
std::fs::write(&codex, "#!/bin/sh\nprintf 'codex-cli 0.1.0\\n'\n").unwrap();
make_executable(&codex);
let mut configured = ResolvedConfig::default();
configured.agents.codex.command = Some(codex.display().to_string());
let required = collect_agents(Some(CodingAgent::Codex), &configured).await;
assert_eq!(required[0].status, Status::Fail);
assert!(required[0].annotation.contains("is unsupported"));
let _environment = EnvScope::set(&[("PATH", Some(temp.path().as_os_str()))]);
let discovered = collect_agents(None, &ResolvedConfig::default()).await;
let optional = discovered
.iter()
.find(|agent| agent.name == "codex")
.unwrap();
assert_eq!(optional.status, Status::Warn);
assert!(optional.annotation.contains("is unsupported"));
std::fs::write(&codex, "#!/bin/sh\nexit 0\n").unwrap();
make_executable(&codex);
let required = collect_agents(Some(CodingAgent::Codex), &configured).await;
assert_eq!(required[0].status, Status::Fail);
assert!(
required[0]
.annotation
.contains("could not determine version")
);
let discovered = collect_agents(None, &ResolvedConfig::default()).await;
let optional = discovered
.iter()
.find(|agent| agent.name == "codex")
.unwrap();
assert_eq!(optional.status, Status::Warn);
assert!(optional.annotation.contains("could not determine version"));
}
#[test]
fn hermes_hook_status_reports_actionable_persistent_diagnosis_failures() {
let temp = tempfile::tempdir().unwrap();
let mut agents = AgentConfigs::default();
agents.hermes.hooks_path = Some(temp.path().join("missing-config.yaml"));
let (status, details) = hook_status(CodingAgent::Hermes, &agents);
assert_eq!(status, Status::Fail);
assert!(details.contains("persistent MCP/hooks"), "{details}");
assert!(details.contains("install hermes --force"), "{details}");
}
#[cfg(unix)]
#[tokio::test]
async fn probe_version_returns_none_for_empty_output_and_spawn_failures() {
let temp = tempfile::tempdir().unwrap();
let quiet = temp.path().join("quiet-agent");
std::fs::write(&quiet, "#!/bin/sh\nexit 0\n").unwrap();
make_executable(&quiet);
let failed = temp.path().join("failed-agent");
std::fs::write(&failed, "#!/bin/sh\nprintf 'codex-cli 99.0.0\\n'\nexit 7\n").unwrap();
make_executable(&failed);
assert_eq!(
probe_version(&[quiet.display().to_string(), "--version".into()]).await,
None
);
assert_eq!(
probe_version(&[failed.display().to_string(), "--version".into()]).await,
None
);
assert_eq!(
probe_version(&[
temp.path().join("missing-agent").display().to_string(),
"--version".into(),
])
.await,
None
);
}
#[test]
fn configuration_and_path_helpers_cover_direct_paths_and_fallbacks() {
let temp = tempfile::tempdir().unwrap();
let home = temp.path().join("home");
std::fs::create_dir_all(&home).unwrap();
let workspace = temp.path().join("workspace");
std::fs::create_dir_all(workspace.join(".nemo-relay")).unwrap();
std::fs::write(
workspace.join(".nemo-relay").join("config.toml"),
"[agents.codex]\ncommand = \"codex\"\n",
)
.unwrap();
let _env = EnvScope::set(&[
("HOME", Some(home.as_os_str())),
("XDG_CONFIG_HOME", None),
("PATH", None),
]);
let info = collect_configuration(
Some(&workspace),
Some(&home),
&ResolvedConfig {
gateway: GatewayConfig {
openai_auth_header: Some("Bearer openai".into()),
anthropic_auth_header: None,
..GatewayConfig::default()
},
..ResolvedConfig::default()
},
&GatewayOverrides::default(),
Check {
name: "Resolution",
status: Status::Pass,
details: "valid".into(),
},
vec!["codex".into()],
&PluginConfigurationDiagnostics {
sources: vec![],
error: None,
resolution: Check {
name: "Plugin resolution",
status: Status::Pass,
details: "valid".into(),
},
},
);
assert_eq!(info.workspace.status, Status::Pass);
assert!(info.global.path.starts_with(&home));
assert_eq!(info.configured_agents, vec!["codex".to_string()]);
assert_eq!(info.upstream_auth.openai, SecretPresence::Configured);
assert_eq!(info.upstream_auth.anthropic, SecretPresence::Unset);
assert_eq!(
crate::process::resolve_executable("definitely-missing"),
None
);
assert_eq!(
crate::process::resolve_executable("/definitely/missing"),
None
);
let binary = temp
.path()
.join(format!("agent-bin{}", std::env::consts::EXE_SUFFIX));
std::fs::write(&binary, "").unwrap();
assert_eq!(
crate::process::resolve_executable(binary.to_str().unwrap()).as_deref(),
Some(binary.as_path())
);
}
#[test]
fn upstream_auth_info_reports_openai_env_fallback_as_configured() {
let _env = EnvScope::set(&[
("OPENAI_API_KEY", Some(std::ffi::OsStr::new("sk-openai"))),
("ANTHROPIC_API_KEY", None),
]);
let info = UpstreamAuthInfo::from_effective_gateway_auth(&GatewayConfig::default());
assert_eq!(info.openai, SecretPresence::Configured);
assert_eq!(info.anthropic, SecretPresence::Unset);
}
#[test]
fn upstream_auth_info_reports_anthropic_env_fallback_as_configured() {
let _env = EnvScope::set(&[
("OPENAI_API_KEY", None),
(
"ANTHROPIC_API_KEY",
Some(std::ffi::OsStr::new("sk-anthropic")),
),
]);
let info = UpstreamAuthInfo::from_effective_gateway_auth(&GatewayConfig::default());
assert_eq!(info.openai, SecretPresence::Unset);
assert_eq!(info.anthropic, SecretPresence::Configured);
}
#[test]
fn upstream_auth_info_treats_whitespace_env_values_as_unset() {
let _env = EnvScope::set(&[
("OPENAI_API_KEY", Some(std::ffi::OsStr::new(" "))),
("ANTHROPIC_API_KEY", Some(std::ffi::OsStr::new("\t "))),
]);
let info = UpstreamAuthInfo::from_effective_gateway_auth(&GatewayConfig::default());
assert_eq!(info.openai, SecretPresence::Unset);
assert_eq!(info.anthropic, SecretPresence::Unset);
}
#[test]
fn format_human_reports_discovered_dynamic_plugins_in_configuration() {
let mut report = empty_report();
report.configuration.dynamic_plugins = vec![
DynamicPluginReferenceInfo {
plugin_id: "acme.worker".into(),
manifest_ref: "/tmp/plugins/acme/relay-plugin.toml".into(),
source: PathBuf::from("/tmp/plugins.toml"),
host_config_status: DynamicPluginHostConfigStatus::Present,
},
DynamicPluginReferenceInfo {
plugin_id: "acme.native".into(),
manifest_ref: "/tmp/plugins/native/relay-plugin.toml".into(),
source: PathBuf::from("/tmp/plugins.toml"),
host_config_status: DynamicPluginHostConfigStatus::Absent,
},
];
let rendered = format_human(&report);
assert!(rendered.contains("Dynamic"));
assert!(rendered.contains("acme.worker (/tmp/plugins/acme/relay-plugin.toml); host config"));
assert!(rendered.contains("acme.native (/tmp/plugins/native/relay-plugin.toml)"));
assert!(rendered.contains("acme.worker resolved from /tmp/plugins/acme/relay-plugin.toml"));
assert!(
rendered
.contains("acme.native discovered via host config only; not enabled by config alone")
);
assert!(rendered.contains("not enabled by config alone"));
}
#[test]
fn observability_component_helpers_cover_disabled_and_default_paths() {
let plugin = serde_json::json!({
"version": 1,
"components": [{
"kind": OBSERVABILITY_PLUGIN_KIND,
"enabled": true,
"config": {
"version": 3,
"atof": { "enabled": true },
"opentelemetry": {
"enabled": true,
"endpoints": [{
"type": "openinference",
"endpoint": "http://127.0.0.1:1"
}]
}
}
}]
});
let config = observability_component_config(&plugin).unwrap();
assert!(section_enabled(config, "atof"));
assert_eq!(section_output_directory(config, "atof"), None);
assert_eq!(
config["opentelemetry"]["endpoints"][0]["endpoint"],
"http://127.0.0.1:1"
);
assert!(
observability_component_config(&serde_json::json!({
"components": [{ "kind": "other", "config": {} }]
}))
.is_none()
);
assert!(observability_file_exporter_check(config, "missing").is_none());
let default_dir = observability_file_exporter_check(config, "atof").unwrap();
assert_eq!(default_dir.status, Status::Info);
assert!(default_dir.details.contains("runtime default"));
}
#[tokio::test]
async fn opentelemetry_doctor_uses_tcp_probe_for_grpc_endpoints() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let endpoint = format!("http://{}", listener.local_addr().unwrap());
let accept = std::thread::spawn(move || {
let _ = accept_bounded(&listener);
});
let config = serde_json::json!({
"opentelemetry": {
"enabled": true,
"endpoints": [{
"type": "gen_ai",
"transport": "grpc",
"endpoint": endpoint
}]
}
});
let checks = live_observability_http_exporter_checks(&config).await;
assert_eq!(checks.len(), 1);
assert_eq!(checks[0].status, Status::Pass);
assert!(
checks[0]
.details
.contains("live gRPC reachability probe connected to the TCP port")
);
assert!(checks[0].details.contains("OTLP handshake not verified"));
assert!(checks[0].details.contains("endpoints[0] (gen_ai)"));
accept.join().unwrap();
}
#[tokio::test]
async fn opentelemetry_doctor_resolves_bare_http_endpoints_and_warns_on_missing_routes() {
assert!(
live_observability_http_exporter_checks(&serde_json::json!({
"opentelemetry": {"enabled": true, "endpoints": "not-a-list"}
}))
.await
.is_empty()
);
let missing = live_observability_http_exporter_checks(&serde_json::json!({
"opentelemetry": {
"enabled": true,
"endpoints": [{"type": "openinference"}]
}
}))
.await;
assert_eq!(missing.len(), 1);
assert_eq!(missing[0].status, Status::Fail);
assert!(missing[0].details.contains("endpoint is required"));
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let endpoint = format!("http://{}", listener.local_addr().unwrap());
let accept = std::thread::spawn(move || {
let mut stream = accept_bounded(&listener);
let _ = read_headers(&mut stream);
stream
.write_all(b"HTTP/1.1 405 Method Not Allowed\r\nContent-Length: 0\r\n\r\n")
.unwrap();
});
let checks = live_observability_http_exporter_checks(&serde_json::json!({
"opentelemetry": {
"enabled": true,
"endpoints": [{"type": "full", "endpoint": endpoint}]
}
}))
.await;
assert_eq!(checks[0].status, Status::Pass);
assert!(checks[0].details.contains("endpoints[0] (full)"));
assert!(
checks[0]
.details
.contains("/v1/traces (live HTTP reachability probe returned HTTP 405)")
);
accept.join().unwrap();
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let endpoint = format!("http://{}/", listener.local_addr().unwrap());
let accept = std::thread::spawn(move || {
let mut stream = accept_bounded(&listener);
let request = read_headers(&mut stream);
stream
.write_all(b"HTTP/1.1 405 Method Not Allowed\r\nContent-Length: 0\r\n\r\n")
.unwrap();
request
});
let checks = live_observability_http_exporter_checks(&serde_json::json!({
"opentelemetry": {
"enabled": true,
"endpoints": [{"type": "full", "endpoint": endpoint}]
}
}))
.await;
assert_eq!(checks[0].status, Status::Pass);
assert!(
checks[0]
.details
.contains("/ (live HTTP reachability probe returned HTTP 405)")
);
let request = accept.join().unwrap();
assert!(request.starts_with("GET / HTTP/1.1"));
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let endpoint = format!("http://{}/wrong", listener.local_addr().unwrap());
let accept = std::thread::spawn(move || {
let mut stream = accept_bounded(&listener);
let _ = read_headers(&mut stream);
stream
.write_all(b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n")
.unwrap();
});
let checks = live_observability_http_exporter_checks(&serde_json::json!({
"opentelemetry": {
"enabled": true,
"endpoints": [{"type": "full", "endpoint": endpoint}]
}
}))
.await;
assert_eq!(checks[0].status, Status::Warn);
assert!(
checks[0]
.details
.contains("/wrong (live HTTP reachability probe returned HTTP 404)")
);
accept.join().unwrap();
}
#[tokio::test]
async fn opentelemetry_doctor_skips_live_network_probes_offline() {
let checks = offline_observability_http_exporter_checks(&serde_json::json!({
"opentelemetry": {
"enabled": true,
"endpoints": [
{
"type": "gen_ai",
"transport": "grpc",
"endpoint": "http://127.0.0.1:4317"
},
{
"type": "openinference",
"endpoint": "http://127.0.0.1:4318"
}
]
}
}))
.await;
assert_eq!(checks.len(), 2);
assert!(checks.iter().all(|check| check.status == Status::Info));
assert!(checks.iter().all(|check| {
check
.details
.contains("live network probe skipped (--offline)")
}));
}
#[tokio::test]
async fn opentelemetry_doctor_offline_still_rejects_malformed_endpoints() {
let checks = offline_observability_http_exporter_checks(&serde_json::json!({
"opentelemetry": {
"enabled": true,
"endpoints": [
{
"type": "gen_ai",
"transport": "grpc",
"endpoint": "http://:4317"
},
{
"type": "full",
"endpoint": "http://:4318"
}
]
}
}))
.await;
assert_eq!(checks.len(), 2);
assert!(checks[0].details.contains("invalid gRPC endpoint"));
assert!(checks[1].details.contains("invalid OTLP HTTP endpoint"));
assert!(checks.iter().all(|check| check.status == Status::Fail));
}
#[tokio::test]
async fn opentelemetry_doctor_offline_rejects_unsupported_endpoint_schemes() {
let checks = offline_observability_http_exporter_checks(&serde_json::json!({
"opentelemetry": {
"enabled": true,
"endpoints": [
{
"type": "gen_ai",
"transport": "grpc",
"endpoint": "ftp://collector.example:4317"
},
{
"type": "full",
"endpoint": "file:///tmp/collector"
}
]
}
}))
.await;
assert_eq!(checks.len(), 2);
assert_eq!(checks[0].status, Status::Fail);
assert_eq!(checks[1].status, Status::Fail);
assert!(
checks[0]
.details
.contains("gRPC endpoint must use http:// or https://")
);
assert!(
checks[1]
.details
.contains("OTLP HTTP endpoint must use http:// or https://")
);
}
#[test]
fn atof_file_checks_preserve_configured_sink_indices() {
let config = serde_json::json!({
"atof": {
"enabled": true,
"sinks": [
{"type": "stream", "url": "http://127.0.0.1/events"},
{"type": "file"}
]
}
});
let checks = observability_atof_file_checks(&config);
assert_eq!(checks.len(), 1);
assert!(checks[0].details.starts_with("sinks[1]"));
}
#[test]
fn check_directory_reports_pass_warn_and_fail() {
let temp = tempfile::tempdir().unwrap();
let pass = check_directory("ATOF dir", temp.path());
assert_eq!(pass.status, Status::Pass);
let missing = check_directory("ATOF dir", &temp.path().join("missing"));
assert_eq!(missing.status, Status::Warn);
let file = temp.path().join("file");
std::fs::write(&file, "").unwrap();
let fail = check_directory("ATOF dir", &file);
assert_eq!(fail.status, Status::Fail);
}
#[tokio::test]
async fn collect_observability_warns_for_missing_atif_dir_without_creating_it() {
let temp = tempfile::tempdir().unwrap();
let missing = temp.path().join("missing-atif");
let gateway = GatewayConfig {
plugin_config: Some(serde_json::json!({
"version": 1,
"components": [{
"kind": "observability",
"enabled": true,
"config": {
"version": 1,
"atif": {
"enabled": true,
"output_directory": missing
}
}
}]
})),
..GatewayConfig::default()
};
let checks = collect_live_observability(&gateway).await;
let atif_check = checks
.iter()
.find(|check| check.name == "ATIF dir")
.expect("ATIF directory check");
assert_eq!(atif_check.status, Status::Warn);
assert!(!missing.exists());
}
#[tokio::test]
async fn collect_observability_registers_adaptive_before_validation() {
let gateway = GatewayConfig {
plugin_config: Some(serde_json::json!({
"version": 1,
"components": [
{
"kind": "observability",
"enabled": true,
"config": { "version": 1 }
},
{
"kind": "adaptive",
"enabled": false,
"config": {
"policy": {
"unknown_component": "warn",
"unknown_field": "warn",
"unsupported_value": "error"
}
}
}
]
})),
..GatewayConfig::default()
};
let checks = collect_live_observability(&gateway).await;
assert!(
!checks.iter().any(|check| check
.details
.contains("plugin component kind 'adaptive' is unsupported")),
"doctor should register adaptive before plugin validation: {checks:?}"
);
}
#[tokio::test]
async fn collect_observability_reports_response_cache_on_when_configured() {
let gateway = GatewayConfig {
plugin_config: Some(serde_json::json!({
"version": 1,
"components": [
{
"kind": "adaptive",
"enabled": true,
"config": {
"response_cache": {
"ttl_seconds": 3600,
"namespace": "doctor-test",
"backend": { "kind": "in_memory" }
}
}
}
]
})),
..GatewayConfig::default()
};
let checks = collect_live_observability(&gateway).await;
let cache = checks
.iter()
.find(|check| check.name == "Response cache")
.expect("a Response cache check should be present");
assert_eq!(cache.status, Status::Pass, "checks: {checks:?}");
assert!(
cache.details.contains("in_memory") && cache.details.contains("reachable"),
"details: {}",
cache.details
);
}
#[tokio::test]
async fn collect_observability_skips_redis_response_cache_probe_offline() {
let gateway = GatewayConfig {
plugin_config: Some(serde_json::json!({
"version": 1,
"components": [
{
"kind": "adaptive",
"enabled": true,
"config": {
"response_cache": {
"ttl_seconds": 3600,
"namespace": "doctor-test",
"backend": {
"kind": "redis",
"config": {
"url": "redis://127.0.0.1:6379",
"key_prefix": "doctor-test:"
}
}
}
}
}
]
})),
..GatewayConfig::default()
};
let checks = collect_offline_observability(&gateway).await;
let cache = checks
.iter()
.find(|check| check.name == "Response cache")
.expect("a Response cache check should be present");
assert_eq!(cache.status, Status::Info, "checks: {checks:?}");
assert_eq!(
cache.details,
"configured; live redis backend probe skipped (--offline)"
);
}
#[tokio::test]
async fn collect_observability_fails_invalid_redis_response_cache_target_offline() {
let gateway = GatewayConfig {
plugin_config: Some(serde_json::json!({
"version": 1,
"components": [
{
"kind": "adaptive",
"enabled": true,
"config": {
"response_cache": {
"ttl_seconds": 3600,
"namespace": "doctor-test",
"backend": {
"kind": "redis",
"config": {
"url": "not-a-redis-url",
"key_prefix": "doctor-test:"
}
}
}
}
}
]
})),
..GatewayConfig::default()
};
let checks = collect_offline_observability(&gateway).await;
let cache = checks
.iter()
.find(|check| check.name == "Response cache")
.expect("a Response cache check should be present");
assert_eq!(cache.status, Status::Fail, "checks: {checks:?}");
assert!(
cache.details.contains("invalid backend target"),
"details: {}",
cache.details
);
assert!(
cache.details.contains("redis client"),
"details: {}",
cache.details
);
assert!(
!cache.details.contains("probe skipped"),
"must not report skipped for an invalid backend target"
);
}
#[tokio::test(start_paused = true)]
async fn response_cache_backend_check_reports_timeout() {
let check = response_cache_backend_check(std::future::pending()).await;
assert_eq!(check.name, "Response cache");
assert_eq!(check.status, Status::Fail);
assert_eq!(
check.details,
"backend not reachable: probe timed out after 2s"
);
}
#[tokio::test]
async fn response_cache_backend_check_preserves_backend_error() {
let check = response_cache_backend_check(async { Err("connection refused".to_string()) }).await;
assert_eq!(check.name, "Response cache");
assert_eq!(check.status, Status::Fail);
assert_eq!(check.details, "backend not reachable: connection refused");
}
#[tokio::test]
async fn collect_observability_reports_response_cache_not_configured_without_section() {
let gateway = GatewayConfig {
plugin_config: Some(serde_json::json!({
"version": 1,
"components": [
{ "kind": "adaptive", "enabled": true, "config": { "version": 1 } }
]
})),
..GatewayConfig::default()
};
let checks = collect_live_observability(&gateway).await;
let cache = checks
.iter()
.find(|check| check.name == "Response cache")
.expect("a Response cache check should be present");
assert_eq!(cache.status, Status::Info);
assert!(
cache.details.contains("not configured"),
"details: {}",
cache.details
);
}
#[tokio::test]
async fn collect_observability_reports_response_cache_fail_when_config_invalid() {
let gateway = GatewayConfig {
plugin_config: Some(serde_json::json!({
"version": 1,
"components": [
{
"kind": "adaptive",
"enabled": true,
"config": { "response_cache": { "ttl_seconds": 0 } }
}
]
})),
..GatewayConfig::default()
};
let checks = collect_live_observability(&gateway).await;
let cache = checks
.iter()
.find(|check| check.name == "Response cache")
.expect("a Response cache check should be present");
assert_eq!(cache.status, Status::Fail);
assert!(
cache.details.contains("config invalid"),
"details: {}",
cache.details
);
assert!(
!cache.details.contains("reachable"),
"must not claim reachable for an invalid config"
);
}
#[tokio::test]
async fn collect_observability_registers_pii_redaction_before_validation() {
let gateway = GatewayConfig {
plugin_config: Some(serde_json::json!({
"version": 1,
"components": [
{
"kind": "observability",
"enabled": true,
"config": { "version": 1 }
},
{
"kind": "pii_redaction",
"enabled": false,
"config": {
"version": 1,
"mode": "builtin",
"policy": {
"unknown_component": "warn",
"unknown_field": "warn",
"unsupported_value": "error"
},
"builtin": {
"action": "remove"
}
}
}
]
})),
..GatewayConfig::default()
};
let checks = collect_live_observability(&gateway).await;
assert!(
!checks.iter().any(|check| check
.details
.contains("plugin component kind 'pii_redaction' is unsupported")),
"doctor should register pii_redaction before plugin validation: {checks:?}"
);
}
#[tokio::test]
async fn collect_observability_reports_invalid_pii_redaction_config() {
let gateway = GatewayConfig {
plugin_config: Some(serde_json::json!({
"version": 1,
"components": [
{
"kind": "pii_redaction",
"enabled": true,
"config": {
"version": 2,
"mode": "builtin",
"builtin": {
"action": "remove"
}
}
}
]
})),
..GatewayConfig::default()
};
let checks = collect_live_observability(&gateway).await;
let diagnostic = checks
.iter()
.find(|check| check.name == "Plugin diagnostic")
.expect("plugin diagnostic check");
assert_eq!(diagnostic.status, Status::Fail);
assert!(diagnostic.details.contains("unsupported_config_version"));
}
#[tokio::test]
async fn collect_observability_probes_atof_streaming_endpoint() {
let (url, body, server_thread) = start_doctor_http_capture_server();
let gateway = GatewayConfig {
plugin_config: Some(serde_json::json!({
"version": 1,
"components": [{
"kind": "observability",
"enabled": true,
"config": {
"version": 3,
"atof": {
"enabled": true,
"sinks": [{
"type": "stream",
"url": url,
"transport": "http_post",
"headers": {"X-Test": "doctor"}
}]
}
}
}]
})),
..GatewayConfig::default()
};
let checks = collect_live_observability(&gateway).await;
let body = tokio::time::timeout(std::time::Duration::from_secs(2), async {
loop {
let captured = body.lock().unwrap().clone();
if captured.contains("\"kind\":\"mark\"")
&& captured.contains("\"name\":\"nemo_relay.doctor.atof_probe\"")
{
break captured;
}
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
})
.await
.expect("doctor probe body should be captured");
let endpoint = checks
.iter()
.find(|check| check.name == "ATOF stream sink")
.expect("ATOF stream sink check");
assert_eq!(endpoint.status, Status::Pass);
assert!(body.contains("\"kind\":\"mark\""));
assert!(body.contains("\"name\":\"nemo_relay.doctor.atof_probe\""));
server_thread.join().unwrap();
}
#[tokio::test]
async fn collect_observability_covers_absent_invalid_and_componentless_configs() {
let absent = collect_live_observability(&GatewayConfig::default()).await;
assert_eq!(absent[0].status, Status::Info);
assert!(absent[0].details.contains("not configured"));
let invalid = collect_live_observability(&GatewayConfig {
plugin_config: Some(serde_json::json!({"version": "bad"})),
..GatewayConfig::default()
})
.await;
assert_eq!(invalid[0].status, Status::Fail);
assert!(invalid[0].details.contains("invalid plugin config"));
let no_observability = collect_live_observability(&GatewayConfig {
plugin_config: Some(serde_json::json!({
"version": 1,
"components": []
})),
..GatewayConfig::default()
})
.await;
assert!(
no_observability
.iter()
.any(|check| check.name == "Observability plugin"
&& check.details.contains("component not configured"))
);
assert!(
no_observability
.iter()
.any(|check| check.name == "Model pricing" && check.details.contains("not configured"))
);
}
#[tokio::test]
async fn collect_observability_rejects_websocket_endpoint_http_scheme() {
let gateway = GatewayConfig {
plugin_config: Some(serde_json::json!({
"version": 1,
"components": [{
"kind": "observability",
"enabled": true,
"config": {
"version": 3,
"atof": {
"enabled": true,
"sinks": [{
"type": "stream",
"url": "http://localhost:9/events",
"transport": "websocket"
}]
}
}
}]
})),
..GatewayConfig::default()
};
let checks = collect_live_observability(&gateway).await;
let endpoint = checks
.iter()
.find(|check| check.name == "ATOF stream sink")
.expect("ATOF stream sink check");
assert_eq!(endpoint.status, Status::Fail);
assert!(endpoint.details.contains("invalid scheme"));
assert!(endpoint.details.contains("must be ws or wss"));
}
#[tokio::test]
async fn atof_endpoint_validation_rejects_missing_url_headers_timeout_and_transport() {
let missing_url = probe_atof_endpoint(0, &serde_json::json!({})).await;
assert_eq!(missing_url.status, Status::Fail);
assert!(missing_url.details.contains("missing url"));
let timeout_zero = probe_atof_endpoint(
1,
&serde_json::json!({
"url": "http://127.0.0.1:1/events",
"timeout_millis": 0
}),
)
.await;
assert_eq!(timeout_zero.status, Status::Fail);
assert!(timeout_zero.details.contains("timeout_millis"));
let bad_headers = probe_atof_endpoint(
2,
&serde_json::json!({
"url": "http://127.0.0.1:1/events",
"headers": []
}),
)
.await;
assert_eq!(bad_headers.status, Status::Fail);
assert!(bad_headers.details.contains("headers must be an object"));
let non_string_header = probe_atof_endpoint(
3,
&serde_json::json!({
"url": "http://127.0.0.1:1/events",
"headers": {"x-test": 1}
}),
)
.await;
assert_eq!(non_string_header.status, Status::Fail);
assert!(
non_string_header
.details
.contains("headers.x-test must be a string")
);
let blank_static_header = probe_atof_endpoint(
4,
&serde_json::json!({
"url": "http://127.0.0.1:1/events",
"headers": {"x-test": " "}
}),
)
.await;
assert_eq!(blank_static_header.status, Status::Fail);
assert!(
blank_static_header
.details
.contains("headers.x-test must not be blank")
);
let _env = EnvScope::set(&[(
"NEMO_RELAY_INVALID_ATOF_HEADER",
Some(std::ffi::OsStr::new("bad\nvalue")),
)]);
let invalid_header_env = probe_atof_endpoint(
5,
&serde_json::json!({
"url": "http://127.0.0.1:1/events",
"header_env": {"x-test": "NEMO_RELAY_INVALID_ATOF_HEADER"}
}),
)
.await;
assert_eq!(invalid_header_env.status, Status::Fail);
assert!(
invalid_header_env
.details
.contains("header_env.x-test invalid")
);
let mixed_case_duplicate = probe_atof_endpoint(
6,
&serde_json::json!({
"url": "http://127.0.0.1:1/events",
"headers": {"Authorization": "Bearer literal"},
"header_env": {"authorization": "PATH"}
}),
)
.await;
assert_eq!(mixed_case_duplicate.status, Status::Fail);
assert!(
mixed_case_duplicate
.details
.contains("cannot appear in both headers and header_env")
);
let duplicate_static_header = probe_atof_endpoint(
7,
&serde_json::json!({
"url": "http://127.0.0.1:1/events",
"headers": {
"Authorization": "Bearer a",
"authorization": "Bearer b"
}
}),
)
.await;
assert_eq!(duplicate_static_header.status, Status::Fail);
assert!(
duplicate_static_header
.details
.contains("appears more than once")
);
let unsupported = probe_atof_endpoint(
8,
&serde_json::json!({
"url": "http://127.0.0.1:1/events",
"transport": "grpc"
}),
)
.await;
assert_eq!(unsupported.status, Status::Fail);
assert!(unsupported.details.contains("unsupported transport"));
}
#[tokio::test]
async fn atof_endpoint_offline_skips_live_network_probe_after_validation() {
let skipped = offline_atof_endpoint(
0,
&serde_json::json!({
"url": "http://127.0.0.1:1/events",
"transport": "http_post"
}),
)
.await;
assert_eq!(skipped.status, Status::Info);
assert_eq!(
skipped.details,
"sinks[0] http_post http://127.0.0.1:1/events: live network probe skipped (--offline)"
);
}
#[tokio::test]
async fn atof_endpoint_offline_still_rejects_invalid_transport_and_scheme() {
let invalid_websocket = offline_atof_endpoint(
0,
&serde_json::json!({
"url": "http://127.0.0.1:1/events",
"transport": "websocket"
}),
)
.await;
assert_eq!(invalid_websocket.status, Status::Fail);
assert!(
invalid_websocket
.details
.contains("invalid scheme (must be ws or wss)")
);
let unsupported_transport = offline_atof_endpoint(
1,
&serde_json::json!({
"url": "http://127.0.0.1:1/events",
"transport": "udp"
}),
)
.await;
assert_eq!(unsupported_transport.status, Status::Fail);
assert!(
unsupported_transport
.details
.contains("unsupported transport")
);
let blank_static_header = offline_atof_endpoint(
2,
&serde_json::json!({
"url": "http://127.0.0.1:1/events",
"headers": {"x-test": " "}
}),
)
.await;
assert_eq!(blank_static_header.status, Status::Fail);
assert!(
blank_static_header
.details
.contains("headers.x-test must not be blank")
);
let duplicate_static_header = offline_atof_endpoint(
3,
&serde_json::json!({
"url": "http://127.0.0.1:1/events",
"headers": {
"Authorization": "Bearer a",
"authorization": "Bearer b"
}
}),
)
.await;
assert_eq!(duplicate_static_header.status, Status::Fail);
assert!(
duplicate_static_header
.details
.contains("appears more than once")
);
}
#[tokio::test]
async fn atof_http_and_websocket_probes_report_failure_branches() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let url = format!("http://{}", listener.local_addr().unwrap());
let handle = std::thread::spawn(move || {
let mut stream = accept_bounded(&listener);
let mut buf = [0_u8; 1024];
let _ = stream.read(&mut buf).unwrap();
stream
.write_all(b"HTTP/1.1 500 Internal Server Error\r\nContent-Length: 0\r\n\r\n")
.unwrap();
});
let failed_http = probe_atof_http_post(
&url,
Vec::new(),
doctor_atof_probe_payload().unwrap(),
std::time::Duration::from_secs(2),
5,
)
.await;
assert_eq!(failed_http.status, Status::Fail);
assert!(failed_http.details.contains("HTTP 500"));
handle.join().unwrap();
let bad_header_name = probe_atof_websocket(
"ws://127.0.0.1:1/events",
vec![("bad header".to_string(), "value".to_string())],
"{}".to_string(),
std::time::Duration::from_millis(10),
6,
)
.await;
assert_eq!(bad_header_name.status, Status::Fail);
let bad_header_value = probe_atof_websocket(
"ws://127.0.0.1:1/events",
vec![("x-test".to_string(), "bad\r\nvalue".to_string())],
"{}".to_string(),
std::time::Duration::from_millis(10),
7,
)
.await;
assert_eq!(bad_header_value.status, Status::Fail);
let bad_websocket_url = probe_atof_websocket(
"ws://[",
Vec::new(),
"{}".to_string(),
std::time::Duration::from_millis(10),
8,
)
.await;
assert_eq!(bad_websocket_url.status, Status::Fail);
}
#[tokio::test]
async fn atof_http_and_websocket_timeout_errors_are_reported() {
let http_listener = TcpListener::bind("127.0.0.1:0").unwrap();
let http_url = format!("http://{}", http_listener.local_addr().unwrap());
let http_handle = std::thread::spawn(move || {
let _stream = accept_bounded(&http_listener);
std::thread::sleep(std::time::Duration::from_millis(75));
});
let http_timeout = probe_atof_http_post(
&http_url,
Vec::new(),
doctor_atof_probe_payload().unwrap(),
std::time::Duration::from_millis(10),
9,
)
.await;
assert_eq!(http_timeout.status, Status::Fail);
assert!(http_timeout.details.contains("http_post"));
assert!(
http_timeout
.details
.to_ascii_lowercase()
.contains("timeout")
|| http_timeout
.details
.to_ascii_lowercase()
.contains("timed out")
|| http_timeout.details.contains("error sending request"),
"timeout detail was: {}",
http_timeout.details
);
http_handle.join().unwrap();
let ws_listener = TcpListener::bind("127.0.0.1:0").unwrap();
let ws_url = format!("ws://{}", ws_listener.local_addr().unwrap());
let ws_handle = std::thread::spawn(move || {
let _stream = accept_bounded(&ws_listener);
std::thread::sleep(std::time::Duration::from_millis(75));
});
let websocket_timeout = probe_atof_websocket(
&ws_url,
Vec::new(),
"{}".to_string(),
std::time::Duration::from_millis(10),
10,
)
.await;
assert_eq!(websocket_timeout.status, Status::Fail);
assert!(websocket_timeout.details.contains("timed out"));
ws_handle.join().unwrap();
}
#[tokio::test]
async fn otlp_http_probe_warns_on_http_errors() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let url = format!("http://{}", listener.local_addr().unwrap());
let handle = std::thread::spawn(move || {
let mut stream = accept_bounded(&listener);
let mut buf = [0_u8; 1024];
let _ = stream.read(&mut buf).unwrap();
stream
.write_all(b"HTTP/1.1 500 Internal Server Error\r\nContent-Length: 0\r\n\r\n")
.unwrap();
});
let check = probe_otlp_http_named("OpenTelemetry endpoint", &url).await;
assert_eq!(check.status, Status::Warn);
assert!(check.details.contains("HTTP 500"));
handle.join().unwrap();
}
#[tokio::test]
async fn otlp_http_probe_passes_success_method_not_allowed_and_ndjson_upload_success() {
let success_listener = TcpListener::bind("127.0.0.1:0").unwrap();
let success_url = format!("http://{}", success_listener.local_addr().unwrap());
let success_handle = std::thread::spawn(move || {
let mut stream = accept_bounded(&success_listener);
let mut buf = [0_u8; 1024];
let _ = stream.read(&mut buf).unwrap();
stream
.write_all(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\n\r\n")
.unwrap();
});
let check = probe_otlp_http_named("OpenTelemetry endpoint", &success_url).await;
assert_eq!(check.status, Status::Pass);
success_handle.join().unwrap();
let method_listener = TcpListener::bind("127.0.0.1:0").unwrap();
let method_url = format!("http://{}", method_listener.local_addr().unwrap());
let method_handle = std::thread::spawn(move || {
let mut stream = accept_bounded(&method_listener);
let mut buf = [0_u8; 1024];
let _ = stream.read(&mut buf).unwrap();
stream
.write_all(b"HTTP/1.1 405 Method Not Allowed\r\nContent-Length: 0\r\n\r\n")
.unwrap();
});
let check = probe_otlp_http_named("OpenTelemetry endpoint", &method_url).await;
assert_eq!(check.status, Status::Pass);
assert!(check.details.contains("HTTP 405"));
method_handle.join().unwrap();
let (url, body, server_thread) = start_doctor_http_capture_server();
let check = probe_atof_ndjson(
&url,
Vec::new(),
doctor_atof_probe_payload().unwrap(),
std::time::Duration::from_secs(2),
11,
)
.await;
assert_eq!(check.status, Status::Pass);
assert!(
body.lock()
.unwrap()
.contains("nemo_relay.doctor.atof_probe")
);
server_thread.join().unwrap();
}
#[tokio::test]
async fn collect_observability_validates_pricing_file_source() {
let temp = tempfile::tempdir().unwrap();
let catalog = temp.path().join("pricing.json");
std::fs::write(
&catalog,
serde_json::json!({
"version": 1,
"entries": [{
"provider": "openai",
"model_id": "gpt-test",
"currency": "USD",
"unit": "per_token",
"rates": {
"input_per_million": 1.0,
"output_per_million": 2.0
},
"prompt_cache": {
"read_accounting": "separate"
},
"pricing_as_of": "2026-06-06",
"pricing_source": "test"
}]
})
.to_string(),
)
.unwrap();
let gateway = GatewayConfig {
plugin_config: Some(serde_json::json!({
"version": 1,
"components": [{
"kind": "pricing",
"config": {
"sources": [{
"type": "file",
"path": catalog
}]
}
}]
})),
..GatewayConfig::default()
};
let checks = collect_live_observability(&gateway).await;
let pricing = checks
.iter()
.find(|check| check.name == "Model pricing source")
.expect("model pricing source check");
assert_eq!(pricing.status, Status::Pass);
assert!(pricing.details.contains("valid (1 entries)"));
}
#[tokio::test]
async fn collect_observability_fails_for_missing_pricing_file_source() {
let missing = tempfile::tempdir()
.unwrap()
.path()
.join("missing-pricing.json");
let gateway = GatewayConfig {
plugin_config: Some(serde_json::json!({
"version": 1,
"components": [{
"kind": "pricing",
"config": {
"sources": [{
"type": "file",
"path": missing
}]
}
}]
})),
..GatewayConfig::default()
};
let checks = collect_live_observability(&gateway).await;
let pricing = checks
.iter()
.find(|check| check.name == "Model pricing source")
.expect("model pricing source check");
assert_eq!(pricing.status, Status::Fail);
assert!(pricing.details.contains("unreadable"));
}
#[tokio::test]
async fn collect_observability_reports_pricing_disabled_empty_inline_and_invalid_catalogs() {
let disabled_config: PluginConfig = serde_json::from_value(serde_json::json!({
"version": 1,
"components": [{
"kind": "pricing",
"enabled": false,
"config": {}
}]
}))
.unwrap();
let mut checks = Vec::new();
collect_pricing_component_checks(&mut checks, &disabled_config);
assert_eq!(checks[0].status, Status::Info);
assert!(checks[0].details.contains("disabled"));
let empty_config: PluginConfig = serde_json::from_value(serde_json::json!({
"version": 1,
"components": [{
"kind": "pricing",
"config": {"sources": []}
}]
}))
.unwrap();
let mut checks = Vec::new();
collect_pricing_component_checks(&mut checks, &empty_config);
assert_eq!(checks[0].status, Status::Info);
assert!(checks[0].details.contains("no sources"));
let inline_config: PluginConfig = serde_json::from_value(serde_json::json!({
"version": 1,
"components": [{
"kind": "pricing",
"config": {
"sources": [{
"type": "inline",
"catalog": {
"version": 1,
"entries": [{
"provider": "test",
"model_id": "model-a",
"rates": {"input_per_million": 1.0, "output_per_million": 2.0},
"prompt_cache": {"read_accounting": "separate"},
"pricing_as_of": "2026-06-06",
"pricing_source": "unit"
}]
}
}]
}
}]
}))
.unwrap();
let mut checks = Vec::new();
collect_pricing_component_checks(&mut checks, &inline_config);
assert_eq!(checks[0].status, Status::Pass);
assert!(checks[0].details.contains("inline:0 valid (1 entries)"));
let temp = tempfile::tempdir().unwrap();
let invalid = temp.path().join("pricing.json");
std::fs::write(&invalid, r#"{"version":1,"entries":[{"bad":true}]}"#).unwrap();
let invalid_check = pricing_source_check(9, &PricingSourceConfig::File { path: invalid });
assert_eq!(invalid_check.status, Status::Fail);
assert!(invalid_check.details.contains("invalid catalog"));
}
#[cfg(unix)]
fn make_executable(path: &std::path::Path) {
use std::os::unix::fs::PermissionsExt;
let mut permissions = std::fs::metadata(path).unwrap().permissions();
permissions.set_mode(0o755);
std::fs::set_permissions(path, permissions).unwrap();
}
#[test]
fn format_agents_human_lists_supported_and_separates_detected() {
let agents = vec![
AgentInfo {
name: "claude",
status: Status::Pass,
configured: true,
command: "claude".into(),
path: Some(PathBuf::from("/opt/homebrew/bin/claude")),
version: Some("2.1.4".into()),
annotation: "hooks: injected during run".into(),
},
AgentInfo {
name: "codex",
status: Status::Info,
configured: false,
command: "codex".into(),
path: None,
version: None,
annotation: "not configured".into(),
},
];
let rendered = format_agents_human(&agents);
assert!(rendered.contains("Supported"));
assert!(rendered.contains("Detected on this machine"));
assert!(rendered.contains("claude\n"));
assert!(rendered.contains("codex\n"));
assert!(rendered.contains("/opt/homebrew/bin/claude"));
let detected_block = rendered.split("Detected on this machine").nth(1).unwrap();
assert!(!detected_block.contains("codex"));
}
#[test]
fn format_agents_json_matches_doctor_agents_shape() {
let agents = vec![AgentInfo {
name: "claude",
status: Status::Pass,
configured: true,
command: "claude".into(),
path: Some(PathBuf::from("/opt/homebrew/bin/claude")),
version: Some("2.1.4".into()),
annotation: "hooks: injected during run".into(),
}];
let json = format_agents_json(&agents).unwrap();
let parsed: serde_json::Value = serde_json::from_str(&json).unwrap();
assert!(parsed.is_array());
assert_eq!(parsed[0]["name"], "claude");
assert_eq!(parsed[0]["status"], "pass");
assert_eq!(parsed[0]["configured"], true);
assert_eq!(parsed[0]["command"], "claude");
assert_eq!(parsed[0]["version"], "2.1.4");
assert_eq!(parsed[0]["path"], "/opt/homebrew/bin/claude");
}