use crate::agent::{AgentKind, install_hook};
use crate::commands::{
DoctorCheck, DoctorCheckStatus, DoctorFixApplied, DoctorResponse, DoctorSummary, config_dir,
load_workers_from_config, send_daemon_command,
};
use crate::state::primitives::IdempotentResult;
use crate::status_display::query_daemon_full_status;
use crate::status_types::{
DaemonFullStatusResponse, RepoConvergenceStatusFromApi, WorkerStatusFromApi, extract_json_body,
};
use crate::ui::context::OutputContext;
use crate::ui::theme::StatusIndicator;
use anyhow::Result;
use directories::ProjectDirs;
use rch_common::rsync_flavor::{ResolvedRsync, RsyncFlavor, RsyncResolveError, resolve_rsync};
use rch_common::{ApiResponse, ReliabilityReasonCode};
use rch_telemetry::TelemetryStorage;
use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, BTreeSet};
use std::io::{self, Read, Write};
#[cfg(unix)]
use std::os::unix::fs::PermissionsExt;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::time::{Duration, Instant};
use which::which;
fn default_socket_path() -> PathBuf {
PathBuf::from(rch_common::default_socket_path())
}
fn configured_or_default_socket_path() -> PathBuf {
crate::commands::configured_socket_path()
.map(PathBuf::from)
.unwrap_or_else(|_| default_socket_path())
}
const MAX_CONFIG_FILE_BYTES: u64 = 16 * 1024 * 1024;
fn read_config_capped(path: &Path) -> std::io::Result<String> {
let file = std::fs::File::open(path)?;
let metadata = file.metadata()?;
if metadata.len() > MAX_CONFIG_FILE_BYTES {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
format!(
"config file {} is {} bytes, exceeds {}-byte cap",
path.display(),
metadata.len(),
MAX_CONFIG_FILE_BYTES
),
));
}
read_config_capped_from_reader(
file,
MAX_CONFIG_FILE_BYTES,
&format!("config file {}", path.display()),
)
}
fn read_config_capped_from_reader<R: Read>(
reader: R,
max_bytes: u64,
source: &str,
) -> std::io::Result<String> {
let mut limited = reader.take(max_bytes.saturating_add(1));
let mut bytes = Vec::new();
limited.read_to_end(&mut bytes)?;
let max_bytes_usize = usize::try_from(max_bytes).unwrap_or(usize::MAX);
if bytes.len() > max_bytes_usize {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
format!("{source} exceeds {max_bytes}-byte cap"),
));
}
String::from_utf8(bytes)
.map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidData, err))
}
type CheckResult = DoctorCheck;
type CheckStatus = DoctorCheckStatus;
type FixApplied = DoctorFixApplied;
macro_rules! print {
($($arg:tt)*) => {{
write_stdout(format_args!($($arg)*), false);
}};
}
macro_rules! println {
() => {{
write_stdout(format_args!(""), true);
}};
($($arg:tt)*) => {{
write_stdout(format_args!($($arg)*), true);
}};
}
fn write_stdout(args: std::fmt::Arguments<'_>, newline: bool) {
let stdout = io::stdout();
let mut out = stdout.lock();
let result = out.write_fmt(args).and_then(|()| {
if newline {
out.write_all(b"\n")
} else {
Ok(())
}
});
if let Err(err) = result {
if err.kind() == io::ErrorKind::BrokenPipe {
std::process::exit(0);
}
let _ = writeln!(
io::stderr().lock(),
"rch doctor: failed to write output: {err}"
);
std::process::exit(1);
}
}
pub struct DoctorOptions {
pub fix: bool,
pub dry_run: bool,
pub reliability: bool,
pub check_schemas: bool,
pub verbose: bool,
pub strict: bool,
pub lenient: bool,
pub scope: ReliabilityScopeSet,
pub watch: bool,
pub watch_interval_secs: u64,
pub transitions_only: bool,
pub watch_snapshot: Option<std::path::PathBuf>,
}
const RELIABILITY_DOCTOR_SCHEMA_VERSION: &str =
rch_common::schema_version(rch_common::SchemaComponent::DoctorReliability);
const EXPECTED_RELIABILITY_DOCTOR_SCHEMA_VERSION: &str = "1.0.0";
const EXPECTED_STATUS_SCHEMA_VERSION: &str = "1.0.0";
const EXPECTED_REPO_UPDATER_CONTRACT_SCHEMA_VERSION: &str = "1.0.0";
const EXPECTED_PROCESS_TRIAGE_CONTRACT_SCHEMA_VERSION: &str = "1.0.0";
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize)]
#[serde(rename_all = "snake_case")]
enum ReliabilitySeverity {
Pass,
Info,
Warning,
Critical,
}
fn reliability_severity_token(severity: ReliabilitySeverity) -> &'static str {
match severity {
ReliabilitySeverity::Pass => "pass",
ReliabilitySeverity::Info => "info",
ReliabilitySeverity::Warning => "warning",
ReliabilitySeverity::Critical => "critical",
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ReliabilityVerdict {
Healthy,
Degraded,
Failing,
}
impl ReliabilityVerdict {
#[must_use]
pub const fn default_exit_code(self) -> i32 {
match self {
Self::Healthy => 0,
Self::Degraded => 1,
Self::Failing => 2,
}
}
#[must_use]
pub const fn exit_code(self, strict: bool, lenient: bool) -> i32 {
if strict {
match self {
Self::Healthy => 0,
Self::Degraded | Self::Failing => 2,
}
} else if lenient {
match self {
Self::Healthy => 0,
Self::Degraded | Self::Failing => 1,
}
} else {
self.default_exit_code()
}
}
#[must_use]
pub const fn label(self) -> &'static str {
match self {
Self::Healthy => "Healthy",
Self::Degraded => "Degraded",
Self::Failing => "Failing",
}
}
}
fn aggregate_verdict(diagnostics: &[ReliabilityDiagnostic]) -> ReliabilityVerdict {
let mut has_warning = false;
for diag in diagnostics {
match diag.severity {
ReliabilitySeverity::Critical => return ReliabilityVerdict::Failing,
ReliabilitySeverity::Warning => has_warning = true,
ReliabilitySeverity::Pass | ReliabilitySeverity::Info => {}
}
}
if has_warning {
ReliabilityVerdict::Degraded
} else {
ReliabilityVerdict::Healthy
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize)]
#[serde(rename_all = "snake_case")]
enum ReliabilityCategory {
Topology,
RepoPresence,
DiskPressure,
ProcessDebt,
HelperCompatibility,
RolloutPosture,
SchemaCompatibility,
MirrorOwnership,
}
impl ReliabilityCategory {
fn as_str(self) -> &'static str {
match self {
Self::Topology => "topology",
Self::RepoPresence => "repo_presence",
Self::DiskPressure => "disk_pressure",
Self::ProcessDebt => "process_debt",
Self::HelperCompatibility => "helper_compatibility",
Self::RolloutPosture => "rollout_posture",
Self::SchemaCompatibility => "schema_compatibility",
Self::MirrorOwnership => "mirror_ownership",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum ReliabilityScope {
All,
Topology,
Ownership,
Convergence,
Pressure,
Triage,
Helpers,
Rollout,
Schema,
}
impl ReliabilityScope {
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::All => "all",
Self::Topology => "topology",
Self::Ownership => "ownership",
Self::Convergence => "convergence",
Self::Pressure => "pressure",
Self::Triage => "triage",
Self::Helpers => "helpers",
Self::Rollout => "rollout",
Self::Schema => "schema",
}
}
}
impl std::str::FromStr for ReliabilityScope {
type Err = String;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let trimmed = s.trim().to_ascii_lowercase();
match trimmed.as_str() {
"all" => Ok(Self::All),
"topology" => Ok(Self::Topology),
"ownership" => Ok(Self::Ownership),
"convergence" => Ok(Self::Convergence),
"pressure" => Ok(Self::Pressure),
"triage" => Ok(Self::Triage),
"helpers" => Ok(Self::Helpers),
"rollout" => Ok(Self::Rollout),
"schema" => Ok(Self::Schema),
other => Err(format!(
"unknown scope value '{other}' (valid: all, topology, ownership, convergence, pressure, triage, helpers, rollout, schema)"
)),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ReliabilityScopeSet(pub Vec<ReliabilityScope>);
impl Default for ReliabilityScopeSet {
fn default() -> Self {
Self(vec![ReliabilityScope::All])
}
}
impl ReliabilityScopeSet {
#[must_use]
pub fn matches(&self, scope: ReliabilityScope) -> bool {
self.0
.iter()
.any(|s| matches!(s, ReliabilityScope::All) || *s == scope)
}
#[must_use]
pub fn as_strings(&self) -> Vec<String> {
self.0.iter().map(|s| s.as_str().to_string()).collect()
}
fn needs_worker_config(&self) -> bool {
self.matches(ReliabilityScope::Topology) || self.matches(ReliabilityScope::Ownership)
}
fn needs_daemon_status(&self) -> bool {
[
ReliabilityScope::Topology,
ReliabilityScope::Pressure,
ReliabilityScope::Triage,
]
.into_iter()
.any(|scope| self.matches(scope))
}
fn needs_repo_convergence_status(&self) -> bool {
self.matches(ReliabilityScope::Convergence)
}
fn needs_rollout_config(&self) -> bool {
self.matches(ReliabilityScope::Rollout)
}
fn runs_schema_probe(&self, check_schemas: bool) -> bool {
self.0.contains(&ReliabilityScope::Schema)
|| (check_schemas && self.matches(ReliabilityScope::Schema))
}
fn probe_names_to_run(&self, check_schemas: bool) -> Vec<&'static str> {
let mut names = Vec::new();
for (scope, name) in [
(ReliabilityScope::Topology, "topology"),
(ReliabilityScope::Ownership, "ownership"),
(ReliabilityScope::Convergence, "convergence"),
(ReliabilityScope::Pressure, "pressure"),
(ReliabilityScope::Triage, "triage"),
(ReliabilityScope::Helpers, "helpers"),
(ReliabilityScope::Rollout, "rollout"),
] {
if self.matches(scope) {
names.push(name);
}
}
if self.runs_schema_probe(check_schemas) {
names.push("schema");
}
names
}
}
impl std::str::FromStr for ReliabilityScopeSet {
type Err = String;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let trimmed = s.trim();
if trimmed.is_empty() {
return Err("scope list is empty (pass --scope=all or one of: topology, ownership, convergence, pressure, triage, helpers, rollout, schema)".to_string());
}
let mut out: Vec<ReliabilityScope> = Vec::new();
for segment in trimmed.split(',') {
let scope: ReliabilityScope = segment.parse()?;
if !out.contains(&scope) {
out.push(scope);
}
}
if out.contains(&ReliabilityScope::All) && out.len() > 1 {
let redundant: Vec<String> = out
.iter()
.filter(|s| **s != ReliabilityScope::All)
.map(|s| s.as_str().to_string())
.collect();
tracing::info!(
target: "rch::doctor::scope",
redundant = ?redundant,
"doctor.scope.all_dominates_redundant",
);
out = vec![ReliabilityScope::All];
}
Ok(Self(out))
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
enum ReliabilityDoctorMode {
Check,
DryRun,
Fix,
}
#[derive(Debug, Clone, Serialize)]
struct ReliabilityDiagnostic {
category: ReliabilityCategory,
check_name: String,
severity: ReliabilitySeverity,
message: String,
code: ReliabilityReasonCode,
#[serde(skip_serializing_if = "Option::is_none")]
details: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
worker_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
remediation_command: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
validation_check: Option<String>,
dry_run_safe: bool,
}
#[derive(Debug, Clone, Serialize)]
struct ReliabilityDoctorSummary {
total_checks: usize,
pass: usize,
info: usize,
warning: usize,
critical: usize,
categories_checked: Vec<ReliabilityCategory>,
overall: ReliabilityVerdict,
}
#[derive(Debug, Clone, Serialize)]
struct ReliabilityRemediationStep {
order: u32,
category: ReliabilityCategory,
code: ReliabilityReasonCode,
description: String,
command: String,
validation: String,
requires_restart: bool,
dry_run_safe: bool,
auto_fixable: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct AutoConfigFlip {
key: &'static str,
value: &'static str,
}
fn auto_config_flip(code: ReliabilityReasonCode) -> Option<AutoConfigFlip> {
match code {
ReliabilityReasonCode::HookAutoStartDisabled => Some(AutoConfigFlip {
key: "self_healing.hook_starts_daemon",
value: "true",
}),
ReliabilityReasonCode::DaemonHookRepairDisabled => Some(AutoConfigFlip {
key: "self_healing.daemon_installs_hooks",
value: "true",
}),
_ => None,
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum AutoFlipPlan {
AlreadySatisfied,
WouldApply,
Apply,
}
fn plan_auto_flip(already: bool, preview: bool) -> AutoFlipPlan {
if already {
AutoFlipPlan::AlreadySatisfied
} else if preview {
AutoFlipPlan::WouldApply
} else {
AutoFlipPlan::Apply
}
}
fn config_flip_satisfied(config: &rch_common::RchConfig, flip: AutoConfigFlip) -> bool {
match flip.key {
"self_healing.hook_starts_daemon" => {
config.self_healing.hook_starts_daemon.to_string() == flip.value
}
"self_healing.daemon_installs_hooks" => {
config.self_healing.daemon_installs_hooks.to_string() == flip.value
}
_ => false,
}
}
#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
enum RemediationOutcomeStatus {
Applied,
AlreadySatisfied,
WouldApply,
Failed,
Manual,
}
impl RemediationOutcomeStatus {
fn label(self) -> &'static str {
match self {
RemediationOutcomeStatus::Applied => "applied",
RemediationOutcomeStatus::AlreadySatisfied => "already-satisfied",
RemediationOutcomeStatus::WouldApply => "would-apply",
RemediationOutcomeStatus::Failed => "failed",
RemediationOutcomeStatus::Manual => "manual",
}
}
}
#[derive(Debug, Clone, Serialize)]
struct ReliabilityRemediationOutcome {
order: u32,
code: ReliabilityReasonCode,
description: String,
command: String,
status: RemediationOutcomeStatus,
#[serde(skip_serializing_if = "Option::is_none")]
detail: Option<String>,
}
#[derive(Debug, Clone, Serialize)]
struct ReliabilityDoctorResponse {
schema_version: String,
mode: ReliabilityDoctorMode,
scope: Vec<String>,
daemon_unreachable: bool,
daemon_unreachable_reasons: Vec<String>,
diagnostics: Vec<ReliabilityDiagnostic>,
summary: ReliabilityDoctorSummary,
remediation_plan: Vec<ReliabilityRemediationStep>,
fix_requested: bool,
#[serde(skip_serializing_if = "Vec::is_empty")]
remediation_outcomes: Vec<ReliabilityRemediationOutcome>,
}
impl ReliabilityDiagnostic {
fn new(
category: ReliabilityCategory,
check_name: impl Into<String>,
severity: ReliabilitySeverity,
message: impl Into<String>,
code: ReliabilityReasonCode,
) -> Self {
Self {
category,
check_name: check_name.into(),
severity,
message: message.into(),
code,
details: None,
worker_id: None,
remediation_command: None,
validation_check: None,
dry_run_safe: true,
}
}
fn with_details(mut self, details: impl Into<String>) -> Self {
self.details = Some(details.into());
self
}
fn with_worker(mut self, worker_id: impl Into<String>) -> Self {
self.worker_id = Some(worker_id.into());
self
}
fn with_remediation(
mut self,
command: impl Into<String>,
validation_check: impl Into<String>,
) -> Self {
self.remediation_command = Some(command.into());
self.validation_check = Some(validation_check.into());
self
}
}
pub(crate) fn local_build_check(
observation: &crate::local_builds::LocalBuildObservation,
) -> CheckResult {
let mut details: Vec<String> = observation
.builds
.iter()
.map(|build| {
format!(
"pid={} comm={} exe={}",
build.pid,
build.comm,
build
.exe
.as_ref()
.map_or_else(|| "unknown".to_string(), |path| path.display().to_string())
)
})
.collect();
if let Some(error) = &observation.scan_error {
details.push(format!("Scan: {error}"));
}
if let Some(error) = &observation.state_error {
details.push(format!("Alarm state: {error}"));
}
let warning = observation.warning();
CheckResult {
category: "local_builds".to_string(),
name: "dispatcher_local_builds".to_string(),
status: if warning.is_some() { CheckStatus::Warning } else { CheckStatus::Pass },
message: warning.unwrap_or_else(|| "No unmanaged local builds on this dispatcher".to_string()),
details: (!details.is_empty()).then(|| details.join("; ")),
suggestion: (!observation.builds.is_empty()).then(||
"Run rch shim status; check PATH order and absolute-path toolchain Cargo invocations".to_string()),
fixable: false,
fix_applied: false,
fix_message: None,
}
}
fn dispatcher_shim_check(role: rch_common::BoxRole, problems: Vec<String>) -> CheckResult {
let required = role == rch_common::BoxRole::Dispatcher;
let unhealthy = required && !problems.is_empty();
CheckResult {
category: "configuration".to_string(),
name: "dispatcher_shim".to_string(),
status: if unhealthy { CheckStatus::Warning } else { CheckStatus::Pass },
message: if unhealthy {
"role=dispatcher: Cargo interception needs repair".to_string()
} else if required {
"role=dispatcher: Cargo shims are current and effective on PATH".to_string()
} else {
format!("role={}: dispatcher shims are not required", role.as_str())
},
details: unhealthy.then(|| problems.join("; ")),
suggestion: unhealthy.then(||
"Run rch shim install and put ~/.rch/shims first on PATH; restart existing agent shells".to_string()),
fixable: false,
fix_applied: false,
fix_message: None,
}
}
fn reliability_local_build_diagnostics(
observation: Option<&crate::local_builds::LocalBuildObservation>,
) -> Vec<ReliabilityDiagnostic> {
let Some(observation) = observation else {
return Vec::new();
};
let check = local_build_check(observation);
let code = if !observation.builds.is_empty() {
ReliabilityReasonCode::LocalBuildsDetected
} else if observation.scan_error.is_some() {
ReliabilityReasonCode::LocalBuildScanUnavailable
} else if observation.state_error.is_some() {
ReliabilityReasonCode::LocalBuildAlarmStateUnavailable
} else {
ReliabilityReasonCode::LocalBuildsAbsent
};
let mut diagnostic = ReliabilityDiagnostic::new(
ReliabilityCategory::ProcessDebt,
check.name,
if check.status == CheckStatus::Warning {
ReliabilitySeverity::Warning
} else {
ReliabilitySeverity::Pass
},
check.message,
code,
);
diagnostic.details = check.details;
vec![diagnostic]
}
pub async fn run_doctor(ctx: &OutputContext, options: DoctorOptions) -> Result<()> {
if options.reliability {
return run_reliability_doctor(ctx, &options).await;
}
let style = ctx.theme();
let mut checks: Vec<CheckResult> = Vec::new();
let mut fixes_applied: Vec<FixApplied> = Vec::new();
if !ctx.is_json() {
println!("{}", style.format_header("RCH Diagnostic Report"));
println!();
}
if let Ok(config) = crate::config::load_config() {
let problems = if config.general.role == rch_common::BoxRole::Dispatcher {
crate::commands::shim::dispatcher_shim_problems()
.unwrap_or_else(|error| vec![format!("Cannot inspect shims: {error:#}")])
} else {
Vec::new()
};
let check = dispatcher_shim_check(config.general.role, problems);
print_check_result(&check, ctx);
checks.push(check);
if let Some(observation) = crate::local_builds::observe(config.general.role) {
let check = local_build_check(&observation);
print_check_result(&check, ctx);
checks.push(check);
}
}
check_prerequisites(&mut checks, ctx, &options);
check_configuration(&mut checks, ctx, &options);
checks.extend(rustc_wrapper_checks());
check_ssh_keys(&mut checks, ctx, &options, &mut fixes_applied);
check_hooks(&mut checks, ctx, &options, &mut fixes_applied);
check_daemon(&mut checks, ctx, &options, &mut fixes_applied);
check_cancellation_health(&mut checks, ctx).await;
check_workers(&mut checks, ctx, &options).await;
check_telemetry_database(&mut checks, ctx, &options);
let fixed = checks.iter().filter(|c| c.fix_applied).count();
let would_fix = if options.fix && options.dry_run {
checks
.iter()
.filter(|c| matches!(c.fix_message.as_deref(), Some(msg) if msg.starts_with("Would ")))
.count()
} else {
0
};
let summary = DoctorSummary {
total: checks.len(),
passed: checks
.iter()
.filter(|c| c.status == CheckStatus::Pass)
.count(),
warnings: checks
.iter()
.filter(|c| c.status == CheckStatus::Warning)
.count(),
failed: checks
.iter()
.filter(|c| c.status == CheckStatus::Fail)
.count(),
fixed,
would_fix,
};
if ctx.is_json() {
let _ = ctx.json(&ApiResponse::ok(
"doctor",
DoctorResponse {
schema_version: rch_common::schema_version(
rch_common::SchemaComponent::DoctorReliability,
)
.to_string(),
checks,
summary,
fixes_applied,
},
));
} else {
println!();
println!("{}", style.format_header("Summary"));
println!();
println!(
" {} {} passed",
StatusIndicator::Success.display(style),
style.highlight(&summary.passed.to_string())
);
if summary.warnings > 0 {
println!(
" {} {} warnings",
StatusIndicator::Warning.display(style),
style.highlight(&summary.warnings.to_string())
);
}
if summary.failed > 0 {
println!(
" {} {} failed",
StatusIndicator::Error.display(style),
style.highlight(&summary.failed.to_string())
);
}
if summary.fixed > 0 {
println!(
" {} {} fixed",
StatusIndicator::Success.display(style),
style.highlight(&summary.fixed.to_string())
);
}
if summary.would_fix > 0 {
println!(
" {} {} would fix",
StatusIndicator::Pending.display(style),
style.highlight(&summary.would_fix.to_string())
);
}
if !fixes_applied.is_empty() {
println!();
println!("{}", style.format_header("Fixes Applied"));
for fix in &fixes_applied {
if fix.success {
println!(
" {} {}: {}",
StatusIndicator::Success.display(style),
style.highlight(&fix.check_name),
style.muted(&fix.action)
);
} else {
println!(
" {} {}: {} - {}",
StatusIndicator::Error.display(style),
style.highlight(&fix.check_name),
style.muted(&fix.action),
style.error(fix.error.as_deref().unwrap_or("unknown error"))
);
}
}
}
println!();
if summary.failed > 0 {
println!(
"{}",
style.format_error("Some checks failed. Run with --fix to attempt auto-repair.")
);
} else if summary.warnings > 0 {
println!(
"{}",
style.format_warning("System is operational with warnings.")
);
} else {
println!("{}", style.format_success("All checks passed!"));
}
}
Ok(())
}
#[derive(Debug, thiserror::Error)]
#[error("command finished with exit code {0} (result already reported)")]
pub(crate) struct DoctorExit(pub i32);
async fn run_reliability_doctor(ctx: &OutputContext, options: &DoctorOptions) -> Result<()> {
if options.watch {
return run_reliability_watch_loop(ctx, options).await;
}
let mut response = collect_reliability_response_once(options).await;
apply_reliability_remediations(&mut response);
if ctx.is_json() {
let _ = ctx.json(&ApiResponse::ok("doctor.reliability", &response));
} else {
print_reliability_doctor_response(ctx, &response);
}
let exit_code = response
.summary
.overall
.exit_code(options.strict, options.lenient);
let metric_scope = match options.scope.0.as_slice() {
[scope] => scope.as_str(),
_ => "other",
};
tracing::info!(
target: "rch::doctor::verdict",
verdict = response.summary.overall.label(),
scope = metric_scope,
daemon_unreachable = response.daemon_unreachable,
exit_code,
strict = options.strict,
lenient = options.lenient,
pass = response.summary.pass,
info = response.summary.info,
warning = response.summary.warning,
critical = response.summary.critical,
"doctor.verdict",
);
if exit_code == 0 {
Ok(())
} else {
let _ = std::io::Write::flush(&mut std::io::stdout());
let _ = std::io::Write::flush(&mut std::io::stderr());
Err(DoctorExit(exit_code).into())
}
}
const PROBE_TIMEOUT: Duration = Duration::from_secs(10);
async fn timed_async_probe<T, E>(
probe: &'static str,
future: impl std::future::Future<Output = Result<T, E>>,
) -> Result<Result<T, E>, tokio::time::error::Elapsed> {
let started = Instant::now();
let result = tokio::time::timeout(PROBE_TIMEOUT, future).await;
let outcome = match &result {
Ok(Ok(_)) => "completed",
Ok(Err(_)) => "inner_error",
Err(_) => "timeout",
};
tracing::info!(
target: "rch::doctor::probe_duration",
probe,
result = outcome,
duration_seconds = started.elapsed().as_secs_f64(),
"doctor.probe.end",
);
result
}
async fn collect_reliability_response_once(options: &DoctorOptions) -> ReliabilityDoctorResponse {
let scope = &options.scope;
let join_start = Instant::now();
let daemon_handle = scope.needs_daemon_status().then(|| {
tokio::spawn(
async move { timed_async_probe("daemon_status", query_daemon_full_status()).await },
)
});
let convergence_handle = scope.needs_repo_convergence_status().then(|| {
tokio::spawn(async move {
timed_async_probe("repo_convergence", query_repo_convergence_status()).await
})
});
let helpers_handle = scope
.matches(ReliabilityScope::Helpers)
.then(|| tokio::task::spawn_blocking(reliability_helper_compatibility_diagnostics));
let config_result = if scope.needs_rollout_config()
|| scope.matches(ReliabilityScope::Ownership)
|| scope.matches(ReliabilityScope::Triage)
{
Some(crate::config::load_config())
} else {
None
};
tracing::debug!(
target: "rch::doctor::config_loads",
loaded = matches!(config_result.as_ref(), Some(Ok(_))),
skipped = !(scope.needs_rollout_config() || scope.matches(ReliabilityScope::Ownership)
|| scope.matches(ReliabilityScope::Triage)),
"doctor.config.load",
);
let local_observation = if scope.matches(ReliabilityScope::Triage) {
config_result
.as_ref()
.and_then(|result| result.as_ref().ok())
.and_then(|config| crate::local_builds::observe(config.general.role))
} else {
None
};
let ownership_canonical_root: Result<PathBuf, String> =
if scope.matches(ReliabilityScope::Ownership) {
match config_result.as_ref() {
Some(Ok(config)) => Ok(config
.path_topology
.to_policy()
.canonical_root()
.to_path_buf()),
Some(Err(error)) => Err(error.to_string()),
None => Err("config load skipped unexpectedly".to_string()),
}
} else {
Err("ownership scope not selected".to_string())
};
let (workers, worker_config_error) = if scope.needs_worker_config() {
match load_workers_from_config() {
Ok(workers) => (Some(workers), None),
Err(err) => (None, Some(err.to_string())),
}
} else {
(None, None)
};
let ownership_handle: Option<
tokio::task::JoinHandle<Vec<crate::hook::ssh::MirrorOwnershipProbe>>,
> = if scope.matches(ReliabilityScope::Ownership) && ownership_canonical_root.is_ok() {
workers.as_ref().filter(|ws| !ws.is_empty()).map(|ws| {
let workers = ws.clone();
let workers_len = workers.len();
let root = ownership_canonical_root
.as_ref()
.expect("checked is_ok above")
.clone();
tokio::spawn(async move {
let mut set = tokio::task::JoinSet::new();
for (idx, worker) in workers.into_iter().enumerate() {
let root = root.clone();
set.spawn(async move {
let probe =
crate::hook::ssh::probe_worker_mirror_ownership(&worker, &root).await;
(idx, probe)
});
}
let mut results = vec![
crate::hook::ssh::MirrorOwnershipProbe::Unprobeable(
"probe task did not report".to_string(),
);
workers_len
];
while let Some(joined) = set.join_next().await {
if let Ok((idx, probe)) = joined
&& let Some(slot) = results.get_mut(idx)
{
*slot = probe;
}
}
results
})
})
} else {
None
};
let (daemon_status, daemon_outcome) =
join_isolated_async_probe("daemon_status", daemon_handle).await;
let (convergence_status, convergence_outcome) =
join_isolated_async_probe("repo_convergence", convergence_handle).await;
let (helper_diags_override, helpers_outcome) =
join_isolated_blocking_probe("helper_compatibility", helpers_handle).await;
let (ownership_results, ownership_outcome) = join_ownership_probe(ownership_handle).await;
let join_elapsed_ms = join_start.elapsed().as_millis() as u64;
tracing::info!(
target: "rch::doctor::probes",
daemon = %daemon_outcome.label(),
repo_convergence = %convergence_outcome.label(),
helpers = %helpers_outcome.label(),
join_elapsed_ms,
"doctor.probes.joined",
);
let probes_to_run = scope.probe_names_to_run(options.check_schemas);
tracing::info!(
target: "rch::doctor::scope",
scope = ?scope.as_strings(),
probes_to_run = ?probes_to_run,
probes_count = probes_to_run.len(),
"doctor.scope.applied",
);
let mut diagnostics = Vec::new();
if scope.matches(ReliabilityScope::Topology) {
diagnostics.extend(reliability_topology_diagnostics(
workers.as_deref(),
daemon_status.as_ref(),
worker_config_error.clone(),
));
}
if scope.matches(ReliabilityScope::Ownership) {
diagnostics.extend(reliability_mirror_ownership_diagnostics(
workers.as_deref(),
worker_config_error,
ownership_results.as_ref(),
ownership_outcome,
ownership_canonical_root.as_ref().err(),
));
}
if scope.matches(ReliabilityScope::Convergence) {
diagnostics.extend(reliability_repo_diagnostics(convergence_status.as_ref()));
}
if scope.matches(ReliabilityScope::Pressure) {
diagnostics.extend(reliability_disk_pressure_diagnostics(
daemon_status.as_ref(),
));
}
if scope.matches(ReliabilityScope::Triage) {
diagnostics.extend(reliability_local_build_diagnostics(
local_observation.as_ref(),
));
diagnostics.extend(reliability_process_debt_diagnostics(daemon_status.as_ref()));
}
if scope.matches(ReliabilityScope::Helpers) {
diagnostics.extend(helper_diagnostics_from_probe_result(
helper_diags_override,
helpers_outcome,
));
}
if scope.matches(ReliabilityScope::Rollout) {
let config_load: Result<&rch_common::RchConfig, String> = match config_result.as_ref() {
Some(Ok(config)) => Ok(config),
Some(Err(error)) => Err(error.to_string()),
None => Err("config load skipped unexpectedly".to_string()),
};
diagnostics.extend(reliability_rollout_posture_diagnostics(
config_load.as_ref().map(|c| *c).map_err(|e| e.as_str()),
));
let config_socket = config_load
.as_ref()
.ok()
.map(|cfg| cfg.general.socket_path.as_str());
diagnostics.extend(reliability_hook_consistency_diagnostics(
rch_common::hooks::installed_rch_hook_command().as_deref(),
config_socket,
&rch_common::default_socket_path(),
));
}
if scope.runs_schema_probe(options.check_schemas) {
diagnostics.extend(reliability_schema_compatibility_diagnostics());
}
let mode = if options.dry_run {
ReliabilityDoctorMode::DryRun
} else if options.fix {
ReliabilityDoctorMode::Fix
} else {
ReliabilityDoctorMode::Check
};
let mut response = build_reliability_doctor_response(mode, scope, diagnostics);
response.fix_requested = options.fix;
response
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ProbeOutcome {
Ok,
Skipped,
InnerError,
Timeout,
Panicked,
Cancelled,
}
impl ProbeOutcome {
fn label(self) -> &'static str {
match self {
Self::Ok => "ok",
Self::Skipped => "skipped",
Self::InnerError => "inner_error",
Self::Timeout => "timeout",
Self::Panicked => "panicked",
Self::Cancelled => "cancelled",
}
}
}
async fn join_isolated_async_probe<T, E>(
probe_name: &'static str,
handle: Option<tokio::task::JoinHandle<Result<Result<T, E>, tokio::time::error::Elapsed>>>,
) -> (Option<T>, ProbeOutcome)
where
T: Send + 'static,
E: std::fmt::Display + Send + 'static,
{
let Some(handle) = handle else {
return (None, ProbeOutcome::Skipped);
};
match handle.await {
Ok(Ok(Ok(value))) => (Some(value), ProbeOutcome::Ok),
Ok(Ok(Err(err))) => {
tracing::debug!(
target: "rch::doctor::probes",
probe = probe_name,
error = %err,
"doctor.probe.inner_error",
);
(None, ProbeOutcome::InnerError)
}
Ok(Err(_elapsed)) => {
tracing::warn!(
target: "rch::doctor::probes",
probe = probe_name,
timeout_secs = PROBE_TIMEOUT.as_secs(),
"doctor.probe.timeout",
);
(None, ProbeOutcome::Timeout)
}
Err(join_err) if join_err.is_panic() => {
let payload = join_err.into_panic();
let panic_msg = payload
.downcast::<String>()
.map(|b| *b)
.or_else(|p| p.downcast::<&'static str>().map(|b| (*b).to_string()))
.unwrap_or_else(|_| "<non-string panic payload>".to_string());
tracing::error!(
target: "rch::doctor::probes",
probe = probe_name,
panic = %panic_msg,
"doctor.probe.panicked",
);
(None, ProbeOutcome::Panicked)
}
Err(join_err) => {
tracing::warn!(
target: "rch::doctor::probes",
probe = probe_name,
error = %join_err,
"doctor.probe.cancelled",
);
(None, ProbeOutcome::Cancelled)
}
}
}
async fn join_ownership_probe(
handle: Option<tokio::task::JoinHandle<Vec<crate::hook::ssh::MirrorOwnershipProbe>>>,
) -> (
Option<Vec<crate::hook::ssh::MirrorOwnershipProbe>>,
ProbeOutcome,
) {
let Some(handle) = handle else {
return (None, ProbeOutcome::Skipped);
};
match handle.await {
Ok(value) => (Some(value), ProbeOutcome::Ok),
Err(join_err) if join_err.is_panic() => {
let payload = join_err.into_panic();
let panic_msg = payload
.downcast::<String>()
.map(|b| *b)
.or_else(|p| p.downcast::<&'static str>().map(|b| (*b).to_string()))
.unwrap_or_else(|_| "<non-string panic payload>".to_string());
tracing::error!(
target: "rch::doctor::probes",
probe = "mirror_ownership",
panic = %panic_msg,
"doctor.probe.panicked",
);
(None, ProbeOutcome::Panicked)
}
Err(join_err) => {
tracing::warn!(
target: "rch::doctor::probes",
probe = "mirror_ownership",
error = %join_err,
"doctor.probe.cancelled",
);
(None, ProbeOutcome::Cancelled)
}
}
}
async fn join_isolated_blocking_probe<T>(
probe_name: &'static str,
handle: Option<tokio::task::JoinHandle<T>>,
) -> (Option<T>, ProbeOutcome)
where
T: Send + 'static,
{
join_isolated_blocking_probe_with_timeout(probe_name, handle, PROBE_TIMEOUT).await
}
async fn join_isolated_blocking_probe_with_timeout<T>(
probe_name: &'static str,
handle: Option<tokio::task::JoinHandle<T>>,
timeout: Duration,
) -> (Option<T>, ProbeOutcome)
where
T: Send + 'static,
{
let Some(handle) = handle else {
return (None, ProbeOutcome::Skipped);
};
match tokio::time::timeout(timeout, handle).await {
Ok(Ok(value)) => (Some(value), ProbeOutcome::Ok),
Ok(Err(join_err)) if join_err.is_panic() => {
let payload = join_err.into_panic();
let panic_msg = payload
.downcast::<String>()
.map(|b| *b)
.or_else(|p| p.downcast::<&'static str>().map(|b| (*b).to_string()))
.unwrap_or_else(|_| "<non-string panic payload>".to_string());
tracing::error!(
target: "rch::doctor::probes",
probe = probe_name,
panic = %panic_msg,
"doctor.probe.panicked",
);
(None, ProbeOutcome::Panicked)
}
Ok(Err(join_err)) => {
tracing::warn!(
target: "rch::doctor::probes",
probe = probe_name,
error = %join_err,
"doctor.probe.cancelled",
);
(None, ProbeOutcome::Cancelled)
}
Err(_elapsed) => {
tracing::warn!(
target: "rch::doctor::probes",
probe = probe_name,
timeout_secs = timeout.as_secs(),
"doctor.probe.timeout",
);
(None, ProbeOutcome::Timeout)
}
}
}
async fn query_repo_convergence_status() -> Result<RepoConvergenceStatusFromApi> {
let response = send_daemon_command("GET /repo-convergence/status\n").await?;
let json = extract_json_body(&response)
.ok_or_else(|| anyhow::anyhow!("Invalid response format from daemon"))?;
serde_json::from_str(json).map_err(Into::into)
}
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
struct DiagnosticKey {
category: ReliabilityCategory,
check_name: String,
worker_id: Option<String>,
}
impl DiagnosticKey {
fn from_diagnostic(d: &ReliabilityDiagnostic) -> Self {
Self {
category: d.category,
check_name: d.check_name.clone(),
worker_id: d.worker_id.clone(),
}
}
fn render(&self) -> String {
match &self.worker_id {
Some(w) => format!(
"{}/{}[worker={}]",
self.category.as_str(),
self.check_name,
w
),
None => format!("{}/{}", self.category.as_str(), self.check_name),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct DiagnosticFingerprint {
severity: ReliabilitySeverity,
code: ReliabilityReasonCode,
message: String,
}
impl DiagnosticFingerprint {
fn from_diagnostic(d: &ReliabilityDiagnostic) -> Self {
Self {
severity: d.severity,
code: d.code,
message: d.message.clone(),
}
}
}
#[derive(Debug, Default)]
struct DiagnosticDiff {
added: Vec<(DiagnosticKey, DiagnosticFingerprint)>,
cleared: Vec<(DiagnosticKey, DiagnosticFingerprint)>,
changed: Vec<(DiagnosticKey, DiagnosticFingerprint, DiagnosticFingerprint)>,
}
impl DiagnosticDiff {
fn has_changes(&self) -> bool {
!self.added.is_empty() || !self.cleared.is_empty() || !self.changed.is_empty()
}
fn total_transitions(&self) -> usize {
self.added.len() + self.cleared.len() + self.changed.len()
}
}
fn diff_fingerprint_maps(
previous: &std::collections::BTreeMap<DiagnosticKey, DiagnosticFingerprint>,
current: &std::collections::BTreeMap<DiagnosticKey, DiagnosticFingerprint>,
) -> DiagnosticDiff {
let mut diff = DiagnosticDiff::default();
for (key, fp) in current {
match previous.get(key) {
None => diff.added.push((key.clone(), fp.clone())),
Some(prev_fp) if prev_fp != fp => {
diff.changed
.push((key.clone(), prev_fp.clone(), fp.clone()));
}
Some(_) => {}
}
}
for (key, fp) in previous {
if !current.contains_key(key) {
diff.cleared.push((key.clone(), fp.clone()));
}
}
diff
}
fn fingerprint_map_of(
response: &ReliabilityDoctorResponse,
) -> std::collections::BTreeMap<DiagnosticKey, DiagnosticFingerprint> {
let mut map = std::collections::BTreeMap::new();
for d in &response.diagnostics {
map.insert(
DiagnosticKey::from_diagnostic(d),
DiagnosticFingerprint::from_diagnostic(d),
);
}
map
}
struct WatchState {
started_at: Instant,
sweep_count: u64,
transitions_count: u64,
last_verdict: Option<ReliabilityVerdict>,
last_fingerprints: std::collections::BTreeMap<DiagnosticKey, DiagnosticFingerprint>,
worst_verdict: Option<ReliabilityVerdict>,
}
fn worst_reliability_verdict(
observed: Option<ReliabilityVerdict>,
candidate: ReliabilityVerdict,
) -> ReliabilityVerdict {
match (observed, candidate) {
(Some(ReliabilityVerdict::Failing), _) | (_, ReliabilityVerdict::Failing) => {
ReliabilityVerdict::Failing
}
(Some(ReliabilityVerdict::Degraded), _) | (_, ReliabilityVerdict::Degraded) => {
ReliabilityVerdict::Degraded
}
(Some(ReliabilityVerdict::Healthy), ReliabilityVerdict::Healthy)
| (None, ReliabilityVerdict::Healthy) => ReliabilityVerdict::Healthy,
}
}
impl WatchState {
fn new() -> Self {
Self {
started_at: Instant::now(),
sweep_count: 0,
transitions_count: 0,
last_verdict: None,
last_fingerprints: std::collections::BTreeMap::new(),
worst_verdict: None,
}
}
fn observe_verdict(&mut self, verdict: ReliabilityVerdict) {
self.worst_verdict = Some(worst_reliability_verdict(self.worst_verdict, verdict));
}
}
#[derive(Debug, Clone, Serialize)]
struct WatchSweepMachineLine<'a> {
command: &'static str,
schema_version: &'a str,
sweep_index: u64,
elapsed_secs: u64,
verdict: &'static str,
verdict_changed: bool,
diff_summary: WatchDiffSummary,
response: &'a ReliabilityDoctorResponse,
}
#[derive(Debug, Clone, Serialize)]
struct WatchDiffSummary {
added: usize,
cleared: usize,
changed: usize,
has_changes: bool,
}
#[derive(Debug, Clone, Serialize)]
struct WatchSnapshot<'a> {
command: &'static str,
schema_version: &'a str,
sweeps_total: u64,
transitions_total: u64,
elapsed_secs: u64,
final_verdict: &'static str,
worst_verdict: &'static str,
final_diagnostics: &'a Vec<ReliabilityDiagnostic>,
}
async fn run_reliability_watch_loop(ctx: &OutputContext, options: &DoctorOptions) -> Result<()> {
let interval_secs = options.watch_interval_secs.max(1);
let interval = Duration::from_secs(interval_secs);
let snapshot_path = options.watch_snapshot.clone();
let style = ctx.theme();
let schema_version =
rch_common::schema_version(rch_common::SchemaComponent::DoctorReliability).to_string();
tracing::info!(
target: "rch::doctor::watch",
interval_secs,
transitions_only = options.transitions_only,
snapshot_path = ?snapshot_path,
scope = ?options.scope.as_strings(),
"doctor.watch.start",
);
if !ctx.is_json() {
eprintln!(
"{} (interval={interval_secs}s, scope={:?}, press Ctrl-C to exit)",
style.format_header("RCH Doctor — continuous watch mode"),
options.scope.as_strings(),
);
if options.transitions_only {
eprintln!(
"{}",
style.muted(" --transitions-only: emitting only when state changes")
);
}
eprintln!();
}
let webhook_dispatcher = match crate::config::load_config() {
Ok(cfg) => crate::doctor_webhooks::WebhookDispatcher::from_config(&cfg.doctor.webhooks),
Err(e) => {
tracing::warn!(
target: "rch::doctor::webhook",
error = %e,
"doctor.webhook.config_load_failed",
);
None
}
};
let mut state = WatchState::new();
let mut ticker = tokio::time::interval(interval);
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
let sigint = tokio::signal::ctrl_c();
tokio::pin!(sigint);
loop {
tokio::select! {
biased;
sig = &mut sigint => {
if let Err(e) = sig {
tracing::warn!(
target: "rch::doctor::watch",
error = %e,
"ctrl_c handler installation failed; exiting watch loop"
);
}
break;
}
_ = ticker.tick() => {
state.sweep_count += 1;
let sweep_started = Instant::now();
let response = collect_reliability_response_once(options).await;
let current_fp = fingerprint_map_of(&response);
let diff = diff_fingerprint_maps(&state.last_fingerprints, ¤t_fp);
let verdict = response.summary.overall;
let verdict_changed = state.last_verdict != Some(verdict);
let any_change = diff.has_changes() || verdict_changed;
if verdict_changed {
state.transitions_count += 1;
}
state.observe_verdict(verdict);
if verdict_changed
&& let Some(dispatcher) = &webhook_dispatcher
{
let from = state.last_verdict.unwrap_or(ReliabilityVerdict::Healthy);
let transition = crate::doctor_webhooks::WebhookTransition {
from,
to: verdict,
host: crate::doctor_webhooks::hostname(),
ts: chrono::Utc::now().to_rfc3339(),
scope: response.scope.clone(),
diagnostics: response
.diagnostics
.iter()
.map(|d| crate::doctor_webhooks::WebhookDiagnostic {
code: d.code.to_string(),
severity: reliability_severity_token(d.severity).to_string(),
category: d.category.as_str().to_string(),
message: d.message.clone(),
})
.collect(),
};
let fired = dispatcher.on_transition(&transition);
if fired > 0 {
tracing::info!(
target: "rch::doctor::webhook",
from = from.label(),
to = verdict.label(),
endpoints = fired,
"doctor.webhook.transition_dispatched",
);
}
}
let sweep_elapsed_ms = sweep_started.elapsed().as_millis();
let should_emit = !options.transitions_only || any_change || state.sweep_count == 1;
tracing::debug!(
target: "rch::doctor::watch",
sweep = state.sweep_count,
verdict = verdict.label(),
verdict_changed,
added = diff.added.len(),
cleared = diff.cleared.len(),
changed = diff.changed.len(),
transition_items = diff.total_transitions(),
sweep_elapsed_ms,
suppressed = !should_emit,
"doctor.watch.sweep",
);
if should_emit {
if ctx.is_json() {
emit_watch_sweep_machine_line(
ctx,
&response,
&diff,
verdict_changed,
&schema_version,
state.sweep_count,
state.started_at,
);
} else {
emit_watch_sweep_human(
ctx,
&response,
&diff,
verdict_changed,
state.sweep_count,
state.started_at,
sweep_elapsed_ms,
);
}
}
state.last_verdict = Some(verdict);
state.last_fingerprints = current_fp;
}
}
}
let final_response_for_snapshot = if snapshot_path.is_some() {
Some(collect_reliability_response_once(options).await)
} else {
None
};
let final_verdict = final_response_for_snapshot.as_ref().map_or_else(
|| state.last_verdict.unwrap_or(ReliabilityVerdict::Healthy),
|r| r.summary.overall,
);
let worst_verdict = worst_reliability_verdict(state.worst_verdict, final_verdict);
let elapsed_secs = state.started_at.elapsed().as_secs();
if !ctx.is_json() {
eprintln!();
eprintln!(
"{} {} sweeps, {} transitions, {}s elapsed — final={}, worst={}",
style.format_header("Watch session ended"),
style.highlight(&state.sweep_count.to_string()),
style.highlight(&state.transitions_count.to_string()),
style.highlight(&elapsed_secs.to_string()),
style.highlight(final_verdict.label()),
style.highlight(worst_verdict.label()),
);
} else {
eprintln!(
r#"{{"command":"doctor.reliability.watch.end","sweeps_total":{},"transitions_total":{},"elapsed_secs":{},"final_verdict":"{}","worst_verdict":"{}"}}"#,
state.sweep_count,
state.transitions_count,
elapsed_secs,
final_verdict.label(),
worst_verdict.label(),
);
}
tracing::info!(
target: "rch::doctor::watch",
sweeps_total = state.sweep_count,
transitions_total = state.transitions_count,
elapsed_secs,
final_verdict = final_verdict.label(),
worst_verdict = worst_verdict.label(),
"doctor.watch.end",
);
if let (Some(path), Some(final_response)) = (snapshot_path, final_response_for_snapshot) {
let snapshot = WatchSnapshot {
command: "doctor.reliability.watch.snapshot",
schema_version: &schema_version,
sweeps_total: state.sweep_count,
transitions_total: state.transitions_count,
elapsed_secs,
final_verdict: final_response.summary.overall.label(),
worst_verdict: worst_verdict.label(),
final_diagnostics: &final_response.diagnostics,
};
let body = serde_json::to_string_pretty(&snapshot)
.map_err(|e| anyhow::anyhow!("serialize watch snapshot: {e}"))?;
let tmp = path.with_extension("json.partial");
tokio::fs::write(&tmp, body.as_bytes())
.await
.map_err(|e| anyhow::anyhow!("write watch snapshot tmp file: {e}"))?;
tokio::fs::rename(&tmp, &path)
.await
.map_err(|e| anyhow::anyhow!("rename watch snapshot to final path: {e}"))?;
tracing::info!(
target: "rch::doctor::watch",
path = %path.display(),
"doctor.watch.snapshot.written",
);
if !ctx.is_json() {
eprintln!(
"{} {}",
style.muted(" snapshot:"),
style.highlight(&path.display().to_string())
);
}
}
Ok(())
}
fn emit_watch_sweep_human(
ctx: &OutputContext,
response: &ReliabilityDoctorResponse,
diff: &DiagnosticDiff,
verdict_changed: bool,
sweep: u64,
started_at: Instant,
sweep_elapsed_ms: u128,
) {
let style = ctx.theme();
let verdict = response.summary.overall;
let verdict_label = verdict.label();
let colored_verdict = match verdict {
ReliabilityVerdict::Healthy => style.format_success(verdict_label),
ReliabilityVerdict::Degraded => style.format_warning(verdict_label),
ReliabilityVerdict::Failing => style.format_error(verdict_label),
};
let elapsed = started_at.elapsed().as_secs();
let arrow = if verdict_changed { " (CHANGED)" } else { "" };
eprintln!(
"[sweep {sweep:>4} t+{elapsed:>4}s] verdict={colored_verdict}{arrow} \
pass={p} info={i} warn={w} crit={c} probe_ms={ms} \
diff: +{added} -{cleared} ~{changed}",
p = response.summary.pass,
i = response.summary.info,
w = response.summary.warning,
c = response.summary.critical,
ms = sweep_elapsed_ms,
added = diff.added.len(),
cleared = diff.cleared.len(),
changed = diff.changed.len(),
);
for (key, fp) in &diff.added {
eprintln!(
" {} {} [{}] {}",
style.format_warning("+ new"),
key.render(),
severity_short(fp.severity),
fp.message,
);
}
for (key, fp) in &diff.cleared {
eprintln!(
" {} {} [{}] {}",
style.format_success("- cleared"),
key.render(),
severity_short(fp.severity),
fp.message,
);
}
for (key, prev, cur) in &diff.changed {
eprintln!(
" {} {} [{}→{}] {} → {}",
style.muted("~ changed"),
key.render(),
severity_short(prev.severity),
severity_short(cur.severity),
prev.message,
cur.message,
);
}
}
fn emit_watch_sweep_machine_line(
ctx: &OutputContext,
response: &ReliabilityDoctorResponse,
diff: &DiagnosticDiff,
verdict_changed: bool,
schema_version: &str,
sweep: u64,
started_at: Instant,
) {
let payload = WatchSweepMachineLine {
command: "doctor.reliability.watch",
schema_version,
sweep_index: sweep,
elapsed_secs: started_at.elapsed().as_secs(),
verdict: response.summary.overall.label(),
verdict_changed,
diff_summary: WatchDiffSummary {
added: diff.added.len(),
cleared: diff.cleared.len(),
changed: diff.changed.len(),
has_changes: diff.has_changes(),
},
response,
};
let _ = ctx.json_compact(&payload);
}
fn severity_short(severity: ReliabilitySeverity) -> &'static str {
match severity {
ReliabilitySeverity::Pass => "PASS",
ReliabilitySeverity::Info => "INFO",
ReliabilitySeverity::Warning => "WARN",
ReliabilitySeverity::Critical => "CRIT",
}
}
fn reliability_topology_diagnostics(
workers: Option<&[rch_common::WorkerConfig]>,
daemon_status: Option<&DaemonFullStatusResponse>,
worker_config_error: Option<String>,
) -> Vec<ReliabilityDiagnostic> {
let mut diagnostics = Vec::new();
if let Some(error) = worker_config_error {
diagnostics.push(
ReliabilityDiagnostic::new(
ReliabilityCategory::Topology,
"workers_config",
ReliabilitySeverity::Critical,
"Worker configuration could not be loaded",
ReliabilityReasonCode::WorkersConfigUnreadable,
)
.with_details(error)
.with_remediation(
"rch config doctor --json",
"rch doctor --reliability --json",
),
);
} else if let Some(workers) = workers {
if workers.is_empty() {
diagnostics.push(
ReliabilityDiagnostic::new(
ReliabilityCategory::Topology,
"workers_config",
ReliabilitySeverity::Critical,
"No workers are configured, so all builds will run locally",
ReliabilityReasonCode::NoWorkersConfigured,
)
.with_remediation("rch workers init", "rch workers list --json"),
);
} else {
let worker_ids = workers
.iter()
.map(|worker| worker.id.to_string())
.collect::<Vec<_>>()
.join(", ");
diagnostics.push(
ReliabilityDiagnostic::new(
ReliabilityCategory::Topology,
"workers_config",
ReliabilitySeverity::Pass,
format!("{} worker(s) configured", workers.len()),
ReliabilityReasonCode::WorkersConfigured,
)
.with_details(worker_ids),
);
}
}
let Some(status) = daemon_status else {
diagnostics.push(
ReliabilityDiagnostic::new(
ReliabilityCategory::Topology,
"daemon_status",
ReliabilitySeverity::Warning,
"Daemon status is unavailable; reliability health is partial",
ReliabilityReasonCode::DaemonStatusUnavailable,
)
.with_remediation("rch daemon start", "rch status --json"),
);
return diagnostics;
};
let daemon = &status.daemon;
let (severity, code, message) = if daemon.workers_total == 0 {
(
ReliabilitySeverity::Critical,
ReliabilityReasonCode::DaemonHasNoWorkers,
"Daemon has no registered workers".to_string(),
)
} else if daemon.workers_healthy == 0 {
(
ReliabilitySeverity::Critical,
ReliabilityReasonCode::AllWorkersUnhealthy,
format!("0/{} workers are healthy", daemon.workers_total),
)
} else if daemon.workers_healthy < daemon.workers_total {
(
ReliabilitySeverity::Warning,
ReliabilityReasonCode::PartialWorkerCapacity,
format!(
"{}/{} workers are healthy",
daemon.workers_healthy, daemon.workers_total
),
)
} else {
(
ReliabilitySeverity::Pass,
ReliabilityReasonCode::WorkersHealthy,
format!("All {} workers are healthy", daemon.workers_total),
)
};
let mut daemon_diag = ReliabilityDiagnostic::new(
ReliabilityCategory::Topology,
"daemon_worker_capacity",
severity,
message,
code,
)
.with_details(format!(
"slots_available={}, slots_total={}, uptime_secs={}",
daemon.slots_available, daemon.slots_total, daemon.uptime_secs
));
if severity != ReliabilitySeverity::Pass {
daemon_diag =
daemon_diag.with_remediation("rch workers probe --all", "rch status --workers --json");
}
diagnostics.push(daemon_diag);
diagnostics.extend(status.workers.iter().map(worker_topology_diagnostic));
diagnostics
}
fn reliability_mirror_ownership_diagnostics(
workers: Option<&[rch_common::WorkerConfig]>,
worker_config_error: Option<String>,
probe_results: Option<&Vec<crate::hook::ssh::MirrorOwnershipProbe>>,
outcome: ProbeOutcome,
canonical_root_error: Option<&String>,
) -> Vec<ReliabilityDiagnostic> {
let validation = "rch doctor --reliability --scope=ownership --json";
if let Some(error) = worker_config_error {
return vec![
ReliabilityDiagnostic::new(
ReliabilityCategory::MirrorOwnership,
"mirror_ownership",
ReliabilitySeverity::Warning,
"Workers configuration unavailable; mirror-tree ownership not probed",
ReliabilityReasonCode::WorkerMirrorOwnershipUnprobeable,
)
.with_details(format!("workers config error: {error}"))
.with_remediation("rch config doctor --json", validation),
];
}
if let Some(error) = canonical_root_error {
return vec![
ReliabilityDiagnostic::new(
ReliabilityCategory::MirrorOwnership,
"mirror_ownership",
ReliabilitySeverity::Warning,
"Config load failed; cannot resolve the canonical mirror root to probe",
ReliabilityReasonCode::WorkerMirrorOwnershipUnprobeable,
)
.with_details(format!("config error: {error}"))
.with_remediation("rch config doctor --json", validation),
];
}
let Some(workers) = workers else {
return vec![];
};
if workers.is_empty() {
return vec![ReliabilityDiagnostic::new(
ReliabilityCategory::MirrorOwnership,
"mirror_ownership",
ReliabilitySeverity::Info,
"No workers are configured; nothing to probe for ownership drift",
ReliabilityReasonCode::MirrorOwnershipNoWorkers,
)];
}
let Some(results) = probe_results else {
return vec![
ReliabilityDiagnostic::new(
ReliabilityCategory::MirrorOwnership,
"mirror_ownership",
ReliabilitySeverity::Warning,
format!(
"Mirror-ownership probe did not complete ({})",
outcome.label()
),
ReliabilityReasonCode::WorkerMirrorOwnershipUnprobeable,
)
.with_remediation("rch workers probe --all", validation),
];
};
workers
.iter()
.zip(results.iter())
.filter_map(|(worker, probe)| {
let worker_id = worker.id.to_string();
match probe {
crate::hook::ssh::MirrorOwnershipProbe::Skipped => None,
crate::hook::ssh::MirrorOwnershipProbe::Healthy => Some(
ReliabilityDiagnostic::new(
ReliabilityCategory::MirrorOwnership,
"mirror_ownership",
ReliabilitySeverity::Pass,
"Mirror tree has no root-owned entries",
ReliabilityReasonCode::WorkerMirrorOwnershipHealthy,
)
.with_worker(worker_id),
),
crate::hook::ssh::MirrorOwnershipProbe::Drift { count } => {
let user = &worker.user;
let host = &worker.host;
let manual_fix = format!(
"ssh {user}@{host} 'sudo find <canonical-root> -xdev -user root -exec chown -h {user} {{}} +'"
);
Some(
ReliabilityDiagnostic::new(
ReliabilityCategory::MirrorOwnership,
"mirror_ownership",
ReliabilitySeverity::Warning,
format!(
"{count} root-owned entr{} under the canonical mirror tree; \
rsync-as-{user} will fail exit 23 until repaired",
if *count == 1 { "y" } else { "ies" }
),
ReliabilityReasonCode::WorkerMirrorOwnershipDrift,
)
.with_worker(worker_id)
.with_details(format!(
"drift_count={count}; self-heals at the next dispatch preflight (bd-8iwkm)"
))
.with_remediation(manual_fix, validation),
)
}
crate::hook::ssh::MirrorOwnershipProbe::CheckUnavailable => Some(
ReliabilityDiagnostic::new(
ReliabilityCategory::MirrorOwnership,
"mirror_ownership",
ReliabilitySeverity::Warning,
"Passwordless sudo unavailable on worker; ownership check \
cannot run and dispatch-time repair fails open",
ReliabilityReasonCode::WorkerMirrorOwnershipCheckUnavailable,
)
.with_worker(worker_id.clone())
.with_remediation(
format!("ssh {worker_id} 'sudo -n true'"),
validation,
),
),
crate::hook::ssh::MirrorOwnershipProbe::Unprobeable(reason) => Some(
ReliabilityDiagnostic::new(
ReliabilityCategory::MirrorOwnership,
"mirror_ownership",
ReliabilitySeverity::Warning,
"Ownership probe could not reach the worker; drift state unknown",
ReliabilityReasonCode::WorkerMirrorOwnershipUnprobeable,
)
.with_worker(worker_id)
.with_details(reason.clone())
.with_remediation("rch workers probe --all", validation),
),
}
})
.collect()
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum ParsedStatus {
Known(KnownStatus),
Unrecognized(String),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum KnownStatus {
Ready,
Degraded,
Unreachable,
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum ParsedCircuit {
Known(KnownCircuit),
Unrecognized(String),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum KnownCircuit {
Closed,
Open,
HalfOpen,
}
fn parse_worker_ready_status(raw: &str) -> ParsedStatus {
let normalized = raw.trim().to_ascii_lowercase();
if matches!(
normalized.as_str(),
"healthy" | "available" | "ready" | "idle" | "running"
) {
ParsedStatus::Known(KnownStatus::Ready)
} else if matches!(
normalized.as_str(),
"unreachable" | "offline" | "error" | "failed"
) {
ParsedStatus::Known(KnownStatus::Unreachable)
} else if matches!(
normalized.as_str(),
"busy" | "degraded" | "draining" | "drained" | "disabled" | "unhealthy"
) {
ParsedStatus::Known(KnownStatus::Degraded)
} else {
ParsedStatus::Unrecognized(normalized)
}
}
fn parse_worker_circuit_state(raw: &str) -> ParsedCircuit {
let normalized = raw.trim().to_ascii_lowercase();
match normalized.as_str() {
"closed" => ParsedCircuit::Known(KnownCircuit::Closed),
"open" => ParsedCircuit::Known(KnownCircuit::Open),
"half_open" | "half-open" => ParsedCircuit::Known(KnownCircuit::HalfOpen),
_ => ParsedCircuit::Unrecognized(normalized),
}
}
fn worker_topology_diagnostic(worker: &WorkerStatusFromApi) -> ReliabilityDiagnostic {
let parsed_status = parse_worker_ready_status(&worker.status);
let parsed_circuit = parse_worker_circuit_state(&worker.circuit_state);
let (severity, code, message) = match (&parsed_circuit, &parsed_status) {
(ParsedCircuit::Unrecognized(raw), _) => (
ReliabilitySeverity::Warning,
ReliabilityReasonCode::WorkerCircuitStateUnrecognized,
format!(
"Worker {} circuit_state is unrecognized ({raw:?})",
worker.id
),
),
(_, ParsedStatus::Unrecognized(raw)) => (
ReliabilitySeverity::Warning,
ReliabilityReasonCode::WorkerStatusUnrecognized,
format!("Worker {} status is unrecognized ({raw:?})", worker.id),
),
(ParsedCircuit::Known(KnownCircuit::Open), _) => (
ReliabilitySeverity::Critical,
ReliabilityReasonCode::WorkerCircuitOpen,
format!("Worker {} circuit is open", worker.id),
),
(_, ParsedStatus::Known(KnownStatus::Unreachable)) => (
ReliabilitySeverity::Critical,
ReliabilityReasonCode::WorkerUnreachable,
format!("Worker {} is {}", worker.id, worker.status),
),
(ParsedCircuit::Known(KnownCircuit::HalfOpen), _) => (
ReliabilitySeverity::Warning,
ReliabilityReasonCode::WorkerDegraded,
format!(
"Worker {} is degraded (status={}, circuit={})",
worker.id, worker.status, worker.circuit_state
),
),
(_, ParsedStatus::Known(KnownStatus::Degraded)) => (
ReliabilitySeverity::Warning,
ReliabilityReasonCode::WorkerDegraded,
format!(
"Worker {} is degraded (status={}, circuit={})",
worker.id, worker.status, worker.circuit_state
),
),
(ParsedCircuit::Known(KnownCircuit::Closed), ParsedStatus::Known(KnownStatus::Ready)) => (
ReliabilitySeverity::Pass,
ReliabilityReasonCode::WorkerReady,
format!("Worker {} is ready", worker.id),
),
};
let mut diagnostic = ReliabilityDiagnostic::new(
ReliabilityCategory::Topology,
"worker_topology",
severity,
message,
code,
)
.with_worker(worker.id.clone())
.with_details(format!(
"host={}, used_slots={}, total_slots={}, speed_score={:.2}, consecutive_failures={}",
worker.host,
worker.used_slots,
worker.total_slots,
worker.speed_score,
worker.consecutive_failures
));
if severity != ReliabilitySeverity::Pass {
diagnostic = diagnostic.with_remediation(
format!("rch workers probe {} --force", worker.id),
"rch status --workers --json",
);
}
if let ParsedStatus::Unrecognized(raw) = &parsed_status {
tracing::warn!(
target: "rch::doctor::parse",
worker = %worker.id,
field = "status",
raw = %raw,
"doctor.parse.unrecognized",
);
}
if let ParsedCircuit::Unrecognized(raw) = &parsed_circuit {
tracing::warn!(
target: "rch::doctor::parse",
worker = %worker.id,
field = "circuit_state",
raw = %raw,
"doctor.parse.unrecognized",
);
}
diagnostic
}
fn reliability_repo_diagnostics(
convergence: Option<&RepoConvergenceStatusFromApi>,
) -> Vec<ReliabilityDiagnostic> {
let Some(convergence) = convergence else {
return vec![
ReliabilityDiagnostic::new(
ReliabilityCategory::RepoPresence,
"repo_convergence",
ReliabilitySeverity::Warning,
"Repo-convergence status is unavailable",
ReliabilityReasonCode::RepoConvergenceUnavailable,
)
.with_remediation("rch daemon start", "rch status --json"),
];
};
let summary = &convergence.summary;
let mut diagnostics = Vec::new();
let (severity, code, message) = if summary.failed > 0 {
(
ReliabilitySeverity::Critical,
ReliabilityReasonCode::RepoConvergenceFailed,
format!("{} worker(s) failed repo convergence", summary.failed),
)
} else if summary.drifting > 0 || summary.stale > 0 {
(
ReliabilitySeverity::Warning,
ReliabilityReasonCode::RepoConvergenceDrift,
format!(
"{} drifting and {} stale worker(s)",
summary.drifting, summary.stale
),
)
} else if summary.total_workers == 0 {
(
ReliabilitySeverity::Info,
ReliabilityReasonCode::RepoConvergenceNoWorkers,
"No worker repo-convergence records were reported".to_string(),
)
} else {
(
ReliabilitySeverity::Pass,
ReliabilityReasonCode::RepoConvergenceReady,
format!("{} worker(s) are repo-converged", summary.ready),
)
};
let mut summary_diag = ReliabilityDiagnostic::new(
ReliabilityCategory::RepoPresence,
"repo_convergence",
severity,
message,
code,
)
.with_details(format!(
"status={}, total={}, ready={}, converging={}, drifting={}, failed={}, stale={}",
convergence.status,
summary.total_workers,
summary.ready,
summary.converging,
summary.drifting,
summary.failed,
summary.stale
));
if matches!(
severity,
ReliabilitySeverity::Critical | ReliabilitySeverity::Warning
) {
summary_diag =
summary_diag.with_remediation("rch workers probe --all", "rch status --json");
}
diagnostics.push(summary_diag);
diagnostics.extend(convergence.workers.iter().filter_map(|worker| {
if worker.missing_repos.is_empty() && worker.drift_state == "ready" {
return None;
}
let severity = if worker.drift_state == "failed" {
ReliabilitySeverity::Critical
} else {
ReliabilitySeverity::Warning
};
let missing = if worker.missing_repos.is_empty() {
"none".to_string()
} else {
worker.missing_repos.join(", ")
};
let mut diagnostic = ReliabilityDiagnostic::new(
ReliabilityCategory::RepoPresence,
"worker_repo_presence",
severity,
format!(
"Worker {} repo state is {}",
worker.worker_id, worker.drift_state
),
ReliabilityReasonCode::WorkerRepoNotReady,
)
.with_worker(worker.worker_id.clone())
.with_details(format!(
"confidence={:.2}, missing_repos={}, attempts_remaining={}, time_budget_ms={}",
worker.drift_confidence,
missing,
worker.attempt_budget_remaining,
worker.time_budget_remaining_ms
));
if let Some(command) = worker.remediation.first() {
diagnostic =
diagnostic.with_remediation(command.clone(), "rch status --workers --json");
} else {
diagnostic = diagnostic
.with_remediation("rch workers probe --all", "rch status --workers --json");
}
Some(diagnostic)
}));
diagnostics
}
fn reliability_disk_pressure_diagnostics(
status: Option<&DaemonFullStatusResponse>,
) -> Vec<ReliabilityDiagnostic> {
let Some(status) = status else {
return vec![
ReliabilityDiagnostic::new(
ReliabilityCategory::DiskPressure,
"disk_pressure",
ReliabilitySeverity::Warning,
"Disk-pressure telemetry is unavailable because daemon status could not be read",
ReliabilityReasonCode::DiskPressureUnavailable,
)
.with_remediation("rch daemon start", "rch status --workers --json"),
];
};
if status.workers.is_empty() {
return vec![ReliabilityDiagnostic::new(
ReliabilityCategory::DiskPressure,
"disk_pressure",
ReliabilitySeverity::Info,
"No workers reported disk-pressure telemetry",
ReliabilityReasonCode::DiskPressureNoWorkers,
)];
}
status
.workers
.iter()
.map(|worker| {
let state = worker
.pressure_state
.as_deref()
.unwrap_or("telemetry_gap");
let (severity, code, message) = match state {
"critical" => (
ReliabilitySeverity::Critical,
ReliabilityReasonCode::WorkerDiskPressureCritical,
format!(
"Worker {} has critical disk pressure ({})",
worker.id,
format_disk_free(worker.pressure_disk_free_gb)
),
),
"warning" => (
ReliabilitySeverity::Warning,
ReliabilityReasonCode::WorkerDiskPressureWarning,
format!(
"Worker {} has elevated disk pressure ({})",
worker.id,
format_disk_free(worker.pressure_disk_free_gb)
),
),
"healthy" => (
ReliabilitySeverity::Pass,
ReliabilityReasonCode::WorkerDiskPressureHealthy,
format!(
"Worker {} disk pressure is healthy ({})",
worker.id,
format_disk_free(worker.pressure_disk_free_gb)
),
),
_ => (
ReliabilitySeverity::Warning,
ReliabilityReasonCode::WorkerDiskPressureTelemetryGap,
format!("Worker {} is missing fresh disk telemetry", worker.id),
),
};
let mut details = String::with_capacity(256);
use std::fmt::Write as _;
let _ = write!(
details,
"state={state}, confidence={}, free_gb=",
worker.pressure_confidence.as_deref().unwrap_or("unknown")
);
push_opt_f64(&mut details, worker.pressure_disk_free_gb, 2);
details.push_str(", total_gb=");
push_opt_f64(&mut details, worker.pressure_disk_total_gb, 2);
details.push_str(", free_ratio=");
push_opt_f64(&mut details, worker.pressure_disk_free_ratio, 3);
details.push_str(", io_util_pct=");
push_opt_f64(&mut details, worker.pressure_disk_io_util_pct, 1);
details.push_str(", memory_pressure=");
push_opt_f64(&mut details, worker.pressure_memory_pressure, 2);
details.push_str(", telemetry_age_secs=");
push_opt_display(&mut details, worker.pressure_telemetry_age_secs);
details.push_str(", telemetry_fresh=");
push_opt_display(&mut details, worker.pressure_telemetry_fresh);
let mut diagnostic = ReliabilityDiagnostic::new(
ReliabilityCategory::DiskPressure,
"worker_disk_pressure",
severity,
message,
code,
)
.with_worker(worker.id.clone())
.with_details(details);
if severity != ReliabilitySeverity::Pass {
let user_q = shell_escape::escape(worker.user.clone().into());
let host_q = shell_escape::escape(worker.host.clone().into());
diagnostic = diagnostic.with_remediation(
format!(
"ssh {user_q}@{host_q} 'df -h / /tmp && du -sh /tmp/rch-* /tmp/rch_target_* 2>/dev/null'",
),
"rch status --workers --json",
);
}
diagnostic
})
.collect()
}
fn format_disk_free(value: Option<f64>) -> String {
value
.map(|gb| format!("{gb:.1} GB free"))
.unwrap_or_else(|| "free space unknown".to_string())
}
fn push_opt_f64(buf: &mut String, value: Option<f64>, precision: usize) {
use std::fmt::Write as _;
match value {
Some(v) => {
let _ = write!(buf, "{v:.precision$}");
}
None => buf.push_str("unknown"),
}
}
fn push_opt_display<T: std::fmt::Display>(buf: &mut String, value: Option<T>) {
use std::fmt::Write as _;
match value {
Some(v) => {
let _ = write!(buf, "{v}");
}
None => buf.push_str("unknown"),
}
}
fn reliability_process_debt_diagnostics(
status: Option<&DaemonFullStatusResponse>,
) -> Vec<ReliabilityDiagnostic> {
let Some(status) = status else {
return vec![
ReliabilityDiagnostic::new(
ReliabilityCategory::ProcessDebt,
"process_debt",
ReliabilitySeverity::Warning,
"Process-debt health is unavailable because daemon status could not be read",
ReliabilityReasonCode::ProcessDebtUnavailable,
)
.with_remediation("rch daemon start", "rch status --jobs --json"),
];
};
let cancellation = evaluate_cancellation_health(status);
let severity = match cancellation.status {
CheckStatus::Pass => ReliabilitySeverity::Pass,
CheckStatus::Warning => ReliabilitySeverity::Warning,
CheckStatus::Fail => ReliabilitySeverity::Critical,
CheckStatus::Skipped => ReliabilitySeverity::Info,
};
let mut diagnostic = ReliabilityDiagnostic::new(
ReliabilityCategory::ProcessDebt,
"cancellation_cleanup",
severity,
cancellation.message,
match severity {
ReliabilitySeverity::Pass => ReliabilityReasonCode::CancellationCleanupHealthy,
ReliabilitySeverity::Info => ReliabilityReasonCode::CancellationCleanupSkipped,
ReliabilitySeverity::Warning => ReliabilityReasonCode::CancellationCleanupDegraded,
ReliabilitySeverity::Critical => ReliabilityReasonCode::CancellationCleanupFailed,
},
);
if let Some(details) = cancellation.details {
diagnostic = diagnostic.with_details(details);
}
if let Some(suggestion) = cancellation.suggestion {
diagnostic = diagnostic.with_remediation(suggestion, "rch status --jobs --json");
}
vec![diagnostic]
}
fn reliability_helper_compatibility_diagnostics() -> Vec<ReliabilityDiagnostic> {
[
("ssh", "SSH transport", ReliabilitySeverity::Critical),
(
"rsync",
"incremental transfer",
ReliabilitySeverity::Critical,
),
("zstd", "compressed transfer", ReliabilitySeverity::Critical),
("cargo", "Rust build fallback", ReliabilitySeverity::Warning),
]
.into_iter()
.map(|(cmd, description, missing_severity)| {
let rsync = (cmd == "rsync").then(|| {
let configured = crate::config::load_config()
.ok()
.and_then(|config| config.transfer.rsync_bin);
resolve_rsync(configured.as_deref())
});
let available = match &rsync {
Some(resolution) => resolution.is_ok(),
None => which(cmd).is_ok(),
};
if available {
let mut diagnostic = ReliabilityDiagnostic::new(
ReliabilityCategory::HelperCompatibility,
cmd,
ReliabilitySeverity::Pass,
format!("{cmd} is available for {description}"),
ReliabilityReasonCode::HelperAvailable,
);
let details = match &rsync {
Some(Ok(resolved)) => Some(resolved.describe()),
_ => command_version(cmd),
};
if let Some(details) = details {
diagnostic = diagnostic.with_details(details);
}
diagnostic
} else {
ReliabilityDiagnostic::new(
ReliabilityCategory::HelperCompatibility,
cmd,
missing_severity,
format!("{cmd} is missing; {description} may fail or fall back"),
ReliabilityReasonCode::HelperMissing,
)
.with_remediation(
format!("Install {cmd} with the system package manager"),
"rch doctor --reliability --json",
)
}
})
.collect()
}
fn helper_diagnostics_from_probe_result(
prefetched: Option<Vec<ReliabilityDiagnostic>>,
outcome: ProbeOutcome,
) -> Vec<ReliabilityDiagnostic> {
if let Some(diagnostics) = prefetched {
return diagnostics;
}
vec![
ReliabilityDiagnostic::new(
ReliabilityCategory::HelperCompatibility,
"helper_probe",
ReliabilitySeverity::Warning,
"Helper compatibility could not be checked within the probe budget",
ReliabilityReasonCode::HelperProbeUnavailable,
)
.with_details(format!(
"probe_outcome={} timeout_secs={}",
outcome.label(),
PROBE_TIMEOUT.as_secs()
))
.with_remediation(
"rch doctor --reliability --scope helpers --json",
"rch doctor --reliability --scope helpers --json",
),
]
}
fn reliability_rollout_posture_diagnostics(
config_load: Result<&rch_common::RchConfig, &str>,
) -> Vec<ReliabilityDiagnostic> {
let mut diagnostics = Vec::new();
match config_load {
Ok(config) => {
let mut hook_diag = if config.self_healing.hook_starts_daemon {
ReliabilityDiagnostic::new(
ReliabilityCategory::RolloutPosture,
"hook_starts_daemon",
ReliabilitySeverity::Pass,
"Hook auto-start is enabled",
ReliabilityReasonCode::HookAutoStartEnabled,
)
} else {
ReliabilityDiagnostic::new(
ReliabilityCategory::RolloutPosture,
"hook_starts_daemon",
ReliabilitySeverity::Warning,
"Hook auto-start is disabled; daemon outages may silently force local builds",
ReliabilityReasonCode::HookAutoStartDisabled,
)
.with_remediation(
"rch config set self_healing.hook_starts_daemon true",
"rch config get self_healing.hook_starts_daemon --json",
)
};
hook_diag = hook_diag.with_details(format!(
"cooldown_secs={}, timeout_secs={}",
config.self_healing.auto_start_cooldown_secs,
config.self_healing.auto_start_timeout_secs
));
diagnostics.push(hook_diag);
if config.self_healing.daemon_installs_hooks {
diagnostics.push(ReliabilityDiagnostic::new(
ReliabilityCategory::RolloutPosture,
"daemon_installs_hooks",
ReliabilitySeverity::Pass,
"Daemon hook repair is enabled",
ReliabilityReasonCode::DaemonHookRepairEnabled,
));
} else {
diagnostics.push(
ReliabilityDiagnostic::new(
ReliabilityCategory::RolloutPosture,
"daemon_installs_hooks",
ReliabilitySeverity::Warning,
"Daemon hook repair is disabled; hook drift may persist",
ReliabilityReasonCode::DaemonHookRepairDisabled,
)
.with_remediation(
"rch config set self_healing.daemon_installs_hooks true",
"rch config get self_healing.daemon_installs_hooks --json",
),
);
}
}
Err(err) => diagnostics.push(
ReliabilityDiagnostic::new(
ReliabilityCategory::RolloutPosture,
"config_load",
ReliabilitySeverity::Warning,
"Configuration could not be loaded; rollout posture is partial",
ReliabilityReasonCode::ConfigLoadFailed,
)
.with_details(err)
.with_remediation(
"rch config doctor --json",
"rch doctor --reliability --json",
),
),
}
diagnostics.push(ReliabilityDiagnostic::new(
ReliabilityCategory::RolloutPosture,
"status_surface",
ReliabilitySeverity::Pass,
"Unified status surface is compiled in",
ReliabilityReasonCode::StatusSurfaceAvailable,
));
diagnostics.push(ReliabilityDiagnostic::new(
ReliabilityCategory::RolloutPosture,
"repo_convergence_gate",
ReliabilitySeverity::Pass,
"Repo-convergence status endpoint is wired into the CLI",
ReliabilityReasonCode::RepoConvergenceSurfaceAvailable,
));
diagnostics.push(ReliabilityDiagnostic::new(
ReliabilityCategory::RolloutPosture,
"disk_pressure_gate",
ReliabilitySeverity::Pass,
"Disk-pressure fields are wired into worker status",
ReliabilityReasonCode::DiskPressureSurfaceAvailable,
));
diagnostics
}
fn reliability_hook_consistency_diagnostics(
installed_hook_command: Option<&str>,
config_socket: Option<&str>,
canonical_socket: &str,
) -> Vec<ReliabilityDiagnostic> {
let mut diagnostics = Vec::new();
match installed_hook_command {
Some(cmd) => diagnostics.push(
ReliabilityDiagnostic::new(
ReliabilityCategory::RolloutPosture,
"hook_installed",
ReliabilitySeverity::Pass,
"Claude Code PreToolUse rch hook is installed",
ReliabilityReasonCode::HookInstalled,
)
.with_details(format!("hook command: {cmd}")),
),
None => diagnostics.push(
ReliabilityDiagnostic::new(
ReliabilityCategory::RolloutPosture,
"hook_installed",
ReliabilitySeverity::Warning,
"Claude Code PreToolUse rch hook is missing; builds will not offload",
ReliabilityReasonCode::HookNotInstalled,
)
.with_remediation("rch hook install", "rch hook status --json"),
),
}
let socket_consistent = match config_socket {
None => true,
Some(configured) => configured.trim() == canonical_socket,
};
if socket_consistent {
diagnostics.push(ReliabilityDiagnostic::new(
ReliabilityCategory::RolloutPosture,
"socket_path",
ReliabilitySeverity::Pass,
"Configured socket path matches the canonical daemon socket",
ReliabilityReasonCode::SocketPathConsistent,
));
} else {
let configured = config_socket.unwrap_or("").trim();
diagnostics.push(
ReliabilityDiagnostic::new(
ReliabilityCategory::RolloutPosture,
"socket_path",
ReliabilitySeverity::Warning,
"Configured socket path diverges from the canonical daemon socket; \
the hook may not reach the daemon",
ReliabilityReasonCode::SocketPathMismatch,
)
.with_details(format!(
"configured={configured}, canonical={canonical_socket}"
))
.with_remediation(
format!("rch config set general.socket_path {canonical_socket}"),
"rch doctor --reliability --json",
),
);
}
diagnostics
}
fn reliability_schema_compatibility_diagnostics() -> Vec<ReliabilityDiagnostic> {
let entries: [(&str, &str, &str, &str); 4] = [
(
"doctor_reliability",
RELIABILITY_DOCTOR_SCHEMA_VERSION,
EXPECTED_RELIABILITY_DOCTOR_SCHEMA_VERSION,
"reliability doctor response",
),
(
"status",
crate::status_types::STATUS_SCHEMA_VERSION,
EXPECTED_STATUS_SCHEMA_VERSION,
"CLI status response",
),
(
"repo_updater_contract",
rch_common::REPO_UPDATER_CONTRACT_SCHEMA_VERSION,
EXPECTED_REPO_UPDATER_CONTRACT_SCHEMA_VERSION,
"repo updater contract",
),
(
"process_triage_contract",
rch_common::e2e::PROCESS_TRIAGE_CONTRACT_SCHEMA_VERSION,
EXPECTED_PROCESS_TRIAGE_CONTRACT_SCHEMA_VERSION,
"process triage contract",
),
];
entries
.into_iter()
.map(|(name, actual, expected, description)| {
schema_compatibility_diagnostic(name, actual, expected, description)
})
.collect()
}
fn schema_compatibility_diagnostic(
name: &str,
actual: &str,
expected: &str,
description: &str,
) -> ReliabilityDiagnostic {
if actual == expected {
ReliabilityDiagnostic::new(
ReliabilityCategory::SchemaCompatibility,
name,
ReliabilitySeverity::Pass,
format!("{description} schema version is compatible"),
ReliabilityReasonCode::SchemaCompatible,
)
.with_details(format!("schema_version={actual} expected={expected}"))
} else {
ReliabilityDiagnostic::new(
ReliabilityCategory::SchemaCompatibility,
name,
ReliabilitySeverity::Critical,
format!("{description} schema version is incompatible"),
ReliabilityReasonCode::SchemaIncompatible,
)
.with_details(format!("expected={expected}, actual={actual}"))
.with_remediation(
"Upgrade rch/rchd/rch-wkr binaries to the same release",
"rch doctor --reliability --check-schemas --json",
)
}
}
const DAEMON_UNREACHABLE_REASON_CODES: &[ReliabilityReasonCode] = &[
ReliabilityReasonCode::DaemonStatusUnavailable,
ReliabilityReasonCode::DiskPressureUnavailable,
ReliabilityReasonCode::ProcessDebtUnavailable,
ReliabilityReasonCode::RepoConvergenceUnavailable,
];
fn build_reliability_doctor_response(
mode: ReliabilityDoctorMode,
scope: &ReliabilityScopeSet,
diagnostics: Vec<ReliabilityDiagnostic>,
) -> ReliabilityDoctorResponse {
let mut categories = BTreeSet::new();
let mut pass = 0;
let mut info = 0;
let mut warning = 0;
let mut critical = 0;
for diagnostic in &diagnostics {
categories.insert(diagnostic.category);
match diagnostic.severity {
ReliabilitySeverity::Pass => pass += 1,
ReliabilitySeverity::Info => info += 1,
ReliabilitySeverity::Warning => warning += 1,
ReliabilitySeverity::Critical => critical += 1,
}
}
let daemon_unreachable_reasons: Vec<String> = diagnostics
.iter()
.filter(|d| DAEMON_UNREACHABLE_REASON_CODES.contains(&d.code))
.map(|d| format!("{}: {}", d.check_name, d.message))
.collect();
let daemon_unreachable = !daemon_unreachable_reasons.is_empty();
let remediation_plan = build_reliability_remediation_plan(&diagnostics);
let summary = ReliabilityDoctorSummary {
total_checks: diagnostics.len(),
pass,
info,
warning,
critical,
categories_checked: categories.into_iter().collect(),
overall: aggregate_verdict(&diagnostics),
};
ReliabilityDoctorResponse {
schema_version: RELIABILITY_DOCTOR_SCHEMA_VERSION.to_string(),
mode,
scope: scope.as_strings(),
daemon_unreachable,
daemon_unreachable_reasons,
diagnostics,
summary,
remediation_plan,
fix_requested: false,
remediation_outcomes: Vec::new(),
}
}
fn build_reliability_remediation_plan(
diagnostics: &[ReliabilityDiagnostic],
) -> Vec<ReliabilityRemediationStep> {
let mut actionable = diagnostics
.iter()
.filter(|diagnostic| {
matches!(
diagnostic.severity,
ReliabilitySeverity::Critical | ReliabilitySeverity::Warning
) && diagnostic.remediation_command.is_some()
})
.collect::<Vec<_>>();
actionable.sort_by_key(|diagnostic| match diagnostic.severity {
ReliabilitySeverity::Critical => 0,
ReliabilitySeverity::Warning => 1,
ReliabilitySeverity::Info => 2,
ReliabilitySeverity::Pass => 3,
});
actionable
.into_iter()
.enumerate()
.map(|(index, diagnostic)| ReliabilityRemediationStep {
order: u32::try_from(index + 1).unwrap_or(u32::MAX),
category: diagnostic.category,
code: diagnostic.code,
description: format!("{}: {}", diagnostic.check_name, diagnostic.message),
command: diagnostic.remediation_command.clone().unwrap_or_default(),
validation: diagnostic
.validation_check
.clone()
.unwrap_or_else(|| "rch doctor --reliability --json".to_string()),
requires_restart: diagnostic.code.requires_restart(),
dry_run_safe: diagnostic.dry_run_safe,
auto_fixable: auto_config_flip(diagnostic.code).is_some(),
})
.collect()
}
fn apply_reliability_remediations(response: &mut ReliabilityDoctorResponse) {
if !response.fix_requested {
return;
}
let preview = matches!(response.mode, ReliabilityDoctorMode::DryRun);
let current = crate::config::load_config().ok();
let mut outcomes = Vec::with_capacity(response.remediation_plan.len());
for step in &response.remediation_plan {
let started = Instant::now();
let outcome = match auto_config_flip(step.code) {
Some(flip) => {
let already = current
.as_ref()
.is_some_and(|cfg| config_flip_satisfied(cfg, flip));
let command = format!("rch config set {} {}", flip.key, flip.value);
let (status, detail) = match plan_auto_flip(already, preview) {
AutoFlipPlan::AlreadySatisfied => {
(RemediationOutcomeStatus::AlreadySatisfied, None)
}
AutoFlipPlan::WouldApply => (
RemediationOutcomeStatus::WouldApply,
Some(format!("would set {} = {}", flip.key, flip.value)),
),
AutoFlipPlan::Apply => {
match crate::commands::default_config_path().and_then(|path| {
crate::commands::apply_config_set(&path, flip.key, flip.value)
}) {
Ok(()) => (
RemediationOutcomeStatus::Applied,
Some(format!("set {} = {}", flip.key, flip.value)),
),
Err(e) => (RemediationOutcomeStatus::Failed, Some(e.to_string())),
}
}
};
ReliabilityRemediationOutcome {
order: step.order,
code: step.code,
description: step.description.clone(),
command,
status,
detail,
}
}
None => ReliabilityRemediationOutcome {
order: step.order,
code: step.code,
description: step.description.clone(),
command: step.command.clone(),
status: RemediationOutcomeStatus::Manual,
detail: None,
},
};
tracing::info!(
target: "rch::doctor::remediation",
order = outcome.order,
code = outcome.code.code(),
status = outcome.status.label(),
duration_seconds = started.elapsed().as_secs_f64(),
preview,
command = %outcome.command,
detail = outcome.detail.as_deref().unwrap_or(""),
"doctor.remediation.step",
);
outcomes.push(outcome);
}
response.remediation_outcomes = outcomes;
}
fn print_reliability_doctor_response(ctx: &OutputContext, response: &ReliabilityDoctorResponse) {
let style = ctx.theme();
println!("{}", style.format_header("RCH Reliability Doctor"));
println!();
if response.daemon_unreachable && !ctx.is_json() {
println!(
" {} [daemon down — local-only checks]",
StatusIndicator::Warning.display(style)
);
}
println!(
" {} schema {}",
StatusIndicator::Info.display(style),
style.value(&response.schema_version)
);
let verdict_indicator = match response.summary.overall {
ReliabilityVerdict::Healthy => StatusIndicator::Success,
ReliabilityVerdict::Degraded => StatusIndicator::Warning,
ReliabilityVerdict::Failing => StatusIndicator::Error,
};
println!(
" {} verdict: {} ({} check(s): {} pass, {} info, {} warning, {} critical)",
verdict_indicator.display(style),
style.highlight(response.summary.overall.label()),
response.summary.total_checks,
response.summary.pass,
response.summary.info,
response.summary.warning,
response.summary.critical
);
println!();
for diagnostic in &response.diagnostics {
let indicator = match diagnostic.severity {
ReliabilitySeverity::Pass => StatusIndicator::Success,
ReliabilitySeverity::Info => StatusIndicator::Info,
ReliabilitySeverity::Warning => StatusIndicator::Warning,
ReliabilitySeverity::Critical => StatusIndicator::Error,
};
println!(
" {} [{}] {}: {}",
indicator.display(style),
diagnostic.category.as_str(),
style.highlight(&diagnostic.check_name),
diagnostic.message
);
if ctx.is_verbose() {
if let Some(details) = &diagnostic.details {
println!(" {}", style.muted(details));
}
if let Some(command) = &diagnostic.remediation_command {
println!(" remediation: {}", style.value(command));
}
}
}
if !response.remediation_plan.is_empty() {
println!();
println!("{}", style.format_header("Remediation Plan"));
for step in &response.remediation_plan {
let auto = if step.auto_fixable {
style.success(" [auto-fixable]")
} else {
style.muted(" [manual]")
};
println!(
" {}. [{}] {}{}",
step.order,
step.category.as_str(),
step.description,
auto
);
println!(" {}", style.value(&step.command));
println!(" validate: {}", style.value(&step.validation));
}
}
if !response.remediation_outcomes.is_empty() {
println!();
let header = match response.mode {
ReliabilityDoctorMode::DryRun => "Fix Preview (--fix --dry-run, no changes applied)",
_ => "Fix Results",
};
println!("{}", style.format_header(header));
for outcome in &response.remediation_outcomes {
let indicator = match outcome.status {
RemediationOutcomeStatus::Applied | RemediationOutcomeStatus::AlreadySatisfied => {
StatusIndicator::Success
}
RemediationOutcomeStatus::WouldApply => StatusIndicator::Info,
RemediationOutcomeStatus::Manual => StatusIndicator::Warning,
RemediationOutcomeStatus::Failed => StatusIndicator::Error,
};
println!(
" {} {}. {} — {}",
indicator.display(style),
outcome.order,
style.highlight(outcome.status.label()),
outcome.description
);
if let Some(detail) = &outcome.detail {
println!(" {}", style.muted(detail));
}
}
}
println!();
if response.summary.critical > 0 {
println!(
"{}",
style.format_error("Reliability-critical issues found.")
);
} else if response.summary.warning > 0 {
println!(
"{}",
style.format_warning("Reliability checks found warnings.")
);
} else {
println!("{}", style.format_success("Reliability checks passed."));
}
if !ctx.is_json() {
println!();
println!(
" {} Run with {} for machine-readable output",
StatusIndicator::Info.display(style),
style.value("--json")
);
println!(
" {} Exit codes: 0 = healthy, 1 = degraded, 2 = failing (with --strict: degraded → 2; with --lenient: failing → 1)",
StatusIndicator::Info.display(style),
);
}
}
#[derive(Debug, Clone)]
#[allow(dead_code)]
pub struct QuickCheckResult {
pub daemon_running: bool,
pub worker_count: usize,
pub workers_healthy: Option<usize>,
pub hook_installed: bool,
pub warnings: Vec<String>,
pub errors: Vec<String>,
}
impl QuickCheckResult {
pub fn is_healthy(&self) -> bool {
self.daemon_running
&& self.worker_count > 0
&& self.hook_installed
&& self.warnings.is_empty()
&& self.errors.is_empty()
&& self.workers_healthy == Some(self.worker_count)
}
#[allow(dead_code)]
pub fn has_issues(&self) -> bool {
!self.warnings.is_empty() || !self.errors.is_empty()
}
}
pub fn run_quick_check() -> QuickCheckResult {
let socket_path = configured_or_default_socket_path();
let daemon_running = socket_path.exists() && daemon_socket_accepts_connections(&socket_path);
let (worker_count, workers_healthy) = match load_workers_from_config() {
Ok(workers) => (workers.len(), None),
Err(_) => (0, None),
};
let hook_installed = {
let home = dirs::home_dir().unwrap_or_else(|| PathBuf::from("/tmp"));
let settings_path = home.join(".claude").join("settings.json");
if settings_path.exists() {
read_config_capped(&settings_path)
.ok()
.and_then(|content| serde_json::from_str::<serde_json::Value>(&content).ok())
.map(|settings| {
settings
.get("hooks")
.and_then(|h| h.get("PreToolUse"))
.is_some()
})
.unwrap_or(false)
} else {
false
}
};
let mut warnings = Vec::new();
let mut errors = Vec::new();
if !daemon_running {
warnings.push("Daemon is not running".to_string());
}
if worker_count == 0 {
warnings.push("No workers configured".to_string());
}
if !hook_installed {
errors.push("Hook not installed".to_string());
}
if worker_count > 0 && workers_healthy.is_none() {
warnings.push(
"Worker health not probed by quick-check; run `rch doctor --reliability` for full status".to_string(),
);
}
let result = QuickCheckResult {
daemon_running,
worker_count,
workers_healthy,
hook_installed,
warnings,
errors,
};
tracing::debug!(
target: "rch::doctor::quick_check",
daemon_running,
worker_count,
workers_healthy = ?result.workers_healthy,
hook_installed,
is_healthy = result.is_healthy(),
"doctor.quick_check.complete",
);
result
}
pub fn print_quick_check_summary(result: &QuickCheckResult, ctx: &OutputContext) {
let style = ctx.theme();
println!();
println!("{}", style.highlight("Quick Health Check"));
println!();
if result.daemon_running {
println!(
" {} Daemon running",
StatusIndicator::Success.display(style)
);
} else {
println!(
" {} Daemon not running",
StatusIndicator::Warning.display(style)
);
}
if result.worker_count > 0 {
println!(
" {} {} worker(s) configured",
StatusIndicator::Success.display(style),
result.worker_count
);
} else {
println!(
" {} No workers configured",
StatusIndicator::Warning.display(style)
);
}
if result.hook_installed {
println!(
" {} Hook installed",
StatusIndicator::Success.display(style)
);
} else {
println!(
" {} Hook not installed",
StatusIndicator::Error.display(style)
);
}
println!();
if result.is_healthy() {
println!(
"{}",
style.format_success("Setup complete! Your next cargo build will compile remotely.")
);
} else if !result.errors.is_empty() {
println!(
"{}",
style.format_error(&format!(
"Issues found: {} error(s). Run 'rch doctor' for details.",
result.errors.len()
))
);
} else if !result.warnings.is_empty() {
println!(
"{}",
style.format_warning(&format!(
"Setup complete with {} warning(s). Run 'rch doctor' for details.",
result.warnings.len()
))
);
}
}
fn check_prerequisites(
checks: &mut Vec<CheckResult>,
ctx: &OutputContext,
_options: &DoctorOptions,
) {
let style = ctx.theme();
if !ctx.is_json() {
println!("{}", style.highlight("Prerequisites"));
println!();
}
let configured_rsync_bin = crate::config::load_config()
.ok()
.and_then(|config| config.transfer.rsync_bin);
let rsync_result = classify_rsync_check(resolve_rsync(configured_rsync_bin.as_deref()));
print_check_result(&rsync_result, ctx);
checks.push(rsync_result);
let zstd_result = check_command_exists("zstd", "Compression tool");
print_check_result(&zstd_result, ctx);
checks.push(zstd_result);
let ssh_result = check_command_exists("ssh", "SSH client");
print_check_result(&ssh_result, ctx);
checks.push(ssh_result);
let rustup_result = check_command_exists("rustup", "Rust toolchain manager");
print_check_result(&rustup_result, ctx);
checks.push(rustup_result);
let cargo_result = check_command_exists("cargo", "Rust build tool");
print_check_result(&cargo_result, ctx);
checks.push(cargo_result);
if !ctx.is_json() {
println!();
}
}
fn modern_rsync_install_hint() -> &'static str {
if cfg!(target_os = "macos") {
"brew install rsync (rch prefers /opt/homebrew/bin/rsync over the stock openrsync automatically)"
} else {
"install rsync 3.x with your package manager (e.g. apt install rsync)"
}
}
fn classify_rsync_check(resolution: Result<ResolvedRsync, RsyncResolveError>) -> CheckResult {
let result = |status: CheckStatus,
message: String,
details: Option<String>,
suggestion: Option<String>| {
CheckResult {
category: "prerequisites".to_string(),
name: "rsync".to_string(),
status,
message,
details,
suggestion,
fixable: false,
fix_applied: false,
fix_message: None,
}
};
let resolved = match resolution {
Ok(resolved) => resolved,
Err(error @ RsyncResolveError::NotFound) => {
return result(
CheckStatus::Fail,
"File synchronization not found".to_string(),
Some(error.to_string()),
Some(format!("Install rsync: {}", modern_rsync_install_hint())),
);
}
Err(error @ RsyncResolveError::OverrideMissing { origin, .. }) => {
return result(
CheckStatus::Fail,
"configured rsync binary is not executable".to_string(),
Some(error.to_string()),
Some(format!(
"Fix or unset {origin} so rch can pick a binary; to get a modern rsync: {}",
modern_rsync_install_hint()
)),
);
}
Err(error @ RsyncResolveError::ProbeFailed { .. }) => {
return result(
CheckStatus::Fail,
"rsync is present but `rsync --version` failed".to_string(),
Some(error.to_string()),
Some(format!("Reinstall rsync: {}", modern_rsync_install_hint())),
);
}
};
let flavor = resolved.flavor;
let capabilities = resolved.capabilities();
let details = if resolved.version_line.is_empty() {
resolved.describe()
} else {
format!("{} — {}", resolved.describe(), resolved.version_line)
};
if flavor == RsyncFlavor::Unknown {
return result(
CheckStatus::Warning,
"rsync flavour could not be determined from `rsync --version`".to_string(),
Some(details),
Some(format!(
"rch will use the rsync 3.x argv; if transfers fail with `unrecognized option`, \
set [transfer] rsync_bin (or RCH_RSYNC_BIN) to a modern rsync: {}",
modern_rsync_install_hint()
)),
);
}
if !flavor.is_supported() {
let (major, minor, patch) = rch_common::rsync_flavor::MIN_SUPPORTED_RSYNC;
return result(
CheckStatus::Fail,
format!("{flavor} is too old (rch needs rsync {major}.{minor}.{patch} or newer)"),
Some(details),
Some(format!("Upgrade rsync: {}", modern_rsync_install_hint())),
);
}
if capabilities.is_compatibility_mode() {
let mut rejected = vec!["`--info=*`"];
if !capabilities.compress_choice {
rejected.push("`--compress-choice=zstd`");
}
if !capabilities.append_verify {
rejected.push("`--append-verify`");
}
let compression = if capabilities.compress_choice {
"zstd compression"
} else {
"zlib compression"
};
return result(
CheckStatus::Pass,
format!("File synchronization is installed ({flavor}, compatibility mode)"),
Some(details),
Some(format!(
"{flavor} rejects {}; rch drives it with `--progress --stats -vv` and {compression}, \
and the zero-build-output detector fails open. For full-speed transfers and \
diagnostics: {}",
rejected.join(", "),
modern_rsync_install_hint()
)),
);
}
result(
CheckStatus::Pass,
"File synchronization is installed".to_string(),
Some(details),
None,
)
}
fn check_command_exists(cmd: &str, description: &str) -> CheckResult {
let exists = which(cmd).is_ok();
let version = exists.then(|| command_version(cmd)).flatten();
CheckResult {
category: "prerequisites".to_string(),
name: cmd.to_string(),
status: if exists {
CheckStatus::Pass
} else {
CheckStatus::Fail
},
message: if exists {
format!("{} is installed", description)
} else {
format!("{} not found", description)
},
details: version,
suggestion: if exists {
None
} else {
Some(format!("Install {} using your package manager", cmd))
},
fixable: !exists,
fix_applied: false,
fix_message: None,
}
}
fn command_version(cmd: &str) -> Option<String> {
use std::process::Stdio;
use std::time::{Duration, Instant};
let (program, mut command) = match cmd {
"rsync" => {
let mut command = Command::new("rsync");
command.arg("--version");
("rsync", command)
}
"zstd" => {
let mut command = Command::new("zstd");
command.arg("--version");
("zstd", command)
}
"ssh" => {
let mut command = Command::new("ssh");
command.arg("-V");
("ssh", command)
}
"rustup" => {
let mut command = Command::new("rustup");
command.arg("--version");
("rustup", command)
}
"cargo" => {
let mut command = Command::new("cargo");
command.arg("--version");
("cargo", command)
}
_ => return None,
};
let timeout = Duration::from_secs(5);
let mut child = command
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.ok()?;
let started = Instant::now();
let exited = loop {
match child.try_wait() {
Ok(Some(status)) => break Some(status),
Ok(None) => {
if started.elapsed() >= timeout {
break None;
}
std::thread::sleep(Duration::from_millis(50));
}
Err(_) => return None,
}
};
if exited.is_none() {
let _ = child.kill();
let _ = child.wait();
tracing::warn!(
target: "rch::doctor",
cmd = %program,
timeout_secs = timeout.as_secs(),
"version-probe subprocess timed out; killed"
);
return None;
}
let output = child.wait_with_output().ok()?;
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr = String::from_utf8_lossy(&output.stderr);
let text = if stdout.trim().is_empty() {
stderr.as_ref()
} else {
stdout.as_ref()
};
text.lines()
.map(str::trim)
.find(|line| !line.is_empty())
.map(str::to_string)
}
fn check_configuration(
checks: &mut Vec<CheckResult>,
ctx: &OutputContext,
_options: &DoctorOptions,
) {
let style = ctx.theme();
if !ctx.is_json() {
println!("{}", style.highlight("Configuration"));
println!();
}
let config_dir_result = check_config_directory();
print_check_result(&config_dir_result, ctx);
checks.push(config_dir_result);
let config_result = check_config_file();
print_check_result(&config_result, ctx);
checks.push(config_result);
let workers_result = check_workers_file();
print_check_result(&workers_result, ctx);
checks.push(workers_result);
for result in check_remediation_results() {
print_check_result(&result, ctx);
checks.push(result);
}
if !ctx.is_json() {
println!();
}
}
fn check_remediation_results() -> Vec<CheckResult> {
let Ok(config) = crate::config::load_config() else {
return Vec::new();
};
let issues = config.remediation.validate();
if issues.is_empty() {
return vec![CheckResult {
category: "configuration".to_string(),
name: "remediation_config".to_string(),
status: CheckStatus::Pass,
message: "Remediation settings valid".to_string(),
details: Some(
"[remediation] knobs within safe ranges (defaults apply where unset)".to_string(),
),
suggestion: None,
fixable: false,
fix_applied: false,
fix_message: None,
}];
}
issues
.into_iter()
.map(|issue| {
let status = match issue.severity {
rch_common::remediation_config::IssueSeverity::Error => CheckStatus::Fail,
rch_common::remediation_config::IssueSeverity::Warning => CheckStatus::Warning,
};
CheckResult {
category: "configuration".to_string(),
name: "remediation_config".to_string(),
status,
message: format!("{}: {}", issue.field, issue.message),
details: None,
suggestion: Some(format!(
"Adjust `{}` under [remediation] (or the matching RCH_REMEDIATION_* env var)",
issue.field
)),
fixable: false,
fix_applied: false,
fix_message: None,
}
})
.collect()
}
fn check_config_directory() -> CheckResult {
match config_dir() {
Some(dir) => {
if dir.exists() {
CheckResult {
category: "configuration".to_string(),
name: "config_directory".to_string(),
status: CheckStatus::Pass,
message: "Config directory exists".to_string(),
details: Some(dir.display().to_string()),
suggestion: None,
fixable: false,
fix_applied: false,
fix_message: None,
}
} else {
CheckResult {
category: "configuration".to_string(),
name: "config_directory".to_string(),
status: CheckStatus::Warning,
message: "Config directory does not exist".to_string(),
details: Some(dir.display().to_string()),
suggestion: Some("Run 'rch config init' to create it".to_string()),
fixable: true,
fix_applied: false,
fix_message: None,
}
}
}
None => CheckResult {
category: "configuration".to_string(),
name: "config_directory".to_string(),
status: CheckStatus::Fail,
message: "Could not determine config directory".to_string(),
details: None,
suggestion: None,
fixable: false,
fix_applied: false,
fix_message: None,
},
}
}
fn check_config_file() -> CheckResult {
let config_path = match config_dir() {
Some(d) => d.join("config.toml"),
None => {
return CheckResult {
category: "configuration".to_string(),
name: "config.toml".to_string(),
status: CheckStatus::Skipped,
message: "Skipped (no config directory)".to_string(),
details: None,
suggestion: None,
fixable: false,
fix_applied: false,
fix_message: None,
};
}
};
if !config_path.exists() {
return CheckResult {
category: "configuration".to_string(),
name: "config.toml".to_string(),
status: CheckStatus::Warning,
message: "config.toml not found (using defaults)".to_string(),
details: Some(config_path.display().to_string()),
suggestion: Some("Run 'rch config init' to create default config".to_string()),
fixable: true,
fix_applied: false,
fix_message: None,
};
}
match read_config_capped(&config_path) {
Ok(content) => match toml::from_str::<toml::Value>(&content) {
Ok(_) => CheckResult {
category: "configuration".to_string(),
name: "config.toml".to_string(),
status: CheckStatus::Pass,
message: "config.toml is valid".to_string(),
details: Some(config_path.display().to_string()),
suggestion: None,
fixable: false,
fix_applied: false,
fix_message: None,
},
Err(e) => CheckResult {
category: "configuration".to_string(),
name: "config.toml".to_string(),
status: CheckStatus::Fail,
message: "config.toml has syntax errors".to_string(),
details: Some(e.to_string()),
suggestion: Some("Fix TOML syntax errors in config file".to_string()),
fixable: false,
fix_applied: false,
fix_message: None,
},
},
Err(e) => CheckResult {
category: "configuration".to_string(),
name: "config.toml".to_string(),
status: CheckStatus::Fail,
message: "Could not read config.toml".to_string(),
details: Some(e.to_string()),
suggestion: None,
fixable: false,
fix_applied: false,
fix_message: None,
},
}
}
fn check_workers_file() -> CheckResult {
let workers_path = match config_dir() {
Some(d) => d.join("workers.toml"),
None => {
return CheckResult {
category: "configuration".to_string(),
name: "workers.toml".to_string(),
status: CheckStatus::Skipped,
message: "Skipped (no config directory)".to_string(),
details: None,
suggestion: None,
fixable: false,
fix_applied: false,
fix_message: None,
};
}
};
if !workers_path.exists() {
return CheckResult {
category: "configuration".to_string(),
name: "workers.toml".to_string(),
status: CheckStatus::Fail,
message: "workers.toml not found".to_string(),
details: Some(workers_path.display().to_string()),
suggestion: Some("Run 'rch config init' to create example workers config".to_string()),
fixable: true,
fix_applied: false,
fix_message: None,
};
}
match read_config_capped(&workers_path) {
Ok(content) => match toml::from_str::<toml::Value>(&content) {
Ok(parsed) => {
let worker_count = parsed
.get("workers")
.and_then(|w| w.as_array())
.map(|a| a.len())
.unwrap_or(0);
if worker_count == 0 {
CheckResult {
category: "configuration".to_string(),
name: "workers.toml".to_string(),
status: CheckStatus::Warning,
message: "workers.toml is valid but has no workers defined".to_string(),
details: Some(workers_path.display().to_string()),
suggestion: Some("Add worker definitions to workers.toml".to_string()),
fixable: false,
fix_applied: false,
fix_message: None,
}
} else {
CheckResult {
category: "configuration".to_string(),
name: "workers.toml".to_string(),
status: CheckStatus::Pass,
message: format!("workers.toml is valid ({} workers)", worker_count),
details: Some(workers_path.display().to_string()),
suggestion: None,
fixable: false,
fix_applied: false,
fix_message: None,
}
}
}
Err(e) => CheckResult {
category: "configuration".to_string(),
name: "workers.toml".to_string(),
status: CheckStatus::Fail,
message: "workers.toml has syntax errors".to_string(),
details: Some(e.to_string()),
suggestion: Some("Fix TOML syntax errors in workers file".to_string()),
fixable: false,
fix_applied: false,
fix_message: None,
},
},
Err(e) => CheckResult {
category: "configuration".to_string(),
name: "workers.toml".to_string(),
status: CheckStatus::Fail,
message: "Could not read workers.toml".to_string(),
details: Some(e.to_string()),
suggestion: None,
fixable: false,
fix_applied: false,
fix_message: None,
},
}
}
fn check_ssh_keys(
checks: &mut Vec<CheckResult>,
ctx: &OutputContext,
options: &DoctorOptions,
fixes_applied: &mut Vec<FixApplied>,
) {
let style = ctx.theme();
if !ctx.is_json() {
println!("{}", style.highlight("SSH Keys"));
println!();
}
let home = dirs::home_dir().unwrap_or_else(|| PathBuf::from("/tmp"));
let ssh_dir = home.join(".ssh");
let key_files = vec![
ssh_dir.join("id_ed25519"),
ssh_dir.join("id_rsa"),
ssh_dir.join("id_ecdsa"),
];
let mut found_key = false;
for key_path in key_files {
if key_path.exists() {
found_key = true;
let result = check_ssh_key_permissions(&key_path, options, fixes_applied);
print_check_result(&result, ctx);
checks.push(result);
}
}
if !found_key {
let default_key = ssh_dir.join("id_ed25519");
let result = CheckResult {
category: "ssh".to_string(),
name: "ssh_keys".to_string(),
status: CheckStatus::Warning,
message: "No standard SSH keys found".to_string(),
details: Some("Checked: ~/.ssh/id_{ed25519,rsa,ecdsa}".to_string()),
suggestion: Some(format!(
"Generate an SSH key: ssh-keygen -t ed25519 -f {} && ssh-add {}",
default_key.display(),
default_key.display()
)),
fixable: false,
fix_applied: false,
fix_message: None,
};
print_check_result(&result, ctx);
checks.push(result);
}
check_worker_identity_files(checks, ctx, options, fixes_applied);
let ssh_config_result = check_ssh_config();
print_check_result(&ssh_config_result, ctx);
checks.push(ssh_config_result);
if !ctx.is_json() {
println!();
}
}
#[cfg(unix)]
fn check_ssh_key_permissions(
key_path: &Path,
options: &DoctorOptions,
fixes_applied: &mut Vec<FixApplied>,
) -> CheckResult {
let key_name = key_path
.file_name()
.map(|n| n.to_string_lossy().to_string())
.unwrap_or_else(|| "unknown".to_string());
match std::fs::metadata(key_path) {
Ok(meta) => {
let mode = meta.permissions().mode();
let perms = mode & 0o777;
if perms == 0o600 || perms == 0o400 {
CheckResult {
category: "ssh".to_string(),
name: key_name,
status: CheckStatus::Pass,
message: format!("SSH key exists with correct permissions (0{:o})", perms),
details: Some(key_path.display().to_string()),
suggestion: None,
fixable: false,
fix_applied: false,
fix_message: None,
}
} else {
if options.fix {
if options.dry_run {
return CheckResult {
category: "ssh".to_string(),
name: key_name,
status: CheckStatus::Warning,
message: format!("SSH key has loose permissions (0{:o})", perms),
details: Some(key_path.display().to_string()),
suggestion: Some(format!("Run: chmod 600 {}", key_path.display())),
fixable: true,
fix_applied: false,
fix_message: Some(format!(
"Would change permissions from 0{:o} to 0600",
perms
)),
};
}
match std::fs::set_permissions(key_path, std::fs::Permissions::from_mode(0o600))
{
Ok(()) => {
fixes_applied.push(FixApplied {
check_name: key_name.clone(),
action: format!("Changed permissions from 0{:o} to 0600", perms),
success: true,
error: None,
});
CheckResult {
category: "ssh".to_string(),
name: key_name,
status: CheckStatus::Pass,
message: "SSH key permissions fixed to 0600".to_string(),
details: Some(key_path.display().to_string()),
suggestion: None,
fixable: false,
fix_applied: true,
fix_message: Some(format!(
"Changed permissions from 0{:o} to 0600",
perms
)),
}
}
Err(e) => {
fixes_applied.push(FixApplied {
check_name: key_name.clone(),
action: "Failed to fix permissions".to_string(),
success: false,
error: Some(e.to_string()),
});
CheckResult {
category: "ssh".to_string(),
name: key_name,
status: CheckStatus::Warning,
message: format!(
"SSH key has loose permissions (0{:o}), fix failed",
perms
),
details: Some(e.to_string()),
suggestion: Some(format!("Run: chmod 600 {}", key_path.display())),
fixable: true,
fix_applied: false,
fix_message: Some(format!("Failed to fix permissions: {}", e)),
}
}
}
} else {
CheckResult {
category: "ssh".to_string(),
name: key_name,
status: CheckStatus::Warning,
message: format!("SSH key has loose permissions (0{:o})", perms),
details: Some(key_path.display().to_string()),
suggestion: Some(format!("Run: chmod 600 {}", key_path.display())),
fixable: true,
fix_applied: false,
fix_message: None,
}
}
}
}
Err(e) => CheckResult {
category: "ssh".to_string(),
name: key_name,
status: CheckStatus::Fail,
message: "Could not read SSH key metadata".to_string(),
details: Some(e.to_string()),
suggestion: None,
fixable: false,
fix_applied: false,
fix_message: None,
},
}
}
#[cfg(not(unix))]
fn check_ssh_key_permissions(
key_path: &Path,
_options: &DoctorOptions,
_fixes_applied: &mut Vec<FixApplied>,
) -> CheckResult {
let key_name = key_path
.file_name()
.map(|n| n.to_string_lossy().to_string())
.unwrap_or_else(|| "unknown".to_string());
CheckResult {
category: "ssh".to_string(),
name: key_name,
status: CheckStatus::Skipped,
message: "SSH key permission checks are not supported on this platform".to_string(),
details: Some(key_path.display().to_string()),
suggestion: None,
fixable: false,
fix_applied: false,
fix_message: None,
}
}
fn check_worker_identity_files(
checks: &mut Vec<CheckResult>,
ctx: &OutputContext,
options: &DoctorOptions,
fixes_applied: &mut Vec<FixApplied>,
) {
let workers = match load_workers_from_config() {
Ok(w) => w,
Err(_) => return,
};
for worker in workers {
let key_path = PathBuf::from(shellexpand::tilde(&worker.identity_file).to_string());
let name = format!("worker_key:{}", worker.id.as_str());
let suggestion = ssh_worker_suggestion(&worker.user, &worker.host, &key_path);
if !key_path.exists() {
let result = CheckResult {
category: "ssh".to_string(),
name,
status: CheckStatus::Warning,
message: format!("Identity file missing for worker {}", worker.id.as_str()),
details: Some(key_path.display().to_string()),
suggestion: Some(suggestion),
fixable: false,
fix_applied: false,
fix_message: None,
};
print_check_result(&result, ctx);
checks.push(result);
continue;
}
let key_result = check_ssh_key_permissions(&key_path, options, fixes_applied);
let status = key_result.status;
let mut message = key_result.message;
message.push_str(&format!(" (worker {})", worker.id.as_str()));
let result = CheckResult {
category: "ssh".to_string(),
name,
status,
message,
details: key_result.details,
suggestion: Some(suggestion),
fixable: key_result.fixable,
fix_applied: key_result.fix_applied,
fix_message: key_result.fix_message,
};
print_check_result(&result, ctx);
checks.push(result);
}
}
fn ssh_worker_suggestion(user: &str, host: &str, key_path: &Path) -> String {
let key_q = shell_escape::escape(key_path.to_string_lossy());
let user_q = shell_escape::escape(user.into());
let host_q = shell_escape::escape(host.into());
format!(
"Copy key: ssh-copy-id -i {key_q} {user_q}@{host_q}; \
Test: ssh -i {key_q} {user_q}@{host_q} echo \"success\"; \
Agent: eval $(ssh-agent) && ssh-add {key_q}; \
Debug: ssh -vvv -i {key_q} {user_q}@{host_q}",
)
}
fn check_ssh_config() -> CheckResult {
let home = dirs::home_dir().unwrap_or_else(|| PathBuf::from("/tmp"));
let ssh_config = home.join(".ssh").join("config");
if ssh_config.exists() {
CheckResult {
category: "ssh".to_string(),
name: "ssh_config".to_string(),
status: CheckStatus::Pass,
message: "SSH config file exists".to_string(),
details: Some(ssh_config.display().to_string()),
suggestion: None,
fixable: false,
fix_applied: false,
fix_message: None,
}
} else {
CheckResult {
category: "ssh".to_string(),
name: "ssh_config".to_string(),
status: CheckStatus::Warning,
message: "No SSH config file".to_string(),
details: Some(ssh_config.display().to_string()),
suggestion: Some(
"Consider creating ~/.ssh/config for custom host settings".to_string(),
),
fixable: false,
fix_applied: false,
fix_message: None,
}
}
}
fn which_rchd_path() -> PathBuf {
if let Ok(exe_path) = std::env::current_exe()
&& let Some(dir) = exe_path.parent()
{
let rchd = dir.join("rchd");
if rchd.exists() {
return rchd;
}
}
which("rchd").unwrap_or_else(|_| PathBuf::from("rchd"))
}
fn start_rchd_via_systemd_user(socket_path: &Path) -> Result<(), String> {
let start = Command::new("systemctl")
.args(["--user", "start", "rchd.service"])
.stdout(Stdio::null())
.stderr(Stdio::null())
.status()
.map_err(|e| format!("systemctl start failed to invoke: {e}"))?;
if !start.success() {
return Err(format!(
"systemctl --user start rchd.service exited {start}"
));
}
if wait_for_socket(socket_path, Duration::from_secs(5)) {
Ok(())
} else {
Err("systemd started rchd but no listener appeared on the socket within 5s".into())
}
}
fn spawn_rchd(rchd_path: &Path, socket_path: &Path) -> Result<(), String> {
let mut cmd = Command::new("nohup");
cmd.arg(rchd_path)
.arg("-s")
.arg(socket_path)
.stdout(Stdio::null())
.stderr(Stdio::null())
.stdin(Stdio::null());
let mut child = cmd.spawn().map_err(|e| match e.kind() {
std::io::ErrorKind::NotFound => "nohup not found while launching rchd".to_string(),
_ => e.to_string(),
})?;
let spawn_deadline = Instant::now() + Duration::from_millis(500);
loop {
if let Some(status) = child.try_wait().map_err(|e| e.to_string())? {
return if status.success() {
Ok(())
} else {
Err(format!(
"rchd launch wrapper exited unsuccessfully: {status}"
))
};
}
if Instant::now() >= spawn_deadline {
break;
}
std::thread::sleep(Duration::from_millis(10));
}
std::thread::spawn(move || {
let _ = child.wait();
});
Ok(())
}
fn wait_for_socket(socket_path: &Path, timeout: Duration) -> bool {
let deadline = Instant::now() + timeout;
while Instant::now() < deadline {
if daemon_socket_accepts_connections(socket_path) {
return true;
}
std::thread::sleep(Duration::from_millis(100));
}
daemon_socket_accepts_connections(socket_path)
}
fn start_daemon_with_binary(
socket_path: &Path,
rchd_path: &Path,
timeout: Duration,
) -> Result<(), String> {
if let Some(parent) = socket_path.parent() {
std::fs::create_dir_all(parent).map_err(|e| e.to_string())?;
}
spawn_rchd(rchd_path, socket_path)?;
if wait_for_socket(socket_path, timeout) {
return Ok(());
}
Err(format!(
"daemon process started but did not accept connections within {}s",
timeout.as_secs()
))
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
struct RustcWrapperState {
wrappers: BTreeMap<String, String>,
}
const RUSTC_WRAPPER_STATE_FILE: &str = "doctor-rustc-wrapper-state.json";
fn rustc_wrapper_state_path() -> Option<PathBuf> {
dirs::cache_dir().map(|d| d.join("rch").join(RUSTC_WRAPPER_STATE_FILE))
}
fn load_rustc_wrapper_state() -> RustcWrapperState {
let Some(path) = rustc_wrapper_state_path() else {
return RustcWrapperState::default();
};
match std::fs::read(&path) {
Ok(bytes) => serde_json::from_slice(&bytes).unwrap_or_default(),
Err(_) => RustcWrapperState::default(),
}
}
fn store_rustc_wrapper_state(state: &RustcWrapperState) -> bool {
let Some(path) = rustc_wrapper_state_path() else {
return false;
};
if let Some(parent) = path.parent() {
let _ = std::fs::create_dir_all(parent);
}
match serde_json::to_vec(state) {
Ok(bytes) => std::fs::write(&path, bytes).is_ok(),
Err(_) => false,
}
}
#[derive(Debug, Clone)]
struct WrapperObservation {
env_var: &'static str,
raw_value: String,
resolved: Option<PathBuf>,
identity: Option<String>,
}
const WRAPPER_IDENTITY_PREFIX_BYTES: u64 = 4 * 1024 * 1024;
fn hash_wrapper_identity(path: &Path) -> Option<String> {
let file = std::fs::File::open(path).ok()?;
let mut hasher = blake3::Hasher::new();
std::io::copy(&mut file.take(WRAPPER_IDENTITY_PREFIX_BYTES), &mut hasher).ok()?;
let hex = hasher.finalize().to_hex();
Some(hex.as_str()[..32].to_string())
}
fn wrapper_binary_path(raw: &str) -> Option<PathBuf> {
if raw.contains('/') {
let candidate = Path::new(raw);
if candidate.is_file() {
Some(candidate.to_path_buf())
} else {
None
}
} else {
which(raw).ok()
}
}
fn observe_wrapper(env_var: &'static str) -> Option<WrapperObservation> {
let raw_value = std::env::var(env_var).ok()?;
if raw_value.trim().is_empty() {
return None;
}
let resolved = wrapper_binary_path(&raw_value);
let identity = resolved.as_deref().and_then(hash_wrapper_identity);
Some(WrapperObservation {
env_var,
raw_value,
resolved,
identity,
})
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum WrapperVerdict {
FirstEnablement,
Unchanged,
IdentityChanged,
}
fn wrapper_verdict(state: &RustcWrapperState, obs: &WrapperObservation) -> WrapperVerdict {
match state.wrappers.get(obs.env_var) {
None => WrapperVerdict::FirstEnablement,
Some(prev) if *prev == obs.identity.as_deref().expect("hashed") => {
WrapperVerdict::Unchanged
}
Some(_) => WrapperVerdict::IdentityChanged,
}
}
fn rustc_wrapper_results(
state: &RustcWrapperState,
observed: &[WrapperObservation],
) -> Vec<CheckResult> {
const MONITORING_DOC: &str = "docs/guides/monitoring.md (RUSTC_WRAPPER section)";
if observed.is_empty() {
return vec![CheckResult {
category: "toolchain".to_string(),
name: "rustc_wrapper".to_string(),
status: CheckStatus::Pass,
message: "RUSTC_WRAPPER not set".to_string(),
details: Some(
"No compile wrapper configured; no wrapper-related fingerprint churn possible."
.to_string(),
),
suggestion: None,
fixable: false,
fix_applied: false,
fix_message: None,
}];
}
let mut results = Vec::new();
let mut new_state = state.clone();
for obs in observed {
let (status, message, details, suggestion);
let fingerprinted = obs.env_var == "RUSTC_WORKSPACE_WRAPPER";
match (&obs.resolved, &obs.identity) {
(None, _) => {
status = CheckStatus::Warning;
message = format!(
"{} set to '{}' but the wrapper binary was not found",
obs.env_var, obs.raw_value
);
details = Some(
"Cargo invokes this binary for every compilation; a missing \
wrapper breaks or stalls builds."
.to_string(),
);
suggestion = Some(
"Fix the path, unset the variable, or install the wrapper binary".to_string(),
);
}
(Some(path), identity) if !fingerprinted => {
status = CheckStatus::Pass;
message = format!("{} wrapper active: '{}'", obs.env_var, obs.raw_value);
details = Some(format!(
"Wrapper binary: {}. Plain RUSTC_WRAPPER is not part of \
Cargo's fingerprints (verified on current cargo): enabling, \
swapping, or removing it does NOT force a rebuild. The \
wrapper must still exec the real compiler.",
path.display()
));
suggestion = None;
new_state.wrappers.remove(obs.env_var);
let _ = identity;
}
(Some(path), Some(_)) => match wrapper_verdict(state, obs) {
WrapperVerdict::FirstEnablement => {
status = CheckStatus::Warning;
message = format!(
"{} wrapper enabled (first observation): '{}'",
obs.env_var, obs.raw_value
);
details = Some(format!(
"Wrapper binary: {}. Cargo fingerprints include this \
variable, so its first use compiles the workspace once \
— expected, NOT a regression. Subsequent builds are \
incremental; removing it falls back to previously \
valid artifacts without recompiling.",
path.display()
));
suggestion = Some(format!("See {MONITORING_DOC}"));
}
WrapperVerdict::Unchanged => {
status = CheckStatus::Pass;
message = format!("{} wrapper unchanged since last doctor run", obs.env_var);
details = Some(format!(
"Wrapper binary: {} — identity matches the recorded \
marker; no rebuild expected.",
path.display()
));
suggestion = None;
}
WrapperVerdict::IdentityChanged => {
status = CheckStatus::Warning;
message = format!(
"{} wrapper value changed since last doctor run",
obs.env_var
);
details = Some(format!(
"Wrapper binary: {}. A new wrapper value compiles the \
workspace once (the value is fingerprinted, not the \
binary contents) — expected after upgrading or \
replacing the wrapper, NOT a regression.",
path.display()
));
suggestion = Some(format!("See {MONITORING_DOC}"));
}
},
(Some(path), None) => {
status = CheckStatus::Warning;
message = format!(
"{} wrapper binary exists but could not be hashed",
obs.env_var
);
details = Some(format!(
"Wrapper binary: {}. Identity tracking skipped for this run.",
path.display()
));
suggestion = Some("Check file permissions".to_string());
}
}
if fingerprinted && let Some(identity) = &obs.identity {
new_state
.wrappers
.insert(obs.env_var.to_string(), identity.clone());
}
results.push(CheckResult {
category: "toolchain".to_string(),
name: "rustc_wrapper".to_string(),
status,
message,
details,
suggestion,
fixable: false,
fix_applied: false,
fix_message: None,
});
}
let _ = store_rustc_wrapper_state(&new_state);
results
}
fn rustc_wrapper_checks() -> Vec<CheckResult> {
let observed: Vec<WrapperObservation> = ["RUSTC_WRAPPER", "RUSTC_WORKSPACE_WRAPPER"]
.into_iter()
.filter_map(observe_wrapper)
.collect();
let state = load_rustc_wrapper_state();
rustc_wrapper_results(&state, &observed)
}
fn start_daemon_for_doctor(socket_path: &Path, timeout: Duration) -> Result<(), String> {
if let Some(parent) = socket_path.parent() {
std::fs::create_dir_all(parent).map_err(|e| e.to_string())?;
}
if cfg!(target_os = "linux") {
match start_rchd_via_systemd_user(socket_path) {
Ok(()) => return Ok(()),
Err(reason) => {
eprintln!(
"rch: not using systemd --user rchd.service ({reason}); \
falling back to direct spawn"
);
}
}
}
start_daemon_with_binary(socket_path, &which_rchd_path(), timeout)
}
fn check_daemon(
checks: &mut Vec<CheckResult>,
ctx: &OutputContext,
options: &DoctorOptions,
fixes_applied: &mut Vec<FixApplied>,
) {
let style = ctx.theme();
if !ctx.is_json() {
println!("{}", style.highlight("Daemon"));
println!();
}
let socket_path = configured_or_default_socket_path();
let socket_exists = socket_path.exists();
let socket_live = socket_exists && daemon_socket_accepts_connections(&socket_path);
let mut result = if socket_live {
CheckResult {
category: "daemon".to_string(),
name: "daemon_socket".to_string(),
status: CheckStatus::Pass,
message: "Daemon is accepting connections".to_string(),
details: Some(socket_path.to_string_lossy().to_string()),
suggestion: None,
fixable: false,
fix_applied: false,
fix_message: None,
}
} else if socket_exists {
CheckResult {
category: "daemon".to_string(),
name: "daemon_socket".to_string(),
status: CheckStatus::Warning,
message: "Daemon socket is stale or unreachable".to_string(),
details: Some(socket_path.to_string_lossy().to_string()),
suggestion: Some("Restart daemon with: rch daemon restart".to_string()),
fixable: true,
fix_applied: false,
fix_message: None,
}
} else {
CheckResult {
category: "daemon".to_string(),
name: "daemon_socket".to_string(),
status: CheckStatus::Warning,
message: "Daemon is not running".to_string(),
details: Some(socket_path.to_string_lossy().to_string()),
suggestion: Some("Start daemon with: rch daemon start".to_string()),
fixable: true,
fix_applied: false,
fix_message: None,
}
};
let mut fix_line: Option<(StatusIndicator, String)> = None;
if options.fix && result.fixable && result.status != CheckStatus::Pass {
if options.dry_run {
let msg = "Would start RCH daemon".to_string();
result.fix_message = Some(msg.clone());
fix_line = Some((StatusIndicator::Pending, format!("Would fix: {}", msg)));
} else {
match start_daemon_for_doctor(&socket_path, Duration::from_secs(3)) {
Ok(()) => {
let msg = "Started RCH daemon".to_string();
result.status = CheckStatus::Pass;
result.message = "Daemon started (fixed)".to_string();
result.details = Some(socket_path.to_string_lossy().to_string());
result.suggestion = None;
result.fixable = false;
result.fix_applied = true;
result.fix_message = Some(msg.clone());
fix_line = Some((StatusIndicator::Success, format!("Fixed: {}", msg)));
fixes_applied.push(FixApplied {
check_name: "daemon_socket".to_string(),
action: msg,
success: true,
error: None,
});
}
Err(e) => {
let msg = format!("Failed to start daemon: {}", e);
result.fix_message = Some(msg.clone());
fix_line = Some((StatusIndicator::Error, msg.clone()));
fixes_applied.push(FixApplied {
check_name: "daemon_socket".to_string(),
action: "Start RCH daemon".to_string(),
success: false,
error: Some(e),
});
}
}
}
}
if let Some((indicator, line)) = fix_line
&& !ctx.is_json()
{
let rendered = match indicator {
StatusIndicator::Success => style.success(&line),
StatusIndicator::Pending => style.muted(&line),
StatusIndicator::Error => style.error(&line),
_ => style.info(&line),
};
println!(" {} {}", indicator.display(style), rendered);
}
print_check_result(&result, ctx);
checks.push(result);
let legacy_socket = Path::new("/tmp/rch.sock");
if socket_path != legacy_socket && legacy_socket.exists() {
let legacy_result = CheckResult {
category: "daemon".to_string(),
name: "legacy_socket_path".to_string(),
status: CheckStatus::Warning,
message: "Legacy /tmp socket detected".to_string(),
details: Some(legacy_socket.display().to_string()),
suggestion: Some(
"Restart the daemon so it binds to the configured socket path".to_string(),
),
fixable: false,
fix_applied: false,
fix_message: None,
};
print_check_result(&legacy_result, ctx);
checks.push(legacy_result);
}
if !ctx.is_json() {
println!();
}
}
fn daemon_socket_accepts_connections(socket_path: &Path) -> bool {
#[cfg(unix)]
{
std::os::unix::net::UnixStream::connect(socket_path).is_ok()
}
#[cfg(not(unix))]
{
socket_path.exists()
}
}
fn evaluate_cancellation_health(
status: &crate::status_types::DaemonFullStatusResponse,
) -> CheckResult {
let mut total = 0usize;
let mut cleanup_failures = 0usize;
let mut sigkill_escalations = 0usize;
let mut unreachable_workers = 0usize;
let mut operations = Vec::new();
for build in &status.recent_builds {
let Some(cancellation) = &build.cancellation else {
continue;
};
total += 1;
operations.push(cancellation.operation_id.clone());
if !cancellation.cleanup_ok {
cleanup_failures += 1;
}
if cancellation.escalation_stage == "sigkill" {
sigkill_escalations += 1;
}
if cancellation
.worker_health
.as_ref()
.is_some_and(|health| health.status == "unreachable")
{
unreachable_workers += 1;
}
}
if total == 0 {
return CheckResult {
category: "cancellation".to_string(),
name: "cancellation_health".to_string(),
status: CheckStatus::Pass,
message: "No recent cancellation events detected".to_string(),
details: None,
suggestion: None,
fixable: false,
fix_applied: false,
fix_message: None,
};
}
let details = Some(format!(
"recent={}, cleanup_failures={}, sigkill_escalations={}, unreachable_workers={}, operations={}",
total,
cleanup_failures,
sigkill_escalations,
unreachable_workers,
operations.join(",")
));
if cleanup_failures > 0 {
return CheckResult {
category: "cancellation".to_string(),
name: "cancellation_health".to_string(),
status: CheckStatus::Fail,
message: format!(
"{} cancellation(s) ended with cleanup failures",
cleanup_failures
),
details,
suggestion: Some(
"Run `rch workers probe --all` and inspect daemon `cancellation_failed` events before retrying affected builds.".to_string(),
),
fixable: false,
fix_applied: false,
fix_message: None,
};
}
if sigkill_escalations > 0 || unreachable_workers > 0 {
return CheckResult {
category: "cancellation".to_string(),
name: "cancellation_health".to_string(),
status: CheckStatus::Warning,
message: format!(
"{} cancellation(s) required escalation and/or involved unreachable workers",
total
),
details,
suggestion: Some(
"Review `rch status --jobs` for stuck phases and verify worker connectivity with `rch workers probe --all`.".to_string(),
),
fixable: false,
fix_applied: false,
fix_message: None,
};
}
CheckResult {
category: "cancellation".to_string(),
name: "cancellation_health".to_string(),
status: CheckStatus::Pass,
message: format!(
"{} recent cancellation(s) completed with deterministic cleanup",
total
),
details,
suggestion: None,
fixable: false,
fix_applied: false,
fix_message: None,
}
}
async fn check_cancellation_health(checks: &mut Vec<CheckResult>, ctx: &OutputContext) {
let style = ctx.theme();
if !ctx.is_json() {
println!("{}", style.highlight("Cancellation Health"));
println!();
}
let socket_path = configured_or_default_socket_path();
let result = if !socket_path.exists() {
CheckResult {
category: "cancellation".to_string(),
name: "cancellation_health".to_string(),
status: CheckStatus::Skipped,
message: "Daemon socket not present; skipping cancellation diagnostics".to_string(),
details: Some(socket_path.display().to_string()),
suggestion: Some("Start daemon with: rch daemon start".to_string()),
fixable: false,
fix_applied: false,
fix_message: None,
}
} else {
match query_daemon_full_status().await {
Ok(status) => evaluate_cancellation_health(&status),
Err(e) => CheckResult {
category: "cancellation".to_string(),
name: "cancellation_health".to_string(),
status: CheckStatus::Warning,
message: "Unable to query daemon status for cancellation diagnostics".to_string(),
details: Some(e.to_string()),
suggestion: Some(
"Ensure daemon is responsive (`rch status`) and retry `rch doctor`."
.to_string(),
),
fixable: false,
fix_applied: false,
fix_message: None,
},
}
};
print_check_result(&result, ctx);
checks.push(result);
if !ctx.is_json() {
println!();
}
}
fn check_hooks(
checks: &mut Vec<CheckResult>,
ctx: &OutputContext,
options: &DoctorOptions,
fixes_applied: &mut Vec<FixApplied>,
) {
let style = ctx.theme();
if !ctx.is_json() {
println!("{}", style.highlight("Hooks"));
println!();
}
let mut claude_result = check_claude_code_hook();
let mut fix_message: Option<String> = None;
let mut fix_applied = false;
let mut fix_line: Option<(StatusIndicator, String)> = None;
if options.fix && claude_result.fixable && claude_result.status != CheckStatus::Pass {
match install_hook(AgentKind::ClaudeCode, options.dry_run) {
Ok(IdempotentResult::Changed) => {
fix_applied = true;
let msg = "Installed Claude Code hook".to_string();
fix_message = Some(msg.clone());
fix_line = Some((StatusIndicator::Success, format!("Fixed: {}", msg)));
fixes_applied.push(FixApplied {
check_name: "claude_code_hook".to_string(),
action: msg.clone(),
success: true,
error: None,
});
claude_result.status = CheckStatus::Pass;
claude_result.message = "Claude Code PreToolUse hook installed (fixed)".to_string();
claude_result.suggestion = None;
claude_result.fixable = false;
}
Ok(IdempotentResult::WouldChange(msg)) => {
fix_message = Some(msg.clone());
fix_line = Some((StatusIndicator::Pending, format!("Would fix: {}", msg)));
}
Ok(IdempotentResult::Unchanged) => {
fix_message = Some("Claude Code hook already installed".to_string());
claude_result.status = CheckStatus::Pass;
claude_result.message = "Claude Code PreToolUse hook already installed".to_string();
claude_result.suggestion = None;
claude_result.fixable = false;
}
Ok(other) => {
let msg = format!("Hook install result: {}", other);
fix_message = Some(msg.clone());
fix_line = Some((StatusIndicator::Success, format!("Fixed: {}", msg)));
if !options.dry_run {
fix_applied = true;
fixes_applied.push(FixApplied {
check_name: "claude_code_hook".to_string(),
action: msg.clone(),
success: true,
error: None,
});
claude_result.status = CheckStatus::Pass;
claude_result.message =
"Claude Code PreToolUse hook installed (fixed)".to_string();
claude_result.suggestion = None;
claude_result.fixable = false;
}
}
Err(e) => {
let msg = format!("Failed to install hook: {}", e);
fix_message = Some(msg.clone());
fix_line = Some((StatusIndicator::Error, msg.clone()));
if !options.dry_run {
fixes_applied.push(FixApplied {
check_name: "claude_code_hook".to_string(),
action: "Install Claude Code hook".to_string(),
success: false,
error: Some(e.to_string()),
});
}
}
}
}
claude_result.fix_applied = fix_applied;
claude_result.fix_message = fix_message;
if let Some((indicator, line)) = fix_line
&& !ctx.is_json()
{
let rendered = match indicator {
StatusIndicator::Success => style.success(&line),
StatusIndicator::Pending => style.muted(&line),
StatusIndicator::Error => style.error(&line),
_ => style.info(&line),
};
println!(" {} {}", indicator.display(style), rendered);
}
print_check_result(&claude_result, ctx);
checks.push(claude_result);
if !ctx.is_json() {
println!();
}
}
fn check_claude_code_hook() -> CheckResult {
let home = dirs::home_dir().unwrap_or_else(|| PathBuf::from("/tmp"));
let settings_path = home.join(".claude").join("settings.json");
if !settings_path.exists() {
return CheckResult {
category: "hooks".to_string(),
name: "claude_code_hook".to_string(),
status: CheckStatus::Warning,
message: "Claude Code settings not found".to_string(),
details: Some(settings_path.display().to_string()),
suggestion: Some("Install hook with: rch hook install".to_string()),
fixable: true,
fix_applied: false,
fix_message: None,
};
}
match read_config_capped(&settings_path) {
Ok(content) => match serde_json::from_str::<serde_json::Value>(&content) {
Ok(settings) => {
let has_hook = settings
.get("hooks")
.and_then(|h| h.get("PreToolUse"))
.is_some();
if has_hook {
CheckResult {
category: "hooks".to_string(),
name: "claude_code_hook".to_string(),
status: CheckStatus::Pass,
message: "Claude Code PreToolUse hook is installed".to_string(),
details: Some(settings_path.display().to_string()),
suggestion: None,
fixable: false,
fix_applied: false,
fix_message: None,
}
} else {
CheckResult {
category: "hooks".to_string(),
name: "claude_code_hook".to_string(),
status: CheckStatus::Warning,
message: "Claude Code PreToolUse hook not configured".to_string(),
details: Some(settings_path.display().to_string()),
suggestion: Some("Install hook with: rch hook install".to_string()),
fixable: true,
fix_applied: false,
fix_message: None,
}
}
}
Err(e) => CheckResult {
category: "hooks".to_string(),
name: "claude_code_hook".to_string(),
status: CheckStatus::Fail,
message: "Could not parse Claude Code settings".to_string(),
details: Some(e.to_string()),
suggestion: None,
fixable: false,
fix_applied: false,
fix_message: None,
},
},
Err(e) => CheckResult {
category: "hooks".to_string(),
name: "claude_code_hook".to_string(),
status: CheckStatus::Fail,
message: "Could not read Claude Code settings".to_string(),
details: Some(e.to_string()),
suggestion: None,
fixable: false,
fix_applied: false,
fix_message: None,
},
}
}
async fn check_workers(
checks: &mut Vec<CheckResult>,
ctx: &OutputContext,
options: &DoctorOptions,
) {
let style = ctx.theme();
if !ctx.is_json() {
println!("{}", style.highlight("Workers"));
println!();
}
let workers = match load_workers_from_config() {
Ok(w) => w,
Err(_) => {
let result = CheckResult {
category: "workers".to_string(),
name: "worker_config".to_string(),
status: CheckStatus::Skipped,
message: "Could not load workers configuration".to_string(),
details: None,
suggestion: Some("Run 'rch config init' to create workers.toml".to_string()),
fixable: false,
fix_applied: false,
fix_message: None,
};
print_check_result(&result, ctx);
checks.push(result);
return;
}
};
if workers.is_empty() {
let result = CheckResult {
category: "workers".to_string(),
name: "worker_count".to_string(),
status: CheckStatus::Warning,
message: "No workers configured".to_string(),
details: None,
suggestion: Some("Add workers to workers.toml".to_string()),
fixable: false,
fix_applied: false,
fix_message: None,
};
print_check_result(&result, ctx);
checks.push(result);
return;
}
let count_result = CheckResult {
category: "workers".to_string(),
name: "worker_count".to_string(),
status: CheckStatus::Pass,
message: format!("{} worker(s) configured", workers.len()),
details: Some(
workers
.iter()
.map(|w| w.id.as_str().to_string())
.collect::<Vec<_>>()
.join(", "),
),
suggestion: None,
fixable: false,
fix_applied: false,
fix_message: None,
};
print_check_result(&count_result, ctx);
checks.push(count_result);
if options.verbose && !ctx.is_json() {
println!(
" {}",
style.muted("(use --verbose to probe worker connectivity)")
);
}
if !ctx.is_json() {
println!();
}
}
fn check_telemetry_database(
checks: &mut Vec<CheckResult>,
ctx: &OutputContext,
options: &DoctorOptions,
) {
let style = ctx.theme();
if !ctx.is_json() {
println!("{}", style.highlight("Telemetry Database"));
println!();
}
let db_path = match ProjectDirs::from("com", "rch", "rch") {
Some(dirs) => dirs.data_local_dir().join("telemetry").join("telemetry.db"),
None => {
let result = CheckResult {
category: "telemetry".to_string(),
name: "telemetry_database".to_string(),
status: CheckStatus::Skipped,
message: "Could not determine telemetry database path".to_string(),
details: None,
suggestion: None,
fixable: false,
fix_applied: false,
fix_message: None,
};
print_check_result(&result, ctx);
checks.push(result);
return;
}
};
if !db_path.exists() {
let result = CheckResult {
category: "telemetry".to_string(),
name: "telemetry_database".to_string(),
status: CheckStatus::Warning,
message: "Telemetry database does not exist yet".to_string(),
details: Some(db_path.display().to_string()),
suggestion: Some("Database will be created when daemon starts".to_string()),
fixable: false,
fix_applied: false,
fix_message: None,
};
print_check_result(&result, ctx);
checks.push(result);
return;
}
match TelemetryStorage::new(&db_path, 30, 24, 365, 100) {
Ok(storage) => {
match storage.integrity_check() {
Ok(()) => {
let details = if options.verbose {
storage.stats().ok().map(|s| {
format!(
"Snapshots: {}, Aggregates: {}, SpeedScores: {}, Tests: {}, Size: {} KB",
s.telemetry_snapshots,
s.hourly_aggregates,
s.speedscore_entries,
s.test_runs,
s.db_size_bytes / 1024
)
})
} else {
Some(db_path.display().to_string())
};
let result = CheckResult {
category: "telemetry".to_string(),
name: "telemetry_database".to_string(),
status: CheckStatus::Pass,
message: "Telemetry database is healthy".to_string(),
details,
suggestion: None,
fixable: false,
fix_applied: false,
fix_message: None,
};
print_check_result(&result, ctx);
checks.push(result);
}
Err(e) => {
let result = CheckResult {
category: "telemetry".to_string(),
name: "telemetry_database".to_string(),
status: CheckStatus::Fail,
message: "Telemetry database integrity check failed".to_string(),
details: Some(e.to_string()),
suggestion: Some(
"Database may be corrupted. Delete and let daemon recreate it"
.to_string(),
),
fixable: false,
fix_applied: false,
fix_message: None,
};
print_check_result(&result, ctx);
checks.push(result);
}
}
}
Err(e) => {
let result = CheckResult {
category: "telemetry".to_string(),
name: "telemetry_database".to_string(),
status: CheckStatus::Fail,
message: "Could not open telemetry database".to_string(),
details: Some(e.to_string()),
suggestion: Some(
"Check file permissions or delete and let daemon recreate it".to_string(),
),
fixable: false,
fix_applied: false,
fix_message: None,
};
print_check_result(&result, ctx);
checks.push(result);
}
}
if !ctx.is_json() {
println!();
}
}
fn print_check_result(result: &CheckResult, ctx: &OutputContext) {
if ctx.is_json() {
return;
}
let style = ctx.theme();
let indicator = match result.status {
CheckStatus::Pass => StatusIndicator::Success,
CheckStatus::Warning => StatusIndicator::Warning,
CheckStatus::Fail => StatusIndicator::Error,
CheckStatus::Skipped => StatusIndicator::Pending,
};
print!(
" {} {} {}",
indicator.display(style),
style.highlight(&result.name),
style.muted("-")
);
match result.status {
CheckStatus::Pass => println!(" {}", style.success(&result.message)),
CheckStatus::Warning => println!(" {}", style.warning(&result.message)),
CheckStatus::Fail => println!(" {}", style.error(&result.message)),
CheckStatus::Skipped => println!(" {}", style.muted(&result.message)),
}
if let Some(ref details) = result.details
&& ctx.is_verbose()
{
println!(" {}", style.muted(details));
}
if let Some(ref suggestion) = result.suggestion {
println!(" {} {}", style.muted("Hint:"), style.info(suggestion));
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
use tempfile::TempDir;
#[test]
fn dispatcher_role_shim_check_is_read_only_and_preserves_findings() {
use rch_common::BoxRole;
let problems = vec![
"missing cargo".to_string(),
"PATH bypasses shim".to_string(),
];
let check = dispatcher_shim_check(BoxRole::Dispatcher, problems.clone());
assert_eq!(check.status, CheckStatus::Warning);
assert_eq!(
check.details.as_deref(),
Some("missing cargo; PATH bypasses shim")
);
assert!(
check
.suggestion
.as_deref()
.unwrap()
.contains("rch shim install")
);
assert!(!check.fixable && !check.fix_applied);
assert_eq!(
dispatcher_shim_check(BoxRole::Dispatcher, vec![]).status,
CheckStatus::Pass
);
for role in [BoxRole::Worker, BoxRole::Hybrid] {
let check = dispatcher_shim_check(role, problems.clone());
assert_eq!(check.status, CheckStatus::Pass);
assert!(check.message.contains(role.as_str()));
assert!(check.details.is_none() && check.suggestion.is_none());
}
}
#[test]
fn local_build_diagnostics_preserve_current_warning_and_failure_details() {
let mut observation = crate::local_builds::LocalBuildObservation {
builds: vec![crate::local_builds::LocalBuild {
pid: 4242,
comm: "cargo-rch-real".to_string(),
exe: None,
}],
scan_error: None,
state_error: Some("cache denied".to_string()),
transition: None,
};
let check = local_build_check(&observation);
assert_eq!(check.status, CheckStatus::Warning);
assert!(check.message.contains("1 local builds on a dispatcher"));
assert!(check.details.as_ref().unwrap().contains("pid=4242"));
assert!(check.details.as_ref().unwrap().contains("cache denied"));
assert!(!check.fixable);
let diagnostics = reliability_local_build_diagnostics(Some(&observation));
assert_eq!(
diagnostics[0].code,
ReliabilityReasonCode::LocalBuildsDetected
);
assert_eq!(diagnostics[0].severity, ReliabilitySeverity::Warning);
observation.builds.clear();
observation.scan_error = Some("proc denied".to_string());
assert_eq!(
reliability_local_build_diagnostics(Some(&observation))[0].code,
ReliabilityReasonCode::LocalBuildScanUnavailable
);
observation.scan_error = None;
assert_eq!(
reliability_local_build_diagnostics(Some(&observation))[0].code,
ReliabilityReasonCode::LocalBuildAlarmStateUnavailable
);
observation.state_error = None;
let diagnostics = reliability_local_build_diagnostics(Some(&observation));
assert_eq!(
diagnostics[0].code,
ReliabilityReasonCode::LocalBuildsAbsent
);
assert_eq!(diagnostics[0].severity, ReliabilitySeverity::Pass);
assert!(reliability_local_build_diagnostics(None).is_empty());
}
fn worker_status(
id: &str,
status: &str,
circuit_state: &str,
) -> crate::status_types::WorkerStatusFromApi {
serde_json::from_value(json!({
"id": id,
"host": "worker.example",
"user": "ubuntu",
"status": status,
"circuit_state": circuit_state,
"used_slots": 0,
"total_slots": 8,
"speed_score": 99.0,
"last_error": null
}))
.expect("worker status fixture should parse")
}
fn diag(severity: ReliabilitySeverity, code: ReliabilityReasonCode) -> ReliabilityDiagnostic {
ReliabilityDiagnostic::new(
ReliabilityCategory::Topology,
"fixture",
severity,
"fixture diagnostic",
code,
)
}
fn daemon_status(
workers_total: usize,
workers_healthy: usize,
workers: serde_json::Value,
) -> crate::status_types::DaemonFullStatusResponse {
serde_json::from_value(json!({
"daemon": {
"pid": 1,
"uptime_secs": 10,
"version": "0.1.0",
"socket_path": "/tmp/rch.sock",
"started_at": "2026-01-01T00:00:00Z",
"workers_total": workers_total,
"workers_healthy": workers_healthy,
"slots_total": 8,
"slots_available": 4
},
"workers": workers,
"active_builds": [],
"queued_builds": [],
"recent_builds": [],
"issues": [],
"alerts": [],
"stats": {
"total_builds": 0,
"success_count": 0,
"failure_count": 0,
"remote_count": 0,
"local_count": 0,
"avg_duration_ms": 0
},
"test_stats": null,
"saved_time": null
}))
.expect("daemon status fixture should parse")
}
fn ownership_worker(id: &str) -> rch_common::WorkerConfig {
serde_json::from_value(json!({
"id": id,
"host": "worker.example",
"user": "ubuntu",
"identity_file": "/tmp/id_ed25519",
"total_slots": 8
}))
.expect("worker config fixture should parse")
}
#[test]
fn mirror_ownership_maps_each_probe_outcome_to_its_code() {
let workers = vec![
ownership_worker("w-clean"),
ownership_worker("w-drift"),
ownership_worker("w-sudo"),
ownership_worker("w-dead"),
ownership_worker("w-mock"),
];
let results = vec![
crate::hook::ssh::MirrorOwnershipProbe::Healthy,
crate::hook::ssh::MirrorOwnershipProbe::Drift { count: 3 },
crate::hook::ssh::MirrorOwnershipProbe::CheckUnavailable,
crate::hook::ssh::MirrorOwnershipProbe::Unprobeable("ssh refused".to_string()),
crate::hook::ssh::MirrorOwnershipProbe::Skipped,
];
let diags = reliability_mirror_ownership_diagnostics(
Some(&workers),
None,
Some(&results),
ProbeOutcome::Ok,
None,
);
let by_code = |code: ReliabilityReasonCode| {
diags.iter().filter(|d| d.code == code).collect::<Vec<_>>()
};
assert_eq!(
by_code(ReliabilityReasonCode::WorkerMirrorOwnershipHealthy).len(),
1
);
let drift = by_code(ReliabilityReasonCode::WorkerMirrorOwnershipDrift);
assert_eq!(drift.len(), 1);
assert_eq!(drift[0].severity, ReliabilitySeverity::Warning);
assert_eq!(drift[0].worker_id.as_deref(), Some("w-drift"));
assert!(
drift[0]
.remediation_command
.as_deref()
.unwrap_or_default()
.contains("chown"),
"drift remediation must name the chown fix"
);
assert_eq!(
by_code(ReliabilityReasonCode::WorkerMirrorOwnershipCheckUnavailable).len(),
1
);
let dead = by_code(ReliabilityReasonCode::WorkerMirrorOwnershipUnprobeable);
assert_eq!(dead.len(), 1);
assert_eq!(dead[0].worker_id.as_deref(), Some("w-dead"));
assert_eq!(diags.len(), 4);
}
#[test]
fn mirror_ownership_fleet_fallback_when_probe_bucket_dies() {
let workers = vec![ownership_worker("w1"), ownership_worker("w2")];
let diags = reliability_mirror_ownership_diagnostics(
Some(&workers),
None,
None,
ProbeOutcome::Timeout,
None,
);
assert_eq!(diags.len(), 1);
assert_eq!(
diags[0].code,
ReliabilityReasonCode::WorkerMirrorOwnershipUnprobeable
);
assert_eq!(diags[0].severity, ReliabilitySeverity::Warning);
assert!(diags[0].message.contains("timeout"), "{}", diags[0].message);
}
#[test]
fn mirror_ownership_reports_empty_fleet_and_config_failures() {
let empty: Vec<rch_common::WorkerConfig> = Vec::new();
let diags = reliability_mirror_ownership_diagnostics(
Some(&empty),
None,
Some(&Vec::new()),
ProbeOutcome::Ok,
None,
);
assert_eq!(diags.len(), 1);
assert_eq!(
diags[0].code,
ReliabilityReasonCode::MirrorOwnershipNoWorkers
);
assert_eq!(diags[0].severity, ReliabilitySeverity::Info);
let config_err = reliability_mirror_ownership_diagnostics(
None,
Some("parse error".to_string()),
None,
ProbeOutcome::Ok,
None,
);
assert_eq!(config_err.len(), 1);
assert_eq!(
config_err[0].code,
ReliabilityReasonCode::WorkerMirrorOwnershipUnprobeable
);
assert!(
config_err[0]
.details
.as_deref()
.unwrap_or_default()
.contains("parse error")
);
let root_err = reliability_mirror_ownership_diagnostics(
None,
None,
None,
ProbeOutcome::Ok,
Some(&"toml broken".to_string()),
);
assert_eq!(root_err.len(), 1);
assert_eq!(
root_err[0].code,
ReliabilityReasonCode::WorkerMirrorOwnershipUnprobeable
);
}
#[test]
fn aggregate_verdict_is_healthy_for_empty_and_pass_info() {
assert_eq!(aggregate_verdict(&[]), ReliabilityVerdict::Healthy);
let diags = vec![
diag(
ReliabilitySeverity::Pass,
ReliabilityReasonCode::StatusSurfaceAvailable,
),
diag(
ReliabilitySeverity::Info,
ReliabilityReasonCode::DiskPressureNoWorkers,
),
];
assert_eq!(aggregate_verdict(&diags), ReliabilityVerdict::Healthy);
}
#[test]
fn aggregate_verdict_warning_is_degraded() {
let diags = vec![
diag(
ReliabilitySeverity::Pass,
ReliabilityReasonCode::StatusSurfaceAvailable,
),
diag(
ReliabilitySeverity::Warning,
ReliabilityReasonCode::HookAutoStartDisabled,
),
];
assert_eq!(aggregate_verdict(&diags), ReliabilityVerdict::Degraded);
}
#[test]
fn aggregate_verdict_critical_dominates_warning_regardless_of_order() {
let warn = diag(
ReliabilitySeverity::Warning,
ReliabilityReasonCode::HookAutoStartDisabled,
);
let crit = diag(
ReliabilitySeverity::Critical,
ReliabilityReasonCode::NoWorkersConfigured,
);
assert_eq!(
aggregate_verdict(&[warn.clone(), crit.clone()]),
ReliabilityVerdict::Failing
);
assert_eq!(
aggregate_verdict(&[crit, warn]),
ReliabilityVerdict::Failing
);
}
#[test]
fn reliability_severity_token_maps_all_severities() {
assert_eq!(
reliability_severity_token(ReliabilitySeverity::Pass),
"pass"
);
assert_eq!(
reliability_severity_token(ReliabilitySeverity::Info),
"info"
);
assert_eq!(
reliability_severity_token(ReliabilitySeverity::Warning),
"warning"
);
assert_eq!(
reliability_severity_token(ReliabilitySeverity::Critical),
"critical"
);
}
#[test]
fn hook_consistency_pass_when_hook_present_and_socket_matches() {
let diags = reliability_hook_consistency_diagnostics(
Some("/usr/local/bin/rch"),
Some("/home/u/.cache/rch/rch.sock"),
"/home/u/.cache/rch/rch.sock",
);
let hook = diags
.iter()
.find(|d| d.check_name == "hook_installed")
.unwrap();
assert_eq!(hook.severity, ReliabilitySeverity::Pass);
assert_eq!(hook.code, ReliabilityReasonCode::HookInstalled);
let socket = diags
.iter()
.find(|d| d.check_name == "socket_path")
.unwrap();
assert_eq!(socket.severity, ReliabilitySeverity::Pass);
assert_eq!(socket.code, ReliabilityReasonCode::SocketPathConsistent);
}
#[test]
fn hook_consistency_warns_with_remediation_when_hook_missing() {
let diags = reliability_hook_consistency_diagnostics(None, None, "/c/rch.sock");
let hook = diags
.iter()
.find(|d| d.check_name == "hook_installed")
.unwrap();
assert_eq!(hook.severity, ReliabilitySeverity::Warning);
assert_eq!(hook.code, ReliabilityReasonCode::HookNotInstalled);
assert_eq!(
hook.remediation_command.as_deref(),
Some("rch hook install")
);
let socket = diags
.iter()
.find(|d| d.check_name == "socket_path")
.unwrap();
assert_eq!(socket.code, ReliabilityReasonCode::SocketPathConsistent);
}
#[test]
fn hook_consistency_warns_on_socket_mismatch_with_realign_remediation() {
let diags = reliability_hook_consistency_diagnostics(
Some("rch"),
Some("/tmp/custom.sock"),
"/home/u/.cache/rch/rch.sock",
);
let socket = diags
.iter()
.find(|d| d.check_name == "socket_path")
.unwrap();
assert_eq!(socket.severity, ReliabilitySeverity::Warning);
assert_eq!(socket.code, ReliabilityReasonCode::SocketPathMismatch);
assert_eq!(
socket.remediation_command.as_deref(),
Some("rch config set general.socket_path /home/u/.cache/rch/rch.sock")
);
}
#[test]
fn hook_consistency_steps_are_manual_not_auto_fixable() {
assert!(auto_config_flip(ReliabilityReasonCode::HookNotInstalled).is_none());
assert!(auto_config_flip(ReliabilityReasonCode::SocketPathMismatch).is_none());
let diags =
reliability_hook_consistency_diagnostics(None, Some("/tmp/x.sock"), "/c/rch.sock");
let plan = build_reliability_remediation_plan(&diags);
assert_eq!(plan.len(), 2, "both warnings become remediation steps");
assert!(plan.iter().all(|s| !s.auto_fixable));
}
#[test]
fn schema_compatibility_diagnostics_are_deterministic_and_categorised() {
let a = reliability_schema_compatibility_diagnostics();
let b = reliability_schema_compatibility_diagnostics();
assert!(!a.is_empty(), "schema probe must emit diagnostics");
assert_eq!(a.len(), b.len(), "schema probe must be deterministic");
assert!(
a.iter()
.all(|d| d.category == ReliabilityCategory::SchemaCompatibility)
);
}
#[test]
fn rollout_posture_passes_when_self_healing_enabled() {
let mut config = rch_common::RchConfig::default();
config.self_healing.hook_starts_daemon = true;
config.self_healing.daemon_installs_hooks = true;
let diags = reliability_rollout_posture_diagnostics(Ok(&config));
let hook = diags
.iter()
.find(|d| d.check_name == "hook_starts_daemon")
.expect("hook diagnostic present");
assert_eq!(hook.severity, ReliabilitySeverity::Pass);
assert!(hook.remediation_command.is_none());
}
#[test]
fn rollout_posture_warns_with_remediation_when_disabled() {
let mut config = rch_common::RchConfig::default();
config.self_healing.hook_starts_daemon = false;
config.self_healing.daemon_installs_hooks = false;
let diags = reliability_rollout_posture_diagnostics(Ok(&config));
let hook = diags
.iter()
.find(|d| d.check_name == "hook_starts_daemon")
.expect("hook diagnostic present");
assert_eq!(hook.severity, ReliabilitySeverity::Warning);
assert_eq!(
hook.remediation_command.as_deref(),
Some("rch config set self_healing.hook_starts_daemon true")
);
assert_eq!(hook.code, ReliabilityReasonCode::HookAutoStartDisabled);
}
#[test]
fn rollout_posture_warns_on_config_load_failure() {
let diags = reliability_rollout_posture_diagnostics(Err("boom"));
let load = diags
.iter()
.find(|d| d.check_name == "config_load")
.expect("config_load diagnostic present");
assert_eq!(load.severity, ReliabilitySeverity::Warning);
assert_eq!(load.code, ReliabilityReasonCode::ConfigLoadFailed);
}
#[test]
fn build_response_counts_severities_and_defaults_fix_fields() {
let scope = ReliabilityScopeSet::default();
let diags = vec![
diag(
ReliabilitySeverity::Pass,
ReliabilityReasonCode::StatusSurfaceAvailable,
),
diag(
ReliabilitySeverity::Warning,
ReliabilityReasonCode::HookAutoStartDisabled,
),
diag(
ReliabilitySeverity::Critical,
ReliabilityReasonCode::NoWorkersConfigured,
),
];
let response =
build_reliability_doctor_response(ReliabilityDoctorMode::Check, &scope, diags);
assert_eq!(response.summary.total_checks, 3);
assert_eq!(response.summary.pass, 1);
assert_eq!(response.summary.warning, 1);
assert_eq!(response.summary.critical, 1);
assert_eq!(response.summary.overall, ReliabilityVerdict::Failing);
assert!(!response.daemon_unreachable);
assert!(!response.fix_requested);
assert!(response.remediation_outcomes.is_empty());
}
#[test]
fn build_response_flags_daemon_unreachable_from_reason_codes() {
let scope = ReliabilityScopeSet::default();
let diags = vec![diag(
ReliabilitySeverity::Warning,
ReliabilityReasonCode::DaemonStatusUnavailable,
)];
let response =
build_reliability_doctor_response(ReliabilityDoctorMode::Check, &scope, diags);
assert!(response.daemon_unreachable);
assert_eq!(response.daemon_unreachable_reasons.len(), 1);
}
#[test]
fn remediation_plan_orders_critical_before_warning_and_skips_unactionable() {
let warn = ReliabilityDiagnostic::new(
ReliabilityCategory::RolloutPosture,
"warn_fix",
ReliabilitySeverity::Warning,
"warn",
ReliabilityReasonCode::HookAutoStartDisabled,
)
.with_remediation("rch config set self_healing.hook_starts_daemon true", "v");
let crit = ReliabilityDiagnostic::new(
ReliabilityCategory::Topology,
"crit_fix",
ReliabilitySeverity::Critical,
"crit",
ReliabilityReasonCode::NoWorkersConfigured,
)
.with_remediation("rch workers init", "v");
let pass = diag(
ReliabilitySeverity::Pass,
ReliabilityReasonCode::StatusSurfaceAvailable,
);
let warn_no_cmd = diag(
ReliabilitySeverity::Warning,
ReliabilityReasonCode::DaemonHookRepairDisabled,
);
let plan = build_reliability_remediation_plan(&[warn, crit, pass, warn_no_cmd]);
assert_eq!(plan.len(), 2, "only actionable diagnostics become steps");
assert_eq!(plan[0].order, 1);
assert_eq!(plan[0].code, ReliabilityReasonCode::NoWorkersConfigured);
assert_eq!(plan[1].order, 2);
assert_eq!(plan[1].code, ReliabilityReasonCode::HookAutoStartDisabled);
assert!(!plan[0].auto_fixable);
assert!(plan[1].auto_fixable);
}
#[test]
fn topology_critical_when_worker_config_unreadable() {
let diags =
reliability_topology_diagnostics(None, None, Some("permission denied".to_string()));
assert!(
diags
.iter()
.any(|d| d.severity == ReliabilitySeverity::Critical
&& d.code == ReliabilityReasonCode::WorkersConfigUnreadable)
);
}
#[test]
fn topology_critical_when_no_workers_configured() {
let workers: Vec<rch_common::WorkerConfig> = Vec::new();
let diags = reliability_topology_diagnostics(Some(&workers), None, None);
assert!(
diags
.iter()
.any(|d| d.severity == ReliabilitySeverity::Critical
&& d.code == ReliabilityReasonCode::NoWorkersConfigured)
);
}
#[test]
fn topology_passes_worker_config_and_warns_on_missing_daemon() {
let worker = rch_common::WorkerConfig {
id: rch_common::WorkerId("worker-a".to_string()),
..Default::default()
};
let workers = vec![worker];
let diags = reliability_topology_diagnostics(Some(&workers), None, None);
assert!(
diags.iter().any(
|d| d.check_name == "workers_config" && d.severity == ReliabilitySeverity::Pass
)
);
assert!(
diags
.iter()
.any(|d| d.severity == ReliabilitySeverity::Warning
&& d.code == ReliabilityReasonCode::DaemonStatusUnavailable)
);
}
#[test]
fn repo_diagnostics_unavailable_when_convergence_missing() {
let diags = reliability_repo_diagnostics(None);
assert!(
diags
.iter()
.any(|d| d.code == ReliabilityReasonCode::RepoConvergenceUnavailable)
);
}
#[test]
fn disk_pressure_unavailable_when_status_missing() {
let diags = reliability_disk_pressure_diagnostics(None);
assert_eq!(diags.len(), 1);
assert_eq!(diags[0].severity, ReliabilitySeverity::Warning);
assert_eq!(
diags[0].code,
ReliabilityReasonCode::DiskPressureUnavailable
);
}
#[test]
fn disk_pressure_info_when_no_workers_report() {
let status = daemon_status(0, 0, json!([]));
let diags = reliability_disk_pressure_diagnostics(Some(&status));
assert_eq!(diags.len(), 1);
assert_eq!(diags[0].severity, ReliabilitySeverity::Info);
assert_eq!(diags[0].code, ReliabilityReasonCode::DiskPressureNoWorkers);
}
#[test]
fn disk_pressure_critical_for_worker_under_critical_pressure() {
let status = daemon_status(
1,
1,
json!([{
"id": "worker-a",
"host": "worker.example",
"user": "ubuntu",
"status": "ready",
"circuit_state": "closed",
"used_slots": 0,
"total_slots": 8,
"speed_score": 99.0,
"last_error": null,
"pressure_state": "critical",
"pressure_reason_code": "disk_free_below_critical_gb"
}]),
);
let diags = reliability_disk_pressure_diagnostics(Some(&status));
assert!(
diags
.iter()
.any(|d| d.severity == ReliabilitySeverity::Critical
&& d.code == ReliabilityReasonCode::WorkerDiskPressureCritical)
);
}
#[test]
fn process_debt_unavailable_when_status_missing() {
let diags = reliability_process_debt_diagnostics(None);
assert!(
diags
.iter()
.any(|d| d.code == ReliabilityReasonCode::ProcessDebtUnavailable)
);
}
#[test]
fn process_debt_emits_diagnostics_for_present_status() {
let status = daemon_status(1, 1, json!([]));
let diags = reliability_process_debt_diagnostics(Some(&status));
assert!(!diags.is_empty());
assert!(
diags
.iter()
.all(|d| d.category == ReliabilityCategory::ProcessDebt)
);
}
#[test]
fn helper_compatibility_diagnostics_cover_helper_category() {
let diags = reliability_helper_compatibility_diagnostics();
assert!(!diags.is_empty());
assert!(
diags
.iter()
.all(|d| d.category == ReliabilityCategory::HelperCompatibility)
);
}
#[test]
fn auto_config_flip_maps_self_healing_codes_only() {
let hook = auto_config_flip(ReliabilityReasonCode::HookAutoStartDisabled)
.expect("hook auto-start disabled is auto-fixable");
assert_eq!(hook.key, "self_healing.hook_starts_daemon");
assert_eq!(hook.value, "true");
let repair = auto_config_flip(ReliabilityReasonCode::DaemonHookRepairDisabled)
.expect("daemon hook repair disabled is auto-fixable");
assert_eq!(repair.key, "self_healing.daemon_installs_hooks");
assert_eq!(repair.value, "true");
assert!(auto_config_flip(ReliabilityReasonCode::NoWorkersConfigured).is_none());
}
#[test]
fn config_flip_satisfied_reads_the_target_key() {
let mut config = rch_common::RchConfig::default();
config.self_healing.hook_starts_daemon = false;
let flip = auto_config_flip(ReliabilityReasonCode::HookAutoStartDisabled).unwrap();
assert!(!config_flip_satisfied(&config, flip));
config.self_healing.hook_starts_daemon = true;
assert!(config_flip_satisfied(&config, flip));
}
#[test]
fn plan_auto_flip_covers_the_intent_execute_matrix() {
assert_eq!(plan_auto_flip(true, false), AutoFlipPlan::AlreadySatisfied);
assert_eq!(plan_auto_flip(true, true), AutoFlipPlan::AlreadySatisfied);
assert_eq!(plan_auto_flip(false, true), AutoFlipPlan::WouldApply);
assert_eq!(plan_auto_flip(false, false), AutoFlipPlan::Apply);
}
fn remediation_step(
order: u32,
code: ReliabilityReasonCode,
command: &str,
) -> ReliabilityRemediationStep {
ReliabilityRemediationStep {
order,
category: ReliabilityCategory::RolloutPosture,
code,
description: format!("step {order}"),
command: command.to_string(),
validation: "rch doctor --reliability --json".to_string(),
requires_restart: false,
dry_run_safe: true,
auto_fixable: auto_config_flip(code).is_some(),
}
}
fn response_with_plan(
mode: ReliabilityDoctorMode,
fix_requested: bool,
plan: Vec<ReliabilityRemediationStep>,
) -> ReliabilityDoctorResponse {
let scope = ReliabilityScopeSet::default();
let mut response = build_reliability_doctor_response(mode, &scope, Vec::new());
response.remediation_plan = plan;
response.fix_requested = fix_requested;
response
}
#[test]
fn executor_is_noop_without_fix_requested() {
let plan = vec![remediation_step(
1,
ReliabilityReasonCode::HookAutoStartDisabled,
"rch config set self_healing.hook_starts_daemon true",
)];
let mut response = response_with_plan(ReliabilityDoctorMode::DryRun, false, plan);
apply_reliability_remediations(&mut response);
assert!(
response.remediation_outcomes.is_empty(),
"no fix requested must yield no outcomes"
);
}
#[test]
fn executor_preview_never_applies_and_classifies_manual_steps() {
let plan = vec![
remediation_step(
1,
ReliabilityReasonCode::HookAutoStartDisabled,
"rch config set self_healing.hook_starts_daemon true",
),
remediation_step(
2,
ReliabilityReasonCode::NoWorkersConfigured,
"rch workers init",
),
];
let mut response = response_with_plan(ReliabilityDoctorMode::DryRun, true, plan);
apply_reliability_remediations(&mut response);
assert_eq!(response.remediation_outcomes.len(), 2);
let auto = &response.remediation_outcomes[0];
assert!(
matches!(
auto.status,
RemediationOutcomeStatus::WouldApply | RemediationOutcomeStatus::AlreadySatisfied
),
"preview auto step must not mutate: {:?}",
auto.status
);
let manual = &response.remediation_outcomes[1];
assert_eq!(manual.status, RemediationOutcomeStatus::Manual);
assert_eq!(manual.command, "rch workers init");
}
#[test]
fn remediation_plan_marks_auto_fixable_steps() {
let diag = ReliabilityDiagnostic::new(
ReliabilityCategory::RolloutPosture,
"hook_starts_daemon",
ReliabilitySeverity::Warning,
"Hook auto-start is disabled",
ReliabilityReasonCode::HookAutoStartDisabled,
)
.with_remediation(
"rch config set self_healing.hook_starts_daemon true",
"rch config get self_healing.hook_starts_daemon --json",
);
let plan = build_reliability_remediation_plan(&[diag]);
assert_eq!(plan.len(), 1);
assert!(plan[0].auto_fixable);
assert_eq!(plan[0].code, ReliabilityReasonCode::HookAutoStartDisabled);
}
#[test]
fn test_worker_topology_known_non_ready_status_is_degraded() {
for status in ["degraded", "draining", "drained", "disabled", "busy"] {
let worker = worker_status("worker-a", status, "closed");
let diagnostic = worker_topology_diagnostic(&worker);
assert_eq!(diagnostic.severity, ReliabilitySeverity::Warning);
assert_eq!(diagnostic.code, ReliabilityReasonCode::WorkerDegraded);
assert!(
diagnostic.message.contains(status),
"message should preserve the known non-ready status, got: {}",
diagnostic.message
);
}
}
#[test]
fn test_worker_topology_unknown_status_reports_protocol_drift() {
let worker = worker_status("worker-a", "parked", "closed");
let diagnostic = worker_topology_diagnostic(&worker);
assert_eq!(diagnostic.severity, ReliabilitySeverity::Warning);
assert_eq!(
diagnostic.code,
ReliabilityReasonCode::WorkerStatusUnrecognized
);
assert!(
diagnostic.message.contains("status is unrecognized"),
"unexpected message: {}",
diagnostic.message
);
}
#[test]
fn test_parse_ready_status_known_ready_variants() {
for v in ["healthy", "available", "ready", "idle", "running"] {
assert_eq!(
parse_worker_ready_status(v),
ParsedStatus::Known(KnownStatus::Ready),
"{v} should parse as Ready"
);
}
}
#[test]
fn test_parse_ready_status_known_degraded_variants() {
for v in [
"busy",
"degraded",
"draining",
"drained",
"disabled",
"unhealthy",
] {
assert_eq!(
parse_worker_ready_status(v),
ParsedStatus::Known(KnownStatus::Degraded),
"{v} should parse as Degraded"
);
}
}
#[test]
fn test_parse_ready_status_known_unreachable_variants() {
for v in ["unreachable", "offline", "error", "failed"] {
assert_eq!(
parse_worker_ready_status(v),
ParsedStatus::Known(KnownStatus::Unreachable),
"{v} should parse as Unreachable"
);
}
}
#[test]
fn test_parse_ready_status_trims_and_lowers() {
for v in [
" healthy ",
"\tHealthy\n",
"READY",
" Ready ",
"\u{00A0}healthy\u{00A0}", ] {
assert_eq!(
parse_worker_ready_status(v),
ParsedStatus::Known(KnownStatus::Ready),
"{v:?} should normalize to Ready"
);
}
}
#[test]
fn test_parse_ready_status_unknown_preserves_normalized_input() {
assert_eq!(
parse_worker_ready_status(" PARKED "),
ParsedStatus::Unrecognized("parked".to_string())
);
}
#[test]
fn test_parse_ready_status_empty_string_unrecognized() {
assert_eq!(
parse_worker_ready_status(""),
ParsedStatus::Unrecognized(String::new())
);
assert_eq!(
parse_worker_ready_status(" \t "),
ParsedStatus::Unrecognized(String::new())
);
}
#[test]
fn test_parse_circuit_state_known_variants() {
assert_eq!(
parse_worker_circuit_state("closed"),
ParsedCircuit::Known(KnownCircuit::Closed)
);
assert_eq!(
parse_worker_circuit_state("open"),
ParsedCircuit::Known(KnownCircuit::Open)
);
assert_eq!(
parse_worker_circuit_state("half_open"),
ParsedCircuit::Known(KnownCircuit::HalfOpen)
);
assert_eq!(
parse_worker_circuit_state("half-open"),
ParsedCircuit::Known(KnownCircuit::HalfOpen)
);
}
#[test]
fn test_parse_circuit_state_trims_and_lowers() {
for v in [" Open ", "\tOPEN\n", "OPEN"] {
assert_eq!(
parse_worker_circuit_state(v),
ParsedCircuit::Known(KnownCircuit::Open),
"{v:?} should normalize to Open"
);
}
}
#[test]
fn test_parse_circuit_state_unknown_preserves_normalized() {
assert_eq!(
parse_worker_circuit_state("OPEN_FORCED"),
ParsedCircuit::Unrecognized("open_forced".to_string())
);
assert_eq!(
parse_worker_circuit_state("closed_for_maintenance"),
ParsedCircuit::Unrecognized("closed_for_maintenance".to_string())
);
assert_eq!(
parse_worker_circuit_state(""),
ParsedCircuit::Unrecognized(String::new())
);
}
#[test]
fn test_worker_topology_unknown_circuit_state_reports_protocol_drift() {
let worker = worker_status("worker-a", "healthy", "OPEN_FORCED");
let diagnostic = worker_topology_diagnostic(&worker);
assert_eq!(diagnostic.severity, ReliabilitySeverity::Warning);
assert_eq!(
diagnostic.code,
ReliabilityReasonCode::WorkerCircuitStateUnrecognized
);
assert!(
diagnostic.message.contains("circuit_state is unrecognized"),
"unexpected message: {}",
diagnostic.message
);
}
#[test]
fn test_worker_topology_circuit_unrecognized_dominates_status() {
let worker = worker_status("worker-a", "parked", "UNKNOWN");
let diagnostic = worker_topology_diagnostic(&worker);
assert_eq!(
diagnostic.code,
ReliabilityReasonCode::WorkerCircuitStateUnrecognized
);
}
#[test]
fn test_read_config_capped_reader_rejects_bytes_past_cap() {
let err = read_config_capped_from_reader(
std::io::Cursor::new(b"abcd".to_vec()),
3,
"test config",
)
.expect_err("reader that yields more than the cap must be rejected");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
assert!(
err.to_string().contains("exceeds 3-byte cap"),
"unexpected error message: {err}"
);
}
#[test]
fn test_read_config_capped_reader_accepts_exact_cap() {
let content =
read_config_capped_from_reader(std::io::Cursor::new(b"abc".to_vec()), 3, "test config")
.expect("reader at the cap should be accepted");
assert_eq!(content, "abc");
}
#[test]
fn test_schema_compatibility_diagnostic_flags_mismatch() {
let diagnostic =
schema_compatibility_diagnostic("status", "2.0.0", "1.0.0", "CLI status response");
assert_eq!(
diagnostic.category,
ReliabilityCategory::SchemaCompatibility
);
assert_eq!(diagnostic.severity, ReliabilitySeverity::Critical);
assert_eq!(diagnostic.code, ReliabilityReasonCode::SchemaIncompatible);
assert_eq!(
diagnostic.details.as_deref(),
Some("expected=1.0.0, actual=2.0.0")
);
assert!(
diagnostic.remediation_command.is_some(),
"schema mismatch should include remediation"
);
}
#[test]
fn test_schema_compatibility_diagnostic_passes_match() {
let diagnostic =
schema_compatibility_diagnostic("status", "1.0.0", "1.0.0", "CLI status response");
assert_eq!(diagnostic.severity, ReliabilitySeverity::Pass);
assert_eq!(diagnostic.code, ReliabilityReasonCode::SchemaCompatible);
assert_eq!(
diagnostic.details.as_deref(),
Some("schema_version=1.0.0 expected=1.0.0")
);
assert!(diagnostic.remediation_command.is_none());
}
#[test]
fn test_schema_compatibility_diagnostics_use_pinned_expected_versions() {
let diagnostics = reliability_schema_compatibility_diagnostics();
assert_eq!(diagnostics.len(), 4);
for check_name in [
"doctor_reliability",
"status",
"repo_updater_contract",
"process_triage_contract",
] {
let diagnostic = diagnostics
.iter()
.find(|diagnostic| diagnostic.check_name == check_name);
assert!(
diagnostic.is_some(),
"missing schema diagnostic for {check_name}"
);
let Some(diagnostic) = diagnostic else {
return;
};
assert_eq!(diagnostic.severity, ReliabilitySeverity::Pass);
assert_eq!(diagnostic.code, ReliabilityReasonCode::SchemaCompatible);
assert!(
diagnostic
.details
.as_deref()
.is_some_and(|details| details.ends_with(" expected=1.0.0")),
"{check_name} should compare against this doctor's pinned expected schema version"
);
}
}
#[test]
fn test_check_command_exists_which() {
let result = check_command_exists("which", "which command");
assert_eq!(result.status, CheckStatus::Pass);
}
#[test]
fn test_check_command_exists_nonexistent() {
let result = check_command_exists("totally_nonexistent_command_12345", "fake command");
assert_eq!(result.status, CheckStatus::Fail);
assert!(result.suggestion.is_some());
}
#[test]
fn test_check_status_serialization() {
let pass = serde_json::to_string(&CheckStatus::Pass).unwrap();
assert_eq!(pass, "\"pass\"");
let fail = serde_json::to_string(&CheckStatus::Fail).unwrap();
assert_eq!(fail, "\"fail\"");
}
#[test]
fn test_doctor_summary() {
let summary = DoctorSummary {
total: 10,
passed: 7,
warnings: 2,
failed: 1,
fixed: 0,
would_fix: 0,
};
let json = serde_json::to_string(&summary).unwrap();
assert!(json.contains("\"total\":10"));
assert!(json.contains("\"passed\":7"));
}
#[test]
fn test_evaluate_cancellation_health_fails_on_cleanup_failure() {
let status: crate::status_types::DaemonFullStatusResponse = serde_json::from_value(json!({
"daemon": {
"pid": 1,
"uptime_secs": 10,
"version": "0.1.0",
"socket_path": "/tmp/rch.sock",
"started_at": "2026-01-01T00:00:00Z",
"workers_total": 1,
"workers_healthy": 1,
"slots_total": 8,
"slots_available": 4
},
"workers": [],
"active_builds": [],
"queued_builds": [],
"recent_builds": [{
"id": 9,
"started_at": "2026-01-01T00:00:00Z",
"completed_at": "2026-01-01T00:00:05Z",
"project_id": "proj",
"worker_id": "worker-a",
"command": "cargo test",
"exit_code": 130,
"duration_ms": 5000,
"location": "remote",
"bytes_transferred": 1024,
"timing": null,
"cancellation": {
"operation_id": "cancel-9",
"origin": "timeout",
"reason_code": "timeout",
"decision_path": ["requested", "term_sent", "remote_kill_sent", "escalated", "completed"],
"escalation_stage": "sigkill",
"escalation_count": 2,
"remote_kill_attempted": true,
"cleanup_ok": false,
"history_cancelled": true,
"final_state": "completed",
"worker_health": {
"status": "unreachable",
"speed_score": 0.0,
"used_slots": 4,
"available_slots": 0,
"pressure_state": "critical",
"pressure_reason_code": "disk_free_below_critical_gb"
}
}
}],
"issues": [],
"alerts": [],
"stats": {
"total_builds": 1,
"success_count": 0,
"failure_count": 1,
"remote_count": 1,
"local_count": 0,
"avg_duration_ms": 5000
},
"test_stats": null,
"saved_time": null
}))
.expect("status json should parse");
let result = evaluate_cancellation_health(&status);
assert_eq!(result.status, CheckStatus::Fail);
assert!(result.message.contains("cleanup failures"));
}
#[test]
fn test_evaluate_cancellation_health_passes_on_clean_cancel() {
let status: crate::status_types::DaemonFullStatusResponse = serde_json::from_value(json!({
"daemon": {
"pid": 1,
"uptime_secs": 10,
"version": "0.1.0",
"socket_path": "/tmp/rch.sock",
"started_at": "2026-01-01T00:00:00Z",
"workers_total": 1,
"workers_healthy": 1,
"slots_total": 8,
"slots_available": 4
},
"workers": [],
"active_builds": [],
"queued_builds": [],
"recent_builds": [{
"id": 10,
"started_at": "2026-01-01T00:00:00Z",
"completed_at": "2026-01-01T00:00:03Z",
"project_id": "proj",
"worker_id": "worker-a",
"command": "cargo check",
"exit_code": 130,
"duration_ms": 3000,
"location": "remote",
"bytes_transferred": 1024,
"timing": null,
"cancellation": {
"operation_id": "cancel-10",
"origin": "user",
"reason_code": "user",
"decision_path": ["requested", "term_sent", "completed"],
"escalation_stage": "term",
"escalation_count": 0,
"remote_kill_attempted": false,
"cleanup_ok": true,
"history_cancelled": true,
"final_state": "completed",
"worker_health": {
"status": "healthy",
"speed_score": 97.2,
"used_slots": 0,
"available_slots": 8,
"pressure_state": "healthy",
"pressure_reason_code": "healthy"
}
}
}],
"issues": [],
"alerts": [],
"stats": {
"total_builds": 1,
"success_count": 0,
"failure_count": 1,
"remote_count": 1,
"local_count": 0,
"avg_duration_ms": 3000
},
"test_stats": null,
"saved_time": null
}))
.expect("status json should parse");
let result = evaluate_cancellation_health(&status);
assert_eq!(result.status, CheckStatus::Pass);
assert!(result.message.contains("deterministic cleanup"));
}
#[test]
fn test_quick_check_result_is_healthy() {
let healthy = QuickCheckResult {
daemon_running: true,
worker_count: 1,
workers_healthy: Some(1),
hook_installed: true,
warnings: vec![],
errors: vec![],
};
assert!(healthy.is_healthy());
let no_daemon = QuickCheckResult {
daemon_running: false,
worker_count: 1,
workers_healthy: Some(1),
hook_installed: true,
warnings: vec![],
errors: vec![],
};
assert!(!no_daemon.is_healthy());
let no_workers = QuickCheckResult {
daemon_running: true,
worker_count: 0,
workers_healthy: None,
hook_installed: true,
warnings: vec![],
errors: vec![],
};
assert!(!no_workers.is_healthy());
let no_hook = QuickCheckResult {
daemon_running: true,
worker_count: 1,
workers_healthy: Some(1),
hook_installed: false,
warnings: vec![],
errors: vec![],
};
assert!(!no_hook.is_healthy());
let with_warning = QuickCheckResult {
daemon_running: true,
worker_count: 1,
workers_healthy: Some(1),
hook_installed: true,
warnings: vec!["Worker health stale".to_string()],
errors: vec![],
};
assert!(
!with_warning.is_healthy(),
"warnings are issues and must not report full quick-check health"
);
}
#[test]
fn test_quick_check_result_has_issues() {
let no_issues = QuickCheckResult {
daemon_running: true,
worker_count: 1,
workers_healthy: Some(1),
hook_installed: true,
warnings: vec![],
errors: vec![],
};
assert!(!no_issues.has_issues());
let with_warnings = QuickCheckResult {
daemon_running: true,
worker_count: 1,
workers_healthy: Some(1),
hook_installed: true,
warnings: vec!["Some warning".to_string()],
errors: vec![],
};
assert!(with_warnings.has_issues());
let with_errors = QuickCheckResult {
daemon_running: true,
worker_count: 1,
workers_healthy: Some(1),
hook_installed: true,
warnings: vec![],
errors: vec!["Some error".to_string()],
};
assert!(with_errors.has_issues());
}
#[test]
fn test_run_quick_check_returns_result() {
let result = run_quick_check();
let _total_issues = result.warnings.len() + result.errors.len();
}
#[test]
fn test_quick_check_unknown_workers_health_is_not_healthy() {
let unknown = QuickCheckResult {
daemon_running: true,
worker_count: 3,
workers_healthy: None, hook_installed: true,
warnings: vec![],
errors: vec![],
};
assert!(
!unknown.is_healthy(),
"is_healthy() must NOT return true when workers_healthy is None"
);
}
#[test]
fn test_quick_check_partial_health_is_not_healthy() {
let partial = QuickCheckResult {
daemon_running: true,
worker_count: 5,
workers_healthy: Some(3),
hook_installed: true,
warnings: vec![],
errors: vec![],
};
assert!(!partial.is_healthy());
}
#[test]
fn test_quick_check_full_health_is_healthy() {
let full = QuickCheckResult {
daemon_running: true,
worker_count: 3,
workers_healthy: Some(3),
hook_installed: true,
warnings: vec![],
errors: vec![],
};
assert!(full.is_healthy());
}
#[test]
fn test_quick_check_does_not_default_to_success() {
let result = run_quick_check();
assert_eq!(
result.workers_healthy, None,
"fast-only run_quick_check must report unknown worker health"
);
assert!(
!result.is_healthy(),
"fast-only check must never report healthy without a probe"
);
}
#[test]
fn test_quick_check_emits_warning_about_unprobed_workers() {
let r = QuickCheckResult {
daemon_running: true,
worker_count: 3,
workers_healthy: None,
hook_installed: true,
warnings: vec![
"Worker health not probed by quick-check; run `rch doctor --reliability` for full status".to_string(),
],
errors: vec![],
};
assert!(
r.warnings.iter().any(|w| w.contains("not probed")),
"expected fast-check to surface 'not probed' warning"
);
assert!(!r.is_healthy());
}
#[test]
fn test_check_config_directory_with_existing_dir() {
let result = check_config_directory();
assert!(
matches!(
result.status,
CheckStatus::Pass | CheckStatus::Warning | CheckStatus::Fail
),
"Config directory check returned unexpected status"
);
assert_eq!(result.category, "configuration");
assert_eq!(result.name, "config_directory");
}
#[test]
fn test_check_config_file_structure() {
let result = check_config_file();
assert_eq!(result.category, "configuration");
assert_eq!(result.name, "config.toml");
assert!(
matches!(
result.status,
CheckStatus::Pass | CheckStatus::Warning | CheckStatus::Fail | CheckStatus::Skipped
),
"Config file check returned unexpected status"
);
if result.status == CheckStatus::Skipped {
assert!(result.message.contains("Skipped"));
}
}
#[test]
fn test_check_workers_file_structure() {
let result = check_workers_file();
assert_eq!(result.category, "configuration");
assert_eq!(result.name, "workers.toml");
assert!(
matches!(
result.status,
CheckStatus::Pass | CheckStatus::Warning | CheckStatus::Fail | CheckStatus::Skipped
),
"Workers file check returned unexpected status"
);
}
#[test]
fn test_check_ssh_config_returns_valid_result() {
let result = check_ssh_config();
assert_eq!(result.category, "ssh");
assert_eq!(result.name, "ssh_config");
assert!(
matches!(result.status, CheckStatus::Pass | CheckStatus::Warning),
"SSH config check should return Pass or Warning, got {:?}",
result.status
);
}
#[test]
fn test_check_claude_code_hook_returns_valid_result() {
let result = check_claude_code_hook();
assert_eq!(result.category, "hooks");
assert_eq!(result.name, "claude_code_hook");
assert!(
matches!(
result.status,
CheckStatus::Pass | CheckStatus::Warning | CheckStatus::Fail
),
"Claude Code hook check returned unexpected status"
);
assert!(result.details.is_some() || result.status == CheckStatus::Fail);
}
#[test]
fn test_check_command_exists_common_tools() {
let tools = [
("ls", "List command"),
("cat", "Concatenate command"),
("echo", "Echo command"),
];
for (cmd, desc) in tools {
let result = check_command_exists(cmd, desc);
assert_eq!(
result.status,
CheckStatus::Pass,
"Expected {} to exist on system",
cmd
);
assert_eq!(result.category, "prerequisites");
assert_eq!(result.name, cmd);
assert!(result.message.contains("installed"));
}
}
#[test]
fn test_check_command_exists_returns_version_info() {
let result = check_command_exists("ls", "List command");
if result.status == CheckStatus::Pass {
let _ = &result.details;
}
}
#[test]
fn test_check_command_exists_provides_suggestion_for_missing() {
let result = check_command_exists("rch_nonexistent_test_cmd_xyz", "fake tool");
assert_eq!(result.status, CheckStatus::Fail);
assert!(
result.suggestion.is_some(),
"Missing command should provide installation suggestion"
);
assert!(
result.suggestion.unwrap().contains("package manager"),
"Suggestion should mention package manager"
);
}
#[test]
fn test_check_result_json_serialization() {
let result = CheckResult {
category: "test".to_string(),
name: "test_check".to_string(),
status: CheckStatus::Pass,
message: "Test passed".to_string(),
details: Some("Extra details".to_string()),
suggestion: None,
fixable: false,
fix_applied: false,
fix_message: None,
};
let json = serde_json::to_string(&result).unwrap();
assert!(json.contains("\"category\":\"test\""));
assert!(json.contains("\"name\":\"test_check\""));
assert!(json.contains("\"status\":\"pass\""));
assert!(json.contains("\"message\":\"Test passed\""));
assert!(json.contains("\"details\":\"Extra details\""));
assert!(!json.contains("\"suggestion\":"));
assert!(!json.contains("\"fix_message\":"));
}
#[test]
fn test_check_result_with_fix_info() {
let result = CheckResult {
category: "ssh".to_string(),
name: "key_permissions".to_string(),
status: CheckStatus::Warning,
message: "Loose permissions".to_string(),
details: Some("/home/user/.ssh/id_ed25519".to_string()),
suggestion: Some("chmod 600 /path/to/key".to_string()),
fixable: true,
fix_applied: true,
fix_message: Some("Changed permissions to 0600".to_string()),
};
let json = serde_json::to_string(&result).unwrap();
assert!(json.contains("\"fixable\":true"));
assert!(json.contains("\"fix_applied\":true"));
assert!(json.contains("\"fix_message\":"));
}
#[test]
fn test_all_check_statuses_serialize() {
let statuses = [
(CheckStatus::Pass, "\"pass\""),
(CheckStatus::Warning, "\"warning\""),
(CheckStatus::Fail, "\"fail\""),
(CheckStatus::Skipped, "\"skipped\""),
];
for (status, expected) in statuses {
let json = serde_json::to_string(&status).unwrap();
assert_eq!(
json, expected,
"CheckStatus::{:?} serialized incorrectly",
status
);
}
}
fn resolved_rsync_fixture(flavor: RsyncFlavor, path: &str) -> ResolvedRsync {
ResolvedRsync {
path: PathBuf::from(path),
flavor,
version_line: "banner".to_string(),
source: rch_common::rsync_flavor::RsyncSource::Path,
shadowed: None,
}
}
#[test]
fn rsync_check_passes_cleanly_on_modern_rsync() {
let check = classify_rsync_check(Ok(resolved_rsync_fixture(
RsyncFlavor::Rsync {
major: 3,
minor: 4,
patch: 1,
},
"/opt/homebrew/bin/rsync",
)));
assert_eq!(check.category, "prerequisites");
assert_eq!(check.name, "rsync");
assert_eq!(check.status, CheckStatus::Pass);
assert!(check.suggestion.is_none(), "{:?}", check.suggestion);
let details = check.details.expect("details");
assert!(details.contains("rsync 3.4.1"), "{details}");
assert!(details.contains("/opt/homebrew/bin/rsync"), "{details}");
assert!(!check.fixable);
}
#[test]
fn rsync_check_reports_shadowed_path_binary() {
let mut resolved = resolved_rsync_fixture(
RsyncFlavor::Rsync {
major: 3,
minor: 4,
patch: 1,
},
"/opt/homebrew/bin/rsync",
);
resolved.source = rch_common::rsync_flavor::RsyncSource::PreferredLocation;
resolved.shadowed = Some(rch_common::rsync_flavor::ShadowedRsync {
path: PathBuf::from("/usr/bin/rsync"),
flavor: RsyncFlavor::OpenRsync { protocol: Some(29) },
});
let check = classify_rsync_check(Ok(resolved));
assert_eq!(check.status, CheckStatus::Pass);
let details = check.details.expect("details");
assert!(
details.contains("PATH rsync is openrsync (protocol 29) at /usr/bin/rsync"),
"{details}"
);
}
#[test]
fn rsync_check_passes_openrsync_in_compatibility_mode_with_remedy() {
let check = classify_rsync_check(Ok(resolved_rsync_fixture(
RsyncFlavor::OpenRsync { protocol: Some(29) },
"/usr/bin/rsync",
)));
assert_eq!(check.status, CheckStatus::Pass);
assert!(
check.message.contains("compatibility mode"),
"{}",
check.message
);
assert!(
check.message.contains("openrsync (protocol 29)"),
"{}",
check.message
);
let suggestion = check.suggestion.expect("remedy hint");
assert!(
suggestion.contains("--progress --stats -vv"),
"{suggestion}"
);
assert!(suggestion.contains("install"), "{suggestion}");
if cfg!(target_os = "macos") {
assert!(suggestion.contains("brew install rsync"), "{suggestion}");
}
}
#[test]
fn rsync_check_warns_on_unrecognized_banner() {
let check = classify_rsync_check(Ok(resolved_rsync_fixture(
RsyncFlavor::Unknown,
"/usr/local/bin/rsync",
)));
assert_eq!(check.status, CheckStatus::Warning);
assert!(
check.message.contains("could not be determined"),
"{}",
check.message
);
let suggestion = check.suggestion.expect("hint");
assert!(suggestion.contains("rsync_bin"), "{suggestion}");
}
#[test]
fn rsync_check_fails_on_too_old_rsync() {
let check = classify_rsync_check(Ok(resolved_rsync_fixture(
RsyncFlavor::Rsync {
major: 2,
minor: 5,
patch: 7,
},
"/usr/bin/rsync",
)));
assert_eq!(check.status, CheckStatus::Fail);
assert!(check.message.contains("too old"), "{}", check.message);
assert!(check.message.contains("2.6.9"), "{}", check.message);
}
#[test]
fn rsync_check_fails_when_missing_or_misconfigured() {
let missing = classify_rsync_check(Err(RsyncResolveError::NotFound));
assert_eq!(missing.status, CheckStatus::Fail);
assert_eq!(missing.message, "File synchronization not found");
assert!(missing.suggestion.expect("hint").contains("Install rsync"));
let bad_override = classify_rsync_check(Err(RsyncResolveError::OverrideMissing {
origin: rch_common::rsync_flavor::RsyncSource::Config,
path: PathBuf::from("/nonexistent/rsync"),
}));
assert_eq!(bad_override.status, CheckStatus::Fail);
assert!(
bad_override.message.contains("configured rsync"),
"{}",
bad_override.message
);
let hint = bad_override.suggestion.expect("hint");
assert!(hint.contains("[transfer] rsync_bin"), "{hint}");
let details = bad_override.details.expect("details");
assert!(details.contains("/nonexistent/rsync"), "{details}");
let broken = classify_rsync_check(Err(RsyncResolveError::ProbeFailed {
path: PathBuf::from("/usr/bin/rsync"),
reason: "timed out after 5s".to_string(),
}));
assert_eq!(broken.status, CheckStatus::Fail);
assert!(broken.message.contains("--version"), "{}", broken.message);
assert!(broken.details.expect("details").contains("timed out"));
}
#[test]
fn test_doctor_response_serialization() {
let response = DoctorResponse {
schema_version: "1.0.0".to_string(),
checks: vec![
CheckResult {
category: "prerequisites".to_string(),
name: "rsync".to_string(),
status: CheckStatus::Pass,
message: "rsync is installed".to_string(),
details: Some("rsync version 3.2.7".to_string()),
suggestion: None,
fixable: false,
fix_applied: false,
fix_message: None,
},
CheckResult {
category: "configuration".to_string(),
name: "config.toml".to_string(),
status: CheckStatus::Warning,
message: "config.toml not found".to_string(),
details: None,
suggestion: Some("Run rch config init".to_string()),
fixable: true,
fix_applied: false,
fix_message: None,
},
],
summary: DoctorSummary {
total: 2,
passed: 1,
warnings: 1,
failed: 0,
fixed: 0,
would_fix: 0,
},
fixes_applied: vec![],
};
let json = serde_json::to_string(&response).unwrap();
assert!(json.contains("\"checks\":["));
assert!(json.contains("\"summary\":{"));
assert!(json.contains("\"fixes_applied\":[]"));
}
#[test]
fn test_doctor_response_with_fixes() {
let response = DoctorResponse {
schema_version: "1.0.0".to_string(),
checks: vec![],
summary: DoctorSummary {
total: 1,
passed: 1,
warnings: 0,
failed: 0,
fixed: 1,
would_fix: 0,
},
fixes_applied: vec![FixApplied {
check_name: "ssh_key_perms".to_string(),
action: "Changed permissions to 0600".to_string(),
success: true,
error: None,
}],
};
let json = serde_json::to_string(&response).unwrap();
assert!(json.contains("\"fixes_applied\":[{"));
assert!(json.contains("\"check_name\":\"ssh_key_perms\""));
assert!(json.contains("\"success\":true"));
}
#[test]
fn test_fix_applied_success() {
let fix = FixApplied {
check_name: "id_ed25519".to_string(),
action: "Changed permissions from 0644 to 0600".to_string(),
success: true,
error: None,
};
let json = serde_json::to_string(&fix).unwrap();
assert!(json.contains("\"success\":true"));
assert!(!json.contains("\"error\""));
}
#[test]
fn test_fix_applied_failure() {
let fix = FixApplied {
check_name: "id_rsa".to_string(),
action: "Attempted to change permissions".to_string(),
success: false,
error: Some("Permission denied".to_string()),
};
let json = serde_json::to_string(&fix).unwrap();
assert!(json.contains("\"success\":false"));
assert!(json.contains("\"error\":\"Permission denied\""));
}
#[test]
fn test_doctor_options_default_values() {
let opts_minimal = DoctorOptions {
fix: false,
dry_run: false,
reliability: false,
check_schemas: false,
verbose: false,
strict: false,
lenient: false,
scope: ReliabilityScopeSet::default(),
watch: false,
watch_interval_secs: 5,
transitions_only: false,
watch_snapshot: None,
};
assert!(!opts_minimal.fix);
assert!(!opts_minimal.dry_run);
let opts_fix = DoctorOptions {
fix: true,
dry_run: false,
reliability: false,
check_schemas: false,
verbose: false,
strict: false,
lenient: false,
scope: ReliabilityScopeSet::default(),
watch: false,
watch_interval_secs: 5,
transitions_only: false,
watch_snapshot: None,
};
assert!(opts_fix.fix);
let opts_dry_run = DoctorOptions {
fix: true,
dry_run: true,
reliability: false,
check_schemas: false,
verbose: false,
strict: false,
lenient: false,
scope: ReliabilityScopeSet::default(),
watch: false,
watch_interval_secs: 5,
transitions_only: false,
watch_snapshot: None,
};
assert!(opts_dry_run.fix);
assert!(opts_dry_run.dry_run);
let opts_verbose = DoctorOptions {
fix: false,
dry_run: false,
reliability: false,
check_schemas: false,
verbose: true,
strict: false,
lenient: false,
scope: ReliabilityScopeSet::default(),
watch: false,
watch_interval_secs: 5,
transitions_only: false,
watch_snapshot: None,
};
assert!(opts_verbose.verbose);
}
#[test]
fn test_doctor_summary_all_passed() {
let summary = DoctorSummary {
total: 15,
passed: 15,
warnings: 0,
failed: 0,
fixed: 0,
would_fix: 0,
};
let json = serde_json::to_string(&summary).unwrap();
assert!(json.contains("\"total\":15"));
assert!(json.contains("\"passed\":15"));
assert!(json.contains("\"failed\":0"));
}
#[test]
fn test_doctor_summary_with_failures() {
let summary = DoctorSummary {
total: 10,
passed: 5,
warnings: 2,
failed: 3,
fixed: 0,
would_fix: 0,
};
assert_eq!(
summary.passed + summary.warnings + summary.failed,
summary.total
);
}
#[test]
fn test_doctor_summary_with_fixes() {
let summary = DoctorSummary {
total: 10,
passed: 8,
warnings: 0,
failed: 0,
fixed: 2,
would_fix: 0,
};
let json = serde_json::to_string(&summary).unwrap();
assert!(json.contains("\"fixed\":2"));
}
#[test]
fn test_doctor_summary_dry_run_would_fix() {
let summary = DoctorSummary {
total: 10,
passed: 7,
warnings: 3,
failed: 0,
fixed: 0,
would_fix: 3,
};
let json = serde_json::to_string(&summary).unwrap();
assert!(json.contains("\"would_fix\":3"));
}
#[test]
fn test_quick_check_result_multiple_issues() {
let result = QuickCheckResult {
daemon_running: false,
worker_count: 0,
workers_healthy: None,
hook_installed: false,
warnings: vec![
"Daemon not running".to_string(),
"No workers configured".to_string(),
],
errors: vec!["Hook not installed".to_string()],
};
assert!(!result.is_healthy());
assert!(result.has_issues());
assert_eq!(result.warnings.len(), 2);
assert_eq!(result.errors.len(), 1);
}
#[test]
fn test_quick_check_result_partial_health() {
let result = QuickCheckResult {
daemon_running: true,
worker_count: 2,
workers_healthy: Some(1), hook_installed: true,
warnings: vec!["Worker css is offline".to_string()],
errors: vec![],
};
assert!(
!result.is_healthy(),
"1/2 workers healthy must NOT report system-healthy"
);
assert!(result.has_issues());
}
#[test]
fn test_ssh_worker_suggestion_format() {
let suggestion = ssh_worker_suggestion(
"ubuntu",
"build-server.local",
Path::new("/home/user/.ssh/id_ed25519"),
);
assert!(
suggestion.contains("ssh-copy-id"),
"Should suggest ssh-copy-id"
);
assert!(
suggestion.contains("ssh -i"),
"Should suggest testing with ssh -i"
);
assert!(
suggestion.contains("ssh-agent") && suggestion.contains("ssh-add"),
"Should suggest ssh-agent setup"
);
assert!(suggestion.contains("-vvv"), "Should suggest verbose debug");
assert!(suggestion.contains("ubuntu@build-server.local"));
}
#[test]
fn test_ssh_worker_suggestion_with_special_path() {
let suggestion =
ssh_worker_suggestion("admin", "192.168.1.100", Path::new("/custom/path/my_key"));
assert!(suggestion.contains("/custom/path/my_key"));
assert!(suggestion.contains("admin@192.168.1.100"));
}
#[test]
fn test_ssh_worker_suggestion_quotes_shell_metachars() {
let suggestion = ssh_worker_suggestion(
"evil; rm -rf ~",
"host\"$(touch /tmp/pwned)",
Path::new("/keys/with spaces/id"),
);
let dangerous_unquoted = "; rm -rf ~"; let user_segment_starts = suggestion.find("evil").expect("user appears");
let prev_char = suggestion[..user_segment_starts]
.chars()
.last()
.expect("preceding char");
assert!(
matches!(prev_char, '\'' | '"'),
"user field must be inside shell quoting; got prev_char={:?} in suggestion={}",
prev_char,
suggestion
);
let parse_check = std::process::Command::new("sh")
.arg("-n")
.arg("-c")
.arg(&suggestion)
.output();
if let Ok(out) = parse_check {
assert!(
out.status.success(),
"shell parse-only check failed for: {}\nstderr: {}",
suggestion,
String::from_utf8_lossy(&out.stderr)
);
}
let _ = dangerous_unquoted; }
#[test]
fn test_default_socket_path_returns_valid_path() {
let path = default_socket_path();
assert!(
!path.as_os_str().is_empty(),
"Socket path should not be empty"
);
let filename = path.file_name().map(|f| f.to_string_lossy().to_string());
assert!(
filename.is_some(),
"Socket path should have a filename component"
);
}
#[test]
fn test_configured_or_default_socket_path_uses_config_override() {
let _guard = rch_common::test_guard!();
struct ResetConfigOverride;
impl Drop for ResetConfigOverride {
fn drop(&mut self) {
crate::config::set_test_config_override(None);
}
}
let mut config = rch_common::RchConfig::default();
config.general.socket_path = "/tmp/rch-doctor-custom.sock".to_string();
crate::config::set_test_config_override(Some(config));
let _reset = ResetConfigOverride;
assert_eq!(
configured_or_default_socket_path(),
PathBuf::from("/tmp/rch-doctor-custom.sock")
);
}
#[cfg(unix)]
#[test]
fn test_daemon_check_warns_on_stale_socket_file() {
use crate::ui::context::{OutputConfig, OutputContext};
let _guard = rch_common::test_guard!();
struct ResetConfigOverride;
impl Drop for ResetConfigOverride {
fn drop(&mut self) {
crate::config::set_test_config_override(None);
}
}
let tmp = TempDir::new().unwrap();
let socket_path = tmp.path().join("stale.sock");
std::fs::write(&socket_path, "not a unix socket").unwrap();
let mut config = rch_common::RchConfig::default();
config.general.socket_path = socket_path.display().to_string();
crate::config::set_test_config_override(Some(config));
let _reset = ResetConfigOverride;
let ctx = OutputContext::new(OutputConfig::default());
let options = DoctorOptions {
fix: false,
dry_run: false,
reliability: false,
check_schemas: false,
verbose: false,
strict: false,
lenient: false,
scope: ReliabilityScopeSet::default(),
watch: false,
watch_interval_secs: 5,
transitions_only: false,
watch_snapshot: None,
};
let mut checks = Vec::new();
let mut fixes_applied = Vec::new();
check_daemon(&mut checks, &ctx, &options, &mut fixes_applied);
let daemon_socket = checks
.iter()
.find(|check| check.name == "daemon_socket")
.expect("daemon socket check");
assert_eq!(daemon_socket.status, CheckStatus::Warning);
assert_eq!(
daemon_socket.message,
"Daemon socket is stale or unreachable"
);
}
#[test]
fn test_prerequisite_checks_run_without_panic() {
use crate::ui::context::{OutputConfig, OutputContext};
let ctx = OutputContext::new(OutputConfig::default());
let options = DoctorOptions {
fix: false,
dry_run: false,
reliability: false,
check_schemas: false,
verbose: false,
strict: false,
lenient: false,
scope: ReliabilityScopeSet::default(),
watch: false,
watch_interval_secs: 5,
transitions_only: false,
watch_snapshot: None,
};
let mut checks = Vec::new();
check_prerequisites(&mut checks, &ctx, &options);
assert!(
checks.len() >= 5,
"Should check at least 5 prerequisite tools, got {}",
checks.len()
);
for check in &checks {
assert_eq!(check.category, "prerequisites");
assert!(!check.name.is_empty());
assert!(!check.message.is_empty());
}
}
#[test]
fn test_configuration_checks_run_without_panic() {
use crate::ui::context::{OutputConfig, OutputContext};
let ctx = OutputContext::new(OutputConfig::default());
let options = DoctorOptions {
fix: false,
dry_run: false,
reliability: false,
check_schemas: false,
verbose: false,
strict: false,
lenient: false,
scope: ReliabilityScopeSet::default(),
watch: false,
watch_interval_secs: 5,
transitions_only: false,
watch_snapshot: None,
};
let mut checks = Vec::new();
check_configuration(&mut checks, &ctx, &options);
assert!(
checks.len() >= 3,
"Should check at least 3 config items, got {}",
checks.len()
);
for check in &checks {
assert_eq!(check.category, "configuration");
}
}
#[test]
fn test_daemon_check_runs_without_panic() {
use crate::ui::context::{OutputConfig, OutputContext};
let ctx = OutputContext::new(OutputConfig::default());
let options = DoctorOptions {
fix: false,
dry_run: false,
reliability: false,
check_schemas: false,
verbose: false,
strict: false,
lenient: false,
scope: ReliabilityScopeSet::default(),
watch: false,
watch_interval_secs: 5,
transitions_only: false,
watch_snapshot: None,
};
let mut fixes_applied = Vec::new();
let mut checks = Vec::new();
check_daemon(&mut checks, &ctx, &options, &mut fixes_applied);
assert!(!checks.is_empty(), "Should have daemon checks");
for check in &checks {
assert_eq!(check.category, "daemon");
}
}
#[test]
fn test_wait_for_socket_times_out() {
let tmp = TempDir::new().unwrap();
let socket_path = tmp.path().join("missing.sock");
assert!(!wait_for_socket(&socket_path, Duration::from_millis(50)));
}
#[cfg(unix)]
#[test]
fn test_wait_for_socket_accepts_live_listener() {
let tmp = TempDir::new().unwrap();
let socket_path = tmp.path().join("live.sock");
let _listener = std::os::unix::net::UnixListener::bind(&socket_path).unwrap();
assert!(wait_for_socket(&socket_path, Duration::from_millis(50)));
}
#[cfg(unix)]
#[test]
fn test_start_daemon_with_fake_rchd_rejects_regular_socket_file() {
let tmp = TempDir::new().unwrap();
let socket_path = tmp.path().join("daemon.sock");
let fake_rchd = tmp.path().join("rchd");
let script = "#!/usr/bin/env sh\n\
sock=\"\"\n\
while [ \"$#\" -gt 0 ]; do\n\
if [ \"$1\" = \"-s\" ] || [ \"$1\" = \"--socket\" ]; then\n\
shift\n\
sock=\"$1\"\n\
fi\n\
shift\n\
done\n\
[ -n \"$sock\" ] || exit 1\n\
: > \"$sock\"\n\
exit 0\n"
.to_string();
std::fs::write(&fake_rchd, script).unwrap();
let mut perms = std::fs::metadata(&fake_rchd).unwrap().permissions();
perms.set_mode(0o755);
std::fs::set_permissions(&fake_rchd, perms).unwrap();
let error = start_daemon_with_binary(&socket_path, &fake_rchd, Duration::from_millis(50))
.unwrap_err();
assert!(error.contains("did not accept connections"));
let deadline = Instant::now() + Duration::from_secs(2);
while !socket_path.is_file() && Instant::now() < deadline {
std::thread::sleep(Duration::from_millis(20));
}
assert!(
socket_path.is_file(),
"fake must receive the -s socket path"
);
}
#[cfg(unix)]
#[test]
fn test_start_daemon_with_failing_fake_rchd_reports_exit() {
let tmp = TempDir::new().unwrap();
let socket_path = tmp.path().join("daemon.sock");
let fake_rchd = tmp.path().join("rchd");
std::fs::write(&fake_rchd, "#!/usr/bin/env sh\nexit 42\n").unwrap();
let mut perms = std::fs::metadata(&fake_rchd).unwrap().permissions();
perms.set_mode(0o755);
std::fs::set_permissions(&fake_rchd, perms).unwrap();
let err =
start_daemon_with_binary(&socket_path, &fake_rchd, Duration::from_secs(1)).unwrap_err();
assert!(
err.contains("exited unsuccessfully") && err.contains("42"),
"unexpected error: {err}"
);
assert!(!socket_path.exists());
}
#[test]
fn test_check_result_fixable_field_semantics() {
let pass_result = CheckResult {
category: "test".to_string(),
name: "passing_check".to_string(),
status: CheckStatus::Pass,
message: "All good".to_string(),
details: None,
suggestion: None,
fixable: false, fix_applied: false,
fix_message: None,
};
assert!(!pass_result.fixable);
let fixable_fail = CheckResult {
category: "test".to_string(),
name: "fixable_issue".to_string(),
status: CheckStatus::Warning,
message: "Permission issue".to_string(),
details: None,
suggestion: Some("Run chmod 600".to_string()),
fixable: true, fix_applied: false,
fix_message: None,
};
assert!(fixable_fail.fixable);
let unfixable_fail = CheckResult {
category: "test".to_string(),
name: "unfixable_issue".to_string(),
status: CheckStatus::Fail,
message: "Missing hardware".to_string(),
details: None,
suggestion: Some("Buy new hardware".to_string()),
fixable: false, fix_applied: false,
fix_message: None,
};
assert!(!unfixable_fail.fixable);
}
#[test]
fn test_check_result_fix_applied_and_message_consistency() {
let fixed_result = CheckResult {
category: "test".to_string(),
name: "fixed_check".to_string(),
status: CheckStatus::Pass,
message: "Fixed!".to_string(),
details: None,
suggestion: None,
fixable: false,
fix_applied: true,
fix_message: Some("Changed X to Y".to_string()),
};
assert!(fixed_result.fix_applied);
assert!(fixed_result.fix_message.is_some());
let not_fixed = CheckResult {
category: "test".to_string(),
name: "not_fixed".to_string(),
status: CheckStatus::Warning,
message: "Issue detected".to_string(),
details: None,
suggestion: Some("Run fix command".to_string()),
fixable: true,
fix_applied: false,
fix_message: None,
};
assert!(!not_fixed.fix_applied);
}
#[test]
fn test_rollout_posture_takes_borrowed_config() {
let config = rch_common::RchConfig::default();
let diags = reliability_rollout_posture_diagnostics(Ok(&config));
assert!(
diags.len() >= 4,
"expected at least 4 diagnostics (got {}): {:?}",
diags.len(),
diags.iter().map(|d| &d.check_name).collect::<Vec<_>>()
);
}
#[test]
fn test_rollout_posture_handles_borrowed_err() {
let diags = reliability_rollout_posture_diagnostics(Err("synthetic toml parse failure"));
let warning_count = diags
.iter()
.filter(|d| {
matches!(d.severity, ReliabilitySeverity::Warning)
&& d.code == ReliabilityReasonCode::ConfigLoadFailed
})
.count();
assert_eq!(
warning_count,
1,
"expected exactly one ConfigLoadFailed diagnostic, got: {:?}",
diags
.iter()
.map(|d| (&d.check_name, &d.code))
.collect::<Vec<_>>()
);
}
#[test]
fn test_rollout_posture_idempotent_for_same_config() {
let config = rch_common::RchConfig::default();
let a = reliability_rollout_posture_diagnostics(Ok(&config));
let b = reliability_rollout_posture_diagnostics(Ok(&config));
assert_eq!(a.len(), b.len());
for (da, db) in a.iter().zip(b.iter()) {
assert_eq!(da.check_name, db.check_name);
assert_eq!(da.severity, db.severity);
assert_eq!(da.message, db.message);
assert_eq!(da.code, db.code);
}
}
#[test]
fn test_rollout_posture_two_configs_isolated() {
let mut config_a = rch_common::RchConfig::default();
config_a.self_healing.hook_starts_daemon = true;
let mut config_b = rch_common::RchConfig::default();
config_b.self_healing.hook_starts_daemon = false;
let a = reliability_rollout_posture_diagnostics(Ok(&config_a));
let b = reliability_rollout_posture_diagnostics(Ok(&config_b));
let a_hook = a
.iter()
.find(|d| d.check_name == "hook_starts_daemon")
.expect("hook diag in a");
let b_hook = b
.iter()
.find(|d| d.check_name == "hook_starts_daemon")
.expect("hook diag in b");
assert_eq!(a_hook.code, ReliabilityReasonCode::HookAutoStartEnabled);
assert_eq!(b_hook.code, ReliabilityReasonCode::HookAutoStartDisabled);
}
fn make_diag(severity: ReliabilitySeverity) -> ReliabilityDiagnostic {
ReliabilityDiagnostic::new(
ReliabilityCategory::Topology,
"synthetic",
severity,
"test",
ReliabilityReasonCode::WorkersConfigured,
)
}
#[test]
fn test_aggregate_verdict_empty_is_healthy() {
assert_eq!(aggregate_verdict(&[]), ReliabilityVerdict::Healthy);
}
#[test]
fn test_aggregate_verdict_all_pass_is_healthy() {
let diags = vec![make_diag(ReliabilitySeverity::Pass); 5];
assert_eq!(aggregate_verdict(&diags), ReliabilityVerdict::Healthy);
}
#[test]
fn test_aggregate_verdict_pass_plus_info_is_healthy() {
let diags = vec![
make_diag(ReliabilitySeverity::Pass),
make_diag(ReliabilitySeverity::Info),
make_diag(ReliabilitySeverity::Pass),
];
assert_eq!(aggregate_verdict(&diags), ReliabilityVerdict::Healthy);
}
#[test]
fn test_aggregate_verdict_one_warning_is_degraded() {
let diags = vec![
make_diag(ReliabilitySeverity::Pass),
make_diag(ReliabilitySeverity::Warning),
make_diag(ReliabilitySeverity::Info),
];
assert_eq!(aggregate_verdict(&diags), ReliabilityVerdict::Degraded);
}
#[test]
fn test_aggregate_verdict_one_critical_is_failing() {
let diags = vec![
make_diag(ReliabilitySeverity::Pass),
make_diag(ReliabilitySeverity::Warning),
make_diag(ReliabilitySeverity::Critical),
];
assert_eq!(aggregate_verdict(&diags), ReliabilityVerdict::Failing);
}
#[test]
fn test_aggregate_verdict_critical_dominates_warning() {
let diags = vec![
make_diag(ReliabilitySeverity::Critical),
make_diag(ReliabilitySeverity::Warning),
];
assert_eq!(aggregate_verdict(&diags), ReliabilityVerdict::Failing);
}
#[test]
fn test_verdict_serde_lowercase_strings() {
let h = serde_json::to_string(&ReliabilityVerdict::Healthy).unwrap();
let d = serde_json::to_string(&ReliabilityVerdict::Degraded).unwrap();
let f = serde_json::to_string(&ReliabilityVerdict::Failing).unwrap();
assert_eq!(h, "\"healthy\"");
assert_eq!(d, "\"degraded\"");
assert_eq!(f, "\"failing\"");
let back: ReliabilityVerdict = serde_json::from_str(&d).unwrap();
assert_eq!(back, ReliabilityVerdict::Degraded);
}
#[test]
fn test_default_exit_code_mapping() {
assert_eq!(ReliabilityVerdict::Healthy.default_exit_code(), 0);
assert_eq!(ReliabilityVerdict::Degraded.default_exit_code(), 1);
assert_eq!(ReliabilityVerdict::Failing.default_exit_code(), 2);
}
#[test]
fn test_strict_promotes_degraded_to_two() {
assert_eq!(ReliabilityVerdict::Healthy.exit_code(true, false), 0);
assert_eq!(ReliabilityVerdict::Degraded.exit_code(true, false), 2);
assert_eq!(ReliabilityVerdict::Failing.exit_code(true, false), 2);
}
#[test]
fn test_lenient_demotes_failing_to_one() {
assert_eq!(ReliabilityVerdict::Healthy.exit_code(false, true), 0);
assert_eq!(ReliabilityVerdict::Degraded.exit_code(false, true), 1);
assert_eq!(ReliabilityVerdict::Failing.exit_code(false, true), 1);
}
#[test]
fn test_default_no_strict_no_lenient_uses_default_mapping() {
for v in [
ReliabilityVerdict::Healthy,
ReliabilityVerdict::Degraded,
ReliabilityVerdict::Failing,
] {
assert_eq!(v.exit_code(false, false), v.default_exit_code());
}
}
#[test]
fn test_verdict_label_matches_variant_name() {
assert_eq!(ReliabilityVerdict::Healthy.label(), "Healthy");
assert_eq!(ReliabilityVerdict::Degraded.label(), "Degraded");
assert_eq!(ReliabilityVerdict::Failing.label(), "Failing");
}
#[test]
fn test_summary_overall_uses_aggregate_verdict() {
let diags = vec![
make_diag(ReliabilitySeverity::Pass),
make_diag(ReliabilitySeverity::Warning),
];
let response = build_reliability_doctor_response(
ReliabilityDoctorMode::Check,
&ReliabilityScopeSet::default(),
diags,
);
assert_eq!(response.summary.overall, ReliabilityVerdict::Degraded);
}
#[test]
fn test_summary_overall_failing_when_critical_present() {
let diags = vec![
make_diag(ReliabilitySeverity::Pass),
make_diag(ReliabilitySeverity::Critical),
];
let response = build_reliability_doctor_response(
ReliabilityDoctorMode::Check,
&ReliabilityScopeSet::default(),
diags,
);
assert_eq!(response.summary.overall, ReliabilityVerdict::Failing);
}
#[test]
fn test_scope_default_is_all() {
let s = ReliabilityScopeSet::default();
assert_eq!(s.0, vec![ReliabilityScope::All]);
assert!(s.matches(ReliabilityScope::Topology));
assert!(s.matches(ReliabilityScope::Pressure));
for v in [
ReliabilityScope::All,
ReliabilityScope::Topology,
ReliabilityScope::Convergence,
ReliabilityScope::Pressure,
ReliabilityScope::Triage,
ReliabilityScope::Helpers,
ReliabilityScope::Rollout,
ReliabilityScope::Schema,
] {
assert!(s.matches(v));
}
}
#[test]
fn test_scope_parses_single_value() {
let s: ReliabilityScopeSet = "topology".parse().unwrap();
assert_eq!(s.0, vec![ReliabilityScope::Topology]);
assert!(s.matches(ReliabilityScope::Topology));
assert!(!s.matches(ReliabilityScope::Pressure));
}
#[test]
fn test_scope_parses_multi_value_csv() {
let s: ReliabilityScopeSet = "topology,pressure".parse().unwrap();
assert_eq!(
s.0,
vec![ReliabilityScope::Topology, ReliabilityScope::Pressure]
);
assert!(s.matches(ReliabilityScope::Topology));
assert!(s.matches(ReliabilityScope::Pressure));
assert!(!s.matches(ReliabilityScope::Helpers));
}
#[test]
fn test_scope_dedups_keeping_first_occurrence() {
let s: ReliabilityScopeSet = "topology,pressure,topology,pressure".parse().unwrap();
assert_eq!(
s.0,
vec![ReliabilityScope::Topology, ReliabilityScope::Pressure]
);
}
#[test]
fn test_scope_all_dominates_when_mixed() {
let s: ReliabilityScopeSet = "all,topology".parse().unwrap();
assert_eq!(s.0, vec![ReliabilityScope::All]);
}
#[test]
fn test_scope_empty_string_errors() {
let err = ""
.parse::<ReliabilityScopeSet>()
.expect_err("empty must err");
assert!(err.contains("scope list is empty"));
}
#[test]
fn test_scope_unknown_segment_errors_with_offender() {
let err = "topology,bogus"
.parse::<ReliabilityScopeSet>()
.expect_err("unknown segment must err");
assert!(
err.contains("bogus"),
"error should name the offender, got: {err}"
);
}
#[test]
fn test_scope_case_insensitive_segment_parse() {
let s: ReliabilityScopeSet = "Topology,PRESSURE".parse().unwrap();
assert_eq!(
s.0,
vec![ReliabilityScope::Topology, ReliabilityScope::Pressure]
);
}
#[test]
fn test_scope_whitespace_trimmed() {
let s: ReliabilityScopeSet = " topology ".parse().unwrap();
assert_eq!(s.0, vec![ReliabilityScope::Topology]);
}
#[test]
fn test_scope_as_strings_stable_order() {
let s = ReliabilityScopeSet(vec![ReliabilityScope::Pressure, ReliabilityScope::Topology]);
assert_eq!(s.as_strings(), vec!["pressure", "topology"]);
}
#[test]
fn test_scope_response_field_records_what_was_asked() {
let scope =
ReliabilityScopeSet(vec![ReliabilityScope::Topology, ReliabilityScope::Pressure]);
let response =
build_reliability_doctor_response(ReliabilityDoctorMode::Check, &scope, vec![]);
assert_eq!(response.scope, vec!["topology", "pressure"]);
}
#[test]
fn test_scope_response_default_is_all_array() {
let response = build_reliability_doctor_response(
ReliabilityDoctorMode::Check,
&ReliabilityScopeSet::default(),
vec![],
);
assert_eq!(response.scope, vec!["all"]);
}
#[test]
fn test_scope_prefetch_dependencies_are_precise() {
let all = ReliabilityScopeSet::default();
assert!(all.needs_worker_config());
assert!(all.needs_daemon_status());
assert!(all.needs_repo_convergence_status());
assert!(all.needs_rollout_config());
let topology: ReliabilityScopeSet = "topology".parse().unwrap();
assert!(topology.needs_worker_config());
assert!(topology.needs_daemon_status());
assert!(!topology.needs_repo_convergence_status());
assert!(!topology.needs_rollout_config());
let pressure: ReliabilityScopeSet = "pressure".parse().unwrap();
assert!(!pressure.needs_worker_config());
assert!(pressure.needs_daemon_status());
assert!(!pressure.needs_repo_convergence_status());
assert!(!pressure.needs_rollout_config());
let convergence: ReliabilityScopeSet = "convergence".parse().unwrap();
assert!(!convergence.needs_worker_config());
assert!(!convergence.needs_daemon_status());
assert!(convergence.needs_repo_convergence_status());
assert!(!convergence.needs_rollout_config());
let rollout: ReliabilityScopeSet = "rollout".parse().unwrap();
assert!(!rollout.needs_worker_config());
assert!(!rollout.needs_daemon_status());
assert!(!rollout.needs_repo_convergence_status());
assert!(rollout.needs_rollout_config());
let local_only: ReliabilityScopeSet = "helpers,schema".parse().unwrap();
assert!(!local_only.needs_worker_config());
assert!(!local_only.needs_daemon_status());
assert!(!local_only.needs_repo_convergence_status());
assert!(!local_only.needs_rollout_config());
}
#[test]
fn test_scope_probe_names_honor_schema_gate() {
let all = ReliabilityScopeSet::default();
assert_eq!(
all.probe_names_to_run(false),
vec![
"topology",
"ownership",
"convergence",
"pressure",
"triage",
"helpers",
"rollout"
]
);
assert_eq!(
all.probe_names_to_run(true),
vec![
"topology",
"ownership",
"convergence",
"pressure",
"triage",
"helpers",
"rollout",
"schema"
]
);
let schema_only: ReliabilityScopeSet = "schema".parse().unwrap();
assert_eq!(schema_only.probe_names_to_run(false), vec!["schema"]);
assert_eq!(schema_only.probe_names_to_run(true), vec!["schema"]);
let helpers_schema: ReliabilityScopeSet = "helpers,schema".parse().unwrap();
assert_eq!(
helpers_schema.probe_names_to_run(false),
vec!["helpers", "schema"]
);
}
fn synthetic_diagnostic(
code: ReliabilityReasonCode,
check_name: &str,
message: &str,
severity: ReliabilitySeverity,
) -> ReliabilityDiagnostic {
ReliabilityDiagnostic::new(
ReliabilityCategory::Topology,
check_name,
severity,
message,
code,
)
}
#[test]
fn test_daemon_unreachable_false_when_all_reachable() {
let diags = vec![
synthetic_diagnostic(
ReliabilityReasonCode::WorkersHealthy,
"daemon_worker_capacity",
"All 7 workers healthy",
ReliabilitySeverity::Pass,
),
synthetic_diagnostic(
ReliabilityReasonCode::WorkerReady,
"worker_topology",
"Worker css ready",
ReliabilitySeverity::Pass,
),
];
let response = build_reliability_doctor_response(
ReliabilityDoctorMode::Check,
&ReliabilityScopeSet::default(),
diags,
);
assert!(!response.daemon_unreachable);
assert!(response.daemon_unreachable_reasons.is_empty());
}
#[test]
fn test_daemon_unreachable_true_when_status_unavailable() {
let diags = vec![synthetic_diagnostic(
ReliabilityReasonCode::DaemonStatusUnavailable,
"daemon_status",
"Daemon status is unavailable",
ReliabilitySeverity::Warning,
)];
let response = build_reliability_doctor_response(
ReliabilityDoctorMode::Check,
&ReliabilityScopeSet::default(),
diags,
);
assert!(response.daemon_unreachable);
assert_eq!(response.daemon_unreachable_reasons.len(), 1);
assert!(
response.daemon_unreachable_reasons[0].contains("daemon_status"),
"reason text should name the probe: {:?}",
response.daemon_unreachable_reasons
);
assert!(
response.daemon_unreachable_reasons[0].contains("unavailable"),
"reason text should include the diagnostic message"
);
}
#[test]
fn test_daemon_unreachable_aggregates_multiple_probes() {
let diags = vec![
synthetic_diagnostic(
ReliabilityReasonCode::DaemonStatusUnavailable,
"daemon_status",
"daemon down",
ReliabilitySeverity::Warning,
),
synthetic_diagnostic(
ReliabilityReasonCode::DiskPressureUnavailable,
"disk_pressure",
"disk surface gone",
ReliabilitySeverity::Warning,
),
synthetic_diagnostic(
ReliabilityReasonCode::ProcessDebtUnavailable,
"process_debt",
"triage unavailable",
ReliabilitySeverity::Warning,
),
synthetic_diagnostic(
ReliabilityReasonCode::WorkersHealthy,
"daemon_worker_capacity",
"ignored",
ReliabilitySeverity::Pass,
),
];
let response = build_reliability_doctor_response(
ReliabilityDoctorMode::Check,
&ReliabilityScopeSet::default(),
diags,
);
assert!(response.daemon_unreachable);
assert_eq!(
response.daemon_unreachable_reasons.len(),
3,
"expected 3 reasons (the 3 unreachable codes), got {:?}",
response.daemon_unreachable_reasons
);
for r in &response.daemon_unreachable_reasons {
assert!(
!r.contains("ignored"),
"Pass diagnostics should NOT contribute to reasons: {r}"
);
}
}
#[test]
fn test_reliability_response_carries_schema_version() {
let response = build_reliability_doctor_response(
ReliabilityDoctorMode::Check,
&ReliabilityScopeSet::default(),
vec![],
);
assert!(!response.schema_version.is_empty());
assert!(
response
.schema_version
.chars()
.all(|c| c.is_ascii_digit() || c == '.'),
"schema_version must be numeric+dots: {:?}",
response.schema_version
);
}
#[test]
fn test_doctor_unreachable_codes_table_is_subset_of_known_codes() {
for c in DAEMON_UNREACHABLE_REASON_CODES {
assert!(
ReliabilityReasonCode::ALL.contains(c),
"DAEMON_UNREACHABLE_REASON_CODES contains {c:?} which is not in ALL"
);
}
let unique: std::collections::HashSet<_> = DAEMON_UNREACHABLE_REASON_CODES.iter().collect();
assert_eq!(unique.len(), DAEMON_UNREACHABLE_REASON_CODES.len());
}
#[test]
fn test_push_opt_f64_some_at_precision_1() {
let mut buf = String::new();
push_opt_f64(&mut buf, Some(7.654), 1);
assert_eq!(buf, "7.7");
}
#[test]
fn test_push_opt_f64_some_at_precision_2() {
let mut buf = String::new();
push_opt_f64(&mut buf, Some(0.123456), 2);
assert_eq!(buf, "0.12");
}
#[test]
fn test_push_opt_f64_some_at_precision_3() {
let mut buf = String::new();
push_opt_f64(&mut buf, Some(0.123456), 3);
assert_eq!(buf, "0.123");
}
#[test]
fn test_push_opt_f64_none_writes_unknown() {
let mut buf = String::new();
push_opt_f64(&mut buf, None, 2);
assert_eq!(buf, "unknown");
}
#[test]
fn test_push_opt_f64_appends_to_existing_content() {
let mut buf = String::from("prefix=");
push_opt_f64(&mut buf, Some(2.5), 1);
assert_eq!(buf, "prefix=2.5");
}
#[test]
fn test_push_opt_display_some_u64() {
let mut buf = String::new();
push_opt_display(&mut buf, Some(42u64));
assert_eq!(buf, "42");
}
#[test]
fn test_push_opt_display_some_bool() {
let mut buf = String::new();
push_opt_display(&mut buf, Some(true));
assert_eq!(buf, "true");
}
#[test]
fn test_push_opt_display_none_writes_unknown() {
let mut buf = String::new();
push_opt_display::<i64>(&mut buf, None);
assert_eq!(buf, "unknown");
}
#[test]
fn test_t11_helpers_match_old_format_behavior() {
let v = Some(123.456);
let old: String = v
.map(|value| format!("{value:.2}"))
.unwrap_or_else(|| "unknown".to_string());
let mut new_buf = String::new();
push_opt_f64(&mut new_buf, v, 2);
assert_eq!(old, new_buf, "precision-2 helper must match old format");
let old: String = v
.map(|value| format!("{value:.3}"))
.unwrap_or_else(|| "unknown".to_string());
let mut new_buf = String::new();
push_opt_f64(&mut new_buf, v, 3);
assert_eq!(old, new_buf, "precision-3 helper must match old format");
let v: Option<u64> = Some(3600);
let old: String = v
.map(|value| value.to_string())
.unwrap_or_else(|| "unknown".to_string());
let mut new_buf = String::new();
push_opt_display(&mut new_buf, v);
assert_eq!(old, new_buf, "Display helper for u64 must match old format");
let none_f64: Option<f64> = None;
let old: String = none_f64
.map(|value| format!("{value:.2}"))
.unwrap_or_else(|| "unknown".to_string());
let mut new_buf = String::new();
push_opt_f64(&mut new_buf, none_f64, 2);
assert_eq!(
old, new_buf,
"None precision-2 helper must match old format"
);
let none_u64: Option<u64> = None;
let old: String = none_u64
.map(|value| value.to_string())
.unwrap_or_else(|| "unknown".to_string());
let mut new_buf = String::new();
push_opt_display(&mut new_buf, none_u64);
assert_eq!(old, new_buf, "None Display helper must match old format");
}
fn make_watch_diag(
category: ReliabilityCategory,
check_name: &str,
severity: ReliabilitySeverity,
code: ReliabilityReasonCode,
message: &str,
worker_id: Option<&str>,
) -> ReliabilityDiagnostic {
let mut d = ReliabilityDiagnostic::new(category, check_name, severity, message, code);
if let Some(w) = worker_id {
d = d.with_worker(w);
}
d
}
fn fp_map_of(
diags: &[ReliabilityDiagnostic],
) -> std::collections::BTreeMap<DiagnosticKey, DiagnosticFingerprint> {
let mut map = std::collections::BTreeMap::new();
for d in diags {
map.insert(
DiagnosticKey::from_diagnostic(d),
DiagnosticFingerprint::from_diagnostic(d),
);
}
map
}
#[test]
fn probe_outcome_label_is_lowercase_and_stable() {
assert_eq!(ProbeOutcome::Ok.label(), "ok");
assert_eq!(ProbeOutcome::Skipped.label(), "skipped");
assert_eq!(ProbeOutcome::InnerError.label(), "inner_error");
assert_eq!(ProbeOutcome::Timeout.label(), "timeout");
assert_eq!(ProbeOutcome::Panicked.label(), "panicked");
assert_eq!(ProbeOutcome::Cancelled.label(), "cancelled");
}
#[tokio::test]
async fn join_isolated_async_probe_returns_skipped_for_none_handle() {
let (result, outcome) =
join_isolated_async_probe::<String, std::io::Error>("ghost", None).await;
assert_eq!(result, None);
assert_eq!(outcome, ProbeOutcome::Skipped);
}
#[tokio::test]
async fn join_isolated_async_probe_returns_ok_for_successful_task() {
let handle = tokio::spawn(async {
tokio::time::timeout(PROBE_TIMEOUT, async {
Ok::<_, std::io::Error>("probe_value".to_string())
})
.await
});
let (result, outcome) = join_isolated_async_probe("ok_probe", Some(handle)).await;
assert_eq!(result.as_deref(), Some("probe_value"));
assert_eq!(outcome, ProbeOutcome::Ok);
}
#[tokio::test]
async fn join_isolated_async_probe_classifies_inner_error() {
let handle = tokio::spawn(async {
tokio::time::timeout(PROBE_TIMEOUT, async {
Err::<String, std::io::Error>(std::io::Error::other("inner fault"))
})
.await
});
let (result, outcome) = join_isolated_async_probe("err_probe", Some(handle)).await;
assert!(result.is_none());
assert_eq!(outcome, ProbeOutcome::InnerError);
}
#[tokio::test]
async fn join_isolated_async_probe_surfaces_timeout() {
let handle = tokio::spawn(async {
tokio::time::timeout(std::time::Duration::from_millis(20), async {
tokio::time::sleep(std::time::Duration::from_secs(60)).await;
Ok::<_, std::io::Error>("never returned".to_string())
})
.await
});
let (result, outcome) = join_isolated_async_probe("slow_probe", Some(handle)).await;
assert!(result.is_none());
assert_eq!(outcome, ProbeOutcome::Timeout);
}
#[tokio::test]
async fn join_isolated_async_probe_isolates_panic() {
let handle = tokio::spawn(async {
tokio::time::timeout(PROBE_TIMEOUT, async {
std::panic::panic_any("probe explosion (intentional, for test)".to_string());
#[expect(unreachable_code)]
Ok::<_, std::io::Error>("never".to_string())
})
.await
});
let (result, outcome) = join_isolated_async_probe("panic_probe", Some(handle)).await;
assert!(result.is_none());
assert_eq!(outcome, ProbeOutcome::Panicked);
}
#[tokio::test]
async fn join_isolated_blocking_probe_returns_value_for_successful_task() {
let handle = tokio::task::spawn_blocking(|| vec!["a".to_string(), "b".to_string()]);
let (result, outcome) = join_isolated_blocking_probe("blocking_ok", Some(handle)).await;
assert_eq!(
result.as_deref(),
Some(&["a".to_string(), "b".to_string()][..])
);
assert_eq!(outcome, ProbeOutcome::Ok);
}
#[tokio::test]
async fn join_isolated_blocking_probe_isolates_panic() {
let handle = tokio::task::spawn_blocking(|| -> Vec<String> {
std::panic::panic_any("blocking probe explosion (intentional, for test)".to_string());
});
let (result, outcome) = join_isolated_blocking_probe("blocking_panic", Some(handle)).await;
assert!(result.is_none());
assert_eq!(outcome, ProbeOutcome::Panicked);
}
#[tokio::test]
async fn join_isolated_blocking_probe_returns_skipped_for_none_handle() {
let (result, outcome): (Option<Vec<String>>, ProbeOutcome) =
join_isolated_blocking_probe("ghost", None).await;
assert!(result.is_none());
assert_eq!(outcome, ProbeOutcome::Skipped);
}
#[tokio::test]
async fn join_isolated_blocking_probe_surfaces_timeout_promptly() {
let start = std::time::Instant::now();
let handle = tokio::task::spawn_blocking(|| {
std::thread::sleep(std::time::Duration::from_millis(80));
vec!["too_late".to_string()]
});
let (result, outcome) = join_isolated_blocking_probe_with_timeout(
"blocking_timeout",
Some(handle),
std::time::Duration::from_millis(10),
)
.await;
assert!(result.is_none());
assert_eq!(outcome, ProbeOutcome::Timeout);
assert!(
start.elapsed() < std::time::Duration::from_millis(60),
"blocking timeout should return promptly"
);
}
#[test]
fn helper_probe_failure_emits_unavailable_diagnostic_without_rerun() {
let diagnostics = helper_diagnostics_from_probe_result(None, ProbeOutcome::Timeout);
assert_eq!(diagnostics.len(), 1);
let diagnostic = &diagnostics[0];
assert_eq!(
diagnostic.category,
ReliabilityCategory::HelperCompatibility
);
assert_eq!(diagnostic.check_name, "helper_probe");
assert_eq!(diagnostic.severity, ReliabilitySeverity::Warning);
assert_eq!(
diagnostic.code,
ReliabilityReasonCode::HelperProbeUnavailable
);
assert!(
diagnostic
.details
.as_deref()
.is_some_and(|details| details.contains("probe_outcome=timeout")),
"details should retain the probe outcome"
);
}
#[tokio::test]
async fn timed_async_probe_exports_measured_outcomes_once() {
use tracing_subscriber::prelude::*;
let metrics = rch_telemetry::metrics::Metrics::new().unwrap();
let subscriber = tracing_subscriber::registry()
.with(rch_telemetry::metrics::MetricsLayer::new(metrics.clone()));
let _subscriber = tracing::subscriber::set_default(subscriber);
let success = timed_async_probe("daemon_status", async {
tokio::time::sleep(Duration::from_millis(5)).await;
Ok::<_, &str>(42)
})
.await;
assert_eq!(success.unwrap().unwrap(), 42);
let error =
timed_async_probe("repo_convergence", async { Err::<(), _>("rpc failed") }).await;
assert_eq!(error.unwrap().unwrap_err(), "rpc failed");
let timeout =
timed_async_probe("daemon_status", std::future::pending::<Result<(), &str>>()).await;
assert!(timeout.is_err());
for (probe, outcome, minimum) in [
("daemon_status", "completed", 0.005),
("repo_convergence", "inner_error", 0.0),
("daemon_status", "timeout", PROBE_TIMEOUT.as_secs_f64()),
] {
let histogram = metrics
.doctor_probe_duration_seconds
.with_label_values(&[probe, outcome]);
assert_eq!(histogram.get_sample_count(), 1, "{probe}/{outcome}");
assert!(histogram.get_sample_sum() >= minimum, "{probe}/{outcome}");
assert!(histogram.get_sample_sum() > 0.0, "{probe}/{outcome}");
}
}
#[tokio::test]
async fn probes_actually_run_in_parallel_not_serial() {
let start = std::time::Instant::now();
let h1 = tokio::spawn(async {
tokio::time::timeout(PROBE_TIMEOUT, async {
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
Ok::<_, std::io::Error>("a".to_string())
})
.await
});
let h2 = tokio::spawn(async {
tokio::time::timeout(PROBE_TIMEOUT, async {
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
Ok::<_, std::io::Error>("b".to_string())
})
.await
});
let h3 = tokio::task::spawn_blocking(|| {
std::thread::sleep(std::time::Duration::from_millis(100));
vec!["c".to_string()]
});
let (r1, _) = join_isolated_async_probe("p1", Some(h1)).await;
let (r2, _) = join_isolated_async_probe("p2", Some(h2)).await;
let (r3, _) = join_isolated_blocking_probe("p3", Some(h3)).await;
let elapsed = start.elapsed();
assert_eq!(r1.as_deref(), Some("a"));
assert_eq!(r2.as_deref(), Some("b"));
assert_eq!(r3.as_deref(), Some(&["c".to_string()][..]));
assert!(
elapsed < std::time::Duration::from_millis(250),
"probes must run in parallel: elapsed={elapsed:?} (serial would be ~300ms+)"
);
}
#[test]
fn diff_empty_vs_empty_is_quiet() {
let prev = std::collections::BTreeMap::new();
let curr = std::collections::BTreeMap::new();
let diff = diff_fingerprint_maps(&prev, &curr);
assert!(!diff.has_changes(), "empty→empty must be quiet");
assert_eq!(diff.added.len(), 0);
assert_eq!(diff.cleared.len(), 0);
assert_eq!(diff.changed.len(), 0);
assert_eq!(diff.total_transitions(), 0);
}
#[test]
fn diff_first_sweep_classifies_everything_as_added() {
let prev = std::collections::BTreeMap::new();
let curr_diags = vec![
make_watch_diag(
ReliabilityCategory::Topology,
"workers_config",
ReliabilitySeverity::Pass,
ReliabilityReasonCode::WorkersConfigured,
"3 workers configured",
None,
),
make_watch_diag(
ReliabilityCategory::DiskPressure,
"disk_pressure",
ReliabilitySeverity::Warning,
ReliabilityReasonCode::PartialWorkerCapacity,
"disk pressure observed",
Some("css"),
),
];
let curr = fp_map_of(&curr_diags);
let diff = diff_fingerprint_maps(&prev, &curr);
assert!(diff.has_changes());
assert_eq!(diff.added.len(), 2, "first sweep must mark all as added");
assert_eq!(diff.cleared.len(), 0);
assert_eq!(diff.changed.len(), 0);
assert_eq!(diff.total_transitions(), 2);
}
#[test]
fn diff_cleared_when_diagnostic_disappears() {
let prev_diags = vec![make_watch_diag(
ReliabilityCategory::Topology,
"workers_config",
ReliabilitySeverity::Critical,
ReliabilityReasonCode::NoWorkersConfigured,
"no workers configured",
None,
)];
let prev = fp_map_of(&prev_diags);
let curr = std::collections::BTreeMap::new();
let diff = diff_fingerprint_maps(&prev, &curr);
assert_eq!(diff.added.len(), 0);
assert_eq!(diff.cleared.len(), 1);
assert_eq!(diff.changed.len(), 0);
}
#[test]
fn diff_severity_flip_is_changed_not_cleared_plus_added() {
let prev_diags = vec![make_watch_diag(
ReliabilityCategory::Topology,
"workers_config",
ReliabilitySeverity::Warning,
ReliabilityReasonCode::PartialWorkerCapacity,
"2/3 workers healthy",
None,
)];
let curr_diags = vec![make_watch_diag(
ReliabilityCategory::Topology,
"workers_config",
ReliabilitySeverity::Critical, ReliabilityReasonCode::AllWorkersUnhealthy,
"0/3 workers healthy",
None,
)];
let prev = fp_map_of(&prev_diags);
let curr = fp_map_of(&curr_diags);
let diff = diff_fingerprint_maps(&prev, &curr);
assert_eq!(diff.added.len(), 0, "must not double-count flip as added");
assert_eq!(
diff.cleared.len(),
0,
"must not double-count flip as cleared"
);
assert_eq!(
diff.changed.len(),
1,
"severity flip is a single 'changed' event"
);
let (key, prev_fp, cur_fp) = &diff.changed[0];
assert_eq!(key.check_name, "workers_config");
assert_eq!(prev_fp.severity, ReliabilitySeverity::Warning);
assert_eq!(cur_fp.severity, ReliabilitySeverity::Critical);
}
#[test]
fn diff_identical_fingerprint_is_quiet() {
let diags = vec![make_watch_diag(
ReliabilityCategory::Topology,
"workers_config",
ReliabilitySeverity::Pass,
ReliabilityReasonCode::WorkersConfigured,
"3 workers configured",
None,
)];
let prev = fp_map_of(&diags);
let curr = fp_map_of(&diags);
let diff = diff_fingerprint_maps(&prev, &curr);
assert!(
!diff.has_changes(),
"identical fingerprints must not flag changes"
);
}
#[test]
fn diff_per_worker_keys_are_independent() {
let prev_diags = vec![
make_watch_diag(
ReliabilityCategory::Topology,
"worker_health",
ReliabilitySeverity::Pass,
ReliabilityReasonCode::WorkersHealthy,
"ok",
Some("css"),
),
make_watch_diag(
ReliabilityCategory::Topology,
"worker_health",
ReliabilitySeverity::Pass,
ReliabilityReasonCode::WorkersHealthy,
"ok",
Some("dlx"),
),
];
let curr_diags = vec![
make_watch_diag(
ReliabilityCategory::Topology,
"worker_health",
ReliabilitySeverity::Pass,
ReliabilityReasonCode::WorkersHealthy,
"ok",
Some("css"),
),
make_watch_diag(
ReliabilityCategory::Topology,
"worker_health",
ReliabilitySeverity::Critical,
ReliabilityReasonCode::AllWorkersUnhealthy,
"down",
Some("dlx"),
),
];
let prev = fp_map_of(&prev_diags);
let curr = fp_map_of(&curr_diags);
let diff = diff_fingerprint_maps(&prev, &curr);
assert_eq!(diff.added.len(), 0);
assert_eq!(diff.cleared.len(), 0);
assert_eq!(
diff.changed.len(),
1,
"only the dlx row should be marked changed"
);
let (key, _, _) = &diff.changed[0];
assert_eq!(key.worker_id.as_deref(), Some("dlx"));
}
#[test]
fn watch_state_observe_verdict_tracks_worst_case() {
let mut state = WatchState::new();
assert!(state.worst_verdict.is_none());
state.observe_verdict(ReliabilityVerdict::Healthy);
assert_eq!(state.worst_verdict, Some(ReliabilityVerdict::Healthy));
state.observe_verdict(ReliabilityVerdict::Degraded);
assert_eq!(state.worst_verdict, Some(ReliabilityVerdict::Degraded));
state.observe_verdict(ReliabilityVerdict::Healthy); assert_eq!(
state.worst_verdict,
Some(ReliabilityVerdict::Degraded),
"recovery must not lower the worst-observed verdict"
);
state.observe_verdict(ReliabilityVerdict::Failing);
assert_eq!(state.worst_verdict, Some(ReliabilityVerdict::Failing));
state.observe_verdict(ReliabilityVerdict::Healthy); assert_eq!(
state.worst_verdict,
Some(ReliabilityVerdict::Failing),
"worst_verdict is sticky once Failing"
);
}
#[test]
fn watch_worst_verdict_includes_exit_snapshot_probe() {
assert_eq!(
worst_reliability_verdict(
Some(ReliabilityVerdict::Healthy),
ReliabilityVerdict::Failing
),
ReliabilityVerdict::Failing,
"a failing exit snapshot must promote worst_verdict"
);
assert_eq!(
worst_reliability_verdict(
Some(ReliabilityVerdict::Degraded),
ReliabilityVerdict::Healthy
),
ReliabilityVerdict::Degraded,
"a recovered exit snapshot must not lower the observed worst verdict"
);
assert_eq!(
worst_reliability_verdict(None, ReliabilityVerdict::Degraded),
ReliabilityVerdict::Degraded,
"snapshot-only sessions must use the exit probe verdict"
);
}
#[test]
fn diagnostic_key_render_includes_worker_when_present() {
let key_no_worker = DiagnosticKey {
category: ReliabilityCategory::Topology,
check_name: "workers_config".to_string(),
worker_id: None,
};
assert_eq!(key_no_worker.render(), "topology/workers_config");
let key_with_worker = DiagnosticKey {
category: ReliabilityCategory::DiskPressure,
check_name: "disk_pressure".to_string(),
worker_id: Some("css".to_string()),
};
assert_eq!(
key_with_worker.render(),
"disk_pressure/disk_pressure[worker=css]"
);
}
#[test]
fn fingerprint_excludes_details_so_rotating_uptime_does_not_churn() {
let d_a = ReliabilityDiagnostic::new(
ReliabilityCategory::Topology,
"daemon_worker_capacity",
ReliabilitySeverity::Pass,
"All 3 workers healthy",
ReliabilityReasonCode::WorkersHealthy,
)
.with_details("uptime_secs=100");
let d_b = ReliabilityDiagnostic::new(
ReliabilityCategory::Topology,
"daemon_worker_capacity",
ReliabilitySeverity::Pass,
"All 3 workers healthy",
ReliabilityReasonCode::WorkersHealthy,
)
.with_details("uptime_secs=200");
let fp_a = DiagnosticFingerprint::from_diagnostic(&d_a);
let fp_b = DiagnosticFingerprint::from_diagnostic(&d_b);
assert_eq!(
fp_a, fp_b,
"rotating details must NOT change the fingerprint"
);
}
}
#[cfg(test)]
mod rustc_wrapper_tests {
use super::*;
fn obs(
env_var: &'static str,
raw: &str,
resolved: Option<&str>,
identity: Option<&str>,
) -> WrapperObservation {
WrapperObservation {
env_var,
raw_value: raw.to_string(),
resolved: resolved.map(PathBuf::from),
identity: identity.map(str::to_string),
}
}
fn single(observation: &WrapperObservation) -> CheckResult {
let state = RustcWrapperState::default();
rustc_wrapper_results(&state, std::slice::from_ref(observation))
.pop()
.expect("one check per observed wrapper")
}
#[test]
fn no_wrappers_yields_single_pass() {
let results = rustc_wrapper_results(&RustcWrapperState::default(), &[]);
assert_eq!(results.len(), 1);
assert_eq!(results[0].status, CheckStatus::Pass);
assert!(results[0].message.contains("not set"));
}
#[test]
fn workspace_first_enablement_warns_and_explains_one_time_rebuild() {
let check = single(&obs(
"RUSTC_WORKSPACE_WRAPPER",
"/usr/local/bin/sccache",
Some("/usr/local/bin/sccache"),
Some("deadbeef"),
));
assert_eq!(check.status, CheckStatus::Warning);
assert!(
check.message.contains("first observation"),
"{:?}",
check.message
);
assert!(
check
.details
.as_deref()
.unwrap_or_default()
.contains("NOT a regression"),
"must explicitly defuse the regression interpretation"
);
}
#[test]
fn plain_wrapper_is_pass_and_not_fingerprinted() {
let check = single(&obs(
"RUSTC_WRAPPER",
"/usr/local/bin/sccache",
Some("/usr/local/bin/sccache"),
Some("deadbeef"),
));
assert_eq!(check.status, CheckStatus::Pass);
assert!(check.message.contains("wrapper active"));
assert!(
check
.details
.as_deref()
.unwrap_or_default()
.contains("does NOT force a rebuild"),
);
}
#[test]
fn plain_wrapper_drops_stale_state_marker() {
let state = RustcWrapperState {
wrappers: BTreeMap::from([("RUSTC_WRAPPER".to_string(), "stale".to_string())]),
};
let results = rustc_wrapper_results(
&state,
&[obs(
"RUSTC_WRAPPER",
"/usr/bin/true",
Some("/usr/bin/true"),
Some("beef"),
)],
);
assert_eq!(results[0].status, CheckStatus::Pass);
}
#[test]
fn workspace_unchanged_wrapper_passes() {
let state = RustcWrapperState {
wrappers: BTreeMap::from([(
"RUSTC_WORKSPACE_WRAPPER".to_string(),
"deadbeef".to_string(),
)]),
};
let results = rustc_wrapper_results(
&state,
&[obs(
"RUSTC_WORKSPACE_WRAPPER",
"/usr/bin/sccache",
Some("/usr/bin/sccache"),
Some("deadbeef"),
)],
);
assert_eq!(results[0].status, CheckStatus::Pass);
assert!(results[0].message.contains("unchanged"));
}
#[test]
fn changed_value_warns() {
let state = RustcWrapperState {
wrappers: BTreeMap::from([(
"RUSTC_WORKSPACE_WRAPPER".to_string(),
"cafebabe".to_string(),
)]),
};
let results = rustc_wrapper_results(
&state,
&[obs(
"RUSTC_WORKSPACE_WRAPPER",
"/usr/bin/sccache",
Some("/usr/bin/sccache"),
Some("deadbeef"),
)],
);
assert_eq!(results[0].status, CheckStatus::Warning);
assert!(results[0].message.contains("value changed"));
}
#[test]
fn missing_binary_warns() {
let check = single(&obs("RUSTC_WRAPPER", "/nope/missing-wrapper", None, None));
assert_eq!(check.status, CheckStatus::Warning);
assert!(check.message.contains("was not found"));
}
#[test]
fn unhashable_workspace_wrapper_warns_and_skips_tracking() {
let state = RustcWrapperState {
wrappers: BTreeMap::from([(
"RUSTC_WORKSPACE_WRAPPER".to_string(),
"mywrapper".to_string(),
)]),
};
let results = rustc_wrapper_results(
&state,
&[obs(
"RUSTC_WORKSPACE_WRAPPER",
"mywrapper",
Some("/bin/mywrapper"),
None,
)],
);
assert_eq!(results[0].status, CheckStatus::Warning);
assert!(results[0].message.contains("could not be hashed"));
}
#[test]
fn workspace_wrapper_is_observed_independently() {
let results = rustc_wrapper_results(
&RustcWrapperState::default(),
&[obs(
"RUSTC_WORKSPACE_WRAPPER",
"/usr/bin/true",
Some("/usr/bin/true"),
Some("beef"),
)],
);
assert_eq!(results.len(), 1);
assert!(results[0].message.contains("RUSTC_WORKSPACE_WRAPPER"));
}
#[test]
fn wrapper_binary_path_resolves_via_which() {
let found = wrapper_binary_path("sh");
assert!(found.is_some(), "which(sh) should resolve");
assert!(!found.unwrap().as_os_str().is_empty());
assert!(wrapper_binary_path("/definitely/not/here").is_none());
assert!(wrapper_binary_path("definitely-not-a-real-binary-xyz").is_none());
}
#[test]
fn identity_hash_is_stable_and_bounded() {
let dir = tempfile::tempdir().expect("dir");
let bin = dir.path().join("wrapper");
std::fs::write(&bin, b"#!/bin/sh\nexec \"$@\"\n").expect("write");
let a = hash_wrapper_identity(&bin).expect("hash");
let b = hash_wrapper_identity(&bin).expect("hash");
assert_eq!(a, b);
assert_eq!(a.len(), 32);
std::fs::write(&bin, b"#!/bin/sh\nexit 0\n").expect("rewrite");
let c = hash_wrapper_identity(&bin).expect("hash");
assert_ne!(a, c, "content change must change identity");
}
}