use std::collections::BTreeMap;
use crate::root_cause_taxonomy::{
AttributionConfidence, EvidenceRef, RootCauseAttribution, RootCauseFamily,
};
use serde::{Deserialize, Serialize};
pub const FLEET_DOCTOR_SCHEMA_VERSION: u32 = 1;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum HostProbeStatus {
Ok,
Partial,
TimedOut,
OldBinarySkew,
CommandNotFound,
Unreachable,
Degraded,
}
impl HostProbeStatus {
pub const fn as_str(self) -> &'static str {
match self {
HostProbeStatus::Ok => "ok",
HostProbeStatus::Partial => "partial",
HostProbeStatus::TimedOut => "timed-out",
HostProbeStatus::OldBinarySkew => "old-binary-skew",
HostProbeStatus::CommandNotFound => "command-not-found",
HostProbeStatus::Unreachable => "unreachable",
HostProbeStatus::Degraded => "degraded",
}
}
pub const fn is_hard_failure(self) -> bool {
matches!(
self,
HostProbeStatus::Unreachable | HostProbeStatus::CommandNotFound
)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum HostOs {
Linux,
#[serde(rename = "macos")]
MacOs,
Windows,
Other,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum PathStyle {
Posix,
Windows,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Platform {
pub os: HostOs,
pub arch: String,
pub path_style: PathStyle,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub tool_notes: Vec<String>,
}
impl Platform {
pub fn linux_x86_64() -> Self {
Self {
os: HostOs::Linux,
arch: "x86_64".to_string(),
path_style: PathStyle::Posix,
tool_notes: Vec::new(),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum CapabilityTier {
Full,
Standard,
Minimal,
Unknown,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum ReadinessState {
Ready,
Degraded,
NotReady,
Unknown,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum SemanticState {
Enabled,
Disabled,
AssetsMissing,
Unknown,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum RemoteSyncState {
Synced,
Stale,
NeverSynced,
Failed,
NotConfigured,
Unknown,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum ArchiveRisk {
Unknown,
Low,
Medium,
High,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct SourceRoot {
pub path: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub agent: Option<String>,
pub archived: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
pub struct SourceCounts {
pub roots: u64,
pub sessions: u64,
pub indexed_sessions: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
pub struct QuarantineState {
pub quarantined: u64,
pub recoverable: u64,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct HostDoctorReport {
pub schema_version: u32,
pub host_alias: String,
pub platform: Platform,
pub status: HostProbeStatus,
pub elapsed_ms: u64,
pub timed_out: bool,
pub unreachable: bool,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub cancelled: bool,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub stale_data: bool,
pub archive_risk: ArchiveRisk,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub skipped_sections: Vec<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cass_version: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub capability_tier: Option<CapabilityTier>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub data_dir: Option<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub source_roots: Vec<SourceRoot>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub source_counts: Option<SourceCounts>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub readiness: Option<ReadinessState>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub semantic: Option<SemanticState>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub quarantine: Option<QuarantineState>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub remote_sync: Option<RemoteSyncState>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub likely_root_cause: Option<RootCauseFamily>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub root_cause: Option<RootCauseAttribution>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub recommended_action: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub connection_error: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub retry_hint: Option<String>,
}
impl HostDoctorReport {
pub fn skeleton(
host_alias: impl Into<String>,
platform: Platform,
status: HostProbeStatus,
elapsed_ms: u64,
) -> Self {
Self {
schema_version: FLEET_DOCTOR_SCHEMA_VERSION,
host_alias: host_alias.into(),
platform,
status,
elapsed_ms,
timed_out: status == HostProbeStatus::TimedOut,
unreachable: status == HostProbeStatus::Unreachable,
cancelled: false,
stale_data: false,
archive_risk: ArchiveRisk::Unknown,
skipped_sections: Vec::new(),
cass_version: None,
capability_tier: None,
data_dir: None,
source_roots: Vec::new(),
source_counts: None,
readiness: None,
semantic: None,
quarantine: None,
remote_sync: None,
likely_root_cause: None,
root_cause: None,
recommended_action: None,
connection_error: None,
retry_hint: None,
}
}
pub fn unreachable(
host_alias: impl Into<String>,
platform: Platform,
elapsed_ms: u64,
recommended_action: impl Into<String>,
) -> Self {
let mut report = Self::skeleton(
host_alias,
platform,
HostProbeStatus::Unreachable,
elapsed_ms,
);
report.likely_root_cause = Some(RootCauseFamily::RemoteTransportAuth);
report.root_cause = Some(transport_root_cause_attribution(report.status, None));
report.recommended_action = Some(recommended_action.into());
report
}
pub fn failed(
host_alias: impl Into<String>,
platform: Platform,
elapsed_ms: u64,
connection_error: impl Into<String>,
) -> Self {
let connection_error = connection_error.into();
let (status, retry_hint) = classify_connection_failure(&connection_error);
let mut report = Self::skeleton(host_alias, platform, status, elapsed_ms);
report.likely_root_cause = Some(RootCauseFamily::RemoteTransportAuth);
report.root_cause = Some(transport_root_cause_attribution(
report.status,
Some(&connection_error),
));
report.recommended_action = Some(retry_hint.to_string());
report.retry_hint = Some(retry_hint.to_string());
report.connection_error = Some(connection_error);
report
}
pub fn cancelled(host_alias: impl Into<String>, platform: Platform, elapsed_ms: u64) -> Self {
let mut report = Self::skeleton(host_alias, platform, HostProbeStatus::Partial, elapsed_ms);
report.cancelled = true;
report.connection_error = Some("probe cancelled by operator".to_string());
report.retry_hint = Some("rerun the fleet probe to collect fresh evidence".to_string());
report.recommended_action = report.retry_hint.clone();
report
}
#[must_use]
pub fn with_last_known(mut self, previous: &HostDoctorReport) -> Self {
if self.status == HostProbeStatus::Ok {
return self;
}
let mut carried = false;
if self.cass_version.is_none() && previous.cass_version.is_some() {
self.cass_version = previous.cass_version.clone();
carried = true;
}
if self.capability_tier.is_none() && previous.capability_tier.is_some() {
self.capability_tier = previous.capability_tier;
carried = true;
}
if self.data_dir.is_none() && previous.data_dir.is_some() {
self.data_dir = previous.data_dir.clone();
carried = true;
}
if self.source_roots.is_empty() && !previous.source_roots.is_empty() {
self.source_roots = previous.source_roots.clone();
carried = true;
}
if self.source_counts.is_none() && previous.source_counts.is_some() {
self.source_counts = previous.source_counts;
carried = true;
}
if carried {
self.stale_data = true;
}
self
}
}
pub fn classify_connection_failure(error: &str) -> (HostProbeStatus, &'static str) {
let lower = error.to_ascii_lowercase();
if lower.contains("could not resolve")
|| lower.contains("name or service not known")
|| lower.contains("nodename nor servname")
|| lower.contains("temporary failure in name resolution")
{
(
HostProbeStatus::Unreachable,
"hostname did not resolve; verify the SSH host alias and DNS, then retry",
)
} else if lower.contains("permission denied")
|| lower.contains("publickey")
|| lower.contains("authentication failed")
|| lower.contains("no valid authentication")
|| lower.contains("too many authentication failures")
{
(
HostProbeStatus::Unreachable,
"SSH authentication was rejected; load the key into ssh-agent or fix the identity file, then retry",
)
} else if lower.contains("host key verification")
|| lower.contains("remote host identification has changed")
{
(
HostProbeStatus::Unreachable,
"host key verification failed; resolve the known_hosts entry (possible key change) before retrying",
)
} else if lower.contains("connection refused") {
(
HostProbeStatus::Unreachable,
"remote refused the connection; confirm sshd is running on the expected port, then retry",
)
} else if lower.contains("banner exchange") {
(
HostProbeStatus::TimedOut,
"SSH banner exchange timed out (slow or overloaded host); retry when the host is responsive",
)
} else if lower.contains("timed out") || lower.contains("timeout") {
(
HostProbeStatus::TimedOut,
"host did not answer within the probe budget; retry when it is online or raise the timeout",
)
} else if lower.contains("no route to host") || lower.contains("network is unreachable") {
(
HostProbeStatus::Unreachable,
"no network route to the host; check connectivity and retry from a reachable fleet node",
)
} else {
(
HostProbeStatus::Unreachable,
"host was unreachable; check the transport and retry from a reachable fleet node",
)
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct FleetSummary {
pub total_hosts: usize,
pub ok: usize,
pub degraded: usize,
pub timed_out: usize,
pub unreachable: usize,
pub cancelled: usize,
pub stale_data: usize,
pub highest_archive_risk: ArchiveRisk,
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub root_cause_distribution: BTreeMap<RootCauseFamily, usize>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub dominant_root_cause: Option<RootCauseFamily>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct FleetDoctorReport {
pub schema_version: u32,
pub hosts: Vec<HostDoctorReport>,
pub summary: FleetSummary,
}
impl FleetDoctorReport {
pub fn from_hosts(hosts: Vec<HostDoctorReport>) -> Self {
let total_hosts = hosts.len();
let mut ok = 0;
let mut degraded = 0;
let mut timed_out = 0;
let mut unreachable = 0;
let mut cancelled = 0;
let mut stale_data = 0;
let mut highest_archive_risk = ArchiveRisk::Unknown;
let mut root_cause_distribution: BTreeMap<RootCauseFamily, usize> = BTreeMap::new();
for host in &hosts {
if let Some(family) = host.likely_root_cause {
*root_cause_distribution.entry(family).or_default() += 1;
}
match host.status {
HostProbeStatus::Ok => ok += 1,
HostProbeStatus::TimedOut => timed_out += 1,
HostProbeStatus::Unreachable | HostProbeStatus::CommandNotFound => {
unreachable += 1;
}
HostProbeStatus::Partial
| HostProbeStatus::OldBinarySkew
| HostProbeStatus::Degraded => degraded += 1,
}
if host.cancelled {
cancelled += 1;
}
if host.stale_data {
stale_data += 1;
}
if host.archive_risk > highest_archive_risk {
highest_archive_risk = host.archive_risk;
}
}
let dominant_root_cause = root_cause_distribution
.iter()
.max_by(|(fa, ca), (fb, cb)| ca.cmp(cb).then_with(|| fb.cmp(fa)))
.map(|(family, _)| *family);
Self {
schema_version: FLEET_DOCTOR_SCHEMA_VERSION,
hosts,
summary: FleetSummary {
total_hosts,
ok,
degraded,
timed_out,
unreachable,
cancelled,
stale_data,
highest_archive_risk,
root_cause_distribution,
dominant_root_cause,
},
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SourceCheck {
pub name: String,
pub status: String,
pub remediation: Option<String>,
}
impl SourceCheck {
pub fn new(
name: impl Into<String>,
status: impl Into<String>,
remediation: Option<String>,
) -> Self {
Self {
name: name.into(),
status: status.into(),
remediation,
}
}
fn is_fail(&self) -> bool {
self.status.eq_ignore_ascii_case("fail")
}
fn matches(&self, needle: &str) -> bool {
self.name.to_ascii_lowercase().contains(needle)
}
}
fn transport_root_cause_attribution(
status: HostProbeStatus,
detail: Option<&str>,
) -> RootCauseAttribution {
let confidence = match status {
HostProbeStatus::Unreachable | HostProbeStatus::CommandNotFound => {
AttributionConfidence::Confirmed
}
HostProbeStatus::TimedOut => AttributionConfidence::Probable,
_ => AttributionConfidence::Possible,
};
let mut evidence = vec![EvidenceRef::new("transport.probe_status", "sources")];
if let Some(detail) = detail {
evidence.push(
EvidenceRef::new("transport.connection_error", "sources")
.with_detail(detail.to_string()),
);
}
RootCauseAttribution::new(
RootCauseFamily::RemoteTransportAuth,
confidence,
"remote transport/auth failure; confirm host reachability before trusting host state",
)
.with_evidence(evidence)
}
pub fn host_report_from_checks(
host_alias: &str,
platform: Platform,
elapsed_ms: u64,
checks: &[SourceCheck],
) -> HostDoctorReport {
let ssh_failed = checks
.iter()
.any(|c| (c.matches("ssh") || c.matches("connect")) && c.is_fail());
let rsync_failed = checks
.iter()
.any(|c| (c.matches("rsync") || c.matches("transport")) && c.is_fail());
let any_fail = checks.iter().any(SourceCheck::is_fail);
let any_warn = checks.iter().any(|c| c.status.eq_ignore_ascii_case("warn"));
let status = if ssh_failed {
HostProbeStatus::Unreachable
} else if rsync_failed {
HostProbeStatus::CommandNotFound
} else if any_fail {
HostProbeStatus::Degraded
} else if any_warn {
HostProbeStatus::Partial
} else {
HostProbeStatus::Ok
};
let recommended_action = checks
.iter()
.find(|c| c.is_fail())
.or_else(|| {
checks
.iter()
.find(|c| c.status.eq_ignore_ascii_case("warn"))
})
.and_then(|c| c.remediation.clone());
let mut report = HostDoctorReport::skeleton(host_alias, platform, status, elapsed_ms);
report.recommended_action = recommended_action;
if matches!(
status,
HostProbeStatus::Unreachable | HostProbeStatus::CommandNotFound
) {
report.likely_root_cause = Some(RootCauseFamily::RemoteTransportAuth);
report.root_cause = Some(transport_root_cause_attribution(status, None));
}
report
}
#[cfg(test)]
mod tests {
use super::*;
fn chk(name: &str, status: &str) -> SourceCheck {
SourceCheck::new(name, status, Some(format!("fix {name}")))
}
#[test]
fn host_report_ok_when_all_checks_pass() {
let checks = [
chk("SSH connectivity", "pass"),
chk("rsync available", "pass"),
];
let r = host_report_from_checks("ts1", Platform::linux_x86_64(), 30, &checks);
assert_eq!(r.status, HostProbeStatus::Ok);
assert_eq!(r.host_alias, "ts1");
assert!(!r.unreachable);
assert!(r.recommended_action.is_none());
assert!(r.likely_root_cause.is_none());
}
#[test]
fn ssh_fail_marks_unreachable_with_transport_root_cause() {
let checks = [
chk("SSH connectivity", "fail"),
chk("rsync available", "pass"),
];
let r = host_report_from_checks("mac-mini-old", Platform::linux_x86_64(), 5000, &checks);
assert_eq!(r.status, HostProbeStatus::Unreachable);
assert!(r.unreachable, "unreachable flag must mirror status");
assert_eq!(
r.likely_root_cause,
Some(RootCauseFamily::RemoteTransportAuth)
);
let rc = r
.root_cause
.as_ref()
.expect("unreachable host carries root_cause");
assert_eq!(rc.family, RootCauseFamily::RemoteTransportAuth);
assert_eq!(rc.confidence, AttributionConfidence::Confirmed);
assert!(rc.recommended_next_probe.is_some());
assert!(
rc.evidence_refs
.iter()
.any(|e| e.kind == "transport.probe_status")
);
assert_eq!(
r.recommended_action.as_deref(),
Some("fix SSH connectivity")
);
assert_eq!(
r.host_alias, "mac-mini-old",
"identity preserved when unreachable"
);
}
#[test]
fn failed_host_carries_full_transport_attribution_with_connection_detail() {
let r = HostDoctorReport::failed(
"css",
Platform::linux_x86_64(),
1200,
"ssh: connect to host css port 22: Connection timed out",
);
let rc = r
.root_cause
.as_ref()
.expect("failed host carries root_cause");
assert_eq!(rc.family, RootCauseFamily::RemoteTransportAuth);
assert_eq!(rc.confidence, AttributionConfidence::Probable);
assert!(
rc.evidence_refs
.iter()
.any(|e| e.kind == "transport.connection_error"
&& e.detail
.as_deref()
.is_some_and(|d| d.contains("Connection timed out"))),
"connection error must be preserved as evidence: {rc:?}"
);
}
#[test]
fn rsync_fail_marks_command_not_found() {
let checks = [
chk("SSH connectivity", "pass"),
chk("rsync available", "fail"),
];
let r = host_report_from_checks("ts2", Platform::linux_x86_64(), 80, &checks);
assert_eq!(r.status, HostProbeStatus::CommandNotFound);
assert!(r.status.is_hard_failure());
assert_eq!(
r.likely_root_cause,
Some(RootCauseFamily::RemoteTransportAuth)
);
}
#[test]
fn other_fail_is_degraded_and_warn_is_partial() {
let degraded = [
chk("SSH connectivity", "pass"),
chk("Remote Path: paths[0]", "fail"),
];
let r = host_report_from_checks("css", Platform::linux_x86_64(), 50, °raded);
assert_eq!(r.status, HostProbeStatus::Degraded);
assert_eq!(
r.recommended_action.as_deref(),
Some("fix Remote Path: paths[0]")
);
let warned = [chk("SSH connectivity", "pass"), chk("storage", "warn")];
let r2 = host_report_from_checks("csd", Platform::linux_x86_64(), 50, &warned);
assert_eq!(r2.status, HostProbeStatus::Partial);
assert_eq!(r2.recommended_action.as_deref(), Some("fix storage"));
}
#[test]
fn host_report_serializes_with_status_and_identity() {
let checks = [chk("SSH connectivity", "fail")];
let r = host_report_from_checks("mac-mini-old", Platform::linux_x86_64(), 5000, &checks);
let value = serde_json::to_value(&r).unwrap();
assert_eq!(value["status"], "unreachable");
assert_eq!(value["unreachable"], true);
assert_eq!(value["host_alias"], "mac-mini-old");
assert_eq!(value["likely_root_cause"], "remote-transport-auth");
let back: HostDoctorReport = serde_json::from_value(value).unwrap();
assert_eq!(back, r);
}
fn populated_ok_host() -> HostDoctorReport {
let mut h =
HostDoctorReport::skeleton("ts1", Platform::linux_x86_64(), HostProbeStatus::Ok, 42);
h.cass_version = Some("0.6.13".to_string());
h.capability_tier = Some(CapabilityTier::Full);
h.data_dir = Some("/home/ubuntu/.cass".to_string());
h.source_roots = vec![SourceRoot {
path: "/home/ubuntu/.claude".to_string(),
agent: Some("claude".to_string()),
archived: true,
}];
h.source_counts = Some(SourceCounts {
roots: 1,
sessions: 500,
indexed_sessions: 500,
});
h.readiness = Some(ReadinessState::Ready);
h.semantic = Some(SemanticState::Enabled);
h.quarantine = Some(QuarantineState {
quarantined: 0,
recoverable: 0,
});
h.remote_sync = Some(RemoteSyncState::Synced);
h.archive_risk = ArchiveRisk::Low;
h
}
#[test]
fn success_scenario_round_trips_with_all_fields() {
let host = populated_ok_host();
let value = serde_json::to_value(&host).expect("serialize");
assert_eq!(value["status"], "ok");
assert_eq!(value["host_alias"], "ts1");
assert_eq!(value["platform"]["os"], "linux");
assert_eq!(value["cass_version"], "0.6.13");
assert_eq!(value["readiness"], "ready");
assert_eq!(value["archive_risk"], "low");
let back: HostDoctorReport = serde_json::from_value(value).expect("deserialize");
assert_eq!(back, host);
}
#[test]
fn partial_scenario_names_skipped_sections() {
let mut h = HostDoctorReport::skeleton(
"css",
Platform::linux_x86_64(),
HostProbeStatus::Partial,
7_900,
);
h.readiness = Some(ReadinessState::Ready);
h.skipped_sections = vec!["semantic".to_string(), "remote_sync".to_string()];
let value = serde_json::to_value(&h).unwrap();
assert_eq!(value["status"], "partial");
assert_eq!(value["skipped_sections"][0], "semantic");
assert!(value.get("semantic").is_none());
assert_eq!(
serde_json::from_value::<HostDoctorReport>(value).unwrap(),
h
);
}
#[test]
fn timeout_scenario_sets_flag_and_keeps_identity() {
let h = HostDoctorReport::skeleton(
"csd",
Platform::linux_x86_64(),
HostProbeStatus::TimedOut,
8_000,
);
assert!(h.timed_out, "TimedOut status must set the scalar flag");
assert!(!h.unreachable);
let value = serde_json::to_value(&h).unwrap();
assert_eq!(value["status"], "timed-out");
assert_eq!(value["timed_out"], true);
assert_eq!(value["host_alias"], "csd", "identity survives timeout");
assert!(
value.get("readiness").is_none(),
"deep state absent on timeout"
);
}
#[test]
fn old_binary_scenario_carries_version_and_action() {
let mut h = HostDoctorReport::skeleton(
"mac-mini-max",
Platform {
os: HostOs::MacOs,
arch: "aarch64".to_string(),
path_style: PathStyle::Posix,
tool_notes: vec![],
},
HostProbeStatus::OldBinarySkew,
120,
);
h.cass_version = Some("0.5.0".to_string());
h.capability_tier = Some(CapabilityTier::Standard);
h.likely_root_cause = Some(RootCauseFamily::OldBinarySkew);
h.recommended_action = Some("upgrade cass to 0.6.13".to_string());
let value = serde_json::to_value(&h).unwrap();
assert_eq!(value["status"], "old-binary-skew");
assert_eq!(value["cass_version"], "0.5.0");
assert_eq!(value["likely_root_cause"], "old-binary-skew");
assert_eq!(
serde_json::from_value::<HostDoctorReport>(value).unwrap(),
h
);
}
#[test]
fn command_not_found_scenario_is_hard_failure() {
let h = HostDoctorReport::skeleton(
"ts2",
Platform::linux_x86_64(),
HostProbeStatus::CommandNotFound,
55,
);
assert!(h.status.is_hard_failure());
let value = serde_json::to_value(&h).unwrap();
assert_eq!(value["status"], "command-not-found");
assert_eq!(value["host_alias"], "ts2");
}
#[test]
fn unreachable_ssh_scenario_preserves_identity_and_attributes_transport() {
let h = HostDoctorReport::unreachable(
"mac-mini-old",
Platform {
os: HostOs::MacOs,
arch: "x86_64".to_string(),
path_style: PathStyle::Posix,
tool_notes: vec![],
},
5_000,
"check SSH reachability and host key for mac-mini-old",
);
assert!(h.unreachable);
assert_eq!(h.status, HostProbeStatus::Unreachable);
assert_eq!(
h.likely_root_cause,
Some(RootCauseFamily::RemoteTransportAuth)
);
let value = serde_json::to_value(&h).unwrap();
assert_eq!(value["status"], "unreachable");
assert_eq!(value["unreachable"], true);
assert_eq!(
value["host_alias"], "mac-mini-old",
"identity survives unreachable"
);
assert!(value["recommended_action"].is_string());
assert!(value.get("readiness").is_none());
}
#[test]
fn classify_connection_failure_covers_probe_taxonomy() {
assert_eq!(
classify_connection_failure("ssh: connect to host ts2 port 22: Connection timed out").0,
HostProbeStatus::TimedOut
);
assert_eq!(
classify_connection_failure(
"kex_exchange_identification: read: Connection timed out during banner exchange",
)
.0,
HostProbeStatus::TimedOut
);
assert_eq!(
classify_connection_failure(
"ssh: Could not resolve hostname mac-mini-old: Name or service not known",
)
.0,
HostProbeStatus::Unreachable
);
assert_eq!(
classify_connection_failure("Permission denied (publickey).").0,
HostProbeStatus::Unreachable
);
assert_eq!(
classify_connection_failure("ssh: connect to host h: Connection refused").0,
HostProbeStatus::Unreachable
);
assert_eq!(
classify_connection_failure("Host key verification failed.").0,
HostProbeStatus::Unreachable
);
for err in [
"Connection timed out",
"Could not resolve hostname",
"Permission denied",
"Connection refused",
"banner exchange",
"No route to host",
"something unexpected",
] {
assert!(!classify_connection_failure(err).1.is_empty());
}
}
#[test]
fn failed_probe_preserves_identity_error_and_retry_hint() {
let dns = HostDoctorReport::failed(
"mac-mini-old",
Platform::linux_x86_64(),
3_200,
"ssh: Could not resolve hostname mac-mini-old: Name or service not known",
);
assert_eq!(dns.status, HostProbeStatus::Unreachable);
assert!(dns.unreachable);
assert!(!dns.timed_out);
assert_eq!(dns.elapsed_ms, 3_200);
let value = serde_json::to_value(&dns).unwrap();
assert_eq!(value["status"], "unreachable");
assert_eq!(
value["host_alias"], "mac-mini-old",
"identity survives failure"
);
assert!(
value["connection_error"]
.as_str()
.unwrap()
.contains("resolve")
);
assert!(value["retry_hint"].is_string());
assert_eq!(
serde_json::from_value::<HostDoctorReport>(value).unwrap(),
dns
);
let auth = HostDoctorReport::failed(
"ts2",
Platform::linux_x86_64(),
800,
"Permission denied (publickey).",
);
assert_eq!(auth.status, HostProbeStatus::Unreachable);
assert!(
auth.connection_error
.as_deref()
.unwrap()
.contains("Permission denied")
);
let banner = HostDoctorReport::failed(
"ts2",
Platform::linux_x86_64(),
9_500,
"kex_exchange_identification: read: Connection timed out during banner exchange",
);
assert_eq!(banner.status, HostProbeStatus::TimedOut);
assert!(banner.timed_out);
assert!(!banner.unreachable);
assert_eq!(banner.elapsed_ms, 9_500);
}
#[test]
fn operator_cancelled_probe_is_distinct_explicit_state() {
let cancelled = HostDoctorReport::cancelled("ts1", Platform::linux_x86_64(), 120);
assert!(cancelled.cancelled);
assert!(!cancelled.timed_out);
assert!(!cancelled.unreachable);
assert_eq!(cancelled.status, HostProbeStatus::Partial);
assert!(
cancelled
.connection_error
.as_deref()
.unwrap()
.contains("cancelled")
);
let value = serde_json::to_value(&cancelled).unwrap();
assert_eq!(value["cancelled"], true);
assert_eq!(value["status"], "partial");
assert_eq!(
serde_json::from_value::<HostDoctorReport>(value).unwrap(),
cancelled
);
}
#[test]
fn with_last_known_surfaces_stale_evidence_for_intermittent_host() {
let mut healthy = populated_ok_host();
healthy.cass_version = Some("0.6.10".to_string());
healthy.data_dir = Some("/home/me/.local/share/cass".to_string());
let now_failed = HostDoctorReport::failed(
&healthy.host_alias,
healthy.platform.clone(),
9_900,
"Connection timed out",
)
.with_last_known(&healthy);
assert_eq!(now_failed.status, HostProbeStatus::TimedOut);
assert!(
now_failed.stale_data,
"carried last-known evidence marks stale"
);
assert_eq!(now_failed.cass_version.as_deref(), Some("0.6.10"));
assert_eq!(
now_failed.data_dir.as_deref(),
Some("/home/me/.local/share/cass")
);
let value = serde_json::to_value(&now_failed).unwrap();
assert_eq!(value["stale_data"], true);
let still_ok = populated_ok_host().with_last_known(&healthy);
assert!(!still_ok.stale_data);
}
#[test]
fn fleet_rollup_counts_unreachable_cancelled_and_stale_separately() {
let mut healthy = populated_ok_host();
healthy.cass_version = Some("0.6.10".to_string());
let hosts = vec![
populated_ok_host(),
HostDoctorReport::failed(
"dns-host",
Platform::linux_x86_64(),
1_000,
"Could not resolve hostname",
),
HostDoctorReport::failed(
"auth-host",
Platform::linux_x86_64(),
800,
"Permission denied (publickey)",
),
HostDoctorReport::failed(
"slow-host",
Platform::linux_x86_64(),
9_000,
"Connection timed out",
),
HostDoctorReport::cancelled("cancel-host", Platform::linux_x86_64(), 50),
HostDoctorReport::failed(
"intermittent",
Platform::linux_x86_64(),
9_900,
"Connection timed out",
)
.with_last_known(&healthy),
];
let report = FleetDoctorReport::from_hosts(hosts);
let s = report.summary;
assert_eq!(s.total_hosts, 6);
assert_eq!(s.ok, 1);
assert_eq!(s.unreachable, 2);
assert_eq!(s.timed_out, 2);
assert_eq!(s.cancelled, 1);
assert_eq!(s.stale_data, 1);
assert_ne!(s.ok, s.total_hosts);
}
#[test]
fn macos_path_and_tool_differences_are_representable() {
let platform = Platform {
os: HostOs::MacOs,
arch: "aarch64".to_string(),
path_style: PathStyle::Posix,
tool_notes: vec!["rsync=bsd".to_string(), "data_dir=~/Library".to_string()],
};
let h = HostDoctorReport::skeleton("mac-mini-max", platform, HostProbeStatus::Ok, 80);
let value = serde_json::to_value(&h).unwrap();
assert_eq!(value["platform"]["os"], "macos");
assert_eq!(value["platform"]["tool_notes"][0], "rsync=bsd");
assert_eq!(
serde_json::from_value::<HostDoctorReport>(value).unwrap(),
h
);
}
#[test]
fn high_archive_risk_is_representable() {
let mut h = populated_ok_host();
h.status = HostProbeStatus::Degraded;
h.archive_risk = ArchiveRisk::High;
h.recommended_action = Some("back up derived archive before re-index".to_string());
let value = serde_json::to_value(&h).unwrap();
assert_eq!(value["archive_risk"], "high");
assert_eq!(
serde_json::from_value::<HostDoctorReport>(value).unwrap(),
h
);
}
#[test]
fn host_identity_is_present_for_every_status() {
for status in [
HostProbeStatus::Ok,
HostProbeStatus::Partial,
HostProbeStatus::TimedOut,
HostProbeStatus::OldBinarySkew,
HostProbeStatus::CommandNotFound,
HostProbeStatus::Unreachable,
HostProbeStatus::Degraded,
] {
let h = HostDoctorReport::skeleton("host-x", Platform::linux_x86_64(), status, 1);
let value = serde_json::to_value(&h).unwrap();
assert_eq!(value["host_alias"], "host-x", "{status:?}: lost host alias");
assert!(value.get("platform").is_some(), "{status:?}: lost platform");
assert_eq!(value["timed_out"], status == HostProbeStatus::TimedOut);
assert_eq!(value["unreachable"], status == HostProbeStatus::Unreachable);
}
}
#[test]
fn probe_status_wire_values_match_as_str() {
for status in [
HostProbeStatus::Ok,
HostProbeStatus::Partial,
HostProbeStatus::TimedOut,
HostProbeStatus::OldBinarySkew,
HostProbeStatus::CommandNotFound,
HostProbeStatus::Unreachable,
HostProbeStatus::Degraded,
] {
let json = serde_json::to_string(&status).unwrap();
assert_eq!(json, format!("\"{}\"", status.as_str()));
let back: HostProbeStatus = serde_json::from_str(&json).unwrap();
assert_eq!(back, status);
}
}
#[test]
fn archive_risk_orders_low_to_high() {
assert!(ArchiveRisk::High > ArchiveRisk::Medium);
assert!(ArchiveRisk::Medium > ArchiveRisk::Low);
assert!(ArchiveRisk::Low > ArchiveRisk::Unknown);
}
#[test]
fn fleet_summary_is_derived_and_takes_max_archive_risk() {
let hosts = vec![
populated_ok_host(),
HostDoctorReport::skeleton(
"csd",
Platform::linux_x86_64(),
HostProbeStatus::TimedOut,
8000,
),
HostDoctorReport::unreachable(
"mac-mini-old",
Platform::linux_x86_64(),
5000,
"check ssh",
),
{
let mut h = HostDoctorReport::skeleton(
"css",
Platform::linux_x86_64(),
HostProbeStatus::Degraded,
100,
);
h.archive_risk = ArchiveRisk::High;
h
},
];
let report = FleetDoctorReport::from_hosts(hosts);
assert_eq!(report.summary.total_hosts, 4);
assert_eq!(report.summary.ok, 1);
assert_eq!(report.summary.timed_out, 1);
assert_eq!(report.summary.unreachable, 1);
assert_eq!(report.summary.degraded, 1);
assert_eq!(report.summary.highest_archive_risk, ArchiveRisk::High);
let value = serde_json::to_value(&report).unwrap();
let back: FleetDoctorReport = serde_json::from_value(value).unwrap();
assert_eq!(back, report);
}
#[test]
fn command_not_found_counts_as_unreachable_in_rollup() {
let hosts = vec![HostDoctorReport::skeleton(
"ts2",
Platform::linux_x86_64(),
HostProbeStatus::CommandNotFound,
10,
)];
let report = FleetDoctorReport::from_hosts(hosts);
assert_eq!(
report.summary.unreachable, 1,
"command-not-found is a hard failure"
);
}
fn host_attributed(alias: &str, family: RootCauseFamily) -> HostDoctorReport {
let mut h = HostDoctorReport::skeleton(
alias,
Platform::linux_x86_64(),
HostProbeStatus::Degraded,
40,
);
h.likely_root_cause = Some(family);
h
}
#[test]
fn fleet_rollup_keeps_dependency_and_storage_dominance_distinct() {
let hosts = vec![
HostDoctorReport::unreachable("css1", Platform::linux_x86_64(), 100, "fix ssh"),
HostDoctorReport::failed(
"css2",
Platform::linux_x86_64(),
120,
"ssh: connect to host css2 port 22: Connection timed out",
),
host_attributed("csd", RootCauseFamily::FrankensqliteStorage),
HostDoctorReport::skeleton("ts1", Platform::linux_x86_64(), HostProbeStatus::Ok, 20),
];
let report = FleetDoctorReport::from_hosts(hosts);
let dist = &report.summary.root_cause_distribution;
assert_eq!(dist.get(&RootCauseFamily::RemoteTransportAuth), Some(&2));
assert_eq!(dist.get(&RootCauseFamily::FrankensqliteStorage), Some(&1));
assert_eq!(dist.values().sum::<usize>(), 3);
assert_eq!(
report.summary.dominant_root_cause,
Some(RootCauseFamily::RemoteTransportAuth)
);
assert_eq!(report.summary.unreachable, 1, "css1 hard-unreachable");
assert_eq!(report.summary.timed_out, 1, "css2 connect-timeout");
}
#[test]
fn fleet_rollup_dominant_flips_when_storage_dominates() {
let hosts = vec![
host_attributed("csd1", RootCauseFamily::FrankensqliteStorage),
host_attributed("csd2", RootCauseFamily::FrankensqliteStorage),
host_attributed("csd3", RootCauseFamily::FrankensqliteStorage),
HostDoctorReport::unreachable("css", Platform::linux_x86_64(), 90, "fix ssh"),
];
let report = FleetDoctorReport::from_hosts(hosts);
assert_eq!(
report.summary.dominant_root_cause,
Some(RootCauseFamily::FrankensqliteStorage)
);
assert_eq!(
report
.summary
.root_cause_distribution
.get(&RootCauseFamily::FrankensqliteStorage),
Some(&3)
);
assert_eq!(
report
.summary
.root_cause_distribution
.get(&RootCauseFamily::RemoteTransportAuth),
Some(&1)
);
}
#[test]
fn fleet_rollup_dominant_tie_breaks_deterministically_to_lowest_family() {
let hosts = vec![
host_attributed("a", RootCauseFamily::CassDerivedState),
HostDoctorReport::unreachable("b", Platform::linux_x86_64(), 50, "fix"),
];
let report = FleetDoctorReport::from_hosts(hosts);
assert_eq!(
report.summary.dominant_root_cause,
Some(RootCauseFamily::CassDerivedState),
"ties resolve to the lowest family for reproducibility"
);
}
#[test]
fn fleet_rollup_unattributed_fleet_has_empty_distribution_and_no_dominant() {
let hosts = vec![
HostDoctorReport::skeleton("ts1", Platform::linux_x86_64(), HostProbeStatus::Ok, 10),
HostDoctorReport::skeleton(
"ts2",
Platform::linux_x86_64(),
HostProbeStatus::Partial,
12,
),
];
let report = FleetDoctorReport::from_hosts(hosts);
assert!(report.summary.root_cause_distribution.is_empty());
assert_eq!(report.summary.dominant_root_cause, None);
}
#[test]
fn fleet_rollup_distribution_serializes_with_kebab_family_keys() {
let report = FleetDoctorReport::from_hosts(vec![HostDoctorReport::unreachable(
"css",
Platform::linux_x86_64(),
50,
"fix",
)]);
let value = serde_json::to_value(&report.summary).unwrap();
assert_eq!(value["root_cause_distribution"]["remote-transport-auth"], 1);
assert_eq!(value["dominant_root_cause"], "remote-transport-auth");
let back: FleetSummary = serde_json::from_value(value).unwrap();
assert_eq!(back, report.summary);
}
}