#![allow(dead_code)]
use crate::error::DaemonError;
#[cfg(not(unix))]
use crate::error::PlatformError;
use crate::status_types::{DaemonFullStatusResponse, extract_json_body, format_duration};
use crate::ui::theme::Theme;
use anyhow::{Context, Result};
use rch_telemetry::TestRunStatsScope;
use std::io::Write;
fn format_duration_ms(ms: u64) -> String {
if ms < 1000 {
format!("{}ms", ms)
} else if ms < 60_000 {
format!("{:.1}s", ms as f64 / 1000.0)
} else if ms < 3_600_000 {
let mins = ms / 60_000;
let secs = (ms % 60_000) / 1000;
if secs == 0 {
format!("{}m", mins)
} else {
format!("{}m {}s", mins, secs)
}
} else {
let hours = ms / 3_600_000;
let mins = (ms % 3_600_000) / 60_000;
if mins == 0 {
format!("{}h", hours)
} else {
format!("{}h {}m", hours, mins)
}
}
}
pub async fn query_daemon_full_status() -> Result<DaemonFullStatusResponse> {
let response = send_status_command().await?;
let json_body =
extract_json_body(&response).ok_or_else(|| anyhow::anyhow!("Invalid response format"))?;
let status: DaemonFullStatusResponse =
serde_json::from_str(json_body).context("Failed to parse status response")?;
Ok(status)
}
#[cfg(not(unix))]
async fn send_status_command() -> Result<String> {
Err(PlatformError::UnixOnly {
feature: "daemon status".to_string(),
})?
}
#[cfg(unix)]
async fn send_status_command() -> Result<String> {
crate::commands::send_daemon_command("GET /status\n").await
}
#[cfg(unix)]
pub async fn drain_worker(worker_id: &str) -> Result<()> {
let command = format!("POST /workers/{}/drain\n", urlencoding::encode(worker_id));
let response = crate::commands::send_daemon_command(&command).await?;
if response.contains("\"status\":\"ok\"") {
Ok(())
} else {
Err(DaemonError::ProtocolError {
message: format!("Drain failed: {}", response),
}
.into())
}
}
#[cfg(not(unix))]
pub async fn drain_worker(_worker_id: &str) -> Result<()> {
Err(PlatformError::UnixOnly {
feature: "worker drain".to_string(),
})?
}
#[cfg(unix)]
pub async fn enable_worker(worker_id: &str) -> Result<()> {
let command = format!("POST /workers/{}/enable\n", urlencoding::encode(worker_id));
let response = crate::commands::send_daemon_command(&command).await?;
if response.contains("\"status\":\"ok\"") {
Ok(())
} else {
Err(DaemonError::ProtocolError {
message: format!("Enable failed: {}", response),
}
.into())
}
}
#[cfg(not(unix))]
pub async fn enable_worker(_worker_id: &str) -> Result<()> {
Err(PlatformError::UnixOnly {
feature: "worker enable".to_string(),
})?
}
#[cfg(unix)]
pub async fn cancel_build(build_id: &str) -> Result<()> {
let command = format!("POST /builds/{}/cancel\n", build_id);
let response = crate::commands::send_daemon_command(&command).await?;
if response.contains("\"status\":\"ok\"") {
Ok(())
} else {
Err(DaemonError::ProtocolError {
message: format!("Cancel failed: {}", response),
}
.into())
}
}
#[cfg(not(unix))]
pub async fn cancel_build(_build_id: &str) -> Result<()> {
Err(PlatformError::UnixOnly {
feature: "build cancel".to_string(),
})?
}
#[cfg(unix)]
pub async fn force_kill_build(build_id: &str) -> Result<()> {
let command = format!("POST /builds/{}/cancel?force=true\n", build_id);
let response = crate::commands::send_daemon_command(&command).await?;
if response.contains("\"status\":\"ok\"") {
Ok(())
} else {
Err(DaemonError::ProtocolError {
message: format!("Force kill failed: {}", response),
}
.into())
}
}
#[cfg(not(unix))]
pub async fn force_kill_build(_build_id: &str) -> Result<()> {
Err(PlatformError::UnixOnly {
feature: "build force kill".to_string(),
})?
}
pub fn render_full_status(
status: &DaemonFullStatusResponse,
show_workers: bool,
show_jobs: bool,
convergence: Option<&crate::status_types::RepoConvergenceStatusFromApi>,
remediation_hints: &[crate::status_types::RemediationHint],
posture: &crate::status_types::SystemPosture,
style: &Theme,
) {
let mut stdout = std::io::stdout();
let _ = render_full_status_to(
&mut stdout,
status,
show_workers,
show_jobs,
convergence,
remediation_hints,
posture,
style,
);
}
#[allow(clippy::too_many_arguments)]
fn render_full_status_to<W: Write>(
out: &mut W,
status: &DaemonFullStatusResponse,
show_workers: bool,
show_jobs: bool,
convergence: Option<&crate::status_types::RepoConvergenceStatusFromApi>,
remediation_hints: &[crate::status_types::RemediationHint],
posture: &crate::status_types::SystemPosture,
style: &Theme,
) -> std::io::Result<()> {
writeln!(out, "{}", style.format_header("RCH Status"))?;
writeln!(out)?;
writeln!(
out,
" {} {} {} (PID {})",
style.key("Daemon"),
style.muted(":"),
style.success("Running"),
style.highlight(&status.daemon.pid.to_string())
)?;
writeln!(
out,
" {} {} {}",
style.key("Uptime"),
style.muted(":"),
style.info(&format_duration(status.daemon.uptime_secs))
)?;
writeln!(
out,
" {} {} {}",
style.key("Version"),
style.muted(":"),
style.info(&status.daemon.version)
)?;
let posture_display = match posture {
crate::status_types::SystemPosture::RemoteReady => style.success("remote-ready"),
crate::status_types::SystemPosture::Degraded => style.warning("degraded"),
crate::status_types::SystemPosture::LocalOnly => style.error("local-only"),
};
writeln!(
out,
" {} {} {} {}",
style.key("Posture"),
style.muted(":"),
posture_display,
style.muted(&format!("({})", posture.description()))
)?;
writeln!(
out,
" {} {} {}/{} healthy, {}/{} slots available",
style.key("Workers"),
style.muted(":"),
style.highlight(&status.daemon.workers_healthy.to_string()),
status.daemon.workers_total,
style.highlight(&status.daemon.slots_available.to_string()),
status.daemon.slots_total
)?;
let success_rate = if status.stats.total_builds > 0 {
(status.stats.success_count as f64 / status.stats.total_builds as f64) * 100.0
} else {
100.0
};
writeln!(
out,
" {} {} {} total, {:.0}% command success rate",
style.key("Builds"),
style.muted(":"),
style.highlight(&status.stats.total_builds.to_string()),
success_rate
)?;
if let Some(test_stats) = &status.test_stats {
let scope = match &test_stats.scope {
TestRunStatsScope::Unknown => "scope unknown".to_string(),
TestRunStatsScope::StoredHistory => "all stored records".to_string(),
TestRunStatsScope::RecentMemory { max_records } => {
format!("recent memory, up to {max_records}")
}
};
let pass_rate = if test_stats.total_runs > 0 {
(test_stats.passed_runs as f64 / test_stats.total_runs as f64) * 100.0
} else {
100.0
};
let avg_duration = if test_stats.avg_duration_ms < 1000 {
format!("{}ms", test_stats.avg_duration_ms)
} else {
format_duration((test_stats.avg_duration_ms + 500) / 1000)
};
writeln!(
out,
" {} ({}) {} {} total, {:.0}% command success rate, avg {}",
style.key("Test commands"),
scope,
style.muted(":"),
style.highlight(&test_stats.total_runs.to_string()),
pass_rate,
style.info(&avg_duration)
)?;
}
let hook_installed = check_hook_installed();
writeln!(
out,
" {} {} {}",
style.key("Hook"),
style.muted(":"),
if hook_installed {
style.success("Installed")
} else {
style.warning("Not installed")
}
)?;
if let Some(saved_time) = &status.saved_time
&& saved_time.builds_counted > 0
{
let basis_known = saved_time.estimate_basis == "observed_local_mean";
let time_saved_str = if basis_known {
format_duration_ms(saved_time.time_saved_ms)
} else {
"n/a (no local baseline)".to_string()
};
let speedup_str = if basis_known && saved_time.avg_speedup > 0.0 {
format!("{:.1}x", saved_time.avg_speedup)
} else {
"-".to_string()
};
writeln!(
out,
" {} {} {} saved ({} speedup, {} builds)",
style.key("Saved"),
style.muted(":"),
style.success(&time_saved_str),
style.info(&speedup_str),
saved_time.builds_counted
)?;
if saved_time.today_saved_ms > 0 {
let today_str = format_duration_ms(saved_time.today_saved_ms);
let week_str = format_duration_ms(saved_time.week_saved_ms);
writeln!(
out,
" {} today, {} this week",
style.highlight(&today_str),
style.info(&week_str)
)?;
}
}
if !status.alerts.is_empty() {
writeln!(out, "\n{}", style.format_header("Alerts"))?;
for alert in &status.alerts {
let is_cleared = alert.state == "cleared_pending_clean";
let severity_style = if is_cleared {
style.muted(&format!("{} (cleared)", alert.severity))
} else {
match alert.severity.as_str() {
"critical" | "error" => style.error(&alert.severity),
"warning" => style.warning(&alert.severity),
_ => style.info(&alert.severity),
}
};
let message = if is_cleared {
style.muted(&alert.message).to_string()
} else {
alert.message.clone()
};
if let Some(worker_id) = &alert.worker_id {
writeln!(
out,
" {} [{}] {} {}",
style.symbols.bullet_filled,
severity_style,
message,
style.muted(&format!("({worker_id})")),
)?;
} else {
writeln!(
out,
" {} [{}] {}",
style.symbols.bullet_filled, severity_style, message
)?;
}
}
}
if !status.issues.is_empty() {
writeln!(out, "\n{}", style.format_header("Issues"))?;
for issue in &status.issues {
let severity_style = match issue.severity.as_str() {
"critical" | "error" => style.error(&issue.severity),
"warning" => style.warning(&issue.severity),
_ => style.info(&issue.severity),
};
writeln!(
out,
" {} [{}] {}",
style.symbols.bullet_filled, severity_style, issue.summary
)?;
if let Some(remediation) = &issue.remediation {
writeln!(
out,
" {} {}",
style.muted("Fix:"),
style.info(remediation)
)?;
}
}
}
if let Some(conv) = convergence {
render_convergence_summary_to(out, conv, style)?;
}
if !remediation_hints.is_empty() {
render_remediation_hints_to(out, remediation_hints, style)?;
}
if show_workers {
render_workers_table_to(out, status, style)?;
}
if show_jobs {
render_builds_section_to(out, status, style)?;
}
Ok(())
}
fn render_workers_table_to<W: Write>(
out: &mut W,
status: &DaemonFullStatusResponse,
style: &Theme,
) -> std::io::Result<()> {
writeln!(out, "\n{}", style.format_header("Workers"))?;
if status.workers.is_empty() {
writeln!(out, " {}", style.muted("(none configured)"))?;
return Ok(());
}
writeln!(
out,
" {:12} {:8} {:10} {:6} {:10} {:8}",
style.key("ID"),
style.key("Status"),
style.key("Circuit"),
style.key("Slots"),
style.key("Speed"),
style.key("Host")
)?;
writeln!(
out,
" {:12} {:8} {:10} {:6} {:10} {:8}",
"────────────", "────────", "──────────", "──────", "──────────", "────────"
)?;
for worker in &status.workers {
let status_display = match worker.status.as_str() {
"healthy" => style.success("healthy"),
"degraded" => style.warning("degraded"),
"draining" => style.warning("draining"),
"drained" => style.info("drained"),
"unreachable" | "unhealthy" => style.error("unreachable"),
"disabled" => style.muted("disabled"),
_ => style.muted(&worker.status),
};
let circuit_display = match worker.circuit_state.as_str() {
"closed" => style.success("closed"),
"open" => {
if let Some(secs) = worker.recovery_in_secs {
style.error(&format!("open ({}s)", secs))
} else {
style.error("open")
}
}
"half_open" => style.warning("half-open"),
_ => style.muted(&worker.circuit_state),
};
let slots = format!("{}/{}", worker.used_slots, worker.total_slots);
let speed = format!("{:.1}", worker.speed_score);
writeln!(
out,
" {:12} {:8} {:10} {:6} {:10} {}@{}",
style.highlight(&worker.id),
status_display,
circuit_display,
slots,
speed,
style.muted(&worker.user),
style.info(&worker.host)
)?;
if worker.circuit_state != "closed" {
render_circuit_details_to(out, worker, style)?;
}
if let Some(ref pressure) = worker.pressure_state
&& pressure != "healthy"
{
render_pressure_details_to(out, worker, style)?;
}
if let Some(ref bypass) = worker.bypass {
render_bypass_details_to(out, bypass, style)?;
}
}
Ok(())
}
fn render_bypass_details_to<W: Write>(
out: &mut W,
bypass: &rch_common::BypassRecord,
style: &Theme,
) -> std::io::Result<()> {
let state = style.warning(bypass.state.label());
writeln!(
out,
" {} {} — {} ({}); consec fail {}",
style.muted("Bypass:"),
state,
bypass.failure_class.label(),
bypass.reason_code.code(),
bypass.consecutive_failures,
)?;
if !bypass.last_diagnostic.is_empty() {
writeln!(out, " {} {}", style.muted("↳"), bypass.last_diagnostic)?;
}
Ok(())
}
pub fn render_fleet_status(report: &rch_common::fleet_status::FleetStatusReport, style: &Theme) {
let mut out = Vec::new();
if render_fleet_status_to(&mut out, report, style).is_ok() {
print!("{}", String::from_utf8_lossy(&out));
}
}
fn render_fleet_status_to<W: Write>(
out: &mut W,
report: &rch_common::fleet_status::FleetStatusReport,
style: &Theme,
) -> std::io::Result<()> {
use std::collections::BTreeMap;
writeln!(out, "{}", style.format_header("Fleet Status"))?;
writeln!(out)?;
let total = report.diff.workers.len();
writeln!(
out,
" {} {}/{}",
style.muted("Ready:"),
report.diff.ready_workers,
total
)?;
let problem = if report.problem_class.as_str() == "healthy" {
style.success(report.problem_class.label())
} else {
style.warning(report.problem_class.label())
};
writeln!(out, " {} {}", style.muted("Problem:"), problem)?;
writeln!(
out,
" {} {}",
style.muted("Summary:"),
report.problem_summary
)?;
if !report.diff.workers.is_empty() {
writeln!(out)?;
writeln!(out, "{}", style.format_header("Workers by state"))?;
let mut groups: BTreeMap<&str, Vec<&str>> = BTreeMap::new();
for row in &report.diff.workers {
groups
.entry(row.state.as_str())
.or_default()
.push(row.worker_id.as_str());
}
for (state, ids) in &groups {
writeln!(
out,
" {} ({}): {}",
style.highlight(state),
ids.len(),
style.muted(&ids.join(", "))
)?;
}
}
if report.has_absence_warnings() {
writeln!(out)?;
writeln!(out, "{}", style.format_header("Absence alerts"))?;
for alert in &report.absence_alerts {
writeln!(
out,
" {} {} absent {}s as {} (> {}s window)",
style.warning("⚠"),
style.highlight(&alert.worker_id),
alert.absent_secs,
alert.state.as_str(),
alert.threshold_secs,
)?;
}
}
Ok(())
}
pub fn render_remediation_view(
view: &rch_common::remediation_view::RemediationView,
style: &Theme,
) {
let mut out = Vec::new();
if render_remediation_view_to(&mut out, view, style).is_ok() {
print!("{}", String::from_utf8_lossy(&out));
}
}
fn render_remediation_view_to<W: Write>(
out: &mut W,
view: &rch_common::remediation_view::RemediationView,
style: &Theme,
) -> std::io::Result<()> {
use rch_common::remediation_view::{BandSeverity, RemediationActionClass};
let class_str = |class: RemediationActionClass, text: &str| match class {
RemediationActionClass::Healthy => style.success(text),
RemediationActionClass::NormalFailOpen => style.muted(text),
RemediationActionClass::SelfHealingInProgress => style.info(text),
RemediationActionClass::OperatorActionRequired => style.error(text),
};
let sev_bullet = |sev: BandSeverity| match sev {
BandSeverity::Ok => style.success("●"),
BandSeverity::Info => style.info("●"),
BandSeverity::Warn => style.warning("●"),
BandSeverity::Critical => style.error("●"),
};
writeln!(out, "{}", style.format_header("Remediation"))?;
writeln!(out)?;
writeln!(
out,
" {} {}",
style.muted("Overall:"),
class_str(view.overall, view.overall.label())
)?;
writeln!(
out,
" {}",
style.muted("operator-action vs self-healing vs normal fail-open")
)?;
writeln!(out)?;
for band in &view.bands {
let tag = match band.action_class {
RemediationActionClass::Healthy => "OK",
RemediationActionClass::NormalFailOpen => "FAIL-OPEN",
RemediationActionClass::SelfHealingInProgress => "HEALING",
RemediationActionClass::OperatorActionRequired => "ACTION",
};
writeln!(
out,
" {} {:<20} {} {}",
sev_bullet(band.severity),
style.value(&band.title),
band.headline,
class_str(band.action_class, &format!("[{tag}]")),
)?;
for detail in &band.detail_lines {
writeln!(out, " {}", style.muted(detail))?;
}
}
writeln!(out)?;
if view.incidents.is_empty() {
writeln!(out, " {}", style.success("Recent incidents: none"))?;
} else {
writeln!(out, "{}", style.format_header("Recent incidents"))?;
for inc in view.incidents.iter().take(5) {
let who = inc
.worker_id
.as_deref()
.map_or(String::new(), |w| format!(" [{w}]"));
writeln!(
out,
" {} {}{} — {} ({}s ago)",
style.key(&inc.reason_code),
inc.event_type,
who,
inc.summary,
inc.age_secs
)?;
}
}
Ok(())
}
fn render_pressure_details_to<W: Write>(
out: &mut W,
worker: &crate::status_types::WorkerStatusFromApi,
style: &Theme,
) -> std::io::Result<()> {
let pressure = worker.pressure_state.as_deref().unwrap_or("unknown");
let pressure_display = match pressure {
"critical" => style.error("CRITICAL"),
"warning" => style.warning("WARNING"),
"telemetry_gap" => style.warning("TELEMETRY GAP"),
_ => style.muted(pressure),
};
write!(out, " {} {}", style.muted("Pressure:"), pressure_display)?;
if let (Some(free), Some(total)) = (worker.pressure_disk_free_gb, worker.pressure_disk_total_gb)
{
let ratio = worker
.pressure_disk_free_ratio
.map(|r| format!(" ({:.0}%)", r * 100.0))
.unwrap_or_default();
write!(out, " — {:.1}/{:.1} GB free{}", free, total, ratio)?;
}
writeln!(out)?;
if let Some(ref reason_code) = worker.pressure_reason_code
&& !reason_code.is_empty()
{
writeln!(
out,
" {} {}",
style.muted("reason:"),
style.muted(reason_code)
)?;
}
Ok(())
}
fn render_convergence_summary_to<W: Write>(
out: &mut W,
convergence: &crate::status_types::RepoConvergenceStatusFromApi,
style: &Theme,
) -> std::io::Result<()> {
writeln!(out, "\n{}", style.format_header("Repo Convergence"))?;
let s = &convergence.summary;
let state_display = if s.failed > 0 {
style.error("failed")
} else if s.drifting > 0 || s.converging > 0 {
style.warning("drifting")
} else if s.stale > 0 && s.ready == 0 {
style.warning("stale")
} else {
style.success("ready")
};
writeln!(
out,
" {} {} {} ({} ready, {} drifting, {} failed, {} stale)",
style.key("State"),
style.muted(":"),
state_display,
s.ready,
s.drifting + s.converging,
s.failed,
s.stale
)?;
for w in &convergence.workers {
if w.drift_state != "ready" {
let drift_display = match w.drift_state.as_str() {
"drifting" => style.warning(&w.drift_state),
"converging" => style.info(&w.drift_state),
"failed" => style.error(&w.drift_state),
"stale" => style.muted(&w.drift_state),
_ => style.muted(&w.drift_state),
};
write!(
out,
" {} {} {}",
style.symbols.bullet_filled,
style.highlight(&w.worker_id),
drift_display
)?;
if !w.missing_repos.is_empty() {
write!(
out,
" (missing: {})",
style.muted(&w.missing_repos.join(", "))
)?;
}
writeln!(out)?;
}
}
Ok(())
}
fn render_remediation_hints_to<W: Write>(
out: &mut W,
hints: &[crate::status_types::RemediationHint],
style: &Theme,
) -> std::io::Result<()> {
writeln!(out, "\n{}", style.format_header("Remediation"))?;
for hint in hints {
let severity_display = match hint.severity.as_str() {
"critical" => style.error("!!"),
"warning" => style.warning("!"),
_ => style.info("i"),
};
let worker_suffix = hint
.worker_id
.as_ref()
.map(|id| format!(" [{}]", id))
.unwrap_or_default();
writeln!(
out,
" {} {}{}",
severity_display,
hint.message,
style.muted(&worker_suffix)
)?;
if !hint.suggested_action.is_empty() {
writeln!(
out,
" {} {}",
style.muted("Fix:"),
style.info(&hint.suggested_action)
)?;
}
}
Ok(())
}
fn render_circuit_details_to<W: Write>(
out: &mut W,
worker: &crate::status_types::WorkerStatusFromApi,
style: &Theme,
) -> std::io::Result<()> {
let (state_name, explanation, help_text) = circuit_state_explanation(
&worker.circuit_state,
worker.consecutive_failures,
worker.recovery_in_secs,
worker.last_error.as_deref(),
);
writeln!(
out,
" {} {}",
style.muted("Circuit:"),
match worker.circuit_state.as_str() {
"open" => style.error(state_name),
"half_open" => style.warning(state_name),
_ => style.muted(state_name),
}
)?;
writeln!(out, " {}", style.muted(&explanation))?;
if !worker.failure_history.is_empty() {
let history_visual = format_failure_history(&worker.failure_history);
writeln!(
out,
" {} {} {}",
style.muted("History:"),
history_visual,
style.muted(&format!("(last {} attempts)", worker.failure_history.len()))
)?;
}
if let Some(ref error) = worker.last_error {
let error_truncated = if error.len() > 60 {
format!(
"{}...",
rch_common::util::truncate_at_char_boundary(error, 57)
)
} else {
error.clone()
};
writeln!(
out,
" {} {}",
style.muted("Reason:"),
style.error(&error_truncated)
)?;
}
if !help_text.is_empty() {
writeln!(
out,
" {} {}",
style.muted("Help:"),
style.info(help_text)
)?;
}
Ok(())
}
fn render_builds_section_to<W: Write>(
out: &mut W,
status: &DaemonFullStatusResponse,
style: &Theme,
) -> std::io::Result<()> {
writeln!(out, "\n{}", style.format_header("Active Builds"))?;
if status.active_builds.is_empty() {
writeln!(out, " {}", style.muted("(no active builds)"))?;
} else {
for build in &status.active_builds {
let cmd_display = if build.command.len() > 50 {
format!(
"{}...",
rch_common::util::truncate_at_char_boundary(&build.command, 47)
)
} else {
build.command.clone()
};
writeln!(
out,
" {} #{} on {} - {}",
style.symbols.bullet_filled,
style.highlight(&build.id.to_string()),
style.info(&build.worker_id),
style.muted(&cmd_display)
)?;
let phase = build
.heartbeat_phase
.as_deref()
.map_or_else(|| "unknown".to_string(), str::to_string);
let heartbeat_age = build
.heartbeat_age_secs
.map_or_else(|| "n/a".to_string(), format_duration);
let progress_age = build
.progress_age_secs
.map_or_else(|| "n/a".to_string(), format_duration);
let progress_counter = build
.heartbeat_counter
.map_or_else(|| "n/a".to_string(), |v| v.to_string());
let percent = build
.heartbeat_percent
.map_or_else(|| "n/a".to_string(), |v| format!("{v:.0}%"));
let slots = build
.slots
.map_or_else(|| "n/a".to_string(), |v| v.to_string());
let detail = build
.heartbeat_detail
.as_deref()
.map_or_else(|| "none".to_string(), str::to_string);
let detector_confidence = build
.detector_confidence
.map_or_else(|| "n/a".to_string(), |v| format!("{v:.2}"));
let detector_hook_alive = build
.detector_hook_alive
.map_or_else(|| "n/a".to_string(), |v| v.to_string());
let detector_hb_stale = build
.detector_heartbeat_stale
.map_or_else(|| "n/a".to_string(), |v| v.to_string());
let detector_progress_stale = build
.detector_progress_stale
.map_or_else(|| "n/a".to_string(), |v| v.to_string());
let detector_build_age = build
.detector_build_age_secs
.map_or_else(|| "n/a".to_string(), format_duration);
let detector_slots = build
.detector_slots_owned
.map_or_else(|| "n/a".to_string(), |v| v.to_string());
writeln!(
out,
" {} phase={} | hb_age={} | progress_age={} | counter={} | pct={} | slots={} | detail={}",
style.muted("heartbeat:"),
style.info(&phase),
style.info(&heartbeat_age),
style.info(&progress_age),
style.info(&progress_counter),
style.info(&percent),
style.info(&slots),
style.muted(&detail)
)?;
writeln!(
out,
" {} confidence={} | hook_alive={} | hb_stale={} | progress_stale={} | build_age={} | slots={}",
style.muted("detector:"),
style.info(&detector_confidence),
style.info(&detector_hook_alive),
style.info(&detector_hb_stale),
style.info(&detector_progress_stale),
style.info(&detector_build_age),
style.info(&detector_slots)
)?;
}
}
writeln!(out, "\n{}", style.format_header("Recent Builds"))?;
if status.recent_builds.is_empty() {
writeln!(out, " {}", style.muted("(no recent builds)"))?;
} else {
for build in status.recent_builds.iter().take(10) {
let status_indicator = if build.exit_code == 0 {
style.success("✓")
} else {
style.error("✗")
};
let duration = format!("{:.1}s", build.duration_ms as f64 / 1000.0);
let location_display = if build.location == "remote" {
style.info(&build.location)
} else {
style.muted(&build.location)
};
let cmd_display = if build.command.len() > 40 {
format!(
"{}...",
rch_common::util::truncate_at_char_boundary(&build.command, 37)
)
} else {
build.command.clone()
};
writeln!(
out,
" {} {} {:8} {} {}",
status_indicator,
style.muted(&duration),
location_display,
build
.worker_id
.as_ref()
.map(|w| style.highlight(w))
.unwrap_or_else(|| style.muted("-")),
style.muted(&cmd_display)
)?;
if let Some(cancellation) = &build.cancellation {
let decision_path = if cancellation.decision_path.is_empty() {
"n/a".to_string()
} else {
cancellation.decision_path.join(" -> ")
};
let cleanup = if cancellation.cleanup_ok {
style.success("ok")
} else {
style.error("failed")
};
let health = cancellation
.worker_health
.as_ref()
.map(|worker| {
format!(
"{} / pressure={} ({})",
worker.status, worker.pressure_state, worker.pressure_reason_code
)
})
.unwrap_or_else(|| "n/a".to_string());
writeln!(
out,
" {} origin={} | stage={} | cleanup={} | operation={}",
style.muted("cancellation:"),
style.info(&cancellation.origin),
style.info(&cancellation.escalation_stage),
cleanup,
style.info(&cancellation.operation_id)
)?;
writeln!(
out,
" {} decision={} | worker={} | final_state={}",
style.muted("details:"),
style.muted(&decision_path),
style.muted(&health),
style.info(&cancellation.final_state)
)?;
if !cancellation.cleanup_ok {
writeln!(
out,
" {} {}",
style.muted("action:"),
style.warning(
"run `rch workers probe --all` and inspect daemon cancellation logs"
)
)?;
} else if cancellation.escalation_stage == "sigkill" {
writeln!(
out,
" {} {}",
style.muted("action:"),
style.warning(
"review stuck-process signals; cancellation required SIGKILL escalation"
)
)?;
}
}
}
}
Ok(())
}
pub fn check_hook_installed() -> bool {
dirs::home_dir()
.map(|h| h.join(".claude").join("settings.json"))
.map(|p| {
if p.exists() {
std::fs::read_to_string(&p)
.ok()
.map(|c| c.contains("PreToolUse"))
.unwrap_or(false)
} else {
false
}
})
.unwrap_or(false)
}
pub fn format_failure_history(history: &[bool]) -> String {
if history.is_empty() {
return "(no history)".to_string();
}
history
.iter()
.map(|&success| if success { '✓' } else { '✗' })
.collect()
}
pub fn circuit_state_explanation(
state: &str,
consecutive_failures: u32,
recovery_in_secs: Option<u64>,
_last_error: Option<&str>,
) -> (&'static str, String, &'static str) {
match state {
"closed" => (
"CLOSED",
"Normal operation - requests are being routed".to_string(),
"",
),
"open" => {
let reason = if consecutive_failures > 0 {
format!("{} consecutive failures", consecutive_failures)
} else {
"repeated failures".to_string()
};
let timing = match recovery_in_secs {
Some(secs) => format!(" (auto-recovery in {}s)", secs),
None => " (cooldown elapsed, awaiting probe)".to_string(),
};
let explanation = format!("Circuit open due to {}{}", reason, timing);
(
"OPEN",
explanation,
"Wait for auto-recovery or run: rch workers probe <id> --force",
)
}
"half_open" => (
"HALF-OPEN",
"Testing recovery - limited requests allowed".to_string(),
"Probing in progress; success will close circuit",
),
_ => ("UNKNOWN", "Unknown circuit state".to_string(), ""),
}
}
pub fn render_basic_status(daemon_running: bool, show_workers: bool, style: &Theme) {
let mut stdout = std::io::stdout();
let _ = render_basic_status_to(&mut stdout, daemon_running, show_workers, style);
}
fn render_basic_status_to<W: Write>(
out: &mut W,
daemon_running: bool,
show_workers: bool,
style: &Theme,
) -> std::io::Result<()> {
writeln!(out, "{}", style.format_header("RCH Status"))?;
writeln!(out)?;
writeln!(
out,
" {} {} {}",
style.key("Daemon"),
style.muted(":"),
if daemon_running {
style.warning("Running (not responding)")
} else {
style.error("Stopped")
}
)?;
if !daemon_running {
writeln!(
out,
" {} {}",
style.symbols.bullet_filled,
style.muted("Start with: rch daemon start")
)?;
}
writeln!(
out,
" {} {} {}",
style.key("Workers"),
style.muted(":"),
style.muted("(check config for details)")
)?;
let hook_installed = check_hook_installed();
writeln!(
out,
" {} {} {}",
style.key("Hook"),
style.muted(":"),
if hook_installed {
style.success("Installed")
} else {
style.warning("Not installed")
}
)?;
if show_workers {
writeln!(out, "\n{}", style.format_header("Workers"))?;
writeln!(
out,
" {} {}",
style.symbols.info,
style.muted("Start daemon for worker status: rch daemon start")
)?;
}
writeln!(out)?;
writeln!(
out,
" {} {}",
style.symbols.info,
style.warning("Start daemon for live status: rch daemon start")
)?;
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::status_types::{
ActiveBuildFromApi, BuildRecordFromApi, BuildStatsFromApi, DaemonInfoFromApi, IssueFromApi,
TestRunStatsFromApi, WorkerStatusFromApi,
};
use rch_common::test_guard;
use tracing::info;
struct ConfigOverrideReset;
impl Drop for ConfigOverrideReset {
fn drop(&mut self) {
crate::config::set_test_config_override(None);
}
}
fn sample_status() -> DaemonFullStatusResponse {
DaemonFullStatusResponse {
daemon: DaemonInfoFromApi {
pid: 4242,
uptime_secs: 90,
version: "0.1.0".to_string(),
socket_path: rch_common::default_socket_path(),
started_at: "2026-01-17T00:00:00Z".to_string(),
workers_total: 2,
workers_healthy: 1,
slots_total: 16,
slots_available: 8,
},
workers: vec![
WorkerStatusFromApi {
id: "worker-a".to_string(),
host: "10.0.0.1".to_string(),
user: "ubuntu".to_string(),
status: "healthy".to_string(),
circuit_state: "closed".to_string(),
used_slots: 2,
total_slots: 8,
speed_score: 92.5,
last_error: None,
consecutive_failures: 0,
recovery_in_secs: None,
failure_history: vec![true, true],
pressure_state: None,
pressure_confidence: None,
pressure_reason_code: None,
pressure_policy_rule: None,
pressure_disk_free_gb: None,
pressure_disk_total_gb: None,
pressure_disk_free_ratio: None,
pressure_build_disk_free_gb: None,
pressure_build_disk_total_gb: None,
pressure_disk_io_util_pct: None,
pressure_memory_pressure: None,
pressure_telemetry_age_secs: None,
pressure_telemetry_fresh: None,
bypass: None,
},
WorkerStatusFromApi {
id: "worker-b".to_string(),
host: "10.0.0.2".to_string(),
user: "ubuntu".to_string(),
status: "unhealthy".to_string(),
circuit_state: "open".to_string(),
used_slots: 8,
total_slots: 8,
speed_score: 12.3,
last_error: Some("SSH timeout".to_string()),
consecutive_failures: 3,
recovery_in_secs: Some(30),
failure_history: vec![false, false, true],
pressure_state: None,
pressure_confidence: None,
pressure_reason_code: None,
pressure_policy_rule: None,
pressure_disk_free_gb: None,
pressure_disk_total_gb: None,
pressure_disk_free_ratio: None,
pressure_build_disk_free_gb: None,
pressure_build_disk_total_gb: None,
pressure_disk_io_util_pct: None,
pressure_memory_pressure: None,
pressure_telemetry_age_secs: None,
pressure_telemetry_fresh: None,
bypass: None,
},
],
active_builds: vec![ActiveBuildFromApi {
id: 1,
project_id: "proj".to_string(),
worker_id: "worker-a".to_string(),
command: "cargo build".to_string(),
started_at: "2026-01-17T00:00:01Z".to_string(),
last_heartbeat_at: Some("2026-01-17T00:00:05Z".to_string()),
heartbeat_age_secs: Some(2),
last_progress_at: Some("2026-01-17T00:00:04Z".to_string()),
progress_age_secs: Some(3),
heartbeat_phase: Some("execute".to_string()),
heartbeat_detail: Some("Compiling".to_string()),
heartbeat_counter: Some(7),
heartbeat_percent: Some(35.0),
slots: Some(4),
detector_hook_alive: Some(false),
detector_heartbeat_stale: Some(true),
detector_progress_stale: Some(true),
detector_confidence: Some(0.91),
detector_build_age_secs: Some(120),
detector_slots_owned: Some(4),
detector_last_evaluated_at: Some("2026-01-17T00:00:05Z".to_string()),
}],
queued_builds: vec![],
recent_builds: vec![
BuildRecordFromApi {
id: 2,
started_at: "2026-01-17T00:00:02Z".to_string(),
completed_at: "2026-01-17T00:00:03Z".to_string(),
project_id: "proj".to_string(),
worker_id: Some("worker-a".to_string()),
command: "cargo test --release".to_string(),
exit_code: 0,
duration_ms: 1250,
location: "remote".to_string(),
bytes_transferred: Some(2048),
timing: None,
cancellation: None,
},
BuildRecordFromApi {
id: 3,
started_at: "2026-01-17T00:00:04Z".to_string(),
completed_at: "2026-01-17T00:00:05Z".to_string(),
project_id: "proj".to_string(),
worker_id: None,
command: "cargo check".to_string(),
exit_code: 1,
duration_ms: 540,
location: "local".to_string(),
bytes_transferred: None,
timing: None,
cancellation: None,
},
],
issues: vec![IssueFromApi {
severity: "warning".to_string(),
summary: "worker-b unreachable".to_string(),
remediation: Some("Check SSH connectivity".to_string()),
}],
alerts: vec![],
stats: BuildStatsFromApi {
total_builds: 2,
success_count: 1,
failure_count: 1,
remote_count: 1,
local_count: 1,
avg_duration_ms: 895,
},
test_stats: Some(TestRunStatsFromApi {
scope: TestRunStatsScope::Unknown,
total_runs: 3,
passed_runs: 2,
failed_runs: 1,
avg_duration_ms: 1200,
runs_by_kind: std::collections::HashMap::from([("cargo_test".to_string(), 3)]),
}),
saved_time: None,
remediation: None,
}
}
#[cfg(unix)]
#[tokio::test]
async fn query_daemon_full_status_uses_configured_socket_path() {
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
use tokio::net::UnixListener;
let _guard = test_guard!();
let temp_dir = tempfile::Builder::new()
.prefix("rch-status-")
.tempdir_in("/tmp")
.expect("short socket tempdir");
let socket_path = temp_dir.path().join("custom-rch.sock");
let listener = UnixListener::bind(&socket_path).expect("bind custom socket");
let mut config = rch_common::RchConfig::default();
config.general.socket_path = socket_path.display().to_string();
crate::config::set_test_config_override(Some(config));
let _reset = ConfigOverrideReset;
let mut expected = sample_status();
expected.daemon.version = "configured-socket-test".to_string();
let response_body = serde_json::to_string(&expected).expect("serialize status");
let server = tokio::spawn(async move {
let (stream, _) = listener.accept().await.expect("accept client");
let (reader, mut writer) = stream.into_split();
let mut reader = BufReader::new(reader);
let mut line = String::new();
reader.read_line(&mut line).await.expect("read request");
assert_eq!(line, "GET /status\n");
let response = format!(
"HTTP/1.0 200 OK\r\nContent-Type: application/json\r\n\r\n{}\n",
response_body
);
writer
.write_all(response.as_bytes())
.await
.expect("write response");
});
let status = query_daemon_full_status()
.await
.expect("query configured daemon status");
server.await.expect("server task");
assert_eq!(status.daemon.version, "configured-socket-test");
}
#[test]
fn test_format_failure_history_empty() {
let _guard = test_guard!();
assert_eq!(format_failure_history(&[]), "(no history)");
}
#[test]
fn test_format_failure_history_all_success() {
let _guard = test_guard!();
assert_eq!(format_failure_history(&[true, true, true]), "✓✓✓");
}
#[test]
fn test_format_failure_history_all_failure() {
let _guard = test_guard!();
assert_eq!(format_failure_history(&[false, false, false]), "✗✗✗");
}
#[test]
fn test_format_failure_history_mixed() {
let _guard = test_guard!();
assert_eq!(
format_failure_history(&[false, false, true, false, true]),
"✗✗✓✗✓"
);
}
#[test]
fn test_circuit_state_explanation_closed() {
let _guard = test_guard!();
let (state, explanation, help) = circuit_state_explanation("closed", 0, None, None);
assert_eq!(state, "CLOSED");
assert!(explanation.contains("Normal operation"));
assert!(help.is_empty());
}
#[test]
fn test_circuit_state_explanation_open_with_recovery() {
let _guard = test_guard!();
let (state, explanation, help) =
circuit_state_explanation("open", 3, Some(45), Some("SSH timeout"));
assert_eq!(state, "OPEN");
assert!(explanation.contains("3 consecutive failures"));
assert!(explanation.contains("auto-recovery in 45s"));
assert!(help.contains("rch workers probe"));
}
#[test]
fn test_circuit_state_explanation_open_no_recovery() {
let _guard = test_guard!();
let (state, explanation, _help) = circuit_state_explanation("open", 5, None, None);
assert_eq!(state, "OPEN");
assert!(explanation.contains("5 consecutive failures"));
assert!(explanation.contains("awaiting probe"));
}
#[test]
fn test_circuit_state_explanation_half_open() {
let _guard = test_guard!();
let (state, explanation, help) = circuit_state_explanation("half_open", 0, None, None);
assert_eq!(state, "HALF-OPEN");
assert!(explanation.contains("Testing recovery"));
assert!(help.contains("success will close"));
}
#[test]
fn test_circuit_state_explanation_unknown() {
let _guard = test_guard!();
let (state, explanation, _help) = circuit_state_explanation("weird_state", 0, None, None);
assert_eq!(state, "UNKNOWN");
assert!(explanation.contains("Unknown"));
}
#[test]
fn test_render_test_command_statistics_scope() {
let _guard = test_guard!();
for (scope, expected) in [
(TestRunStatsScope::Unknown, "scope unknown"),
(TestRunStatsScope::StoredHistory, "all stored records"),
(
TestRunStatsScope::RecentMemory { max_records: 200 },
"recent memory, up to 200",
),
] {
let mut status = sample_status();
status.test_stats.as_mut().expect("test stats").scope = scope;
let style = Theme::new(false, true, false);
let mut buf = Vec::new();
render_full_status_to(
&mut buf,
&status,
false,
false,
None,
&[],
&crate::status_types::SystemPosture::RemoteReady,
&style,
)
.expect("render");
let output = String::from_utf8(buf).expect("utf8 output");
assert!(output.contains(&format!("Test commands ({expected})")));
assert!(output.contains("3 total, 67% command success rate"));
assert!(!output.contains("lifetime"));
}
}
#[test]
fn test_render_full_status_includes_workers_and_circuits() {
let _guard = test_guard!();
info!("TEST: test_render_full_status_includes_workers_and_circuits");
let status = sample_status();
let style = Theme::new(false, true, false);
let mut buf = Vec::new();
render_full_status_to(
&mut buf,
&status,
true,
false,
None,
&[],
&crate::status_types::SystemPosture::RemoteReady,
&style,
)
.expect("render");
let output = String::from_utf8(buf).expect("utf8 output");
assert!(output.contains("worker-a"));
assert!(output.contains("healthy"));
assert!(output.contains("worker-b"));
assert!(output.contains("unreachable"));
assert!(output.contains("open (30s)"));
info!("PASS: workers and circuit states rendered");
}
#[test]
fn test_render_full_status_includes_build_sections() {
let _guard = test_guard!();
info!("TEST: test_render_full_status_includes_build_sections");
let status = sample_status();
let style = Theme::new(false, true, false);
let mut buf = Vec::new();
render_full_status_to(
&mut buf,
&status,
false,
true,
None,
&[],
&crate::status_types::SystemPosture::RemoteReady,
&style,
)
.expect("render");
let output = String::from_utf8(buf).expect("utf8 output");
assert!(output.contains("Active Builds"));
assert!(output.contains("Recent Builds"));
assert!(output.contains("cargo build"));
assert!(output.contains("cargo test --release"));
assert!(output.contains("heartbeat:"));
assert!(output.contains("detector:"));
assert!(output.contains("phase="));
assert!(output.contains("✓"));
assert!(output.contains("✗"));
info!("PASS: build sections rendered with status indicators");
}
#[test]
fn test_render_full_status_includes_cancellation_details() {
let _guard = test_guard!();
let mut status = sample_status();
status.recent_builds[1].cancellation = Some(rch_common::BuildCancellationMetadata {
operation_id: "cancel-3".to_string(),
origin: "timeout".to_string(),
reason_code: "timeout".to_string(),
decision_path: vec![
"requested".to_string(),
"term_sent".to_string(),
"remote_kill_sent".to_string(),
"escalated".to_string(),
"completed".to_string(),
],
escalation_stage: "sigkill".to_string(),
escalation_count: 2,
remote_kill_attempted: true,
cleanup_ok: false,
history_cancelled: true,
final_state: "completed".to_string(),
worker_health: Some(rch_common::BuildCancellationWorkerHealth {
status: "unreachable".to_string(),
speed_score: 0.0,
used_slots: 8,
available_slots: 0,
pressure_state: "critical".to_string(),
pressure_reason_code: "disk_free_below_critical_gb".to_string(),
}),
});
let style = Theme::new(false, true, false);
let mut buf = Vec::new();
render_full_status_to(
&mut buf,
&status,
false,
true,
None,
&[],
&crate::status_types::SystemPosture::RemoteReady,
&style,
)
.expect("render");
let output = String::from_utf8(buf).expect("utf8 output");
assert!(output.contains("cancellation:"));
assert!(output.contains("cancel-3"));
assert!(output.contains("sigkill"));
assert!(output.contains("failed"));
}
#[test]
fn test_render_full_status_includes_circuit_details() {
let _guard = test_guard!();
info!("TEST: test_render_full_status_includes_circuit_details");
let status = sample_status();
let style = Theme::new(false, true, false);
let mut buf = Vec::new();
render_full_status_to(
&mut buf,
&status,
true,
false,
None,
&[],
&crate::status_types::SystemPosture::RemoteReady,
&style,
)
.expect("render");
let output = String::from_utf8(buf).expect("utf8 output");
assert!(output.contains("Circuit:"));
assert!(output.contains("History:"));
assert!(output.contains("Reason:"));
assert!(output.contains("Help:"));
info!("PASS: circuit details rendered");
}
#[test]
fn test_render_basic_status_stopped() {
let _guard = test_guard!();
info!("TEST: test_render_basic_status_stopped");
let style = Theme::new(false, true, false);
let mut buf = Vec::new();
render_basic_status_to(&mut buf, false, true, &style).expect("render");
let output = String::from_utf8(buf).expect("utf8 output");
assert!(output.contains("Stopped"));
assert!(output.contains("Start with: rch daemon start"));
assert!(output.contains("Start daemon for worker status"));
info!("PASS: basic status stopped message rendered");
}
#[test]
fn test_render_basic_status_running() {
let _guard = test_guard!();
info!("TEST: test_render_basic_status_running");
let style = Theme::new(false, true, false);
let mut buf = Vec::new();
render_basic_status_to(&mut buf, true, false, &style).expect("render");
let output = String::from_utf8(buf).expect("utf8 output");
assert!(output.contains("Running (not responding)"));
assert!(!output.contains("Start with: rch daemon start"));
info!("PASS: basic status running message rendered");
}
}