use std::{
collections::HashMap,
io::{IsTerminal as _, Write},
sync::Arc,
time::Duration,
};
use bestool_canopy::schema::{CheckResult, CheckSeverity, HealthCheck, StatusPayload};
use clap::Parser;
use futures::FutureExt as _;
use miette::{IntoDiagnostic, Result, WrapErr, miette};
use serde_json::Value;
use tokio::sync::mpsc;
use tracing::{debug, warn};
use bestool_alertd::{
SweepResult, SweepTargets,
check::{Check, CheckOutcome, CheckStatus, OverallResult},
checks, overall_from_payload, perform_sweep,
progress::ProgressSender,
resolve_sweep_targets,
subject::{ApplicationKind, ApplicationRef, Subject},
sweep::{SplitSeverities, validate_selection},
};
use super::{TamanuArgs, try_find_tamanu};
use crate::actions::Context;
mod order;
mod render;
mod tui;
#[derive(Debug, Clone, Parser)]
#[clap(verbatim_doc_comment)]
pub struct DoctorArgs {
#[arg(long)]
pub json: bool,
#[arg(long = "check", value_name = "NAME")]
pub only: Vec<String>,
#[arg(long = "skip", value_name = "NAME")]
pub skip: Vec<String>,
#[arg(long, short = 'a')]
pub all: bool,
#[arg(long)]
pub fresh: bool,
#[arg(long)]
pub no_daemon: bool,
#[arg(long, requires = "only")]
pub heal: bool,
}
const HEAL_SETTLE_BOUND: Duration = Duration::from_secs(15 * 60);
pub enum SweepSource {
DaemonCached { computed_at: jiff::Timestamp },
DaemonStreamed,
Local,
}
pub async fn run(args: DoctorArgs, ctx: Context) -> Result<()> {
let tamanu = ctx.require::<TamanuArgs>();
let ansi = ansi_supported();
let use_colours = tamanu.use_colours && ansi;
let targets = resolve_sweep_targets(try_find_tamanu(tamanu).await?)?;
if targets.is_none() {
warn!("no Tamanu on this host; running host-level checks only");
}
let http_client = crate::http::client();
let live_tty = !args.json && ansi && std::io::stdout().is_terminal();
let (sweep, source, interrupted) = if args.no_daemon || args.heal {
let outcome = run_local_sweep(targets.clone(), http_client.clone(), &args, live_tty).await?;
(outcome.sweep, SweepSource::Local, outcome.interrupted)
} else if args.fresh {
match run_daemon_recompute(&http_client, &args, live_tty).await {
Ok(outcome) => (outcome.sweep, SweepSource::DaemonStreamed, outcome.interrupted),
Err(err) => {
warn!(%err, "alertd did not answer; ran the checks locally instead of on the daemon");
let outcome =
run_local_sweep(targets.clone(), http_client.clone(), &args, live_tty).await?;
(outcome.sweep, SweepSource::Local, outcome.interrupted)
}
}
} else {
match fetch_daemon_latest(&http_client).await {
Ok((sweep, computed_at)) => (sweep, SweepSource::DaemonCached { computed_at }, false),
Err(err) => {
debug!(%err, "daemon latest unavailable, falling back to local");
let outcome =
run_local_sweep(targets.clone(), http_client.clone(), &args, live_tty).await?;
(outcome.sweep, SweepSource::Local, outcome.interrupted)
}
}
};
emit_output(&args, &sweep, &source, use_colours)?;
if interrupted {
std::process::exit(130);
}
if sweep.overall == OverallResult::Failing {
std::process::exit(1);
}
Ok(())
}
struct SweepOutcome {
sweep: SweepResult,
interrupted: bool,
}
async fn run_local_sweep(
targets: Option<SweepTargets>,
http_client: reqwest::Client,
args: &DoctorArgs,
live_tty: bool,
) -> Result<SweepOutcome> {
validate_check_selection(&args.only, &args.skip)?;
let (progress, tui_handle) = setup_progress(live_tty, SweepSource::Local);
let canopy = async {
tokio::time::timeout(CANOPY_CLIENT_TIMEOUT, canopy_client())
.await
.ok()
.flatten()
.map(Arc::new)
}
.boxed()
.shared();
let severities_handle = tokio::spawn({
let canopy = canopy.clone();
async move { fetch_check_severities(canopy.await).await }
});
let sweep_args_only = args.only.clone();
let sweep_args_skip = args.skip.clone();
let enable_heal = args.heal;
let sweep_handle = tokio::spawn(async move {
let pg_pool = async {
match targets.as_ref() {
Some(t) => bestool_postgres::pool::create_pool_sized(
&t.database_url,
"bestool-tamanu-doctor",
bestool_alertd::checks::POOL_SIZE,
bestool_postgres::pool::Prompt::Allowed,
)
.await
.inspect_err(|err| debug!(%err, "no DB pool for this sweep; DB checks will skip"))
.ok(),
None => None,
}
};
let (pg_pool, canopy) = tokio::join!(pg_pool, canopy);
perform_sweep(
env!("CARGO_PKG_VERSION"),
targets,
http_client,
&sweep_args_only,
&sweep_args_skip,
None,
progress,
canopy,
enable_heal,
pg_pool,
)
.await
});
let interrupted = if let Some(handle) = tui_handle {
let outcome = handle.await.into_diagnostic()??;
if outcome.interrupted {
sweep_handle.abort();
severities_handle.abort();
let mut synthetic = synthetic_sweep(outcome.results);
synthetic.payload = empty_payload();
return Ok(SweepOutcome {
sweep: synthetic,
interrupted: true,
});
}
false
} else {
false
};
let mut sweep = sweep_handle.await.into_diagnostic()??;
if enable_heal {
bestool_alertd::heal::settle(HEAL_SETTLE_BOUND).await;
}
if let Some(severities) = severities_handle.await.ok().flatten() {
sweep.apply_severities(&SplitSeverities::flat(severities));
}
Ok(SweepOutcome { sweep, interrupted })
}
async fn fetch_check_severities(
canopy: Option<Arc<bestool_canopy::CanopyClient>>,
) -> Option<HashMap<String, CheckSeverity>> {
let client = canopy?;
tokio::time::timeout(Duration::from_secs(10), async {
let machine_id = bestool_tamanu::server_info::get_or_create_machine_id()
.await
.ok()?;
client.status_check_severities(&machine_id).await.ok()
})
.await
.ok()
.flatten()
}
const CANOPY_CLIENT_TIMEOUT: Duration = Duration::from_secs(10);
async fn canopy_client() -> Option<bestool_canopy::CanopyClient> {
let reg = bestool_canopy::registration::load().await.ok().flatten()?;
let device_key = reg.device_key.as_deref()?;
let base_url = reg
.api_url
.as_deref()
.unwrap_or(bestool_canopy::DEFAULT_CANOPY_URL)
.parse()
.ok()?;
let tailscale_url = bestool_canopy::TAILSCALE_URL.parse().ok()?;
bestool_canopy::connect_to(
base_url,
tailscale_url,
Some(device_key),
crate::http::client_builder,
)
.await
.ok()?
}
async fn run_daemon_recompute(
http: &reqwest::Client,
args: &DoctorArgs,
live_tty: bool,
) -> Result<SweepOutcome> {
let response = crate::http::daemon_get(http, "/tasks/doctor/recompute", |r| {
r.timeout(std::time::Duration::from_secs(5))
})
.await
.into_diagnostic()
.wrap_err("contacting local alertd")?;
if !response.status().is_success() {
return Err(miette!(
"alertd /tasks/doctor/recompute returned {}",
response.status()
));
}
validate_check_selection(&args.only, &args.skip)?;
let (progress, tui_handle) = setup_progress(live_tty, SweepSource::DaemonStreamed);
let stream_handle = tokio::spawn(drain_recompute_stream(response, progress));
let interrupted = if let Some(handle) = tui_handle {
let outcome = handle.await.into_diagnostic()??;
if outcome.interrupted {
stream_handle.abort();
return Ok(SweepOutcome {
sweep: synthetic_sweep(outcome.results),
interrupted: true,
});
}
false
} else {
false
};
let streamed = stream_handle.await.into_diagnostic()??;
let overall = overall_from_payload(&streamed.payload);
Ok(SweepOutcome {
sweep: SweepResult {
machine_id: streamed.machine_id,
results: streamed.results,
overall,
payload: streamed.payload,
pg_version: None,
},
interrupted,
})
}
struct StreamedSweep {
payload: StatusPayload,
machine_id: Option<String>,
results: Vec<CheckOutcome>,
}
async fn drain_recompute_stream(
response: reqwest::Response,
progress: Option<ProgressSender>,
) -> Result<StreamedSweep> {
use bestool_alertd::progress::DoctorEvent;
use futures::StreamExt as _;
let registry = checks::all();
let resolve_name = |s: &str| registry.iter().find(|e| e.name == s).map(|e| e.name);
let mut stream = response.bytes_stream();
let mut buffer = Vec::<u8>::new();
let mut final_payload: Option<StatusPayload> = None;
let mut machine_id: Option<String> = None;
let mut results: Vec<CheckOutcome> = Vec::new();
while let Some(chunk) = stream.next().await {
let chunk = chunk
.into_diagnostic()
.wrap_err("reading alertd recompute stream")?;
buffer.extend_from_slice(&chunk);
while let Some(nl) = buffer.iter().position(|&b| b == b'\n') {
let line: Vec<u8> = buffer.drain(..=nl).collect();
let line = &line[..line.len() - 1];
if line.is_empty() {
continue;
}
let value: Value = match serde_json::from_slice(line) {
Ok(v) => v,
Err(err) => {
warn!(%err, "could not parse alertd recompute line");
continue;
}
};
match value.get("event").and_then(Value::as_str) {
Some("planned") => {
if let Some(tx) = progress.as_ref()
&& let Some(checks) = value.get("checks").and_then(Value::as_array)
{
let names = checks
.iter()
.filter_map(Value::as_str)
.map(str::to_string)
.collect();
let _ = tx.send(DoctorEvent::Planned(names));
}
}
Some("check") => {
if let Some(check_json) = value.get("check")
&& let Some(outcome) =
CheckOutcome::from_streaming_json(check_json, resolve_name)
{
if let Some(tx) = progress.as_ref() {
let _ = tx.send(DoctorEvent::Completed(outcome.clone()));
}
results.push(outcome);
}
}
Some("done") => {
let raw = value
.get("payload")
.cloned()
.ok_or_else(|| miette!("alertd recompute done event carried no payload"))?;
final_payload = Some(
serde_json::from_value(raw)
.into_diagnostic()
.wrap_err("decoding the alertd recompute payload")?,
);
machine_id = value
.get("machineId")
.and_then(Value::as_str)
.map(str::to_string);
}
Some("error") => {
let msg = value
.get("message")
.and_then(Value::as_str)
.unwrap_or("unknown");
return Err(miette!("alertd recompute reported error: {msg}"));
}
_ => {}
}
}
}
let payload = final_payload
.ok_or_else(|| miette!("alertd recompute stream ended without a done event"))?;
Ok(StreamedSweep {
payload,
machine_id,
results,
})
}
async fn fetch_daemon_latest(http: &reqwest::Client) -> Result<(SweepResult, jiff::Timestamp)> {
let response = crate::http::daemon_get(http, "/tasks/doctor/latest", |r| {
r.timeout(std::time::Duration::from_secs(3))
})
.await
.into_diagnostic()
.wrap_err("contacting local alertd")?;
if !response.status().is_success() {
return Err(miette!(
"alertd /tasks/doctor/latest returned {}",
response.status()
));
}
let payload: Value = response
.json()
.await
.into_diagnostic()
.wrap_err("decoding alertd latest payload")?;
let computed_at: jiff::Timestamp = payload
.get("computedAt")
.and_then(Value::as_str)
.ok_or_else(|| miette!("alertd latest payload missing computedAt"))?
.parse()
.into_diagnostic()
.wrap_err("parsing computedAt timestamp")?;
let inner: StatusPayload = payload
.get("payload")
.cloned()
.ok_or_else(|| miette!("alertd latest payload missing payload"))
.and_then(|p| {
serde_json::from_value(p)
.into_diagnostic()
.wrap_err("decoding alertd latest status payload")
})?;
let machine_id = payload
.get("machineId")
.and_then(Value::as_str)
.map(str::to_string);
let overall = overall_from_payload(&inner);
let results = results_from_wire(&inner);
Ok((
SweepResult {
machine_id,
results,
overall,
payload: inner,
pg_version: None,
},
computed_at,
))
}
fn results_from_wire(payload: &StatusPayload) -> Vec<CheckOutcome> {
let registry = checks::all();
let names: HashMap<&str, &'static str> = registry.iter().map(|e| (e.name, e.name)).collect();
let mut targets: Vec<(Subject, &Vec<HealthCheck>)> = Vec::new();
if !payload.health.is_empty() {
targets.push((Subject::Machine, &payload.health));
}
if let Some(machine) = payload.machine.as_ref()
&& let Some(health) = machine.health.as_ref()
{
targets.push((Subject::Machine, health));
}
for (key, report) in payload.applications.iter().flatten() {
let Some(kind) = ApplicationKind::ALL
.into_iter()
.find(|kind| kind.type_slug() == report.type_)
else {
continue;
};
if let Some(health) = report.health.as_ref() {
let app = ApplicationRef {
kind,
key: key.clone(),
};
targets.push((Subject::Application(app), health));
}
}
let mut results = Vec::new();
for (subject, health) in targets {
for entry in health {
let Some(name) = names.get(entry.check.as_str()).copied() else {
continue;
};
let Some(result) = entry.result.as_ref() else {
continue;
};
let text = |key: &str| {
entry
.extra
.get(key)
.and_then(Value::as_str)
.unwrap_or_default()
.to_string()
};
let status = match result {
CheckResult::Passed => CheckStatus::Pass,
CheckResult::Skipped => CheckStatus::Skip(text("reason")),
CheckResult::Warning => CheckStatus::Warning(text("reason")),
CheckResult::Failed => CheckStatus::Fail(text("reason")),
CheckResult::Broken => CheckStatus::Broken(text("reason")),
};
results.push(CheckOutcome {
subject: subject.clone(),
check: Check {
name,
status,
summary: text("summary"),
details: serde_json::Map::new(),
payload_extras: serde_json::Map::new(),
stats: Vec::new(),
},
on_wire: true,
});
}
}
results
}
fn ansi_supported() -> bool {
#[cfg(windows)]
{
crossterm::ansi_support::supports_ansi()
}
#[cfg(not(windows))]
{
true
}
}
fn setup_progress(
live_tty: bool,
source: SweepSource,
) -> (
Option<ProgressSender>,
Option<tokio::task::JoinHandle<Result<tui::TuiOutcome>>>,
) {
if !live_tty {
return (None, None);
}
let (tx, rx) = mpsc::unbounded_channel();
let handle = tokio::task::spawn_blocking(move || tui::run_tui(source, rx));
(Some(tx), Some(handle))
}
fn synthetic_sweep(results: Vec<CheckOutcome>) -> SweepResult {
let overall =
OverallResult::from_checks(&results.iter().map(|o| o.check.clone()).collect::<Vec<_>>());
SweepResult {
machine_id: None,
results,
overall,
payload: empty_payload(),
pg_version: None,
}
}
fn empty_payload() -> StatusPayload {
StatusPayload::builder().health(Vec::new()).build()
}
fn validate_check_selection(only: &[String], skip: &[String]) -> Result<()> {
let registry = checks::all();
validate_selection(®istry, only, "--check")?;
validate_selection(®istry, skip, "--skip")
}
fn emit_output(
args: &DoctorArgs,
sweep: &SweepResult,
source: &SweepSource,
use_colours: bool,
) -> Result<()> {
let stdout = std::io::stdout();
let mut out = stdout.lock();
if args.json {
let mut wrapped = serde_json::Map::new();
wrapped.insert(
"wire".into(),
serde_json::to_value(&sweep.payload).into_diagnostic()?,
);
match source {
SweepSource::Local => {
wrapped.insert("source".into(), Value::String("local".into()));
}
SweepSource::DaemonStreamed => {
wrapped.insert("source".into(), Value::String("daemon-streamed".into()));
}
SweepSource::DaemonCached { computed_at } => {
wrapped.insert("source".into(), Value::String("daemon-cached".into()));
wrapped.insert("computedAt".into(), Value::String(computed_at.to_string()));
}
}
serde_json::to_writer_pretty(&mut out, &Value::Object(wrapped)).into_diagnostic()?;
writeln!(out).into_diagnostic()?;
return Ok(());
}
let sorted = order::filter_and_sort(&sweep.results, true);
render::render_plain(&mut out, &sorted, args.all, sweep.overall, source, use_colours)
.into_diagnostic()?;
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn no_selection_flags_is_accepted() {
assert!(validate_check_selection(&[], &[]).is_ok());
}
#[test]
fn the_cli_accepts_a_qualified_name_the_sweep_accepts() {
assert!(validate_check_selection(&["postgres:connect".into()], &[]).is_ok());
assert!(validate_check_selection(&["machine:memory".into()], &[]).is_ok());
let err = validate_check_selection(&["connect".into()], &[]).unwrap_err();
assert!(format!("{err}").contains("postgres:connect"));
}
#[test]
fn skip_rejects_a_bare_name_on_the_same_terms_as_check() {
let err = validate_check_selection(&[], &["memory".into()]).unwrap_err();
let msg = format!("{err}");
assert!(msg.contains("--skip"), "{msg}");
assert!(msg.contains("machine:memory"), "{msg}");
}
#[test]
fn unknown_skip_is_error() {
let err = validate_check_selection(&[], &["does_not_exist".into()]).unwrap_err();
assert!(format!("{err}").contains("does_not_exist"));
}
#[test]
fn synthetic_sweep_marks_overall_from_results() {
let results = vec![CheckOutcome {
subject: Subject::Machine,
check: Check::fail("a", "bad", "r"),
on_wire: true,
}];
let sweep = synthetic_sweep(results);
assert_eq!(sweep.overall, OverallResult::Failing);
}
#[test]
fn doctor_args_all_short_flag() {
use clap::Parser;
let parsed = DoctorArgs::parse_from(["doctor", "-a"]);
assert!(parsed.all);
}
#[test]
fn doctor_args_all_long_flag() {
use clap::Parser;
let parsed = DoctorArgs::parse_from(["doctor", "--all"]);
assert!(parsed.all);
}
#[test]
fn doctor_args_default_filters_replay() {
use clap::Parser;
let parsed = DoctorArgs::parse_from(["doctor"]);
assert!(!parsed.all);
}
#[test]
fn results_from_wire_reconstructs_per_check_entries() {
let payload: StatusPayload = serde_json::from_value(serde_json::json!({
"health": [],
"machine": {
"detail": {},
"health": [
{ "check": "disk_free", "result": "passed" },
{ "check": "unknown_check_name", "result": "failed" },
],
},
}))
.unwrap();
let results = results_from_wire(&payload);
assert_eq!(results.len(), 1);
assert_eq!(results[0].check.name, "disk_free");
assert_eq!(results[0].subject, Subject::Machine);
assert!(matches!(results[0].check.status, CheckStatus::Pass));
}
#[test]
fn results_from_wire_keeps_summary_and_reason() {
let payload: StatusPayload = serde_json::from_value(serde_json::json!({
"health": [],
"machine": {
"detail": {},
"health": [
{
"check": "time_sync",
"result": "warning",
"summary": "clock drifting",
"reason": "offset 3s from NTP",
},
{ "check": "disk_free", "result": "passed", "summary": "42% free" },
],
},
}))
.unwrap();
let results = results_from_wire(&payload);
let by_name = |name: &str| results.iter().find(|o| o.check.name == name).unwrap();
let time_sync = &by_name("time_sync").check;
assert_eq!(time_sync.summary, "clock drifting");
assert!(
matches!(&time_sync.status, CheckStatus::Warning(r) if r == "offset 3s from NTP"),
"{:?}",
time_sync.status
);
assert_eq!(by_name("disk_free").check.summary, "42% free");
}
#[test]
fn results_from_wire_reads_both_grains() {
let payload: StatusPayload = serde_json::from_value(serde_json::json!({
"health": [],
"machine": {
"detail": {},
"health": [{ "check": "disk_free", "result": "passed" }],
},
"applications": {
"host-tamanu-central": {
"type": "tamanu-central",
"detail": {},
"health": [{ "check": "migrations", "result": "failed" }],
},
},
}))
.unwrap();
let results = results_from_wire(&payload);
assert_eq!(results.len(), 2);
let subject_of = |name: &str| {
results
.iter()
.find(|o| o.check.name == name)
.map(|o| o.subject.clone())
.unwrap()
};
assert_eq!(subject_of("disk_free"), Subject::Machine);
assert_eq!(
subject_of("migrations"),
Subject::Application(ApplicationRef::tamanu(ApplicationKind::TamanuCentral))
);
}
#[test]
fn a_pre_split_payload_is_read_rather_than_called_healthy() {
let payload: StatusPayload = serde_json::from_value(serde_json::json!({
"health": [
{ "check": "disk_free", "result": "failed" },
{ "check": "memory", "result": "passed" },
],
}))
.unwrap();
assert_eq!(
bestool_alertd::overall_from_payload(&payload),
OverallResult::Failing,
);
let results = results_from_wire(&payload);
assert_eq!(results.len(), 2);
assert!(results.iter().all(|o| o.subject == Subject::Machine));
}
#[test]
fn results_from_wire_empty_when_no_targets() {
let payload = empty_payload();
assert!(results_from_wire(&payload).is_empty());
}
}