use std::{sync::Arc, time::Duration};
use miette::{Result, miette};
use tokio::sync::{mpsc, oneshot};
use tracing::{error, info};
use crate::{
DaemonConfig, LogError, canopy::CanopyClient, context::InternalContext, http_server, metrics,
tasks::TaskContext,
};
enum DaemonEvent {
Shutdown,
WatchdogTimeout,
}
pub async fn run(daemon_config: DaemonConfig) -> Result<()> {
let (_shutdown_tx, shutdown_rx) = oneshot::channel();
run_with_shutdown(daemon_config, shutdown_rx).await
}
pub async fn run_with_shutdown(
daemon_config: DaemonConfig,
external_shutdown: oneshot::Receiver<()>,
) -> Result<()> {
info!("starting alertd daemon");
metrics::init_metrics();
metrics::record_activity();
let pool = daemon_config.pg_pool.clone();
let canopy_client = match CanopyClient::new(
daemon_config.tamanu_version.clone(),
daemon_config.device_key_pem.as_ref().map(|r| r.0.as_str()),
crate::http_builder,
)
.await
{
Ok(Some(client)) => {
if client.is_tailscale().await {
info!("canopy client ready via tailscale");
} else {
info!("canopy client ready via mTLS");
}
let client = Arc::new(client);
let renew = client.clone();
tokio::spawn(async move {
let mut interval = tokio::time::interval(crate::canopy::CERT_RENEW_AFTER);
interval.tick().await; loop {
interval.tick().await;
if !renew.is_tailscale().await {
info!("renewing canopy mTLS certificate");
if let Err(err) = renew.renew().await {
error!("failed to renew canopy cert: {}", LogError(&err));
}
}
}
});
Some(client)
}
Ok(None) => {
info!(
"no canopy auth path available (no tailscale, no device key); canopy posting will be skipped"
);
None
}
Err(err) => {
error!("failed to build canopy client: {}", LogError(&err));
None
}
};
let ctx = Arc::new(InternalContext {
pg_pool: pool,
http_client: crate::http_client(),
canopy_client,
});
let (event_tx, mut event_rx) = mpsc::channel(100);
if !daemon_config.no_server {
let ctx_for_server = ctx.clone();
let background_tasks_for_server = daemon_config.background_tasks.clone();
let server_addrs = daemon_config.server_addrs.clone();
let watchdog_timeout = daemon_config.watchdog_timeout;
tokio::spawn(async move {
http_server::start_server(
ctx_for_server,
server_addrs,
watchdog_timeout,
&background_tasks_for_server,
)
.await;
});
}
let signal_tx = event_tx.clone();
tokio::spawn(async move {
match tokio::signal::ctrl_c().await {
Ok(()) => {
info!("received SIGINT, shutting down");
let _ = signal_tx.send(DaemonEvent::Shutdown).await;
}
Err(err) => {
error!("unable to listen for shutdown signal: {}", err);
}
}
});
let external_signal_tx = event_tx.clone();
tokio::spawn(async move {
let _ = external_shutdown.await;
info!("received external shutdown signal");
let _ = external_signal_tx.send(DaemonEvent::Shutdown).await;
});
#[cfg(unix)]
{
use tokio::signal::unix::{SignalKind, signal};
let signal_tx_term = event_tx.clone();
tokio::spawn(async move {
let mut sigterm =
signal(SignalKind::terminate()).expect("failed to setup SIGTERM handler");
sigterm.recv().await;
info!("received SIGTERM, shutting down");
let _ = signal_tx_term.send(DaemonEvent::Shutdown).await;
});
}
for task in &daemon_config.background_tasks {
let task = task.clone();
let task_ctx = TaskContext::from_internal(&ctx);
info!(name = task.name(), interval = ?task.interval(), "registering background task");
tokio::spawn(async move {
let mut tick = tokio::time::interval(task.interval());
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
loop {
tick.tick().await;
metrics::record_activity();
if let Err(err) = task.run(&task_ctx).await {
error!(
name = task.name(),
"background task failed: {}",
LogError(&err)
);
}
}
});
}
if let Some(watchdog_timeout) = daemon_config.watchdog_timeout {
let watchdog_tx = event_tx.clone();
tokio::spawn(async move {
let grace = watchdog_timeout.max(Duration::from_secs(60));
tokio::time::sleep(grace).await;
let mut check_interval = tokio::time::interval(Duration::from_secs(30));
check_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
loop {
check_interval.tick().await;
let last = metrics::last_activity_timestamp();
let now = jiff::Timestamp::now().as_second();
let elapsed = Duration::from_secs(now.saturating_sub(last) as u64);
if elapsed > watchdog_timeout {
error!(
?elapsed,
?watchdog_timeout,
"watchdog: no task activity detected within timeout, shutting down"
);
let _ = watchdog_tx.send(DaemonEvent::WatchdogTimeout).await;
break;
}
}
});
}
info!("daemon started successfully");
match event_rx.recv().await {
Some(DaemonEvent::Shutdown) | None => {
info!("daemon stopped");
Ok(())
}
Some(DaemonEvent::WatchdogTimeout) => {
error!("daemon exiting due to watchdog timeout");
Err(miette!("watchdog timeout: no task activity detected"))
}
}
}