use std::{sync::Arc, time::Duration};
use miette::{Result, miette};
use tokio::sync::{mpsc, oneshot};
use tracing::{error, info};
use crate::{
DaemonConfig, LogError, canopy::CanopyClient, context::InternalContext, http_server, metrics,
tasks::TaskContext,
};
enum DaemonEvent {
Shutdown,
Restart,
WatchdogTimeout,
}
#[derive(Clone)]
pub struct DaemonControl {
reload: Arc<tokio::sync::watch::Sender<u64>>,
events: mpsc::Sender<DaemonEvent>,
}
impl DaemonControl {
pub(crate) fn reload(&self) {
self.reload.send_modify(|n| *n = n.wrapping_add(1));
}
pub(crate) async fn request_restart(&self) {
let _ = self.events.send(DaemonEvent::Restart).await;
}
#[cfg(test)]
pub(crate) fn detached() -> Self {
let (reload, _) = tokio::sync::watch::channel(0);
let (events, _) = mpsc::channel(1);
Self {
reload: Arc::new(reload),
events,
}
}
}
#[derive(Clone, Debug)]
pub struct RestartTrigger {
events: mpsc::Sender<DaemonEvent>,
}
impl RestartTrigger {
pub async fn request_restart(&self) {
let _ = self.events.send(DaemonEvent::Restart).await;
}
}
pub async fn run(daemon_config: DaemonConfig) -> Result<()> {
let (_shutdown_tx, shutdown_rx) = oneshot::channel();
run_with_shutdown(daemon_config, shutdown_rx).await
}
pub async fn run_with_shutdown(
daemon_config: DaemonConfig,
external_shutdown: oneshot::Receiver<()>,
) -> Result<()> {
info!("starting alertd daemon");
metrics::init_metrics();
metrics::record_activity();
let pool = daemon_config.pg_pool.clone();
let canopy_client = match CanopyClient::new(
daemon_config.device_key_pem.as_ref().map(|r| r.0.as_str()),
crate::http_builder,
)
.await
{
Ok(Some(client)) => {
if client.is_tailscale().await {
info!("canopy client ready via tailscale");
} else {
info!("canopy client ready via mTLS");
}
let client = Arc::new(client);
let renew = client.clone();
tokio::spawn(async move {
let mut interval = tokio::time::interval(crate::canopy::CERT_RENEW_AFTER);
interval.tick().await; loop {
interval.tick().await;
if !renew.is_tailscale().await {
info!("renewing canopy mTLS certificate");
if let Err(err) = renew.renew().await {
error!("failed to renew canopy cert: {}", LogError(&err));
}
}
}
});
Some(client)
}
Ok(None) => {
info!(
"no canopy auth path available (no tailscale, no device key); canopy posting will be skipped"
);
None
}
Err(err) => {
error!("failed to build canopy client: {}", LogError(&err));
None
}
};
let (reload_tx, reload_rx) = tokio::sync::watch::channel(0u64);
let reload_tx = Arc::new(reload_tx);
let (event_tx, mut event_rx) = mpsc::channel(100);
let ctx = Arc::new(InternalContext {
pg_pool: pool,
http_client: crate::http_client(),
canopy_client,
reload: reload_rx,
restart: Some(RestartTrigger {
events: event_tx.clone(),
}),
});
let control = DaemonControl {
reload: reload_tx.clone(),
events: event_tx.clone(),
};
if !daemon_config.no_server {
let ctx_for_server = ctx.clone();
let background_tasks_for_server = daemon_config.background_tasks.clone();
let server_addrs = daemon_config.server_addrs.clone();
let watchdog_timeout = daemon_config.watchdog_timeout;
let backups = daemon_config.backups.clone();
tokio::spawn(async move {
http_server::start_server(
ctx_for_server,
server_addrs,
watchdog_timeout,
&background_tasks_for_server,
control,
backups,
)
.await;
});
}
let signal_tx = event_tx.clone();
tokio::spawn(async move {
match tokio::signal::ctrl_c().await {
Ok(()) => {
info!("received SIGINT, shutting down");
let _ = signal_tx.send(DaemonEvent::Shutdown).await;
}
Err(err) => {
error!("unable to listen for shutdown signal: {}", err);
}
}
});
let external_signal_tx = event_tx.clone();
tokio::spawn(async move {
let _ = external_shutdown.await;
info!("received external shutdown signal");
let _ = external_signal_tx.send(DaemonEvent::Shutdown).await;
});
#[cfg(unix)]
{
use tokio::signal::unix::{SignalKind, signal};
let signal_tx_term = event_tx.clone();
tokio::spawn(async move {
let mut sigterm =
signal(SignalKind::terminate()).expect("failed to setup SIGTERM handler");
sigterm.recv().await;
info!("received SIGTERM, shutting down");
let _ = signal_tx_term.send(DaemonEvent::Shutdown).await;
});
tokio::spawn(async move {
let mut sighup = signal(SignalKind::hangup()).expect("failed to setup SIGHUP handler");
let mut sigusr1 =
signal(SignalKind::user_defined1()).expect("failed to setup SIGUSR1 handler");
loop {
tokio::select! {
_ = sighup.recv() => {}
_ = sigusr1.recv() => {}
}
info!("received reload signal; refreshing");
let mut reloading = vec![sd_notify::NotifyState::Reloading];
if let Ok(stamp) = sd_notify::NotifyState::monotonic_usec_now() {
reloading.push(stamp);
}
let _ = sd_notify::notify(&reloading);
reload_tx.send_modify(|n| *n = n.wrapping_add(1));
let _ = sd_notify::notify(&[sd_notify::NotifyState::Ready]);
}
});
}
#[cfg(not(unix))]
let _ = reload_tx;
for task in &daemon_config.background_tasks {
let task = task.clone();
let task_ctx = TaskContext::from_internal(&ctx);
info!(name = task.name(), interval = ?task.interval(), "registering background task");
tokio::spawn(async move {
let mut tick = tokio::time::interval(task.interval());
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
loop {
tick.tick().await;
metrics::record_activity();
if let Err(err) = task.run(&task_ctx).await {
error!(
name = task.name(),
"background task failed: {}",
LogError(&err)
);
}
}
});
}
if let Some(watchdog_timeout) = daemon_config.watchdog_timeout {
let watchdog_tx = event_tx.clone();
tokio::spawn(async move {
let grace = watchdog_timeout.max(Duration::from_secs(60));
tokio::time::sleep(grace).await;
let mut check_interval = tokio::time::interval(Duration::from_secs(30));
check_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
loop {
check_interval.tick().await;
let last = metrics::last_activity_timestamp();
let now = jiff::Timestamp::now().as_second();
let elapsed = Duration::from_secs(now.saturating_sub(last) as u64);
if elapsed > watchdog_timeout {
error!(
?elapsed,
?watchdog_timeout,
"watchdog: no task activity detected within timeout, shutting down"
);
let _ = watchdog_tx.send(DaemonEvent::WatchdogTimeout).await;
break;
}
}
});
}
info!("daemon started successfully");
#[cfg(unix)]
{
let canopy = match &ctx.canopy_client {
Some(client) if client.is_tailscale().await => "canopy via tailscale",
Some(_) => "canopy via mTLS",
None => "canopy not connected",
};
let status = format!(
"monitoring; bestool {}; {canopy}",
daemon_config.binary_version
);
let _ = sd_notify::notify(&[
sd_notify::NotifyState::Ready,
sd_notify::NotifyState::Status(&status),
]);
}
let event = event_rx.recv().await;
#[cfg(unix)]
let _ = sd_notify::notify(&[sd_notify::NotifyState::Stopping]);
match event {
Some(DaemonEvent::Shutdown) | None => {
info!("daemon stopped");
Ok(())
}
Some(DaemonEvent::Restart) => {
info!("restart requested; exiting for the service manager to restart");
Err(miette!("restart requested"))
}
Some(DaemonEvent::WatchdogTimeout) => {
error!("daemon exiting due to watchdog timeout");
Err(miette!("watchdog timeout: no task activity detected"))
}
}
}