use std::sync::{Arc, RwLock};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use tracing::{debug, error, info, warn};
use crate::docker::DockerClient;
use crate::model::Dashboard;
use crate::notify::Notifier;
use crate::store::Store;
use crate::trend::Bucketer;
pub type SharedDashboard = Arc<RwLock<Option<Dashboard>>>;
pub struct Config {
pub interval: Duration,
pub raw_retention: Duration,
pub trend_bucket_secs: u64,
pub trend_retention: Duration,
pub prune_interval: Duration,
}
pub async fn run(
host: String,
mut docker: DockerClient,
cpu_count: usize,
store: Store,
config: Config,
shared: SharedDashboard,
mut notifier: Option<Notifier>,
) {
info!(
host = %host,
interval = ?config.interval,
raw_retention = ?config.raw_retention,
trend_bucket_secs = config.trend_bucket_secs,
prune_interval = ?config.prune_interval,
"starting metrics collector"
);
let mut ticker = tokio::time::interval(config.interval);
let mut bucketer = Bucketer::new(config.trend_bucket_secs, host.clone());
let raw_retention_ms = duration_ms(config.raw_retention);
let trend_retention_ms = duration_ms(config.trend_retention);
let prune_every_cycles = prune_cadence(config.prune_interval, config.interval);
let mut cycle: u64 = 0;
let mut primed = false;
loop {
ticker.tick().await;
let containers = match docker.collect().await {
Ok(containers) => containers,
Err(err) => {
error!(%err, "collecting container metrics failed; keeping last snapshot");
continue;
}
};
if !primed {
primed = true;
debug!("priming cycle done; publishing starts next cycle");
continue;
}
let ts_ms = now_unix_ms();
if let Some(n) = notifier.as_mut() {
n.observe(ts_ms, &containers);
}
let flushed = bucketer.push(ts_ms, &containers);
if !flushed.is_empty() {
debug!(
container_trends = flushed.containers.len(),
"trend bucket closed"
);
}
let dashboard = Dashboard {
generated_at_unix_ms: ts_ms,
cpu_count,
containers,
};
publish(&shared, dashboard.clone());
let prune_cutoffs = cycle.is_multiple_of(prune_every_cycles).then(|| {
(
ts_ms.saturating_sub(raw_retention_ms),
ts_ms.saturating_sub(trend_retention_ms),
)
});
cycle += 1;
persist(
&store,
host.clone(),
ts_ms,
Arc::new(dashboard),
flushed,
prune_cutoffs,
)
.await;
}
}
pub(crate) fn publish(shared: &SharedDashboard, dashboard: Dashboard) {
match shared.write() {
Ok(mut guard) => *guard = Some(dashboard),
Err(poisoned) => *poisoned.into_inner() = Some(dashboard),
}
}
async fn persist(
store: &Store,
host: String,
ts_ms: u64,
dashboard: Arc<Dashboard>,
flushed: crate::trend::Flushed,
prune_cutoffs: Option<(u64, u64)>,
) {
let store = store.clone();
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> {
store.insert_samples(&host, ts_ms, &dashboard.containers)?;
store.insert_container_trends(&flushed.containers)?;
if let Some((raw_cutoff_ms, trend_cutoff_ms)) = prune_cutoffs {
let removed_raw = store.prune_raw(raw_cutoff_ms)?;
let removed_trends = store.prune_trends(trend_cutoff_ms)?;
debug!(%host, removed_raw, removed_trends, "pruned aged-out data");
}
Ok(())
})
.await;
match result {
Ok(Ok(())) => {}
Ok(Err(err)) => warn!(%err, "persisting metrics failed"),
Err(join_err) => warn!(%join_err, "persistence task panicked"),
}
}
fn now_unix_ms() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_or(0, |d| u64::try_from(d.as_millis()).unwrap_or(u64::MAX))
}
fn duration_ms(d: Duration) -> u64 {
u64::try_from(d.as_millis()).unwrap_or(u64::MAX)
}
fn prune_cadence(prune_interval: Duration, interval: Duration) -> u64 {
(prune_interval.as_secs() / interval.as_secs().max(1)).max(1)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn prune_cadence_batches_by_interval() {
let cad = |p, i| prune_cadence(Duration::from_secs(p), Duration::from_secs(i));
assert_eq!(cad(3600, 3), 1200);
assert_eq!(cad(3600, 1), 3600);
assert_eq!(cad(5, 5), 1);
assert_eq!(cad(1, 3), 1);
}
}