use std::collections::BTreeSet;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use futures_util::StreamExt;
use tokio::sync::Notify;
use tokio::task::JoinHandle;
use tracing::{debug, warn};
use crate::docker::DockerApi;
use crate::sync::lock;
const MAX_NOTED: usize = 64;
const FIRST_BACKOFF: Duration = Duration::from_secs(1);
const MAX_BACKOFF: Duration = Duration::from_secs(30);
const SETTLE: Duration = Duration::from_secs(1);
const STEADY: Duration = Duration::from_mins(1);
#[derive(Debug, Default)]
pub(super) struct Exits {
noted: Mutex<Noted>,
wake: Notify,
}
#[derive(Debug, Default, PartialEq, Eq)]
pub(super) struct Noted {
pub(super) containers: BTreeSet<String>,
pub(super) any: bool,
}
impl Noted {
pub(super) fn is_empty(&self) -> bool {
!self.any && self.containers.is_empty()
}
}
impl Exits {
pub(super) fn stopped(&self, container: String) {
{
let mut noted = lock(&self.noted);
if noted.containers.len() < MAX_NOTED {
noted.containers.insert(container);
} else {
noted.any = true;
}
}
self.wake.notify_one();
}
pub(super) fn any(&self) {
lock(&self.noted).any = true;
self.wake.notify_one();
}
pub(super) fn noted_stopped(&self, container: &str) -> bool {
lock(&self.noted).containers.contains(container)
}
pub(super) fn inspecting_all(&self) {
lock(&self.noted).any = false;
}
pub(super) fn take(&self) -> Noted {
std::mem::take(&mut *lock(&self.noted))
}
pub(super) async fn noted(&self) {
self.wake.notified().await;
}
}
#[derive(Debug)]
pub(super) struct Watcher(JoinHandle<()>);
impl Watcher {
pub(super) fn start<D: DockerApi + 'static>(docker: Arc<D>, exits: Arc<Exits>) -> Watcher {
Watcher(tokio::spawn(follow(docker, exits)))
}
}
impl Drop for Watcher {
fn drop(&mut self) {
self.0.abort();
}
}
async fn follow<D: DockerApi>(docker: Arc<D>, exits: Arc<Exits>) {
let mut backoff = FIRST_BACKOFF;
let mut failing = false;
loop {
let followed = tokio::time::Instant::now();
let mut events = docker.exits();
let settle = tokio::time::sleep(SETTLE);
tokio::pin!(settle);
let mut settled = false;
let ended = loop {
tokio::select! {
() = &mut settle, if !settled => {
settled = true;
exits.any();
}
event = events.next() => match event {
Some(Ok(container)) => exits.stopped(container),
Some(Err(error)) => break error.to_string(),
None => break "the stream ended".to_owned(),
},
}
};
if followed.elapsed() >= STEADY {
backoff = FIRST_BACKOFF;
failing = false;
}
if failing {
debug!("cannot follow Docker's events still: {ended}");
} else {
warn!(
"cannot follow Docker's events ({ended}); runners that end are found at the next poll until they \
are followed again"
);
failing = true;
}
tokio::time::sleep(backoff).await;
backoff = (backoff * 2).min(MAX_BACKOFF);
}
}