use std::collections::BTreeMap;
use std::time::Duration;
use tokio::time::Instant;
use super::Occupant;
use crate::scheduler::SlotRef;
const UNREAD_BACKOFF_MAX: Duration = Duration::from_secs(600);
fn unread_backoff(poll_interval: Duration, undecided: u32) -> Duration {
poll_interval
.saturating_mul(1 << undecided.saturating_sub(1).min(16))
.min(UNREAD_BACKOFF_MAX)
}
#[derive(Debug)]
pub(super) struct Unread {
pub(super) slot: SlotRef,
pub(super) occupant: Occupant,
undecided: u32,
}
impl Unread {
pub(super) fn new(slot: SlotRef, occupant: Occupant) -> Unread {
Unread {
slot,
occupant,
undecided: 0,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(super) struct ToRemove {
pub(super) id: String,
pub(super) name: String,
}
#[derive(Debug)]
enum Left {
Unread { unread: Box<Unread>, due: Instant },
Removing { name: String },
}
#[derive(Debug, Default)]
pub(super) struct Leftovers(BTreeMap<String, Left>);
impl Leftovers {
pub(super) fn removing(&mut self, id: String, name: String) {
self.0.insert(id, Left::Removing { name });
}
pub(super) fn undecided(&mut self, mut unread: Unread, poll_interval: Duration) -> Duration {
unread.undecided += 1;
let wait = unread_backoff(poll_interval, unread.undecided);
self.0.insert(
unread.occupant.container.clone(),
Left::Unread {
unread: Box::new(unread),
due: Instant::now() + wait,
},
);
wait
}
pub(super) fn take_due(&mut self, now: Instant) -> Vec<Unread> {
let due: Vec<String> = self
.0
.iter()
.filter(|(_, left)| matches!(left, Left::Unread { due, .. } if *due <= now))
.map(|(id, _)| id.clone())
.collect();
due.iter()
.filter_map(|id| match self.0.remove(id) {
Some(Left::Unread { unread, .. }) => Some(*unread),
_ => None,
})
.collect()
}
pub(super) fn to_remove(&self) -> Vec<ToRemove> {
self.0
.iter()
.filter_map(|(id, left)| match left {
Left::Removing { name } => Some(ToRemove {
id: id.clone(),
name: name.clone(),
}),
Left::Unread { .. } => None,
})
.collect()
}
pub(super) fn removed(&mut self, id: &str) {
self.0.remove(id);
}
}
#[cfg(test)]
impl Leftovers {
pub(super) fn is_removing(&self, id: &str) -> bool {
matches!(self.0.get(id), Some(Left::Removing { .. }))
}
pub(super) fn unread(&self) -> usize {
self.0
.values()
.filter(|left| matches!(left, Left::Unread { .. }))
.count()
}
pub(super) fn make_due(&mut self) {
let now = Instant::now();
for left in self.0.values_mut() {
if let Left::Unread { due, .. } = left {
*due = now;
}
}
}
}
#[cfg(test)]
mod tests {
use std::time::SystemTime;
use super::super::Engagement;
use super::*;
const POLL: Duration = Duration::from_secs(30);
fn unread(id: &str) -> Unread {
let occupant = Occupant {
repository: "o/app".to_owned(),
runner_name: format!("runner-{id}"),
runner_id: 1,
container: id.to_owned(),
started_at: SystemTime::UNIX_EPOCH,
memory: 0,
cpus: 0,
admission: None,
labels: None,
image: None,
spec: None,
engagement: Engagement::adopted(None, None),
};
let slot = SlotRef {
profile: "small".to_owned(),
index: 0,
};
Unread::new(slot, occupant)
}
#[test]
fn a_container_is_to_read_or_to_remove_never_both() {
let mut leftovers = Leftovers::default();
leftovers.undecided(unread("a"), POLL);
leftovers.removing("a".to_owned(), "runner-a".to_owned());
assert_eq!(leftovers.unread(), 0, "removing ends the wait to read it");
assert!(leftovers.is_removing("a"), "a is to remove");
assert_eq!(leftovers.to_remove().len(), 1, "and listed once");
}
#[tokio::test(start_paused = true)]
async fn each_time_docker_cannot_tell_the_wait_doubles_up_to_ten_minutes() {
let mut leftovers = Leftovers::default();
let mut waits = Vec::new();
let mut left = unread("a");
for _ in 0..7 {
waits.push(leftovers.undecided(left, POLL).as_secs());
leftovers.make_due();
left = leftovers
.take_due(Instant::now())
.pop()
.expect("just made due");
}
assert_eq!(waits, [30, 60, 120, 240, 480, 600, 600]);
}
#[tokio::test(start_paused = true)]
async fn only_those_whose_backoff_is_over_are_taken_and_what_stays_is_not_to_remove() {
let mut leftovers = Leftovers::default();
leftovers.undecided(unread("soon"), POLL);
tokio::time::advance(Duration::from_secs(1)).await;
leftovers.undecided(unread("later"), POLL);
leftovers.removing("gone".to_owned(), "runner-gone".to_owned());
let at_its_end = Instant::now() + POLL - Duration::from_secs(1);
let taken = leftovers.take_due(at_its_end);
assert_eq!(taken.len(), 1, "the backoff of the later one is not over");
assert_eq!(taken[0].occupant.container, "soon", "the earlier is");
assert_eq!(leftovers.unread(), 1, "the later one stays to read");
assert_eq!(
leftovers.to_remove(),
[ToRemove {
id: "gone".to_owned(),
name: "runner-gone".to_owned()
}],
"only what is to remove is listed so"
);
}
#[tokio::test(start_paused = true)]
async fn a_taken_container_keeps_its_count_of_failures() {
let mut leftovers = Leftovers::default();
leftovers.undecided(unread("a"), POLL);
leftovers.make_due();
let taken = leftovers.take_due(Instant::now()).pop().expect("due");
assert_eq!(leftovers.undecided(taken, POLL).as_secs(), 60, "its second");
}
#[test]
fn a_removed_container_is_forgotten_and_an_unknown_one_changes_nothing() {
let mut leftovers = Leftovers::default();
assert!(leftovers.to_remove().is_empty(), "none yet");
leftovers.removing("a".to_owned(), "runner-a".to_owned());
leftovers.removed("unknown");
assert!(leftovers.is_removing("a"), "another's removal leaves it");
leftovers.removed("a");
assert!(!leftovers.is_removing("a"), "a is gone");
}
}