use std::collections::{BTreeMap, HashMap};
use std::sync::Mutex;
use std::time::Duration;
use tokio::time::Instant;
use crate::api::RerunRefused;
use crate::sync::lock;
pub(super) const FOLLOWED: Duration = Duration::from_hours(6);
pub(super) const TRIES_MAX: u32 = 5;
pub(super) const HELD: Duration = Duration::from_hours(1);
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub(super) struct RunKey {
pub(super) repository: String,
pub(super) run: u64,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(super) struct Due {
pub(super) key: RunKey,
pub(super) attempt: u32,
pub(super) job: u64,
}
#[derive(Debug, Clone, Copy)]
enum Stage {
Due {
at: Instant,
failed: u32,
firm: bool,
},
Settled,
}
#[derive(Debug, Clone, Copy)]
struct Fault {
attempt: u32,
job: u64,
seen: Instant,
stage: Stage,
}
#[derive(Debug, Clone)]
struct Refusal {
revision: u64,
until: Instant,
refused: RerunRefused,
}
#[derive(Debug, Clone)]
pub(super) struct Held {
pub(super) until: Instant,
pub(super) refused: RerunRefused,
}
#[derive(Default)]
struct Book {
runs: HashMap<RunKey, Fault>,
refusals: BTreeMap<String, Refusal>,
cooldown: Option<Instant>,
}
#[derive(Default)]
pub(super) struct Faults {
book: Mutex<Book>,
}
impl Faults {
pub(super) fn seen(&self, key: RunKey, attempt: u32, job: u64, now: Instant) {
lock(&self.book).seen(key, attempt, job, now);
}
pub(super) fn due(&self, now: Instant) -> Vec<Due> {
lock(&self.book).due(now)
}
pub(super) fn next(&self) -> Option<Instant> {
lock(&self.book).next()
}
pub(super) fn settle(&self, due: &Due) {
lock(&self.book).settle(due);
}
pub(super) fn wait(&self, due: &Due, at: Instant) {
lock(&self.book).wait(due, at);
}
pub(super) fn failed(&self, due: &Due, at: Instant) -> bool {
lock(&self.book).failed(due, at)
}
pub(super) fn hold(&self, due: &Due, at: Instant) {
lock(&self.book).hold(due, at);
}
pub(super) fn cool(&self, until: Instant) {
lock(&self.book).cool(until);
}
pub(super) fn cooling(&self, now: Instant) -> Option<Instant> {
lock(&self.book).cooling(now)
}
pub(super) fn refuse(
&self,
repository: &str,
revision: u64,
refused: RerunRefused,
now: Instant,
) -> Instant {
lock(&self.book).refuse(repository, revision, refused, now)
}
pub(super) fn held(&self, repository: &str, revision: u64, now: Instant) -> Option<Held> {
lock(&self.book).held(repository, revision, now)
}
}
impl Book {
fn seen(&mut self, key: RunKey, attempt: u32, job: u64, now: Instant) {
if let Some(fault) = self.runs.get_mut(&key)
&& fault.attempt >= attempt
{
if fault.attempt == attempt
&& let Stage::Due {
at,
failed,
firm: false,
} = fault.stage
{
fault.stage = Stage::Due {
at: at.min(now),
failed,
firm: false,
};
}
return;
}
self.runs.insert(
key,
Fault {
attempt,
job,
seen: now,
stage: Stage::Due {
at: now,
failed: 0,
firm: false,
},
},
);
}
fn due(&mut self, now: Instant) -> Vec<Due> {
self.runs
.retain(|_, fault| now.saturating_duration_since(fault.seen) < FOLLOWED);
let mut due: Vec<Due> = self
.runs
.iter()
.filter(|(_, fault)| matches!(fault.stage, Stage::Due { at, .. } if at <= now))
.map(|(key, fault)| Due {
key: key.clone(),
attempt: fault.attempt,
job: fault.job,
})
.collect();
due.sort_by(|a, b| a.key.cmp(&b.key));
due
}
fn next(&self) -> Option<Instant> {
self.runs
.values()
.filter_map(|fault| match fault.stage {
Stage::Due { at, .. } => Some(at),
Stage::Settled => None,
})
.min()
}
fn settle(&mut self, due: &Due) {
self.with(due, |fault| fault.stage = Stage::Settled);
}
fn wait(&mut self, due: &Due, at: Instant) {
self.with(due, |fault| {
if let Stage::Due { failed, .. } = fault.stage {
fault.stage = Stage::Due {
at,
failed,
firm: false,
};
}
});
}
fn hold(&mut self, due: &Due, at: Instant) {
self.with(due, |fault| {
if let Stage::Due { failed, .. } = fault.stage {
fault.stage = Stage::Due {
at,
failed,
firm: true,
};
}
});
}
fn cool(&mut self, until: Instant) {
self.cooldown = Some(self.cooldown.map_or(until, |was| was.max(until)));
}
fn cooling(&mut self, now: Instant) -> Option<Instant> {
match self.cooldown {
Some(until) if now < until => Some(until),
_ => {
self.cooldown = None;
None
}
}
}
fn failed(&mut self, due: &Due, at: Instant) -> bool {
let mut again = false;
self.with(due, |fault| {
if let Stage::Due { failed, .. } = fault.stage {
let failed = failed + 1;
again = failed < TRIES_MAX;
fault.stage = if again {
Stage::Due {
at,
failed,
firm: true,
}
} else {
Stage::Settled
};
}
});
again
}
fn refuse(
&mut self,
repository: &str,
revision: u64,
refused: RerunRefused,
now: Instant,
) -> Instant {
let until = now + HELD;
self.refusals.insert(
repository.to_owned(),
Refusal {
revision,
until,
refused,
},
);
until
}
fn held(&mut self, repository: &str, revision: u64, now: Instant) -> Option<Held> {
let refusal = self.refusals.get(repository)?;
if refusal.revision == revision && now < refusal.until {
return Some(Held {
until: refusal.until,
refused: refusal.refused.clone(),
});
}
self.refusals.remove(repository);
None
}
fn with(&mut self, due: &Due, change: impl FnOnce(&mut Fault)) {
if let Some(fault) = self
.runs
.get_mut(&due.key)
.filter(|fault| fault.attempt == due.attempt)
{
change(fault);
}
}
}