use std::sync::atomic::AtomicBool;
use std::sync::atomic::AtomicUsize;
use std::sync::atomic::Ordering;
use std::time::Duration;
use std::time::Instant;
use crate::io::fill_control::FillController;
use crate::io::fill_control::IoObservation;
const SLOW_IO_CHECK_CAP: Duration = Duration::from_millis(500);
pub struct IoRecovery {
timeout: Option<Duration>,
pending: AtomicUsize,
stopped: AtomicBool,
}
impl IoRecovery {
pub const fn new(timeout: Option<Duration>) -> Self {
Self {
timeout,
pending: AtomicUsize::new(0),
stopped: AtomicBool::new(false),
}
}
pub fn is_recovering(&self) -> bool {
self.pending.load(Ordering::Acquire) != 0
}
pub fn stop(&self) {
self.stopped.store(true, Ordering::Release);
}
fn next_deadline(&self, original: Instant) -> Option<Instant> {
if self.stopped.load(Ordering::Acquire) {
return None;
}
let now = Instant::now();
let deadline = now.checked_add(Duration::from_secs(1))?;
let deadline = match self.timeout {
Some(timeout) => {
let end = original.checked_add(timeout)?;
if now >= end {
return None;
}
deadline.min(end)
}
None => deadline,
};
Some(deadline)
}
}
pub struct BackgroundIoAttempt<'a> {
io_recovery: &'a IoRecovery,
fill_control: Option<&'a FillController>,
recovering: bool,
slow: bool,
started: Option<Instant>,
io_timeout: Duration,
observation: Option<IoObservation<'a>>,
}
impl<'a> BackgroundIoAttempt<'a> {
pub const fn new(
io_recovery: &'a IoRecovery,
fill_control: Option<&'a FillController>,
) -> Self {
Self {
io_recovery,
fill_control,
recovering: false,
slow: false,
started: None,
io_timeout: Duration::ZERO,
observation: None,
}
}
pub fn start(&mut self, bytes: u64, timeout: Duration) {
self.started = Some(Instant::now());
self.io_timeout = timeout;
if let Some(fill) = self.fill_control {
self.observation = fill.observe_io(bytes);
}
}
pub fn wait_deadline(&self, original: Instant) -> Instant {
if self.fill_control.is_none() || self.slow {
return original;
}
let Some(start) = self.started else {
return original;
};
(start + (self.io_timeout / 4).min(SLOW_IO_CHECK_CAP)).min(original)
}
pub fn note_slow(&mut self) {
if self.slow {
return;
}
self.slow = true;
if let Some(observation) = &mut self.observation {
observation.note_slow();
}
}
pub fn clear_slow(&mut self) {
if !self.slow {
return;
}
self.slow = false;
if let Some(observation) = &mut self.observation {
observation.clear_slow();
}
}
pub fn next_deadline(&mut self, original: Instant) -> Option<Instant> {
let deadline = self.io_recovery.next_deadline(original)?;
if !self.recovering {
self.recovering = true;
if let Some(fill) = self.fill_control {
fill.hold_pause();
}
if self.io_recovery.pending.fetch_add(1, Ordering::AcqRel) == 0 {
log::warn!(target: "cache2::health", event = "cache_io_recovery_started";
"background I/O timed out; pausing cache fills while retaining owned requests");
}
}
Some(deadline)
}
pub fn finish(mut self) {
if let Some(observation) = self.observation.take() {
observation.finish();
}
if self.recovering {
let last = self.io_recovery.pending.fetch_sub(1, Ordering::AcqRel) == 1;
if let Some(fill) = self.fill_control {
fill.release_pause();
}
if last {
log::info!(target: "cache2::health", event = "cache_io_recovery_completed";
"all timed-out background operations recovered and passed validation");
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::FillControlOptions;
use crate::FillLimits;
use crate::FillPressure;
#[test]
fn failed_publication_keeps_recovery_fenced_after_io_observation_ends() {
let fill = FillController::new(
FillControlOptions::Adaptive(FillLimits::new(64_000, 100)),
2,
4096,
)
.unwrap()
.unwrap();
let recovery = IoRecovery::new(None);
let mut failed = BackgroundIoAttempt::new(&recovery, Some(&fill));
failed.start(4096, Duration::from_secs(1));
failed.note_slow();
assert_eq!(fill.snapshot().pressure, FillPressure::Paused);
assert!(!recovery.is_recovering());
assert!(failed.next_deadline(Instant::now()).is_some());
failed.clear_slow();
drop(failed);
assert_eq!(fill.snapshot().outstanding_operations, 0);
assert!(recovery.is_recovering());
assert!(!fill.try_admit_fill());
let mut successful = BackgroundIoAttempt::new(&recovery, Some(&fill));
successful.start(4096, Duration::from_secs(1));
assert!(successful.next_deadline(Instant::now()).is_some());
successful.finish();
assert!(recovery.is_recovering());
assert!(!fill.try_admit_fill());
}
}