//! The watchdog keeping a load under its memory budget.
use std::sync::atomic::Ordering;
use std::sync::{Arc, Weak};
use crate::memory_budget::_helpers::watch::{process_held, run_stop_due, stop_largest, stop_run};
use crate::memory_budget::constants::{HOLD_PERCENT, STOP_PERCENT, WATCH_INTERVAL};
use crate::memory_budget::models::{MemoryBudget, MemoryGuard};
/// A guard over `budget`, read by a thread of its own every 10 ms until the guard is dropped: what the process and its MotherDuck bridges hold, against the budget. From 80% of it no table starts and pieces are cut smaller; from 90% the table holding the most large values is stopped, one at a time: its DuckDB statements are interrupted (rolling back) and it fails, naming the fix, while the other tables load on. From 95%, when no table holds large values to stop, or the stopped one didn't bring memory back under 90% within half a second, the whole run stops (see [`stop_run`]).
pub fn start_watchdog(budget: MemoryBudget) -> Arc<MemoryGuard> {
let guard = Arc::new(MemoryGuard {
budget,
tables: std::sync::Mutex::default(),
children: std::sync::Mutex::default(),
holding: std::sync::atomic::AtomicBool::new(false),
released: tokio::sync::Notify::new(),
peak: std::sync::atomic::AtomicU64::new(0),
run_stop: std::sync::Mutex::default(),
run_stopped: tokio_util::sync::CancellationToken::new(),
});
let watched = Arc::downgrade(&guard);
let started = std::thread::Builder::new()
.name("sqlload-memory".to_owned())
.spawn(move || watch(&watched));
if let Err(error) = started {
tracing::warn!(%error, "the memory watchdog could not start; the load runs without it");
}
guard
}
/// Read the memory every interval until the guard is gone. `over` counts the readings since memory reached 95% without coming back under 90%.
fn watch(watched: &Weak<MemoryGuard>) {
let mut over: u32 = 0;
loop {
std::thread::sleep(WATCH_INTERVAL);
let Some(guard) = watched.upgrade() else {
return;
};
let held = match process_held(&guard) {
Ok(held) => held,
Err(error) => {
tracing::warn!(%error, "the memory watchdog stopped: it can't read the process's memory");
return;
}
};
#[cfg(feature = "container-tests")]
let held = crate::memory_budget::_helpers::watch::forced_trip(&guard).unwrap_or(held);
guard.peak.fetch_max(held, Ordering::Relaxed);
let percent = u128::from(held) * 100 / u128::from(guard.budget.bytes.max(1));
let holding = percent >= u128::from(HOLD_PERCENT);
if guard.holding.swap(holding, Ordering::Relaxed) && !holding {
guard.released.notify_waiters();
}
let stopping = percent >= u128::from(STOP_PERCENT) && stop_largest(&guard, held);
let stops;
(over, stops) = run_stop_due(percent, over, stopping);
if stops {
stop_run(&guard, held, stopping);
}
}
}