use camber::runtime::{self, RuntimeBuilder};
use camber::runtime_test_support::{RuntimeCheckpoint, RuntimeController, runtime_schedule};
use camber::{Resource, RuntimeError};
use std::future::Future;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc::{Receiver, Sender, channel};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use super::http::{POLL_INTERVAL, poll_until, remaining};
const WORKER_THREADS: usize = 4;
pub const BOUND: Duration = Duration::from_secs(10);
pub const SHORT_DRAIN: Duration = Duration::from_millis(500);
pub const PERPETUAL: Duration = Duration::from_secs(3600);
pub fn block_on_detached<F: Future>(future: F) -> F::Output {
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap()
.block_on(future)
}
pub fn ignore_hook() -> Result<(), RuntimeError> {
Ok(())
}
pub struct RecordingResource<H, S> {
name: &'static str,
on_health: H,
on_shutdown: S,
}
impl<H, S> RecordingResource<H, S>
where
H: Fn() -> Result<(), RuntimeError> + Send + Sync + 'static,
S: Fn() -> Result<(), RuntimeError> + Send + Sync + 'static,
{
pub fn new(name: &'static str, on_health: H, on_shutdown: S) -> Self {
Self {
name,
on_health,
on_shutdown,
}
}
}
impl<H, S> Resource for RecordingResource<H, S>
where
H: Fn() -> Result<(), RuntimeError> + Send + Sync + 'static,
S: Fn() -> Result<(), RuntimeError> + Send + Sync + 'static,
{
fn name(&self) -> &str {
self.name
}
fn health_check(&self) -> Result<(), RuntimeError> {
(self.on_health)()
}
fn shutdown(&self) -> Result<(), RuntimeError> {
(self.on_shutdown)()
}
}
pub fn registry_len(controller: &RuntimeController) -> usize {
controller.scope_registry_len().unwrap_or(usize::MAX)
}
pub fn wait_registry_at_most(
controller: &RuntimeController,
remaining: usize,
bound: Duration,
) -> bool {
poll_until(bound, || registry_len(controller) <= remaining)
}
const FORCED_ABORT_MESSAGE: &str = "task channel closed";
pub fn assert_forced_abort<T: std::fmt::Debug>(outcome: &Result<T, RuntimeError>) {
assert!(
matches!(outcome, Err(RuntimeError::TaskPanicked(message)) if &**message == FORCED_ABORT_MESSAGE),
"the forced abort did not deliver the documented closed-channel result: {outcome:?}"
);
}
pub struct WedgedHandle<T>(Arc<Mutex<Option<T>>>);
impl<T> WedgedHandle<T> {
pub fn new() -> Self {
Self(Arc::new(Mutex::new(None)))
}
pub fn record(&self, handle: T) {
*self
.0
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(handle);
}
pub fn take(&self) -> T {
self.take_expecting("the closure never handed its wedged handle out")
}
pub fn take_expecting(&self, missing: &str) -> T {
match self
.0
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.take()
{
Some(value) => value,
None => panic!("{missing}"),
}
}
}
impl<T> Default for WedgedHandle<T> {
fn default() -> Self {
Self::new()
}
}
impl<T> Clone for WedgedHandle<T> {
fn clone(&self) -> Self {
Self(Arc::clone(&self.0))
}
}
pub async fn join_bounded<H>(handle: H, bound: Duration) -> H::Output
where
H: std::future::IntoFuture,
{
match tokio::time::timeout(bound, handle.into_future()).await {
Ok(output) => output,
Err(_) => panic!(
"{} did not resolve within {bound:?}",
std::any::type_name::<H>()
),
}
}
fn release_when_paused(
controller: &RuntimeController,
checkpoint: RuntimeCheckpoint,
bound: Duration,
) -> bool {
poll_until(bound, || controller.release(checkpoint).is_ok())
}
pub fn wait_paused_bounded(
controller: &RuntimeController,
checkpoint: RuntimeCheckpoint,
bound: Duration,
) -> bool {
poll_until(bound, || controller.is_paused(checkpoint))
}
struct ReleaseOnDrop<'a> {
controller: &'a RuntimeController,
checkpoint: RuntimeCheckpoint,
released: bool,
}
impl ReleaseOnDrop<'_> {
fn release(mut self, bound: Duration) -> bool {
let released = release_when_paused(self.controller, self.checkpoint, bound);
self.released = released;
released
}
}
impl Drop for ReleaseOnDrop<'_> {
fn drop(&mut self) {
match self.released {
true => {}
false => self.controller.disarm(),
}
}
}
pub fn probe_paused_window<P, T>(
controller: &RuntimeController,
checkpoint: RuntimeCheckpoint,
bound: Duration,
probe: P,
) -> Option<T>
where
P: FnOnce() -> T,
{
let deadline = Instant::now() + bound;
let guard = ReleaseOnDrop {
controller,
checkpoint,
released: false,
};
match wait_paused_bounded(controller, checkpoint, remaining(deadline)) {
false => None,
true => {
let observed = probe();
assert!(
guard.release(remaining(deadline)),
"the paused window was read but production could not be released from it"
);
Some(observed)
}
}
}
pub struct ArmedGate<'a> {
controller: &'a RuntimeController,
armed: Sender<()>,
}
impl ArmedGate<'_> {
pub fn arm(&self, checkpoint: RuntimeCheckpoint) {
self.controller
.pause_once(checkpoint)
.expect("the runtime schedule refused to arm the next checkpoint");
self.armed
.send(())
.expect("the observer thread stopped listening for the armed handshake");
}
pub fn controller(&self) -> &RuntimeController {
self.controller
}
}
pub struct ArmedWatch<'a> {
controller: &'a RuntimeController,
armed: Receiver<()>,
}
impl ArmedWatch<'_> {
pub fn wait_armed(&self) {
self.wait_armed_within(BOUND);
}
pub fn wait_armed_within(&self, bound: Duration) {
self.armed
.recv_timeout(bound)
.expect("the runtime closure never armed its next checkpoint");
}
pub fn probe<P, R>(&self, checkpoint: RuntimeCheckpoint, probe: P) -> Option<R>
where
P: FnOnce(&RuntimeController) -> R,
{
probe_paused_window(self.controller, checkpoint, BOUND, || {
probe(self.controller)
})
}
pub fn controller(&self) -> &RuntimeController {
self.controller
}
}
struct LowerOnDrop<'a>(&'a AtomicBool);
impl Drop for LowerOnDrop<'_> {
fn drop(&mut self) {
self.0.store(false, Ordering::Release);
}
}
fn abandon_until_run_returns(controller: &RuntimeController, in_progress: &AtomicBool) {
while in_progress.load(Ordering::Acquire) {
controller.disarm();
std::thread::sleep(POLL_INTERVAL);
}
}
pub struct Abandon<'a> {
controller: &'a RuntimeController,
in_progress: &'a AtomicBool,
on_abandon: &'a (dyn Fn() + Sync),
}
impl Abandon<'_> {
fn run(&self) {
(self.on_abandon)();
abandon_until_run_returns(self.controller, self.in_progress);
}
}
fn observe_while_running<O, R, T>(
controller: &RuntimeController,
on_abandon: &(dyn Fn() + Sync),
observe: O,
run: impl FnOnce() -> Result<T, RuntimeError>,
) -> (Result<T, RuntimeError>, R)
where
O: FnOnce(&Abandon<'_>) -> R + Send,
R: Send,
{
let run_in_progress = AtomicBool::new(true);
let (result, observed) = std::thread::scope(|scope| {
let abandon = Abandon {
controller,
in_progress: &run_in_progress,
on_abandon,
};
let observer = scope.spawn(move || {
let observed =
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| observe(&abandon)));
if observed.is_err() {
abandon.run();
}
observed
});
let lowered = LowerOnDrop(&run_in_progress);
let result = run();
drop(lowered);
(result, observer.join().unwrap())
});
match observed {
Ok(observed) => (result, observed),
Err(payload) => std::panic::resume_unwind(payload),
}
}
pub fn observe_armed_sequence<C, B, O, T, R>(
configure: C,
body: B,
observe: O,
) -> (Result<T, RuntimeError>, R)
where
C: FnOnce(RuntimeBuilder) -> RuntimeBuilder,
B: FnOnce(&ArmedGate<'_>) -> T,
O: FnOnce(&ArmedWatch<'_>) -> R + Send,
R: Send,
{
let controller = runtime_schedule();
let (armed_tx, armed_rx) = channel::<()>();
let builder = configure(runtime::builder().worker_threads(WORKER_THREADS))
.with_test_schedule(&controller);
let watch = ArmedWatch {
controller: &controller,
armed: armed_rx,
};
let gate = ArmedGate {
controller: &controller,
armed: armed_tx,
};
observe_while_running(
&controller,
&|| {},
move |_abandon| observe(&watch),
|| builder.run(|| body(&gate)),
)
}
pub fn observe_armed_window<C, B, P, T, R>(
configure: C,
checkpoint: RuntimeCheckpoint,
body: B,
probe: P,
) -> (Result<T, RuntimeError>, Option<R>)
where
C: FnOnce(RuntimeBuilder) -> RuntimeBuilder,
B: FnOnce(&RuntimeController) -> T,
P: FnOnce(&RuntimeController) -> R + Send,
R: Send,
{
observe_armed_sequence(
configure,
|gate| {
let value = body(gate.controller());
gate.arm(checkpoint);
value
},
|watch| {
watch.wait_armed();
watch.probe(checkpoint, probe)
},
)
}
#[cfg(unix)]
pub const RUNTIME_OWNED_CHILDREN: usize = 1;
#[cfg(not(unix))]
pub const RUNTIME_OWNED_CHILDREN: usize = 0;
const DRAIN_ESCALATION: Duration = Duration::from_secs(1);
pub struct ScopeOwnedProof {
pub occupants: usize,
pub entries_at_drain: Option<usize>,
pub drained: Result<(), RuntimeError>,
}
impl ScopeOwnedProof {
pub fn assert_owned(&self, subsystem: &str, expected_occupants: usize) {
match &self.drained {
Ok(()) => {}
Err(error) => {
panic!("{subsystem} did not exit on ScopeClosing: the drain escalated ({error})")
}
}
assert_eq!(
self.occupants, expected_occupants,
"{subsystem} was not a root-scope child while it ran"
);
match self.entries_at_drain {
None => panic!("{subsystem}: the drain never paused at its holder-only window"),
Some(entries) => assert_eq!(
entries, 1,
"{subsystem} was still registered when only the holder should remain"
),
}
}
}
fn scope_owned_builder<C>(configure: C, controller: &RuntimeController) -> RuntimeBuilder
where
C: FnOnce(RuntimeBuilder) -> RuntimeBuilder,
{
configure(
runtime::builder()
.worker_threads(WORKER_THREADS)
.shutdown_timeout(DRAIN_ESCALATION),
)
.with_test_schedule(controller)
}
fn reported_scope_owned<T>(
reported_rx: Receiver<(usize, T)>,
bound: Duration,
drained: Result<(), RuntimeError>,
entries_at_drain: Option<usize>,
) -> (ScopeOwnedProof, T) {
let (occupants, value) = match reported_rx.recv_timeout(bound) {
Ok(reported) => reported,
Err(_) => panic!("the runtime closure never reported its occupancy: {drained:?}"),
};
(
ScopeOwnedProof {
occupants,
entries_at_drain,
drained,
},
value,
)
}
pub fn prove_scope_owned<C, B, T>(bound: Duration, configure: C, body: B) -> (ScopeOwnedProof, T)
where
C: FnOnce(RuntimeBuilder) -> RuntimeBuilder,
B: FnOnce(Arc<RuntimeController>) -> T,
{
let controller = Arc::new(runtime_schedule());
let holder_only = RuntimeCheckpoint::ScopeWaitObserved(1);
let observer_controller = Arc::clone(&controller);
let closure_controller = Arc::clone(&controller);
let body_controller = Arc::clone(&controller);
let hold = Arc::new(tokio::sync::Notify::new());
let observer_hold = Arc::clone(&hold);
let abandon_hold = Arc::clone(&hold);
let closure_hold = Arc::clone(&hold);
let (armed_tx, armed_rx) = channel::<()>();
let (reported_tx, reported_rx) = channel::<(usize, T)>();
let builder = scope_owned_builder(configure, &controller);
let (drained, entries_at_drain) = observe_while_running(
&controller,
&move || abandon_hold.notify_one(),
move |abandon| {
let deadline = Instant::now() + bound;
let entries = match armed_rx.recv_timeout(remaining(deadline)) {
Err(_) => None,
Ok(()) => probe_paused_window(
&observer_controller,
holder_only,
remaining(deadline),
|| {
let entries = registry_len(&observer_controller);
observer_hold.notify_one();
entries
},
),
};
match entries {
Some(_) => {}
None => abandon.run(),
}
entries
},
|| {
builder.run(move || {
camber::spawn_async(async move { closure_hold.notified().await });
let value = body(body_controller);
let occupants = closure_controller
.scope_registry_len()
.expect("the runtime schedule could not read the root scope registry");
reported_tx
.send((occupants, value))
.expect("the caller stopped listening for the closure's occupancy");
closure_controller
.pause_once(holder_only)
.expect("the runtime schedule refused to arm the holder-only window");
armed_tx
.send(())
.expect("the observer thread stopped listening for the armed handshake");
})
},
);
reported_scope_owned(reported_rx, bound, drained, entries_at_drain)
}