footing 0.5.1

Feature-gated foundations for Rust backend services
Documentation
use std::{sync::Arc, time::Duration};

use async_trait::async_trait;
use tokio::sync::{Mutex, mpsc, watch};

use crate::clock::{Clock, ManualClock};

use super::*;

type TickResult = Result<WorkerTick, ()>;

struct ControlledWorker(Mutex<mpsc::UnboundedReceiver<TickResult>>);

#[async_trait]
impl Worker for ControlledWorker {
    fn name(&self) -> &str {
        "observed"
    }

    async fn run_once(&self, _context: WorkerContext) -> Result<WorkerTick, BoxWorkerError> {
        self.0
            .lock()
            .await
            .recv()
            .await
            .expect("the test must provide a tick result")
            .map_err(|()| std::io::Error::other("private tick failure").into())
    }
}

fn controlled() -> (ControlledWorker, mpsc::UnboundedSender<TickResult>) {
    let (sender, receiver) = mpsc::unbounded_channel();
    (ControlledWorker(Mutex::new(receiver)), sender)
}

async fn phase(
    receiver: &mut watch::Receiver<WorkerStatus>,
    expected: WorkerPhase,
) -> WorkerStatus {
    loop {
        let status = *receiver.borrow_and_update();
        if status.phase() == expected {
            return status;
        }
        receiver
            .changed()
            .await
            .expect("runner stopped before the expected phase");
    }
}

async fn advance(clock: &ManualClock, duration: Duration) {
    clock.advance(duration).unwrap();
    tokio::time::advance(duration).await;
}

#[tokio::test(start_paused = true)]
async fn status_tracks_running_and_waiting_until_graceful_shutdown() {
    let clock = ManualClock::new();
    let started_at = clock.now();
    let wakeup = WorkerWakeup::new();
    let mut options = WorkerRunOptions::new()
        .with_clock(clock.shared())
        .with_wakeup(wakeup.clone());
    let mut status = options.subscribe_status();
    let second_reader = options.subscribe_status();
    assert_eq!(status.borrow().phase(), WorkerPhase::Pending);
    let config = WorkerSettings::default()
        .with_startup_delay(Duration::from_secs(2))
        .with_poll_interval(Duration::from_secs(10))
        .with_min_queue_interval(Duration::from_secs(1))
        .build()
        .unwrap();
    let (trigger, shutdown) = shutdown_channel();
    let (worker, commands) = controlled();
    let task = tokio::spawn(run_worker_until_shutdown_with_options(
        worker, config, shutdown, options,
    ));

    assert_eq!(
        phase(&mut status, WorkerPhase::StartupDelay)
            .await
            .phase_since(),
        started_at
    );
    advance(&clock, Duration::from_secs(2)).await;
    let running = phase(&mut status, WorkerPhase::Running).await;
    assert_eq!(running.last_tick_started_at(), Some(clock.now()));
    assert_eq!(running.last_tick_finished_at(), None);
    assert_eq!(running.last_tick_outcome(), None);

    advance(&clock, Duration::from_secs(3)).await;
    commands.send(Ok(WorkerTick::new(1))).unwrap();
    let waiting = phase(&mut status, WorkerPhase::Poll).await;
    assert_eq!(
        waiting.last_tick_started_at(),
        running.last_tick_started_at()
    );
    assert_eq!(waiting.last_tick_finished_at(), Some(clock.now()));
    assert_eq!(
        waiting.last_tick_outcome(),
        Some(WorkerTickOutcome::Success)
    );
    assert_eq!(waiting.phase_since(), clock.now());

    advance(&clock, Duration::from_secs(10)).await;
    let running = phase(&mut status, WorkerPhase::Running).await;
    assert_eq!(running.last_tick_started_at(), Some(clock.now()));
    assert_eq!(
        running.last_tick_finished_at(),
        waiting.last_tick_finished_at()
    );
    commands
        .send(Ok(WorkerTick::new(0).with_schedule(
            WorkerSchedule::WaitForWork {
                recheck_after: None,
            },
        )))
        .unwrap();
    phase(&mut status, WorkerPhase::QueueIdle).await;
    wakeup.notify();
    phase(&mut status, WorkerPhase::QueueSpacing).await;
    advance(&clock, Duration::from_secs(1)).await;
    phase(&mut status, WorkerPhase::Running).await;

    commands
        .send(Ok(
            WorkerTick::new(0).with_schedule(WorkerSchedule::After(Duration::from_secs(4)))
        ))
        .unwrap();
    phase(&mut status, WorkerPhase::Delay).await;
    advance(&clock, Duration::from_secs(4)).await;
    phase(&mut status, WorkerPhase::Running).await;
    commands
        .send(Ok(
            WorkerTick::new(0).with_schedule(WorkerSchedule::Continue)
        ))
        .unwrap();
    phase(&mut status, WorkerPhase::QueueSpacing).await;
    advance(&clock, Duration::from_secs(1)).await;
    phase(&mut status, WorkerPhase::Running).await;

    trigger.shutdown();
    phase(&mut status, WorkerPhase::Shutdown).await;
    commands.send(Ok(WorkerTick::new(0))).unwrap();
    assert_eq!(task.await.unwrap().unwrap().ticks(), 5);
    let stopped = phase(&mut status, WorkerPhase::Stopped).await;
    assert_eq!(stopped.last_tick_finished_at(), Some(clock.now()));
    assert_eq!(*second_reader.borrow(), stopped);
    assert!(status.changed().await.is_err());
}

#[tokio::test(start_paused = true)]
async fn status_distinguishes_failure_backoff_and_external_cancellation() {
    let clock = ManualClock::new();
    let wakeup = WorkerWakeup::new();
    let mut options = WorkerRunOptions::new()
        .with_clock(clock.shared())
        .with_wakeup(wakeup.clone());
    let mut status = options.subscribe_status();
    let config = WorkerSettings::default()
        .with_failure_backoff(BackoffSettings::new(
            Duration::from_secs(5),
            Duration::from_secs(5),
            1,
        ))
        .build()
        .unwrap();
    let (_trigger, shutdown) = shutdown_channel();
    let (worker, commands) = controlled();
    let task = tokio::spawn(run_worker_until_shutdown_with_options(
        worker, config, shutdown, options,
    ));
    phase(&mut status, WorkerPhase::Running).await;
    commands.send(Err(())).unwrap();
    let backoff = phase(&mut status, WorkerPhase::Backoff).await;
    assert_eq!(
        backoff.last_tick_outcome(),
        Some(WorkerTickOutcome::Failure)
    );
    assert_eq!(backoff.last_tick_finished_at(), Some(clock.now()));
    wakeup.notify();
    tokio::task::yield_now().await;
    assert_eq!(status.borrow().phase(), WorkerPhase::Backoff);

    advance(&clock, Duration::from_secs(5)).await;
    let running = phase(&mut status, WorkerPhase::Running).await;
    assert_eq!(running.last_tick_started_at(), Some(clock.now()));
    task.abort();
    assert!(task.await.unwrap_err().is_cancelled());
    let cancelled = phase(&mut status, WorkerPhase::Cancelled).await;
    assert_eq!(cancelled.phase_since(), clock.now());
    assert_eq!(
        cancelled.last_tick_finished_at(),
        backoff.last_tick_finished_at()
    );
    assert_eq!(
        cancelled.last_tick_outcome(),
        Some(WorkerTickOutcome::Failure)
    );
    assert!(status.changed().await.is_err());
}

#[tokio::test(start_paused = true)]
async fn status_records_errors_and_disabled_runners_without_starting_a_tick() {
    for config in [
        WorkerSettings::default()
            .with_lease(Some(WorkerLeaseSettings::new(
                Duration::from_secs(30),
                Duration::from_secs(10),
                Duration::from_secs(5),
            )))
            .build()
            .unwrap(),
        WorkerConfig::disabled(),
    ] {
        let mut options = WorkerRunOptions::new();
        let mut status = options.subscribe_status();
        let (_trigger, shutdown) = shutdown_channel();
        let (worker, _commands) = controlled();
        let enabled = config.enabled();
        let result =
            run_worker_until_shutdown_with_options(worker, config, shutdown, options).await;
        let expected = if enabled {
            assert!(matches!(result, Err(WorkerRunError::MissingLeaseStore)));
            WorkerPhase::Failed
        } else {
            assert_eq!(result.unwrap().ticks(), 0);
            WorkerPhase::Stopped
        };
        let final_status = phase(&mut status, expected).await;
        assert_eq!(final_status.last_tick_started_at(), None);
        assert_eq!(final_status.last_tick_finished_at(), None);
        assert!(status.changed().await.is_err());
    }
}

#[tokio::test(start_paused = true)]
async fn shutdown_timeout_marks_failure_without_completing_the_interrupted_tick() {
    let mut options = WorkerRunOptions::new();
    let mut status = options.subscribe_status();
    let (trigger, shutdown) = shutdown_channel();
    let (worker, _commands) = controlled();
    let task = tokio::spawn(run_worker_until_shutdown_with_options(
        worker,
        WorkerSettings::default()
            .with_shutdown_timeout(Duration::from_secs(2))
            .build()
            .unwrap(),
        shutdown,
        options,
    ));
    phase(&mut status, WorkerPhase::Running).await;
    trigger.shutdown();
    phase(&mut status, WorkerPhase::Shutdown).await;
    tokio::time::advance(Duration::from_secs(2)).await;
    assert!(matches!(
        task.await.unwrap(),
        Err(WorkerRunError::ShutdownTimedOut)
    ));
    let failed = phase(&mut status, WorkerPhase::Failed).await;
    assert!(failed.last_tick_started_at().is_some());
    assert_eq!(failed.last_tick_finished_at(), None);
    assert_eq!(failed.last_tick_outcome(), None);
    assert!(status.changed().await.is_err());
}

#[test]
fn unused_options_and_unpolled_runner_publish_cancellation() {
    let clock = ManualClock::new();
    let mut options = WorkerRunOptions::new();
    let status = options.subscribe_status();
    options = options.with_clock(clock.shared());
    assert_eq!(status.borrow().phase_since(), clock.now());
    clock.advance(Duration::from_secs(2)).unwrap();
    drop(options);
    assert_eq!(status.borrow().phase(), WorkerPhase::Cancelled);
    assert_eq!(status.borrow().phase_since(), clock.now());
    assert!(status.has_changed().is_err());

    let mut options = WorkerRunOptions::new();
    let status = options.subscribe_status();
    let (_trigger, shutdown) = shutdown_channel();
    let (worker, _commands) = controlled();
    let future =
        run_worker_until_shutdown_with_options(worker, WorkerConfig::default(), shutdown, options);
    drop(future);
    assert_eq!(status.borrow().phase(), WorkerPhase::Cancelled);
    assert_eq!(status.borrow().last_tick_started_at(), None);
    assert!(status.has_changed().is_err());
}

struct BusyLeaseStore;

#[async_trait]
impl WorkerLeaseStore for BusyLeaseStore {
    async fn try_acquire(
        &self,
        _: &str,
        _: &str,
        _: WorkerLeaseConfig,
    ) -> Result<Option<WorkerLease>, WorkerLeaseError> {
        Ok(None)
    }

    async fn renew(
        &self,
        _: &WorkerLease,
        _: WorkerLeaseConfig,
    ) -> Result<Option<WorkerLease>, WorkerLeaseError> {
        Ok(None)
    }

    async fn release(&self, _: &WorkerLease) -> Result<(), WorkerLeaseError> {
        Ok(())
    }
}

#[tokio::test(start_paused = true)]
async fn lease_contention_is_observable_without_reporting_an_execution() {
    let clock = ManualClock::new();
    let mut options = WorkerRunOptions::new()
        .with_clock(clock.shared())
        .with_lease_store("owner", Arc::new(BusyLeaseStore));
    let mut status = options.subscribe_status();
    let config = WorkerSettings::default()
        .with_lease(Some(WorkerLeaseSettings::new(
            Duration::from_secs(30),
            Duration::from_secs(10),
            Duration::from_secs(5),
        )))
        .build()
        .unwrap();
    let (trigger, shutdown) = shutdown_channel();
    let (worker, _commands) = controlled();
    let task = tokio::spawn(run_worker_until_shutdown_with_options(
        worker, config, shutdown, options,
    ));
    let waiting = phase(&mut status, WorkerPhase::LeaseWait).await;
    assert_eq!(waiting.last_tick_started_at(), None);
    advance(&clock, Duration::from_secs(10)).await;
    tokio::task::yield_now().await;
    assert_eq!(status.borrow().phase(), WorkerPhase::LeaseWait);
    assert_eq!(status.borrow().phase_since(), waiting.phase_since());
    trigger.shutdown();
    assert_eq!(task.await.unwrap().unwrap().ticks(), 0);
    phase(&mut status, WorkerPhase::Stopped).await;
    assert!(status.changed().await.is_err());
}