hatchet-client 0.1.0

Rust SDK for Hatchet: run workers and trigger tasks
Documentation
//! The worker's heartbeat loop.
//!
//! Network-agnostic on purpose: `run` is handed a function that sends one
//! heartbeat, so the timing can be tested without a socket.
//!
//! The cadence is [`INTERVAL`] after each beat *completes*, which is what the
//! official Python SDK does; the Go SDK instead measures start to start. The
//! difference only shows on a slow beat, and waiting from the end is the safer
//! of the two: a fixed cadence would follow a slow beat with one right behind
//! it, which tells the engine nothing it does not already know and piles on
//! load exactly when the engine is least able to take it. The cost is that a
//! slow beat delays the next one, which is why [`TIMEOUT`] is short.

use std::time::Duration;

use tokio_util::sync::CancellationToken;

/// How long the worker waits between telling the engine it is still alive. The
/// engine marks a worker inactive well after this, so a few missed beats are
/// survivable.
pub(crate) const INTERVAL: Duration = Duration::from_secs(4);

/// How long one beat is given before it is abandoned.
///
/// Shorter than [`INTERVAL`]: a heartbeat slower than its own interval is
/// worthless — the engine has already heard nothing for longer than the
/// silence the beat was meant to prevent, and with the wait starting after the
/// beat, a stalled one on the 30-second unary deadline would open a silence of
/// 34 seconds. Giving up early and beating again is strictly better. The
/// official Python SDK bounds it the same way, at the same five seconds.
pub(crate) const TIMEOUT: Duration = Duration::from_secs(5);

/// Beats once immediately, then [`INTERVAL`] after each beat finishes, until
/// `stop` fires.
///
/// A failed beat is logged and forgotten: the engine will either see the next
/// one or time the worker out, and tearing the worker down over a single
/// failed heartbeat would be worse than missing it.
///
/// The wait starts when a beat *finishes*, not when it was due, so beats are
/// always at least [`INTERVAL`] apart as the engine sees them. A fixed cadence
/// would instead follow a slow beat with one right behind it — of no use to
/// the engine, and most load exactly when the engine is least able to take it.
pub(crate) async fn run<F, Fut>(mut beat: F, stop: CancellationToken)
where
    F: FnMut() -> Fut,
    Fut: Future<Output = Result<(), tonic::Status>>,
{
    loop {
        tokio::select! {
            () = stop.cancelled() => return,
            result = beat() => {
                if let Err(status) = result {
                    tracing::warn!(
                        code = ?status.code(),
                        message = status.message(),
                        "worker heartbeat failed",
                    );
                }
            }
        }
        tokio::select! {
            () = stop.cancelled() => return,
            () = tokio::time::sleep(INTERVAL) => {}
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::sync::atomic::{AtomicUsize, Ordering};
    use std::sync::{Arc, Mutex};
    use std::time::Duration;
    use tokio::time::Instant;
    use tokio_util::sync::CancellationToken;
    use tonic::Status;

    #[tokio::test(start_paused = true)]
    async fn beats_immediately_and_then_every_four_seconds() {
        let start = Instant::now();
        let beats: Arc<Mutex<Vec<Duration>>> = Arc::new(Mutex::new(Vec::new()));
        let stop = CancellationToken::new();

        let handle = tokio::spawn({
            let beats = Arc::clone(&beats);
            let stop = stop.clone();
            async move {
                run(
                    move || {
                        let beats = Arc::clone(&beats);
                        async move {
                            beats.lock().unwrap().push(start.elapsed());
                            Ok(())
                        }
                    },
                    stop,
                )
                .await;
            }
        });

        tokio::time::sleep(Duration::from_secs(9)).await;
        stop.cancel();
        handle.await.unwrap();

        let recorded = beats.lock().unwrap().clone();
        assert_eq!(
            recorded,
            [
                Duration::ZERO,
                Duration::from_secs(4),
                Duration::from_secs(8)
            ]
        );
    }

    /// The cadence is measured from the end of a beat, so a slow one pushes
    /// the next one back rather than being chased by it.
    #[tokio::test(start_paused = true)]
    async fn a_slow_beat_pushes_the_next_one_back() {
        let start = Instant::now();
        let starts: Arc<Mutex<Vec<Duration>>> = Arc::new(Mutex::new(Vec::new()));
        let stop = CancellationToken::new();

        let handle = tokio::spawn({
            let starts = Arc::clone(&starts);
            let stop = stop.clone();
            async move {
                run(
                    move || {
                        let starts = Arc::clone(&starts);
                        async move {
                            starts.lock().unwrap().push(start.elapsed());
                            // A beat that takes a second of its own.
                            tokio::time::sleep(Duration::from_secs(1)).await;
                            Ok(())
                        }
                    },
                    stop,
                )
                .await;
            }
        });

        // Long enough for the third beat to have started and finished, and
        // short of the fourth, which is due at 15s.
        tokio::time::sleep(Duration::from_secs(12)).await;
        stop.cancel();
        handle.await.unwrap();

        let recorded = starts.lock().unwrap().clone();
        assert_eq!(
            recorded,
            [
                Duration::ZERO,
                Duration::from_secs(5),
                Duration::from_secs(10)
            ],
            "a one-second beat plus a four-second wait is five seconds apart",
        );
    }

    #[tokio::test(start_paused = true)]
    async fn a_failed_beat_does_not_stop_the_loop() {
        let calls = Arc::new(AtomicUsize::new(0));
        let stop = CancellationToken::new();

        let handle = tokio::spawn({
            let calls = Arc::clone(&calls);
            let stop = stop.clone();
            async move {
                run(
                    move || {
                        let calls = Arc::clone(&calls);
                        async move {
                            if calls.fetch_add(1, Ordering::SeqCst) == 0 {
                                Err(Status::unavailable("engine restarting"))
                            } else {
                                Ok(())
                            }
                        }
                    },
                    stop,
                )
                .await;
            }
        });

        tokio::time::sleep(Duration::from_secs(9)).await;
        stop.cancel();
        handle.await.unwrap();

        assert_eq!(calls.load(Ordering::SeqCst), 3);
    }

    #[tokio::test(start_paused = true)]
    async fn stops_when_cancelled() {
        let calls = Arc::new(AtomicUsize::new(0));
        let stop = CancellationToken::new();

        let handle = tokio::spawn({
            let calls = Arc::clone(&calls);
            let stop = stop.clone();
            async move {
                run(
                    move || {
                        let calls = Arc::clone(&calls);
                        async move {
                            calls.fetch_add(1, Ordering::SeqCst);
                            Ok(())
                        }
                    },
                    stop,
                )
                .await;
            }
        });

        tokio::time::sleep(Duration::from_secs(1)).await;
        stop.cancel();
        handle.await.unwrap();
        assert_eq!(calls.load(Ordering::SeqCst), 1);

        tokio::time::sleep(Duration::from_secs(10)).await;
        assert_eq!(calls.load(Ordering::SeqCst), 1);
    }
}