hatchet-client 0.1.0

Rust SDK for Hatchet: run workers and trigger tasks
Documentation
//! Splitting a large batch of triggers into `BulkTriggerWorkflow` calls the
//! engine will accept, and sending them.
//!
//! [`chunk_ranges`] is the pure part: it knows nothing about gRPC, only about
//! encoded sizes, so it is unit-tested without a client or an engine. Sending
//! the chunks ([`trigger_many`]) is what [`Task::run_many`](crate::Task::run_many)
//! and [`Workflow::run_many`](crate::Workflow::run_many) share.

use std::ops::Range;

use hatchet_proto::prost;
use hatchet_proto::root::BulkTriggerWorkflowRequest;
use hatchet_proto::v1;

use crate::error::{Error, Result};
use crate::{Client, retry};

/// The most requests one `BulkTriggerWorkflow` chunk carries.
pub(crate) const MAX_CHUNK_ITEMS: usize = 1000;

/// The most encoded bytes one `BulkTriggerWorkflow` chunk carries — matching
/// the official Go SDK's own chunking (`3 * 1024 * 1024`, i.e. 3 MiB).
pub(crate) const MAX_CHUNK_BYTES: usize = 3 * 1024 * 1024;

/// Splits `sizes` — the encoded byte length of each request, in order — into
/// chunk ranges that each respect [`MAX_CHUNK_ITEMS`] and [`MAX_CHUNK_BYTES`].
///
/// A chunk always takes at least one item, even when that one item alone is
/// already over [`MAX_CHUNK_BYTES`]: it then travels alone rather than being
/// rejected or split. A byte sum that lands exactly on [`MAX_CHUNK_BYTES`]
/// still fits in one chunk — the limit only closes a chunk when the *next*
/// item would push the sum *over* it.
pub(crate) fn chunk_ranges(sizes: &[usize]) -> Vec<Range<usize>> {
    let mut ranges = Vec::new();
    let mut start = 0;
    let mut count = 0usize;
    let mut bytes = 0usize;

    for (index, &size) in sizes.iter().enumerate() {
        if count > 0 && (count >= MAX_CHUNK_ITEMS || bytes + size > MAX_CHUNK_BYTES) {
            ranges.push(start..index);
            start = index;
            count = 0;
            bytes = 0;
        }
        count += 1;
        bytes += size;
    }
    if start < sizes.len() {
        ranges.push(start..sizes.len());
    }
    ranges
}

/// Triggers every request in `requests` through as many `BulkTriggerWorkflow`
/// calls as [`chunk_ranges`] says are needed, and hands back the run ids in
/// input order.
///
/// An empty `requests` makes no call and returns `Ok(Vec::new())`.
///
/// # Errors
///
/// [`Error::PartialBulkTrigger`] when a chunk fails after at least one run id
/// was already obtained — from an earlier chunk, or from the short answer of
/// this very chunk, since ids the engine *did* return name runs it really
/// started. Those ids are carried in the error rather than lost. When nothing
/// has been started yet, the plain underlying error (an [`Error::Grpc`] or
/// [`Error::BulkTriggerMismatch`]) is returned instead: there is nothing to
/// report alongside it.
pub(crate) async fn trigger_many(
    client: &Client,
    requests: Vec<v1::TriggerWorkflowRequest>,
) -> Result<Vec<String>> {
    let sizes: Vec<usize> = requests.iter().map(prost::Message::encoded_len).collect();
    let mut run_ids = Vec::with_capacity(requests.len());

    for range in chunk_ranges(&sizes) {
        let chunk = requests[range].to_vec();
        let expected = chunk.len();
        let bulk_request = BulkTriggerWorkflowRequest { workflows: chunk };

        let outcome = retry::unary(|| {
            let mut workflows = client.workflows_grpc();
            let request = retry::timed(bulk_request.clone());
            async move { workflows.bulk_trigger_workflow(request).await }
        })
        .await
        .map_err(|status| crate::collision::from_trigger(Error::from(status)));

        let error = match outcome {
            Ok(response) => {
                let ids = response.into_inner().workflow_run_ids;
                let got = ids.len();
                // Whatever the engine answered with names runs it started, so
                // the ids are kept even when there are too few of them: they
                // then ride out inside `PartialBulkTrigger` instead of being
                // dropped on the floor.
                run_ids.extend(ids);
                if got == expected {
                    continue;
                }
                Error::BulkTriggerMismatch { expected, got }
            }
            Err(error) => error,
        };

        return Err(if run_ids.is_empty() {
            error
        } else {
            Error::PartialBulkTrigger {
                run_ids,
                source: Box::new(error),
            }
        });
    }

    Ok(run_ids)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn an_empty_input_makes_no_chunk() {
        assert_eq!(chunk_ranges(&[]), Vec::<Range<usize>>::new());
    }

    #[test]
    fn two_thousand_five_hundred_small_items_split_into_three_chunks() {
        let sizes = vec![10; 2500];
        assert_eq!(chunk_ranges(&sizes), [0..1000, 1000..2000, 2000..2500]);
    }

    #[test]
    fn exactly_a_thousand_items_is_one_chunk() {
        let sizes = vec![10; 1000];
        assert_eq!(chunk_ranges(&sizes), vec![0..1000]);
    }

    #[test]
    fn a_thousand_and_one_items_is_two_chunks() {
        let sizes = vec![10; 1001];
        assert_eq!(chunk_ranges(&sizes), [0..1000, 1000..1001]);
    }

    #[test]
    fn seven_one_mebibyte_items_close_a_chunk_just_before_exceeding_three_mebibytes() {
        let mebibyte = 1024 * 1024;
        let sizes = vec![mebibyte; 7];
        assert_eq!(chunk_ranges(&sizes), [0..3, 3..6, 6..7]);
    }

    #[test]
    fn a_single_item_over_the_byte_limit_travels_alone() {
        let sizes = vec![10, MAX_CHUNK_BYTES + 1, 10];
        assert_eq!(chunk_ranges(&sizes), [0..1, 1..2, 2..3]);
    }

    #[test]
    fn a_sum_of_exactly_the_byte_limit_fits_in_one_chunk() {
        let sizes = vec![MAX_CHUNK_BYTES / 2, MAX_CHUNK_BYTES / 2];
        assert_eq!(chunk_ranges(&sizes), vec![0..2]);
    }
}