use std::ops::Range;
use hatchet_proto::prost;
use hatchet_proto::root::BulkTriggerWorkflowRequest;
use hatchet_proto::v1;
use crate::error::{Error, Result};
use crate::{Client, retry};
pub(crate) const MAX_CHUNK_ITEMS: usize = 1000;
pub(crate) const MAX_CHUNK_BYTES: usize = 3 * 1024 * 1024;
pub(crate) fn chunk_ranges(sizes: &[usize]) -> Vec<Range<usize>> {
let mut ranges = Vec::new();
let mut start = 0;
let mut count = 0usize;
let mut bytes = 0usize;
for (index, &size) in sizes.iter().enumerate() {
if count > 0 && (count >= MAX_CHUNK_ITEMS || bytes + size > MAX_CHUNK_BYTES) {
ranges.push(start..index);
start = index;
count = 0;
bytes = 0;
}
count += 1;
bytes += size;
}
if start < sizes.len() {
ranges.push(start..sizes.len());
}
ranges
}
pub(crate) async fn trigger_many(
client: &Client,
requests: Vec<v1::TriggerWorkflowRequest>,
) -> Result<Vec<String>> {
let sizes: Vec<usize> = requests.iter().map(prost::Message::encoded_len).collect();
let mut run_ids = Vec::with_capacity(requests.len());
for range in chunk_ranges(&sizes) {
let chunk = requests[range].to_vec();
let expected = chunk.len();
let bulk_request = BulkTriggerWorkflowRequest { workflows: chunk };
let outcome = retry::unary(|| {
let mut workflows = client.workflows_grpc();
let request = retry::timed(bulk_request.clone());
async move { workflows.bulk_trigger_workflow(request).await }
})
.await
.map_err(|status| crate::collision::from_trigger(Error::from(status)));
let error = match outcome {
Ok(response) => {
let ids = response.into_inner().workflow_run_ids;
let got = ids.len();
run_ids.extend(ids);
if got == expected {
continue;
}
Error::BulkTriggerMismatch { expected, got }
}
Err(error) => error,
};
return Err(if run_ids.is_empty() {
error
} else {
Error::PartialBulkTrigger {
run_ids,
source: Box::new(error),
}
});
}
Ok(run_ids)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn an_empty_input_makes_no_chunk() {
assert_eq!(chunk_ranges(&[]), Vec::<Range<usize>>::new());
}
#[test]
fn two_thousand_five_hundred_small_items_split_into_three_chunks() {
let sizes = vec![10; 2500];
assert_eq!(chunk_ranges(&sizes), [0..1000, 1000..2000, 2000..2500]);
}
#[test]
fn exactly_a_thousand_items_is_one_chunk() {
let sizes = vec![10; 1000];
assert_eq!(chunk_ranges(&sizes), vec![0..1000]);
}
#[test]
fn a_thousand_and_one_items_is_two_chunks() {
let sizes = vec![10; 1001];
assert_eq!(chunk_ranges(&sizes), [0..1000, 1000..1001]);
}
#[test]
fn seven_one_mebibyte_items_close_a_chunk_just_before_exceeding_three_mebibytes() {
let mebibyte = 1024 * 1024;
let sizes = vec![mebibyte; 7];
assert_eq!(chunk_ranges(&sizes), [0..3, 3..6, 6..7]);
}
#[test]
fn a_single_item_over_the_byte_limit_travels_alone() {
let sizes = vec![10, MAX_CHUNK_BYTES + 1, 10];
assert_eq!(chunk_ranges(&sizes), [0..1, 1..2, 2..3]);
}
#[test]
fn a_sum_of_exactly_the_byte_limit_fits_in_one_chunk() {
let sizes = vec![MAX_CHUNK_BYTES / 2, MAX_CHUNK_BYTES / 2];
assert_eq!(chunk_ranges(&sizes), vec![0..2]);
}
}