use std::collections::{BTreeMap, HashMap};
use std::pin::Pin;
use std::task::{Context as TaskContext, Poll};
use std::time::Duration;
use hatchet_proto::root::{
ResourceEventType, SubscribeToWorkflowEventsRequest, WorkflowEvent,
dispatcher_client::DispatcherClient,
};
use tokio::sync::mpsc;
use tokio_stream::Stream;
use tokio_util::sync::{CancellationToken, DropGuard};
use tonic::{Code, Status};
use crate::Client;
use crate::error::{Error, Result};
use crate::retry::{
STREAM_MAX_CONSECUTIVE_OTHER_FAILURES, StreamErrorClass, classify_stream_end, full_jitter,
stream_backoff_cap,
};
use crate::transport::GrpcService;
const CHUNK_BUFFER: usize = 64;
const OPEN_TIMEOUT: Duration = Duration::from_secs(5);
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct StreamChunk {
pub run_id: String,
pub task_run_id: String,
pub index: Option<i64>,
pub payload: String,
}
#[must_use = "an output stream delivers nothing until it is read"]
#[derive(Debug)]
pub struct OutputStream {
chunks: mpsc::Receiver<Result<StreamChunk>>,
_cancel: DropGuard,
}
impl OutputStream {
pub async fn next(&mut self) -> Option<Result<StreamChunk>> {
self.chunks.recv().await
}
}
impl Stream for OutputStream {
type Item = Result<StreamChunk>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Option<Self::Item>> {
self.chunks.poll_recv(cx)
}
}
type EventStream = Pin<Box<dyn Stream<Item = std::result::Result<WorkflowEvent, Status>> + Send>>;
pub(crate) async fn open(
client: &Client,
request: SubscribeToWorkflowEventsRequest,
) -> Result<OutputStream> {
let connect = connecting(client.dispatcher_grpc(), request);
open_with(connect, |attempt| {
full_jitter(stream_backoff_cap(attempt), &mut rand::rng())
})
.await
}
async fn open_with<C, Fut, S, B>(mut connect: C, backoff: B) -> Result<OutputStream>
where
C: FnMut() -> Fut + Send + 'static,
Fut: Future<Output = std::result::Result<S, Status>> + Send + 'static,
S: Stream<Item = std::result::Result<WorkflowEvent, Status>> + Send + Unpin + 'static,
B: FnMut(u32) -> Duration + Send + 'static,
{
let cancel = CancellationToken::new();
let abandoned = cancel.clone().drop_guard();
let mut connecting = tokio::spawn({
let cancel = cancel.clone();
let attempt = connect();
async move {
tokio::select! {
() = cancel.cancelled() => {
Err(Status::cancelled("the subscribe call was abandoned"))
}
connected = attempt => connected,
}
}
});
let first = match tokio::time::timeout(OPEN_TIMEOUT, &mut connecting).await {
Ok(Ok(Ok(stream))) => First::Open(stream),
Ok(Ok(Err(status))) if classify_stream_end(Some(&status)) == StreamErrorClass::Fatal => {
return Err(Error::Grpc(Box::new(status)));
}
Ok(Ok(Err(_))) => First::None,
Ok(Err(_)) => First::None,
Err(_) => First::Connecting(connecting),
};
let (chunks, received) = mpsc::channel(CHUNK_BUFFER);
tokio::spawn(run(connect, first, chunks, cancel, backoff));
Ok(OutputStream {
chunks: received,
_cancel: abandoned.disarm().drop_guard(),
})
}
fn connecting(
dispatcher: DispatcherClient<GrpcService>,
request: SubscribeToWorkflowEventsRequest,
) -> impl FnMut() -> Pin<Box<dyn Future<Output = std::result::Result<EventStream, Status>> + Send>>
{
move || {
let mut dispatcher = dispatcher.clone();
let request = request.clone();
Box::pin(async move {
let events = dispatcher
.subscribe_to_workflow_events(request)
.await?
.into_inner();
Ok(Box::pin(events) as EventStream)
})
}
}
enum First<S> {
None,
Open(S),
Connecting(tokio::task::JoinHandle<std::result::Result<S, Status>>),
}
async fn run<C, Fut, S, B>(
mut connect: C,
first: First<S>,
chunks: mpsc::Sender<Result<StreamChunk>>,
cancel: CancellationToken,
mut backoff: B,
) where
C: FnMut() -> Fut,
Fut: Future<Output = std::result::Result<S, Status>>,
S: Stream<Item = std::result::Result<WorkflowEvent, Status>> + Unpin,
B: FnMut(u32) -> Duration,
{
let mut attempt: u32 = 0;
let mut consecutive_other: u32 = 0;
let mut first = first;
let mut ordering = Reorder::default();
loop {
let connected = match std::mem::replace(&mut first, First::None) {
First::Open(stream) => Ok(stream),
First::Connecting(mut connecting) => tokio::select! {
() = cancel.cancelled() => {
connecting.abort();
return;
}
joined = &mut connecting => match joined {
Ok(connected) => connected,
Err(_) => Err(Status::internal("the subscribe call panicked")),
},
},
First::None => {
tokio::select! {
() = cancel.cancelled() => return,
connected = connect() => connected,
}
}
};
let outcome = match connected {
Ok(events) => serve(events, &chunks, &cancel, &mut ordering).await,
Err(status) => Outcome::Ended {
status: Some(status),
received: false,
},
};
let (status, received) = match outcome {
Outcome::Done => return,
Outcome::Ended { status, received } => (status, received),
};
if received {
attempt = 0;
consecutive_other = 0;
}
let give_up = match classify_stream_end(status.as_ref()) {
StreamErrorClass::StopClean => false,
StreamErrorClass::Transient
if status.as_ref().map(Status::code) == Some(Code::Internal) =>
{
consecutive_other += 1;
consecutive_other >= STREAM_MAX_CONSECUTIVE_OTHER_FAILURES
}
StreamErrorClass::Transient => false,
StreamErrorClass::Fatal => true,
StreamErrorClass::Other => {
consecutive_other += 1;
consecutive_other >= STREAM_MAX_CONSECUTIVE_OTHER_FAILURES
}
};
if give_up {
let mut held = Vec::new();
ordering.flush(&mut held);
for item in held {
if chunks.send(item).await.is_err() {
return;
}
}
let status = status.unwrap_or_else(|| Status::unknown("the output stream ended"));
tracing::warn!(
code = ?status.code(),
message = status.message(),
consecutive_other,
"giving up on a run's output stream",
);
let _ = chunks
.send(Err(Error::ResultStream(Box::new(status))))
.await;
return;
}
let delay = backoff(attempt);
attempt = attempt.saturating_add(1);
log_reconnect(attempt, consecutive_other, delay, status.as_ref());
tokio::select! {
() = cancel.cancelled() => return,
() = tokio::time::sleep(delay) => {}
}
}
}
enum Outcome {
Done,
Ended {
status: Option<Status>,
received: bool,
},
}
async fn serve<S>(
mut events: S,
chunks: &mpsc::Sender<Result<StreamChunk>>,
cancel: &CancellationToken,
ordering: &mut Reorder,
) -> Outcome
where
S: Stream<Item = std::result::Result<WorkflowEvent, Status>> + Unpin,
{
use tokio_stream::StreamExt;
let mut received = false;
let mut ready: Vec<Result<StreamChunk>> = Vec::new();
loop {
let item = tokio::select! {
() = cancel.cancelled() => return Outcome::Done,
item = events.next() => item,
};
match item {
Some(Ok(event)) => {
let hangup = event.hangup;
ready.clear();
if let Some(chunk) = chunk_of(event) {
ordering.accept(chunk, &mut ready);
}
if hangup {
ordering.flush(&mut ready);
}
for item in ready.drain(..) {
received |= item.is_ok();
if chunks.send(item).await.is_err() {
return Outcome::Done;
}
}
if hangup {
tracing::debug!("the engine hung up on a run's output stream");
return Outcome::Done;
}
}
Some(Err(status)) => {
return Outcome::Ended {
status: Some(status),
received,
};
}
None => {
return Outcome::Ended {
status: None,
received,
};
}
}
}
}
fn chunk_of(event: WorkflowEvent) -> Option<StreamChunk> {
if event.event_type != ResourceEventType::Stream as i32 {
return None;
}
Some(StreamChunk {
run_id: event.workflow_run_id,
task_run_id: event.resource_id,
index: event.event_index,
payload: event.event_payload,
})
}
const REORDER_BUFFER: usize = 1;
struct Sequence {
next: i64,
held: BTreeMap<i64, StreamChunk>,
}
#[derive(Default)]
struct Reorder {
tasks: HashMap<String, Sequence>,
}
impl Reorder {
fn accept(&mut self, chunk: StreamChunk, out: &mut Vec<Result<StreamChunk>>) {
let Some(index) = chunk.index else {
out.push(Ok(chunk));
return;
};
let task_run_id = chunk.task_run_id.clone();
let sequence = self.tasks.entry(task_run_id).or_insert(Sequence {
next: index,
held: BTreeMap::new(),
});
if index < sequence.next {
tracing::debug!(
task_run_id = %chunk.task_run_id,
index,
expected = sequence.next,
"dropping a chunk of a run's output stream that was already delivered",
);
return;
}
if index == sequence.next {
out.push(Ok(chunk));
sequence.next += 1;
while let Some(next) = sequence.held.remove(&sequence.next) {
out.push(Ok(next));
sequence.next += 1;
}
return;
}
let task_run_id = chunk.task_run_id.clone();
sequence.held.insert(index, chunk);
if sequence.held.len() > REORDER_BUFFER {
release(&task_run_id, sequence, out);
}
}
fn flush(&mut self, out: &mut Vec<Result<StreamChunk>>) {
for (task_run_id, sequence) in &mut self.tasks {
release(task_run_id, sequence, out);
}
}
}
fn release(task_run_id: &str, sequence: &mut Sequence, out: &mut Vec<Result<StreamChunk>>) {
for (index, chunk) in std::mem::take(&mut sequence.held) {
if index > sequence.next {
tracing::warn!(
task_run_id,
expected = sequence.next,
got = index,
"a run's output stream skipped chunks",
);
out.push(Err(Error::StreamGap {
task_run_id: task_run_id.to_string(),
expected: sequence.next,
got: index,
}));
}
sequence.next = index + 1;
out.push(Ok(chunk));
}
}
fn log_reconnect(attempt: u32, consecutive_other: u32, delay: Duration, status: Option<&Status>) {
let code = status.map(Status::code);
let message = status.map(Status::message).unwrap_or_default();
if attempt == 1 || attempt.is_multiple_of(5) {
tracing::warn!(
attempt,
consecutive_other,
?delay,
?code,
message,
"reconnecting to a run's output stream",
);
} else {
tracing::debug!(
attempt,
consecutive_other,
?delay,
?code,
message,
"reconnecting to a run's output stream",
);
}
}
#[cfg(test)]
mod tests {
use std::collections::VecDeque;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use tokio::sync::mpsc;
use tokio::time::Instant;
use tokio_stream::wrappers::ReceiverStream;
use super::*;
const NEVER: Duration = Duration::from_secs(10 * 60);
type ScriptedStream = ReceiverStream<std::result::Result<WorkflowEvent, Status>>;
type ScriptedSender = mpsc::Sender<std::result::Result<WorkflowEvent, Status>>;
enum Step {
Stream(mpsc::Receiver<std::result::Result<WorkflowEvent, Status>>),
Fail(Status),
}
fn connection() -> (ScriptedSender, Step) {
let (tx, rx) = mpsc::channel(8);
(tx, Step::Stream(rx))
}
#[derive(Clone)]
struct Script {
steps: Arc<Mutex<VecDeque<Step>>>,
calls: Arc<AtomicUsize>,
opened: mpsc::UnboundedSender<Instant>,
}
impl Script {
fn new(steps: Vec<Step>) -> (Self, mpsc::UnboundedReceiver<Instant>) {
let (opened, opens) = mpsc::unbounded_channel();
(
Self {
steps: Arc::new(Mutex::new(steps.into())),
calls: Arc::new(AtomicUsize::new(0)),
opened,
},
opens,
)
}
async fn connect(self) -> std::result::Result<ScriptedStream, Status> {
self.calls.fetch_add(1, Ordering::SeqCst);
let _ = self.opened.send(Instant::now());
let step = self
.steps
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.pop_front();
match step {
Some(Step::Stream(events)) => Ok(ReceiverStream::new(events)),
Some(Step::Fail(status)) => Err(status),
None => std::future::pending().await,
}
}
fn calls(&self) -> usize {
self.calls.load(Ordering::SeqCst)
}
}
async fn opened(
steps: Vec<Step>,
backoff: impl FnMut(u32) -> Duration + Send + 'static,
) -> (OutputStream, Script, mpsc::UnboundedReceiver<Instant>) {
let (script, opens) = Script::new(steps);
let stream = open_with(
{
let script = script.clone();
move || script.clone().connect()
},
backoff,
)
.await
.expect("a scripted first connection is never fatal");
(stream, script, opens)
}
fn chunk(index: i64, payload: &str) -> WorkflowEvent {
WorkflowEvent {
workflow_run_id: "run-1".to_string(),
resource_id: "task-1".to_string(),
event_type: ResourceEventType::Stream as i32,
event_payload: payload.to_string(),
event_index: Some(index),
..WorkflowEvent::default()
}
}
async fn next_open(opens: &mut mpsc::UnboundedReceiver<Instant>) -> Instant {
tokio::time::timeout(NEVER, opens.recv())
.await
.expect("timed out waiting for a connection")
.expect("the loop dropped the recording channel")
}
async fn next_item(stream: &mut OutputStream) -> Option<Result<StreamChunk>> {
tokio::time::timeout(NEVER, stream.next())
.await
.expect("timed out waiting for the stream")
}
fn accepted(ordering: &mut Reorder, chunk: StreamChunk) -> Vec<Result<StreamChunk>> {
let mut out = Vec::new();
ordering.accept(chunk, &mut out);
out
}
fn shape(items: Vec<Result<StreamChunk>>) -> Vec<String> {
items
.into_iter()
.map(|item| match item {
Ok(chunk) => chunk.payload,
Err(Error::StreamGap { expected, got, .. }) => format!("gap({expected}..{got})"),
Err(other) => format!("error({other})"),
})
.collect()
}
fn numbered(task: &str, index: i64) -> StreamChunk {
StreamChunk {
run_id: "run-1".to_string(),
task_run_id: task.to_string(),
index: Some(index),
payload: format!("chunk-{index}"),
}
}
#[test]
fn chunks_that_arrive_in_order_pass_straight_through() {
let mut ordering = Reorder::default();
for n in 0..3 {
assert_eq!(
shape(accepted(&mut ordering, numbered("task-1", n))),
[format!("chunk-{n}")],
);
}
}
#[test]
fn an_early_chunk_waits_for_the_ones_before_it() {
let mut ordering = Reorder::default();
assert_eq!(
shape(accepted(&mut ordering, numbered("task-1", 0))),
["chunk-0"],
);
assert!(shape(accepted(&mut ordering, numbered("task-1", 2))).is_empty());
assert_eq!(
shape(accepted(&mut ordering, numbered("task-1", 1))),
["chunk-1", "chunk-2"],
);
}
#[test]
fn a_hole_is_reported_once_the_buffer_is_over_full_and_the_rest_follows() {
let mut ordering = Reorder::default();
accepted(&mut ordering, numbered("task-1", 0));
let mut released = Vec::new();
for n in 2..=(2 + REORDER_BUFFER as i64) {
released.extend(shape(accepted(&mut ordering, numbered("task-1", n))));
}
let expected: Vec<String> = std::iter::once("gap(1..2)".to_string())
.chain((2..=(2 + REORDER_BUFFER as i64)).map(|n| format!("chunk-{n}")))
.collect();
assert_eq!(released, expected);
assert_eq!(
shape(accepted(
&mut ordering,
numbered("task-1", 3 + REORDER_BUFFER as i64),
)),
[format!("chunk-{}", 3 + REORDER_BUFFER as i64)],
);
}
#[test]
fn every_hole_in_a_released_batch_is_reported() {
let mut ordering = Reorder::default();
accepted(&mut ordering, numbered("task-1", 0));
assert!(shape(accepted(&mut ordering, numbered("task-1", 3))).is_empty());
assert_eq!(
shape(accepted(&mut ordering, numbered("task-1", 7))),
["gap(1..3)", "chunk-3", "gap(4..7)", "chunk-7"],
);
assert_eq!(
shape(accepted(&mut ordering, numbered("task-1", 8))),
["chunk-8"],
);
}
#[test]
fn a_chunk_that_was_already_delivered_is_dropped() {
let mut ordering = Reorder::default();
accepted(&mut ordering, numbered("task-1", 0));
accepted(&mut ordering, numbered("task-1", 1));
assert!(shape(accepted(&mut ordering, numbered("task-1", 0))).is_empty());
assert!(shape(accepted(&mut ordering, numbered("task-1", 1))).is_empty());
assert_eq!(
shape(accepted(&mut ordering, numbered("task-1", 2))),
["chunk-2"],
);
}
#[test]
fn each_task_run_has_its_own_sequence() {
let mut ordering = Reorder::default();
assert_eq!(
shape(accepted(&mut ordering, numbered("task-1", 0))),
["chunk-0"],
);
assert_eq!(
shape(accepted(&mut ordering, numbered("task-2", 0))),
["chunk-0"],
);
assert!(shape(accepted(&mut ordering, numbered("task-1", 2))).is_empty());
assert_eq!(
shape(accepted(&mut ordering, numbered("task-2", 1))),
["chunk-1"],
);
}
#[test]
fn a_chunk_without_an_index_goes_straight_through() {
let mut ordering = Reorder::default();
accepted(&mut ordering, numbered("task-1", 0));
assert!(shape(accepted(&mut ordering, numbered("task-1", 2))).is_empty());
let unnumbered = StreamChunk {
index: None,
payload: "no index".to_string(),
..numbered("task-1", 0)
};
assert_eq!(shape(accepted(&mut ordering, unnumbered)), ["no index"]);
assert_eq!(
shape(accepted(&mut ordering, numbered("task-1", 1))),
["chunk-1", "chunk-2"],
);
}
#[test]
fn a_late_subscription_starts_where_it_joined() {
let mut ordering = Reorder::default();
assert_eq!(
shape(accepted(&mut ordering, numbered("task-1", 40))),
["chunk-40"],
);
assert_eq!(
shape(accepted(&mut ordering, numbered("task-1", 41))),
["chunk-41"],
);
}
#[test]
fn flushing_releases_what_is_held_and_reports_the_hole() {
let mut ordering = Reorder::default();
accepted(&mut ordering, numbered("task-1", 0));
accepted(&mut ordering, numbered("task-1", 2));
let mut out = Vec::new();
ordering.flush(&mut out);
assert_eq!(shape(out), ["gap(1..2)", "chunk-2"]);
let mut out = Vec::new();
ordering.flush(&mut out);
assert!(out.is_empty());
}
#[tokio::test(start_paused = true)]
async fn a_reconnect_waits_for_the_backoff() {
let (first, one) = connection();
let (second, two) = connection();
let (mut chunks, _script, mut opens) =
opened(vec![one, two], |_| Duration::from_secs(7)).await;
let opened_at = next_open(&mut opens).await;
drop(first);
let reopened_at = next_open(&mut opens).await;
assert!(
reopened_at.duration_since(opened_at) >= Duration::from_secs(7),
"reconnected after {:?}",
reopened_at.duration_since(opened_at),
);
second
.send(Ok(chunk(0, "after")))
.await
.expect("the second connection is open");
let chunk = next_item(&mut chunks)
.await
.expect("a chunk")
.expect("not an error");
assert_eq!(chunk.payload, "after");
}
#[tokio::test(start_paused = true)]
async fn ten_unclassified_failures_in_a_row_give_up() {
let (first, one) = connection();
let mut steps = vec![one];
steps.extend((0..10).map(|_| Step::Fail(Status::unknown("what?"))));
let (mut chunks, script, _opens) = opened(steps, |_| Duration::ZERO).await;
drop(first);
let error = next_item(&mut chunks)
.await
.expect("the give-up is the last item")
.expect_err("giving up is an error");
assert!(matches!(error, Error::ResultStream(_)), "{error:?}");
assert!(next_item(&mut chunks).await.is_none(), "and then it ends");
assert_eq!(script.calls(), 11, "one good connection and ten failures");
}
#[tokio::test(start_paused = true)]
async fn a_delivered_chunk_resets_the_give_up_counter() {
let (first, one) = connection();
let (middle, two) = connection();
let (last, three) = connection();
let mut steps = vec![one];
steps.extend((0..9).map(|_| Step::Fail(Status::unknown("what?"))));
steps.push(two);
steps.extend((0..9).map(|_| Step::Fail(Status::unknown("what?"))));
steps.push(three);
let (mut chunks, script, _opens) = opened(steps, |_| Duration::ZERO).await;
drop(first);
middle
.send(Ok(chunk(0, "healthy")))
.await
.expect("the middle connection is open");
assert_eq!(
next_item(&mut chunks)
.await
.expect("a chunk")
.expect("not an error")
.payload,
"healthy",
);
drop(middle);
last.send(Ok(chunk(1, "still here")))
.await
.expect("the last connection is open");
assert_eq!(
next_item(&mut chunks)
.await
.expect("a chunk")
.expect("not an error")
.payload,
"still here",
);
assert_eq!(script.calls(), 21);
}
}