hatchet_client/runs/stream.rs
1//! Reading a run's output stream: the chunks
2//! [`Context::put_stream`](crate::Context::put_stream) publishes.
3//!
4//! One [`OutputStream`] owns one `SubscribeToWorkflowEvents` subscription and
5//! one background task. That is the opposite of the run *result* listener next
6//! door, which multiplexes every wait on one client onto a single stream — and
7//! the difference is in the wire, not in taste: a result subscription takes a
8//! *stream* of run ids, so more runs can be added to one connection, while an
9//! event subscription's filter is a single request message, so one filter
10//! needs one connection.
11//!
12//! Like the result listener, the loop here is network-agnostic: [`run`] is
13//! handed a function that opens one stream, so the filter, the reconnect
14//! policy and the give-up rule can be tested against a scripted stream rather
15//! than a socket.
16
17use std::collections::{BTreeMap, HashMap};
18use std::pin::Pin;
19use std::task::{Context as TaskContext, Poll};
20use std::time::Duration;
21
22use hatchet_proto::root::{
23 ResourceEventType, SubscribeToWorkflowEventsRequest, WorkflowEvent,
24 dispatcher_client::DispatcherClient,
25};
26use tokio::sync::mpsc;
27use tokio_stream::Stream;
28use tokio_util::sync::{CancellationToken, DropGuard};
29use tonic::{Code, Status};
30
31use crate::Client;
32use crate::error::{Error, Result};
33use crate::retry::{
34 STREAM_MAX_CONSECUTIVE_OTHER_FAILURES, StreamErrorClass, classify_stream_end, full_jitter,
35 stream_backoff_cap,
36};
37use crate::transport::GrpcService;
38
39/// How many chunks are held for a consumer that is not reading.
40///
41/// Beyond this the background task waits, which stops reading the socket,
42/// which is what applies back pressure to the engine — the alternative is an
43/// unbounded queue that a slow consumer turns into unbounded memory. Payloads
44/// are whatever the handler published (128 KiB arrived intact in testing), so
45/// this bounds the memory a stalled consumer costs at that many payloads.
46const CHUNK_BUFFER: usize = 64;
47
48/// How long [`open`] waits for the engine to answer the subscribe call before
49/// handing back a stream that is still being opened in the background.
50///
51/// It is a *courtesy*, not a guarantee: a subscription the engine answers
52/// straight away is open before the caller triggers anything, which is what
53/// makes "subscribe, then trigger" worth recommending. Both supported engines
54/// have been observed not sending response headers on a fresh stream until
55/// they have something to say — for over a minute when nothing matched the
56/// filter — so waiting for them without a bound would hang the caller.
57///
58/// Running out of it never *abandons* the attempt: the call goes on in the
59/// background task and becomes the stream's first connection. The engine has
60/// registered the subscription by then, whatever its headers say, so dropping
61/// the call would close a subscription that was already receiving.
62const OPEN_TIMEOUT: Duration = Duration::from_secs(5);
63
64/// One chunk of a run's output, as
65/// [`Context::put_stream`](crate::Context::put_stream) published it.
66///
67/// The payload is whatever the handler wrote. It goes out as bytes and comes
68/// back as a `String`, because that is what the two sides of the wire are
69/// (`PutStreamEventRequest.message` is `bytes`, `WorkflowEvent.event_payload`
70/// is `string`); see [`put_stream`](crate::Context::put_stream) for what that
71/// does to bytes that are not UTF-8.
72///
73/// ```no_run
74/// # async fn example(client: hatchet_client::Client) -> hatchet_client::Result<()> {
75/// let mut chunks = client.runs().stream("d627fab0-385f-4eb4-98f9-59a783bc8631").await?;
76/// if let Some(Ok(chunk)) = chunks.next().await {
77/// println!("run {} task {}: [{:?}] {}", chunk.run_id, chunk.task_run_id, chunk.index, chunk.payload);
78/// }
79/// # Ok(())
80/// # }
81/// ```
82#[derive(Clone, Debug, PartialEq, Eq)]
83pub struct StreamChunk {
84 /// The run the chunk belongs to. Worth reading when one subscription
85 /// covers several runs — see
86 /// [`RunsClient::stream_by_metadata`](crate::RunsClient::stream_by_metadata).
87 pub run_id: String,
88 /// The *task* run that published it. A workflow run has one sequence of
89 /// chunks per task, so this is what tells two interleaved outputs apart.
90 pub task_run_id: String,
91 /// The chunk's number in its task run's sequence, counted from zero, when
92 /// the engine sent one. `None` means the engine did not — the field is
93 /// optional on the wire — and such chunks are handed over in arrival order
94 /// without any ordering guarantee.
95 pub index: Option<i64>,
96 /// What the handler published.
97 pub payload: String,
98}
99
100/// A run's output stream: the chunks its handlers publish, in order.
101///
102/// Made by [`RunsClient::stream`](crate::RunsClient::stream),
103/// [`RunsClient::stream_by_metadata`](crate::RunsClient::stream_by_metadata),
104/// [`RunHandle::stream`](crate::RunHandle::stream) and
105/// [`WorkflowRunHandle::stream`](crate::WorkflowRunHandle::stream).
106///
107/// It implements [`futures_core::Stream`](tokio_stream::Stream), so every
108/// combinator that takes one works; [`next`](OutputStream::next) is here too
109/// so the common case needs no extra crate and no trait import.
110///
111/// **Dropping it ends the subscription.** The background task that owns the
112/// connection is cancelled, and the engine sees the stream close.
113///
114/// Delivery is not guaranteed: the engine has been observed dropping a chunk
115/// silently. A gap in a task run's chunk numbering is reported as one
116/// [`Error::StreamGap`] item and the stream carries on.
117///
118/// ```no_run
119/// # async fn example(client: hatchet_client::Client) -> hatchet_client::Result<()> {
120/// let mut chunks = client.runs().stream("d627fab0-385f-4eb4-98f9-59a783bc8631").await?;
121/// while let Some(chunk) = chunks.next().await {
122/// match chunk {
123/// Ok(chunk) => print!("{}", chunk.payload),
124/// // One hole in the output; the rest still arrives.
125/// Err(hatchet_client::Error::StreamGap { .. }) => print!("…"),
126/// Err(error) => return Err(error),
127/// }
128/// }
129/// # Ok(())
130/// # }
131/// ```
132#[must_use = "an output stream delivers nothing until it is read"]
133#[derive(Debug)]
134pub struct OutputStream {
135 chunks: mpsc::Receiver<Result<StreamChunk>>,
136 /// Cancels the background task when this stream is dropped. Holding the
137 /// receiver alone would not: the task would keep the subscription open
138 /// until it next tried to send.
139 _cancel: DropGuard,
140}
141
142impl OutputStream {
143 /// The next chunk, or `None` once the stream is over — the engine hung up,
144 /// or it failed in a way reconnecting cannot fix (and then the last item
145 /// was that error).
146 ///
147 /// Cancel-safe: dropping the returned future loses nothing.
148 ///
149 /// ```no_run
150 /// # async fn example(client: hatchet_client::Client) -> hatchet_client::Result<()> {
151 /// let mut chunks = client.runs().stream("d627fab0-385f-4eb4-98f9-59a783bc8631").await?;
152 /// while let Some(chunk) = chunks.next().await {
153 /// print!("{}", chunk?.payload);
154 /// }
155 /// # Ok(())
156 /// # }
157 /// ```
158 pub async fn next(&mut self) -> Option<Result<StreamChunk>> {
159 self.chunks.recv().await
160 }
161}
162
163impl Stream for OutputStream {
164 type Item = Result<StreamChunk>;
165
166 fn poll_next(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Option<Self::Item>> {
167 self.chunks.poll_recv(cx)
168 }
169}
170
171/// The events one subscription is answered with.
172type EventStream = Pin<Box<dyn Stream<Item = std::result::Result<WorkflowEvent, Status>> + Send>>;
173
174/// Opens one subscription with `request` and hands back the stream it feeds.
175///
176/// The first connection is awaited for at most [`OPEN_TIMEOUT`] so that a
177/// subscription the engine answers promptly is really open when this returns;
178/// if it is not answered in time, *that same attempt* is handed to the
179/// background task, which goes on waiting for it. A first connection that
180/// fails in a way reconnecting cannot fix (a rejected token or an unknown run
181/// id, most of all) is returned as an error rather than becoming a stream that
182/// quietly yields nothing.
183pub(crate) async fn open(
184 client: &Client,
185 request: SubscribeToWorkflowEventsRequest,
186) -> Result<OutputStream> {
187 // `dispatcher_grpc` already applies the client's gzip setting and the auth
188 // header, so nothing about this connection is special.
189 let connect = connecting(client.dispatcher_grpc(), request);
190 open_with(connect, |attempt| {
191 full_jitter(stream_backoff_cap(attempt), &mut rand::rng())
192 })
193 .await
194}
195
196/// [`open`] with the delay before reconnect number `attempt` (0-based) chosen
197/// by `backoff` rather than by the SDK's policy.
198///
199/// The seam exists for the tests, exactly as it does on the result listener:
200/// under full jitter the real delay may be any length including zero, so no
201/// assertion on elapsed time can tell a backoff that is there from one that
202/// was deleted.
203async fn open_with<C, Fut, S, B>(mut connect: C, backoff: B) -> Result<OutputStream>
204where
205 C: FnMut() -> Fut + Send + 'static,
206 Fut: Future<Output = std::result::Result<S, Status>> + Send + 'static,
207 S: Stream<Item = std::result::Result<WorkflowEvent, Status>> + Send + Unpin + 'static,
208 B: FnMut(u32) -> Duration + Send + 'static,
209{
210 let cancel = CancellationToken::new();
211 // Spawned rather than awaited in place: an attempt this call stops waiting
212 // for is handed to the background task, and dropping the future instead
213 // would reset the HTTP/2 stream and unregister a subscription the engine
214 // has already made — losing whatever is published before a replacement
215 // lands, with no gap to report it by.
216 //
217 // Outliving *this call* is not the same as outliving the caller, though:
218 // the guard below cancels the attempt if this function is dropped before
219 // it hands it over — a `timeout` around `stream()`, a `select!` that loses
220 // — rather than leaving a subscription registered with nobody to read it.
221 let abandoned = cancel.clone().drop_guard();
222 let mut connecting = tokio::spawn({
223 let cancel = cancel.clone();
224 let attempt = connect();
225 async move {
226 tokio::select! {
227 () = cancel.cancelled() => {
228 Err(Status::cancelled("the subscribe call was abandoned"))
229 }
230 connected = attempt => connected,
231 }
232 }
233 });
234 let first = match tokio::time::timeout(OPEN_TIMEOUT, &mut connecting).await {
235 Ok(Ok(Ok(stream))) => First::Open(stream),
236 Ok(Ok(Err(status))) if classify_stream_end(Some(&status)) == StreamErrorClass::Fatal => {
237 return Err(Error::Grpc(Box::new(status)));
238 }
239 // Worth another attempt, and the background task is where attempts
240 // live.
241 Ok(Ok(Err(_))) => First::None,
242 // The attempt panicked, which is a bug in this SDK rather than
243 // anything the engine did; the loop's own attempts are what is left.
244 Ok(Err(_)) => First::None,
245 // The engine has not answered yet. Not an error: it may simply have
246 // nothing to say about this run so far, which is the normal case —
247 // both supported engines hold back response headers until they have
248 // an event. The attempt goes on in the background task.
249 Err(_) => First::Connecting(connecting),
250 };
251 let (chunks, received) = mpsc::channel(CHUNK_BUFFER);
252 tokio::spawn(run(connect, first, chunks, cancel, backoff));
253 Ok(OutputStream {
254 chunks: received,
255 // The attempt belongs to the background task now, so the guard that
256 // watched this call hands its job over to the one the consumer holds.
257 _cancel: abandoned.disarm().drop_guard(),
258 })
259}
260
261/// The connect function for a real engine: opens one
262/// `SubscribeToWorkflowEvents` stream with the same request every time.
263///
264/// No `set_timeout` and no [`retry::unary`](crate::retry::unary), for the same
265/// reasons as the other two streams in this SDK: a deadline would cut off a
266/// run that is simply taking its time, and reconnecting is this loop's own
267/// job.
268fn connecting(
269 dispatcher: DispatcherClient<GrpcService>,
270 request: SubscribeToWorkflowEventsRequest,
271) -> impl FnMut() -> Pin<Box<dyn Future<Output = std::result::Result<EventStream, Status>> + Send>>
272{
273 move || {
274 let mut dispatcher = dispatcher.clone();
275 let request = request.clone();
276 Box::pin(async move {
277 let events = dispatcher
278 .subscribe_to_workflow_events(request)
279 .await?
280 .into_inner();
281 Ok(Box::pin(events) as EventStream)
282 })
283 }
284}
285
286/// The connection attempt [`open_with`] already made, handed to [`run`] so
287/// that one call to `open` never registers two subscriptions.
288enum First<S> {
289 /// Nothing to take over: the loop opens its own.
290 None,
291 /// Answered before [`OPEN_TIMEOUT`] was up, and open.
292 Open(S),
293 /// Still unanswered when [`OPEN_TIMEOUT`] was up. Awaiting the handle is
294 /// what takes it over; aborting it is what closes it.
295 Connecting(tokio::task::JoinHandle<std::result::Result<S, Status>>),
296}
297
298/// Holds one subscription open, reopening it for as long as the engine keeps
299/// dropping it, until the engine hangs up, the consumer goes away, or a
300/// failure that reconnecting cannot fix.
301///
302/// `first` is the attempt [`open`] already made — open, or still being
303/// answered — so a subscription the engine has already registered is used
304/// rather than opened a second time.
305async fn run<C, Fut, S, B>(
306 mut connect: C,
307 first: First<S>,
308 chunks: mpsc::Sender<Result<StreamChunk>>,
309 cancel: CancellationToken,
310 mut backoff: B,
311) where
312 C: FnMut() -> Fut,
313 Fut: Future<Output = std::result::Result<S, Status>>,
314 S: Stream<Item = std::result::Result<WorkflowEvent, Status>> + Unpin,
315 B: FnMut(u32) -> Duration,
316{
317 // The backoff exponent and the run of failures that count towards giving
318 // up. Both are reset by a single delivered chunk: a stream that delivered
319 // output was healthy, whatever it did afterwards.
320 let mut attempt: u32 = 0;
321 let mut consecutive_other: u32 = 0;
322 let mut first = first;
323 // Kept across reconnects: a new subscription may replay chunks the old one
324 // delivered, and this is what remembers them.
325 let mut ordering = Reorder::default();
326
327 loop {
328 let connected = match std::mem::replace(&mut first, First::None) {
329 First::Open(stream) => Ok(stream),
330 // Aborted rather than dropped when the consumer goes away:
331 // dropping a `JoinHandle` detaches the task, which would leave the
332 // subscribe call running with nobody to read it.
333 First::Connecting(mut connecting) => tokio::select! {
334 () = cancel.cancelled() => {
335 connecting.abort();
336 return;
337 }
338 joined = &mut connecting => match joined {
339 Ok(connected) => connected,
340 Err(_) => Err(Status::internal("the subscribe call panicked")),
341 },
342 },
343 First::None => {
344 tokio::select! {
345 () = cancel.cancelled() => return,
346 connected = connect() => connected,
347 }
348 }
349 };
350 let outcome = match connected {
351 Ok(events) => serve(events, &chunks, &cancel, &mut ordering).await,
352 // No difference, for what happens next, between "the stream could
353 // not be opened" and "it closed immediately".
354 Err(status) => Outcome::Ended {
355 status: Some(status),
356 received: false,
357 },
358 };
359 let (status, received) = match outcome {
360 // The consumer dropped the stream, or the engine hung up: either
361 // way this subscription is over for good.
362 Outcome::Done => return,
363 Outcome::Ended { status, received } => (status, received),
364 };
365 if received {
366 attempt = 0;
367 consecutive_other = 0;
368 }
369
370 let give_up = match classify_stream_end(status.as_ref()) {
371 // `Cancelled` is a clean stop for the worker's listener, which
372 // shares `classify_stream_end`; here a cancelled subscription the
373 // consumer still holds is just one more stream to reopen.
374 StreamErrorClass::StopClean => false,
375 // `Internal` is transient in general — a restarting engine sends
376 // it — but it is also what a chunk the engine cannot marshal ends
377 // *every* subscription with (a non-UTF-8 `put_stream` payload;
378 // see `Context::put_stream`), and that one does not heal: the
379 // chunk is replayed to the next subscription and breaks it the
380 // same way. Counting it here, and only when the connection
381 // delivered nothing, keeps a restarting engine reconnecting for
382 // as long as it takes while a poisoned stream still reaches the
383 // consumer as an error instead of silence. The classification
384 // table stays as it is: the other two streams in this SDK have
385 // their own give-up rules.
386 StreamErrorClass::Transient
387 if status.as_ref().map(Status::code) == Some(Code::Internal) =>
388 {
389 consecutive_other += 1;
390 consecutive_other >= STREAM_MAX_CONSECUTIVE_OTHER_FAILURES
391 }
392 StreamErrorClass::Transient => false,
393 StreamErrorClass::Fatal => true,
394 StreamErrorClass::Other => {
395 consecutive_other += 1;
396 consecutive_other >= STREAM_MAX_CONSECUTIVE_OTHER_FAILURES
397 }
398 };
399 if give_up {
400 // What was held goes out first: giving up is no reason to throw
401 // away chunks that did arrive.
402 let mut held = Vec::new();
403 ordering.flush(&mut held);
404 for item in held {
405 if chunks.send(item).await.is_err() {
406 return;
407 }
408 }
409 let status = status.unwrap_or_else(|| Status::unknown("the output stream ended"));
410 tracing::warn!(
411 code = ?status.code(),
412 message = status.message(),
413 consecutive_other,
414 "giving up on a run's output stream",
415 );
416 // The last thing the consumer is told, so the failure is not
417 // indistinguishable from a run that produced nothing.
418 let _ = chunks
419 .send(Err(Error::ResultStream(Box::new(status))))
420 .await;
421 return;
422 }
423
424 let delay = backoff(attempt);
425 attempt = attempt.saturating_add(1);
426 log_reconnect(attempt, consecutive_other, delay, status.as_ref());
427 tokio::select! {
428 () = cancel.cancelled() => return,
429 () = tokio::time::sleep(delay) => {}
430 }
431 }
432}
433
434/// How one connection ended.
435enum Outcome {
436 /// Nothing more will be delivered: the engine hung up, or the consumer
437 /// went away.
438 Done,
439 /// The stream ended; `status` is `None` when it ended cleanly.
440 Ended {
441 status: Option<Status>,
442 /// Whether this connection delivered at least one chunk.
443 received: bool,
444 },
445}
446
447/// Hands every chunk on `events` to the consumer until the stream ends, the
448/// engine hangs up, or the consumer goes away.
449async fn serve<S>(
450 mut events: S,
451 chunks: &mpsc::Sender<Result<StreamChunk>>,
452 cancel: &CancellationToken,
453 ordering: &mut Reorder,
454) -> Outcome
455where
456 S: Stream<Item = std::result::Result<WorkflowEvent, Status>> + Unpin,
457{
458 use tokio_stream::StreamExt;
459
460 let mut received = false;
461 // Reused rather than allocated per event: the ordinary chunk releases
462 // exactly one item.
463 let mut ready: Vec<Result<StreamChunk>> = Vec::new();
464 loop {
465 let item = tokio::select! {
466 () = cancel.cancelled() => return Outcome::Done,
467 item = events.next() => item,
468 };
469 match item {
470 Some(Ok(event)) => {
471 // The hang-up flag is read before the event is consumed, and
472 // acted on after it is delivered: the engine is allowed to put
473 // it on a chunk, and dropping that chunk would lose the last
474 // thing the run said.
475 let hangup = event.hangup;
476 ready.clear();
477 if let Some(chunk) = chunk_of(event) {
478 ordering.accept(chunk, &mut ready);
479 }
480 if hangup {
481 // Everything held goes out before the stream ends — the
482 // hang-up is the last chance to release it.
483 ordering.flush(&mut ready);
484 }
485 for item in ready.drain(..) {
486 // What resets the backoff is a chunk *delivered*; a
487 // reported gap is not evidence that the stream is healthy.
488 received |= item.is_ok();
489 if chunks.send(item).await.is_err() {
490 return Outcome::Done;
491 }
492 }
493 if hangup {
494 tracing::debug!("the engine hung up on a run's output stream");
495 return Outcome::Done;
496 }
497 }
498 Some(Err(status)) => {
499 return Outcome::Ended {
500 status: Some(status),
501 received,
502 };
503 }
504 None => {
505 return Outcome::Ended {
506 status: None,
507 received,
508 };
509 }
510 }
511 }
512}
513
514/// The chunk in `event`, or `None` for an event that is not one.
515///
516/// The filter is the event *type* alone, as the official Go SDK's is
517/// (`pkg/client/listener.go:422`): the same subscription carries a run's
518/// lifecycle events (STARTED, COMPLETED, …) and this is what tells them apart.
519/// `resource_type` is deliberately not looked at — the engine decides what it
520/// puts there, and a second condition could only ever make chunks disappear.
521fn chunk_of(event: WorkflowEvent) -> Option<StreamChunk> {
522 if event.event_type != ResourceEventType::Stream as i32 {
523 return None;
524 }
525 Some(StreamChunk {
526 run_id: event.workflow_run_id,
527 task_run_id: event.resource_id,
528 index: event.event_index,
529 payload: event.event_payload,
530 })
531}
532
533/// How many chunks of one task run are held back while an earlier one is
534/// missing.
535///
536/// Measured rather than guessed: against both supported engines, a task run's
537/// chunks arrived in index order every time, with no duplicates, including a
538/// hundred pushed back to back. Holding chunks costs latency and memory, so
539/// the buffer is only as deep as the engine was observed to need — and the gap
540/// report below is what makes a shallow buffer safe, because a hole is then
541/// *said* rather than waited out.
542const REORDER_BUFFER: usize = 1;
543
544/// One task run's chunk sequence.
545struct Sequence {
546 /// The index to deliver next.
547 next: i64,
548 /// Chunks that arrived before their turn, by index.
549 held: BTreeMap<i64, StreamChunk>,
550}
551
552/// Puts the chunks of one subscription back into `event_index` order, per task
553/// run.
554///
555/// It lives across reconnects on purpose: a new subscription may replay chunks
556/// this one has already delivered, and remembering where each task run got to
557/// is what turns those into duplicates to drop rather than output to hand over
558/// twice.
559///
560/// Three rules, and the reasons they are not the obvious ones:
561///
562/// * **The first index seen starts the sequence.** A subscription opened after
563/// the run started has not "lost" the chunks before it — it was never there
564/// for them — and reporting them as a gap would make the ordinary case look
565/// broken.
566/// * **An index at or below the last one delivered is dropped.** Handing a
567/// consumer the same chunk twice is worse than telling it one is missing.
568/// * **A hole is waited out only so far.** Past [`REORDER_BUFFER`] chunks
569/// behind it, what was held is released in index order and every hole that
570/// release steps over — there can be more than one — is reported as its own
571/// [`Error::StreamGap`](crate::Error::StreamGap). Nothing waits for ever.
572#[derive(Default)]
573struct Reorder {
574 tasks: HashMap<String, Sequence>,
575}
576
577impl Reorder {
578 /// Takes one chunk and appends whatever is now ready, in order, to `out`.
579 fn accept(&mut self, chunk: StreamChunk, out: &mut Vec<Result<StreamChunk>>) {
580 let Some(index) = chunk.index else {
581 // Nothing to order it by, and nothing it can be ordered against:
582 // an engine that does not fill `event_index` gets arrival order,
583 // which is what it gave us.
584 out.push(Ok(chunk));
585 return;
586 };
587 let task_run_id = chunk.task_run_id.clone();
588 let sequence = self.tasks.entry(task_run_id).or_insert(Sequence {
589 next: index,
590 held: BTreeMap::new(),
591 });
592 if index < sequence.next {
593 tracing::debug!(
594 task_run_id = %chunk.task_run_id,
595 index,
596 expected = sequence.next,
597 "dropping a chunk of a run's output stream that was already delivered",
598 );
599 return;
600 }
601 if index == sequence.next {
602 out.push(Ok(chunk));
603 sequence.next += 1;
604 // Whatever was waiting behind it can go now.
605 while let Some(next) = sequence.held.remove(&sequence.next) {
606 out.push(Ok(next));
607 sequence.next += 1;
608 }
609 return;
610 }
611 let task_run_id = chunk.task_run_id.clone();
612 sequence.held.insert(index, chunk);
613 if sequence.held.len() > REORDER_BUFFER {
614 release(&task_run_id, sequence, out);
615 }
616 }
617
618 /// Appends everything still held, in index order, to `out` — what the end
619 /// of a subscription releases.
620 fn flush(&mut self, out: &mut Vec<Result<StreamChunk>>) {
621 for (task_run_id, sequence) in &mut self.tasks {
622 release(task_run_id, sequence, out);
623 }
624 }
625}
626
627/// Releases everything `sequence` holds, in index order, reporting every hole
628/// it steps over.
629///
630/// One report per hole rather than one per batch: a batch can have more than
631/// one, and a second hole passed over in silence is exactly the loss
632/// [`Error::StreamGap`](crate::Error::StreamGap) exists to prevent.
633fn release(task_run_id: &str, sequence: &mut Sequence, out: &mut Vec<Result<StreamChunk>>) {
634 // `BTreeMap` iterates in key order, which is index order.
635 for (index, chunk) in std::mem::take(&mut sequence.held) {
636 if index > sequence.next {
637 tracing::warn!(
638 task_run_id,
639 expected = sequence.next,
640 got = index,
641 "a run's output stream skipped chunks",
642 );
643 out.push(Err(Error::StreamGap {
644 task_run_id: task_run_id.to_string(),
645 expected: sequence.next,
646 got: index,
647 }));
648 }
649 sequence.next = index + 1;
650 out.push(Ok(chunk));
651 }
652}
653
654/// Reconnecting is normal, so only the first attempt and every fifth one are
655/// worth a warning; the rest would drown a long outage in log lines.
656fn log_reconnect(attempt: u32, consecutive_other: u32, delay: Duration, status: Option<&Status>) {
657 let code = status.map(Status::code);
658 let message = status.map(Status::message).unwrap_or_default();
659 if attempt == 1 || attempt.is_multiple_of(5) {
660 tracing::warn!(
661 attempt,
662 consecutive_other,
663 ?delay,
664 ?code,
665 message,
666 "reconnecting to a run's output stream",
667 );
668 } else {
669 tracing::debug!(
670 attempt,
671 consecutive_other,
672 ?delay,
673 ?code,
674 message,
675 "reconnecting to a run's output stream",
676 );
677 }
678}
679
680#[cfg(test)]
681mod tests {
682 use std::collections::VecDeque;
683 use std::sync::atomic::{AtomicUsize, Ordering};
684 use std::sync::{Arc, Mutex};
685
686 use tokio::sync::mpsc;
687 use tokio::time::Instant;
688 use tokio_stream::wrappers::ReceiverStream;
689
690 use super::*;
691
692 /// Nothing here may block for ever: a regression should fail the test
693 /// rather than hang the suite. The clock is paused, so a wait that is
694 /// going to be answered costs no real time.
695 const NEVER: Duration = Duration::from_secs(10 * 60);
696
697 type ScriptedStream = ReceiverStream<std::result::Result<WorkflowEvent, Status>>;
698 type ScriptedSender = mpsc::Sender<std::result::Result<WorkflowEvent, Status>>;
699
700 /// One scripted connection: a stream the test drives, or a failure to
701 /// connect at all.
702 enum Step {
703 Stream(mpsc::Receiver<std::result::Result<WorkflowEvent, Status>>),
704 Fail(Status),
705 }
706
707 /// A connection the test drives: the script's step, and the sender that
708 /// feeds it. Dropping the sender ends that connection cleanly.
709 fn connection() -> (ScriptedSender, Step) {
710 let (tx, rx) = mpsc::channel(8);
711 (tx, Step::Stream(rx))
712 }
713
714 /// A `connect` function driven by a script, which also records when each
715 /// connection was opened. Once the script runs out it blocks for ever, so
716 /// the loop sits waiting rather than spinning.
717 #[derive(Clone)]
718 struct Script {
719 steps: Arc<Mutex<VecDeque<Step>>>,
720 calls: Arc<AtomicUsize>,
721 opened: mpsc::UnboundedSender<Instant>,
722 }
723
724 impl Script {
725 fn new(steps: Vec<Step>) -> (Self, mpsc::UnboundedReceiver<Instant>) {
726 let (opened, opens) = mpsc::unbounded_channel();
727 (
728 Self {
729 steps: Arc::new(Mutex::new(steps.into())),
730 calls: Arc::new(AtomicUsize::new(0)),
731 opened,
732 },
733 opens,
734 )
735 }
736
737 async fn connect(self) -> std::result::Result<ScriptedStream, Status> {
738 self.calls.fetch_add(1, Ordering::SeqCst);
739 let _ = self.opened.send(Instant::now());
740 let step = self
741 .steps
742 .lock()
743 .unwrap_or_else(|poisoned| poisoned.into_inner())
744 .pop_front();
745 match step {
746 Some(Step::Stream(events)) => Ok(ReceiverStream::new(events)),
747 Some(Step::Fail(status)) => Err(status),
748 None => std::future::pending().await,
749 }
750 }
751
752 fn calls(&self) -> usize {
753 self.calls.load(Ordering::SeqCst)
754 }
755 }
756
757 /// An output stream over `steps`, whose reconnect delay is
758 /// `backoff(attempt)` instead of the policy's jittered one.
759 async fn opened(
760 steps: Vec<Step>,
761 backoff: impl FnMut(u32) -> Duration + Send + 'static,
762 ) -> (OutputStream, Script, mpsc::UnboundedReceiver<Instant>) {
763 let (script, opens) = Script::new(steps);
764 let stream = open_with(
765 {
766 let script = script.clone();
767 move || script.clone().connect()
768 },
769 backoff,
770 )
771 .await
772 .expect("a scripted first connection is never fatal");
773 (stream, script, opens)
774 }
775
776 fn chunk(index: i64, payload: &str) -> WorkflowEvent {
777 WorkflowEvent {
778 workflow_run_id: "run-1".to_string(),
779 resource_id: "task-1".to_string(),
780 event_type: ResourceEventType::Stream as i32,
781 event_payload: payload.to_string(),
782 event_index: Some(index),
783 ..WorkflowEvent::default()
784 }
785 }
786
787 /// When the next connection was opened, or a failed test rather than a
788 /// hung one.
789 async fn next_open(opens: &mut mpsc::UnboundedReceiver<Instant>) -> Instant {
790 tokio::time::timeout(NEVER, opens.recv())
791 .await
792 .expect("timed out waiting for a connection")
793 .expect("the loop dropped the recording channel")
794 }
795
796 /// The next item, or a failed test rather than a hung one.
797 async fn next_item(stream: &mut OutputStream) -> Option<Result<StreamChunk>> {
798 tokio::time::timeout(NEVER, stream.next())
799 .await
800 .expect("timed out waiting for the stream")
801 }
802
803 /// The ordering state, and what one call to `accept` released.
804 fn accepted(ordering: &mut Reorder, chunk: StreamChunk) -> Vec<Result<StreamChunk>> {
805 let mut out = Vec::new();
806 ordering.accept(chunk, &mut out);
807 out
808 }
809
810 /// The payloads of the chunks in `items`, and a `"gap(expected..got)"`
811 /// marker for each gap — so a test can assert on one flat sequence.
812 fn shape(items: Vec<Result<StreamChunk>>) -> Vec<String> {
813 items
814 .into_iter()
815 .map(|item| match item {
816 Ok(chunk) => chunk.payload,
817 Err(Error::StreamGap { expected, got, .. }) => format!("gap({expected}..{got})"),
818 Err(other) => format!("error({other})"),
819 })
820 .collect()
821 }
822
823 fn numbered(task: &str, index: i64) -> StreamChunk {
824 StreamChunk {
825 run_id: "run-1".to_string(),
826 task_run_id: task.to_string(),
827 index: Some(index),
828 payload: format!("chunk-{index}"),
829 }
830 }
831
832 #[test]
833 fn chunks_that_arrive_in_order_pass_straight_through() {
834 let mut ordering = Reorder::default();
835 for n in 0..3 {
836 assert_eq!(
837 shape(accepted(&mut ordering, numbered("task-1", n))),
838 [format!("chunk-{n}")],
839 );
840 }
841 }
842
843 /// Criterion 2, without a socket: 2, 0, 1 comes out 0, 1, 2.
844 #[test]
845 fn an_early_chunk_waits_for_the_ones_before_it() {
846 let mut ordering = Reorder::default();
847 // The first index seen is where the sequence starts, so this test
848 // begins at 0 on purpose: 2 arriving first would simply *be* the start
849 // (see `a_late_subscription_starts_where_it_joined`).
850 assert_eq!(
851 shape(accepted(&mut ordering, numbered("task-1", 0))),
852 ["chunk-0"],
853 );
854 assert!(shape(accepted(&mut ordering, numbered("task-1", 2))).is_empty());
855 assert_eq!(
856 shape(accepted(&mut ordering, numbered("task-1", 1))),
857 ["chunk-1", "chunk-2"],
858 );
859 }
860
861 /// Criterion 3: a chunk that never comes does not block the stream for
862 /// ever. Once more than `REORDER_BUFFER` chunks are waiting behind the
863 /// hole, the hole is reported and what was held is released in order.
864 #[test]
865 fn a_hole_is_reported_once_the_buffer_is_over_full_and_the_rest_follows() {
866 let mut ordering = Reorder::default();
867 accepted(&mut ordering, numbered("task-1", 0));
868 // Index 1 never arrives. Everything after it waits…
869 let mut released = Vec::new();
870 for n in 2..=(2 + REORDER_BUFFER as i64) {
871 released.extend(shape(accepted(&mut ordering, numbered("task-1", n))));
872 }
873 // …until the buffer is over full, and then the gap is reported once
874 // and the held chunks follow it in index order.
875 let expected: Vec<String> = std::iter::once("gap(1..2)".to_string())
876 .chain((2..=(2 + REORDER_BUFFER as i64)).map(|n| format!("chunk-{n}")))
877 .collect();
878 assert_eq!(released, expected);
879 // And the sequence carries on from where the engine is.
880 assert_eq!(
881 shape(accepted(
882 &mut ordering,
883 numbered("task-1", 3 + REORDER_BUFFER as i64),
884 )),
885 [format!("chunk-{}", 3 + REORDER_BUFFER as i64)],
886 );
887 }
888
889 /// Two holes in one batch are two reports: the consumer is told about
890 /// every stretch that is missing, not only the first one. With
891 /// `REORDER_BUFFER = 1` the release happens with two chunks held, so
892 /// `{3, 7}` behind `next = 1` is exactly the case that reaches it.
893 #[test]
894 fn every_hole_in_a_released_batch_is_reported() {
895 let mut ordering = Reorder::default();
896 // The sequence starts at 0, so 1 is what is expected next.
897 accepted(&mut ordering, numbered("task-1", 0));
898 assert!(shape(accepted(&mut ordering, numbered("task-1", 3))).is_empty());
899 assert_eq!(
900 shape(accepted(&mut ordering, numbered("task-1", 7))),
901 ["gap(1..3)", "chunk-3", "gap(4..7)", "chunk-7"],
902 );
903 // And the sequence carries on from the last index released.
904 assert_eq!(
905 shape(accepted(&mut ordering, numbered("task-1", 8))),
906 ["chunk-8"],
907 );
908 }
909
910 /// A chunk that was already delivered is dropped rather than handed over
911 /// twice — reconnecting can replay, and a duplicate is worse for a
912 /// consumer than a reported gap.
913 #[test]
914 fn a_chunk_that_was_already_delivered_is_dropped() {
915 let mut ordering = Reorder::default();
916 accepted(&mut ordering, numbered("task-1", 0));
917 accepted(&mut ordering, numbered("task-1", 1));
918 assert!(shape(accepted(&mut ordering, numbered("task-1", 0))).is_empty());
919 assert!(shape(accepted(&mut ordering, numbered("task-1", 1))).is_empty());
920 assert_eq!(
921 shape(accepted(&mut ordering, numbered("task-1", 2))),
922 ["chunk-2"],
923 );
924 }
925
926 /// Indexes are per task run, so two tasks streaming at once do not hold
927 /// each other up.
928 #[test]
929 fn each_task_run_has_its_own_sequence() {
930 let mut ordering = Reorder::default();
931 assert_eq!(
932 shape(accepted(&mut ordering, numbered("task-1", 0))),
933 ["chunk-0"],
934 );
935 // Task two's 0 is not task one's 1, and does not wait for anything.
936 assert_eq!(
937 shape(accepted(&mut ordering, numbered("task-2", 0))),
938 ["chunk-0"],
939 );
940 assert!(shape(accepted(&mut ordering, numbered("task-1", 2))).is_empty());
941 // Task two is unaffected by task one's hole.
942 assert_eq!(
943 shape(accepted(&mut ordering, numbered("task-2", 1))),
944 ["chunk-1"],
945 );
946 }
947
948 /// An engine that does not fill `event_index` still delivers: those chunks
949 /// go straight through, and they neither wait for nor disturb a sequence.
950 #[test]
951 fn a_chunk_without_an_index_goes_straight_through() {
952 let mut ordering = Reorder::default();
953 accepted(&mut ordering, numbered("task-1", 0));
954 assert!(shape(accepted(&mut ordering, numbered("task-1", 2))).is_empty());
955 let unnumbered = StreamChunk {
956 index: None,
957 payload: "no index".to_string(),
958 ..numbered("task-1", 0)
959 };
960 assert_eq!(shape(accepted(&mut ordering, unnumbered)), ["no index"]);
961 // The held chunk is still held, and 1 still completes it.
962 assert_eq!(
963 shape(accepted(&mut ordering, numbered("task-1", 1))),
964 ["chunk-1", "chunk-2"],
965 );
966 }
967
968 /// A subscription opened after the run started joins at whatever index it
969 /// first sees. Reporting everything before that as a gap would make the
970 /// ordinary case — subscribe after trigger — look broken.
971 #[test]
972 fn a_late_subscription_starts_where_it_joined() {
973 let mut ordering = Reorder::default();
974 assert_eq!(
975 shape(accepted(&mut ordering, numbered("task-1", 40))),
976 ["chunk-40"],
977 );
978 assert_eq!(
979 shape(accepted(&mut ordering, numbered("task-1", 41))),
980 ["chunk-41"],
981 );
982 }
983
984 /// The end of the stream releases what is still held, gap and all: chunks
985 /// that did arrive are not thrown away because one did not.
986 #[test]
987 fn flushing_releases_what_is_held_and_reports_the_hole() {
988 let mut ordering = Reorder::default();
989 accepted(&mut ordering, numbered("task-1", 0));
990 accepted(&mut ordering, numbered("task-1", 2));
991 let mut out = Vec::new();
992 ordering.flush(&mut out);
993 assert_eq!(shape(out), ["gap(1..2)", "chunk-2"]);
994 // Flushing twice releases nothing more.
995 let mut out = Vec::new();
996 ordering.flush(&mut out);
997 assert!(out.is_empty());
998 }
999
1000 /// Under full jitter the real delay may be any length including zero, so
1001 /// only a fixed backoff can tell a reconnect delay that is there from one
1002 /// that was deleted.
1003 #[tokio::test(start_paused = true)]
1004 async fn a_reconnect_waits_for_the_backoff() {
1005 let (first, one) = connection();
1006 let (second, two) = connection();
1007 let (mut chunks, _script, mut opens) =
1008 opened(vec![one, two], |_| Duration::from_secs(7)).await;
1009 let opened_at = next_open(&mut opens).await;
1010
1011 // A clean end of stream: worth reconnecting, after the backoff.
1012 drop(first);
1013
1014 let reopened_at = next_open(&mut opens).await;
1015 assert!(
1016 reopened_at.duration_since(opened_at) >= Duration::from_secs(7),
1017 "reconnected after {:?}",
1018 reopened_at.duration_since(opened_at),
1019 );
1020 second
1021 .send(Ok(chunk(0, "after")))
1022 .await
1023 .expect("the second connection is open");
1024 let chunk = next_item(&mut chunks)
1025 .await
1026 .expect("a chunk")
1027 .expect("not an error");
1028 assert_eq!(chunk.payload, "after");
1029 }
1030
1031 /// Ten unclassified failures in a row is the SDK's give-up rule
1032 /// (`STREAM_MAX_CONSECUTIVE_OTHER_FAILURES`), and giving up is told to the
1033 /// consumer rather than looking like a run with no output.
1034 #[tokio::test(start_paused = true)]
1035 async fn ten_unclassified_failures_in_a_row_give_up() {
1036 let (first, one) = connection();
1037 let mut steps = vec![one];
1038 steps.extend((0..10).map(|_| Step::Fail(Status::unknown("what?"))));
1039 let (mut chunks, script, _opens) = opened(steps, |_| Duration::ZERO).await;
1040
1041 // A clean end is transient, so it does not count towards the ten.
1042 drop(first);
1043
1044 let error = next_item(&mut chunks)
1045 .await
1046 .expect("the give-up is the last item")
1047 .expect_err("giving up is an error");
1048 assert!(matches!(error, Error::ResultStream(_)), "{error:?}");
1049 assert!(next_item(&mut chunks).await.is_none(), "and then it ends");
1050 assert_eq!(script.calls(), 11, "one good connection and ten failures");
1051 }
1052
1053 /// A connection that delivered a chunk was healthy, whatever it did
1054 /// afterwards: the give-up counter starts again from it.
1055 #[tokio::test(start_paused = true)]
1056 async fn a_delivered_chunk_resets_the_give_up_counter() {
1057 let (first, one) = connection();
1058 let (middle, two) = connection();
1059 let (last, three) = connection();
1060 let mut steps = vec![one];
1061 steps.extend((0..9).map(|_| Step::Fail(Status::unknown("what?"))));
1062 steps.push(two);
1063 steps.extend((0..9).map(|_| Step::Fail(Status::unknown("what?"))));
1064 steps.push(three);
1065 let (mut chunks, script, _opens) = opened(steps, |_| Duration::ZERO).await;
1066
1067 drop(first);
1068 middle
1069 .send(Ok(chunk(0, "healthy")))
1070 .await
1071 .expect("the middle connection is open");
1072 assert_eq!(
1073 next_item(&mut chunks)
1074 .await
1075 .expect("a chunk")
1076 .expect("not an error")
1077 .payload,
1078 "healthy",
1079 );
1080 drop(middle);
1081
1082 // Nine more failures after that: still under the limit, so the stream
1083 // is alive and the last connection delivers.
1084 last.send(Ok(chunk(1, "still here")))
1085 .await
1086 .expect("the last connection is open");
1087 assert_eq!(
1088 next_item(&mut chunks)
1089 .await
1090 .expect("a chunk")
1091 .expect("not an error")
1092 .payload,
1093 "still here",
1094 );
1095 assert_eq!(script.calls(), 21);
1096 }
1097}