Skip to main content

hatchet_client/runs/
stream.rs

1//! Reading a run's output stream: the chunks
2//! [`Context::put_stream`](crate::Context::put_stream) publishes.
3//!
4//! One [`OutputStream`] owns one `SubscribeToWorkflowEvents` subscription and
5//! one background task. That is the opposite of the run *result* listener next
6//! door, which multiplexes every wait on one client onto a single stream — and
7//! the difference is in the wire, not in taste: a result subscription takes a
8//! *stream* of run ids, so more runs can be added to one connection, while an
9//! event subscription's filter is a single request message, so one filter
10//! needs one connection.
11//!
12//! Like the result listener, the loop here is network-agnostic: [`run`] is
13//! handed a function that opens one stream, so the filter, the reconnect
14//! policy and the give-up rule can be tested against a scripted stream rather
15//! than a socket.
16
17use std::collections::{BTreeMap, HashMap};
18use std::pin::Pin;
19use std::task::{Context as TaskContext, Poll};
20use std::time::Duration;
21
22use hatchet_proto::root::{
23    ResourceEventType, SubscribeToWorkflowEventsRequest, WorkflowEvent,
24    dispatcher_client::DispatcherClient,
25};
26use tokio::sync::mpsc;
27use tokio_stream::Stream;
28use tokio_util::sync::{CancellationToken, DropGuard};
29use tonic::{Code, Status};
30
31use crate::Client;
32use crate::error::{Error, Result};
33use crate::retry::{
34    STREAM_MAX_CONSECUTIVE_OTHER_FAILURES, StreamErrorClass, classify_stream_end, full_jitter,
35    stream_backoff_cap,
36};
37use crate::transport::GrpcService;
38
39/// How many chunks are held for a consumer that is not reading.
40///
41/// Beyond this the background task waits, which stops reading the socket,
42/// which is what applies back pressure to the engine — the alternative is an
43/// unbounded queue that a slow consumer turns into unbounded memory. Payloads
44/// are whatever the handler published (128 KiB arrived intact in testing), so
45/// this bounds the memory a stalled consumer costs at that many payloads.
46const CHUNK_BUFFER: usize = 64;
47
48/// How long [`open`] waits for the engine to answer the subscribe call before
49/// handing back a stream that is still being opened in the background.
50///
51/// It is a *courtesy*, not a guarantee: a subscription the engine answers
52/// straight away is open before the caller triggers anything, which is what
53/// makes "subscribe, then trigger" worth recommending. Both supported engines
54/// have been observed not sending response headers on a fresh stream until
55/// they have something to say — for over a minute when nothing matched the
56/// filter — so waiting for them without a bound would hang the caller.
57///
58/// Running out of it never *abandons* the attempt: the call goes on in the
59/// background task and becomes the stream's first connection. The engine has
60/// registered the subscription by then, whatever its headers say, so dropping
61/// the call would close a subscription that was already receiving.
62const OPEN_TIMEOUT: Duration = Duration::from_secs(5);
63
64/// One chunk of a run's output, as
65/// [`Context::put_stream`](crate::Context::put_stream) published it.
66///
67/// The payload is whatever the handler wrote. It goes out as bytes and comes
68/// back as a `String`, because that is what the two sides of the wire are
69/// (`PutStreamEventRequest.message` is `bytes`, `WorkflowEvent.event_payload`
70/// is `string`); see [`put_stream`](crate::Context::put_stream) for what that
71/// does to bytes that are not UTF-8.
72///
73/// ```no_run
74/// # async fn example(client: hatchet_client::Client) -> hatchet_client::Result<()> {
75/// let mut chunks = client.runs().stream("d627fab0-385f-4eb4-98f9-59a783bc8631").await?;
76/// if let Some(Ok(chunk)) = chunks.next().await {
77///     println!("run {} task {}: [{:?}] {}", chunk.run_id, chunk.task_run_id, chunk.index, chunk.payload);
78/// }
79/// # Ok(())
80/// # }
81/// ```
82#[derive(Clone, Debug, PartialEq, Eq)]
83pub struct StreamChunk {
84    /// The run the chunk belongs to. Worth reading when one subscription
85    /// covers several runs — see
86    /// [`RunsClient::stream_by_metadata`](crate::RunsClient::stream_by_metadata).
87    pub run_id: String,
88    /// The *task* run that published it. A workflow run has one sequence of
89    /// chunks per task, so this is what tells two interleaved outputs apart.
90    pub task_run_id: String,
91    /// The chunk's number in its task run's sequence, counted from zero, when
92    /// the engine sent one. `None` means the engine did not — the field is
93    /// optional on the wire — and such chunks are handed over in arrival order
94    /// without any ordering guarantee.
95    pub index: Option<i64>,
96    /// What the handler published.
97    pub payload: String,
98}
99
100/// A run's output stream: the chunks its handlers publish, in order.
101///
102/// Made by [`RunsClient::stream`](crate::RunsClient::stream),
103/// [`RunsClient::stream_by_metadata`](crate::RunsClient::stream_by_metadata),
104/// [`RunHandle::stream`](crate::RunHandle::stream) and
105/// [`WorkflowRunHandle::stream`](crate::WorkflowRunHandle::stream).
106///
107/// It implements [`futures_core::Stream`](tokio_stream::Stream), so every
108/// combinator that takes one works; [`next`](OutputStream::next) is here too
109/// so the common case needs no extra crate and no trait import.
110///
111/// **Dropping it ends the subscription.** The background task that owns the
112/// connection is cancelled, and the engine sees the stream close.
113///
114/// Delivery is not guaranteed: the engine has been observed dropping a chunk
115/// silently. A gap in a task run's chunk numbering is reported as one
116/// [`Error::StreamGap`] item and the stream carries on.
117///
118/// ```no_run
119/// # async fn example(client: hatchet_client::Client) -> hatchet_client::Result<()> {
120/// let mut chunks = client.runs().stream("d627fab0-385f-4eb4-98f9-59a783bc8631").await?;
121/// while let Some(chunk) = chunks.next().await {
122///     match chunk {
123///         Ok(chunk) => print!("{}", chunk.payload),
124///         // One hole in the output; the rest still arrives.
125///         Err(hatchet_client::Error::StreamGap { .. }) => print!("…"),
126///         Err(error) => return Err(error),
127///     }
128/// }
129/// # Ok(())
130/// # }
131/// ```
132#[must_use = "an output stream delivers nothing until it is read"]
133#[derive(Debug)]
134pub struct OutputStream {
135    chunks: mpsc::Receiver<Result<StreamChunk>>,
136    /// Cancels the background task when this stream is dropped. Holding the
137    /// receiver alone would not: the task would keep the subscription open
138    /// until it next tried to send.
139    _cancel: DropGuard,
140}
141
142impl OutputStream {
143    /// The next chunk, or `None` once the stream is over — the engine hung up,
144    /// or it failed in a way reconnecting cannot fix (and then the last item
145    /// was that error).
146    ///
147    /// Cancel-safe: dropping the returned future loses nothing.
148    ///
149    /// ```no_run
150    /// # async fn example(client: hatchet_client::Client) -> hatchet_client::Result<()> {
151    /// let mut chunks = client.runs().stream("d627fab0-385f-4eb4-98f9-59a783bc8631").await?;
152    /// while let Some(chunk) = chunks.next().await {
153    ///     print!("{}", chunk?.payload);
154    /// }
155    /// # Ok(())
156    /// # }
157    /// ```
158    pub async fn next(&mut self) -> Option<Result<StreamChunk>> {
159        self.chunks.recv().await
160    }
161}
162
163impl Stream for OutputStream {
164    type Item = Result<StreamChunk>;
165
166    fn poll_next(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Option<Self::Item>> {
167        self.chunks.poll_recv(cx)
168    }
169}
170
171/// The events one subscription is answered with.
172type EventStream = Pin<Box<dyn Stream<Item = std::result::Result<WorkflowEvent, Status>> + Send>>;
173
174/// Opens one subscription with `request` and hands back the stream it feeds.
175///
176/// The first connection is awaited for at most [`OPEN_TIMEOUT`] so that a
177/// subscription the engine answers promptly is really open when this returns;
178/// if it is not answered in time, *that same attempt* is handed to the
179/// background task, which goes on waiting for it. A first connection that
180/// fails in a way reconnecting cannot fix (a rejected token or an unknown run
181/// id, most of all) is returned as an error rather than becoming a stream that
182/// quietly yields nothing.
183pub(crate) async fn open(
184    client: &Client,
185    request: SubscribeToWorkflowEventsRequest,
186) -> Result<OutputStream> {
187    // `dispatcher_grpc` already applies the client's gzip setting and the auth
188    // header, so nothing about this connection is special.
189    let connect = connecting(client.dispatcher_grpc(), request);
190    open_with(connect, |attempt| {
191        full_jitter(stream_backoff_cap(attempt), &mut rand::rng())
192    })
193    .await
194}
195
196/// [`open`] with the delay before reconnect number `attempt` (0-based) chosen
197/// by `backoff` rather than by the SDK's policy.
198///
199/// The seam exists for the tests, exactly as it does on the result listener:
200/// under full jitter the real delay may be any length including zero, so no
201/// assertion on elapsed time can tell a backoff that is there from one that
202/// was deleted.
203async fn open_with<C, Fut, S, B>(mut connect: C, backoff: B) -> Result<OutputStream>
204where
205    C: FnMut() -> Fut + Send + 'static,
206    Fut: Future<Output = std::result::Result<S, Status>> + Send + 'static,
207    S: Stream<Item = std::result::Result<WorkflowEvent, Status>> + Send + Unpin + 'static,
208    B: FnMut(u32) -> Duration + Send + 'static,
209{
210    let cancel = CancellationToken::new();
211    // Spawned rather than awaited in place: an attempt this call stops waiting
212    // for is handed to the background task, and dropping the future instead
213    // would reset the HTTP/2 stream and unregister a subscription the engine
214    // has already made — losing whatever is published before a replacement
215    // lands, with no gap to report it by.
216    //
217    // Outliving *this call* is not the same as outliving the caller, though:
218    // the guard below cancels the attempt if this function is dropped before
219    // it hands it over — a `timeout` around `stream()`, a `select!` that loses
220    // — rather than leaving a subscription registered with nobody to read it.
221    let abandoned = cancel.clone().drop_guard();
222    let mut connecting = tokio::spawn({
223        let cancel = cancel.clone();
224        let attempt = connect();
225        async move {
226            tokio::select! {
227                () = cancel.cancelled() => {
228                    Err(Status::cancelled("the subscribe call was abandoned"))
229                }
230                connected = attempt => connected,
231            }
232        }
233    });
234    let first = match tokio::time::timeout(OPEN_TIMEOUT, &mut connecting).await {
235        Ok(Ok(Ok(stream))) => First::Open(stream),
236        Ok(Ok(Err(status))) if classify_stream_end(Some(&status)) == StreamErrorClass::Fatal => {
237            return Err(Error::Grpc(Box::new(status)));
238        }
239        // Worth another attempt, and the background task is where attempts
240        // live.
241        Ok(Ok(Err(_))) => First::None,
242        // The attempt panicked, which is a bug in this SDK rather than
243        // anything the engine did; the loop's own attempts are what is left.
244        Ok(Err(_)) => First::None,
245        // The engine has not answered yet. Not an error: it may simply have
246        // nothing to say about this run so far, which is the normal case —
247        // both supported engines hold back response headers until they have
248        // an event. The attempt goes on in the background task.
249        Err(_) => First::Connecting(connecting),
250    };
251    let (chunks, received) = mpsc::channel(CHUNK_BUFFER);
252    tokio::spawn(run(connect, first, chunks, cancel, backoff));
253    Ok(OutputStream {
254        chunks: received,
255        // The attempt belongs to the background task now, so the guard that
256        // watched this call hands its job over to the one the consumer holds.
257        _cancel: abandoned.disarm().drop_guard(),
258    })
259}
260
261/// The connect function for a real engine: opens one
262/// `SubscribeToWorkflowEvents` stream with the same request every time.
263///
264/// No `set_timeout` and no [`retry::unary`](crate::retry::unary), for the same
265/// reasons as the other two streams in this SDK: a deadline would cut off a
266/// run that is simply taking its time, and reconnecting is this loop's own
267/// job.
268fn connecting(
269    dispatcher: DispatcherClient<GrpcService>,
270    request: SubscribeToWorkflowEventsRequest,
271) -> impl FnMut() -> Pin<Box<dyn Future<Output = std::result::Result<EventStream, Status>> + Send>>
272{
273    move || {
274        let mut dispatcher = dispatcher.clone();
275        let request = request.clone();
276        Box::pin(async move {
277            let events = dispatcher
278                .subscribe_to_workflow_events(request)
279                .await?
280                .into_inner();
281            Ok(Box::pin(events) as EventStream)
282        })
283    }
284}
285
286/// The connection attempt [`open_with`] already made, handed to [`run`] so
287/// that one call to `open` never registers two subscriptions.
288enum First<S> {
289    /// Nothing to take over: the loop opens its own.
290    None,
291    /// Answered before [`OPEN_TIMEOUT`] was up, and open.
292    Open(S),
293    /// Still unanswered when [`OPEN_TIMEOUT`] was up. Awaiting the handle is
294    /// what takes it over; aborting it is what closes it.
295    Connecting(tokio::task::JoinHandle<std::result::Result<S, Status>>),
296}
297
298/// Holds one subscription open, reopening it for as long as the engine keeps
299/// dropping it, until the engine hangs up, the consumer goes away, or a
300/// failure that reconnecting cannot fix.
301///
302/// `first` is the attempt [`open`] already made — open, or still being
303/// answered — so a subscription the engine has already registered is used
304/// rather than opened a second time.
305async fn run<C, Fut, S, B>(
306    mut connect: C,
307    first: First<S>,
308    chunks: mpsc::Sender<Result<StreamChunk>>,
309    cancel: CancellationToken,
310    mut backoff: B,
311) where
312    C: FnMut() -> Fut,
313    Fut: Future<Output = std::result::Result<S, Status>>,
314    S: Stream<Item = std::result::Result<WorkflowEvent, Status>> + Unpin,
315    B: FnMut(u32) -> Duration,
316{
317    // The backoff exponent and the run of failures that count towards giving
318    // up. Both are reset by a single delivered chunk: a stream that delivered
319    // output was healthy, whatever it did afterwards.
320    let mut attempt: u32 = 0;
321    let mut consecutive_other: u32 = 0;
322    let mut first = first;
323    // Kept across reconnects: a new subscription may replay chunks the old one
324    // delivered, and this is what remembers them.
325    let mut ordering = Reorder::default();
326
327    loop {
328        let connected = match std::mem::replace(&mut first, First::None) {
329            First::Open(stream) => Ok(stream),
330            // Aborted rather than dropped when the consumer goes away:
331            // dropping a `JoinHandle` detaches the task, which would leave the
332            // subscribe call running with nobody to read it.
333            First::Connecting(mut connecting) => tokio::select! {
334                () = cancel.cancelled() => {
335                    connecting.abort();
336                    return;
337                }
338                joined = &mut connecting => match joined {
339                    Ok(connected) => connected,
340                    Err(_) => Err(Status::internal("the subscribe call panicked")),
341                },
342            },
343            First::None => {
344                tokio::select! {
345                    () = cancel.cancelled() => return,
346                    connected = connect() => connected,
347                }
348            }
349        };
350        let outcome = match connected {
351            Ok(events) => serve(events, &chunks, &cancel, &mut ordering).await,
352            // No difference, for what happens next, between "the stream could
353            // not be opened" and "it closed immediately".
354            Err(status) => Outcome::Ended {
355                status: Some(status),
356                received: false,
357            },
358        };
359        let (status, received) = match outcome {
360            // The consumer dropped the stream, or the engine hung up: either
361            // way this subscription is over for good.
362            Outcome::Done => return,
363            Outcome::Ended { status, received } => (status, received),
364        };
365        if received {
366            attempt = 0;
367            consecutive_other = 0;
368        }
369
370        let give_up = match classify_stream_end(status.as_ref()) {
371            // `Cancelled` is a clean stop for the worker's listener, which
372            // shares `classify_stream_end`; here a cancelled subscription the
373            // consumer still holds is just one more stream to reopen.
374            StreamErrorClass::StopClean => false,
375            // `Internal` is transient in general — a restarting engine sends
376            // it — but it is also what a chunk the engine cannot marshal ends
377            // *every* subscription with (a non-UTF-8 `put_stream` payload;
378            // see `Context::put_stream`), and that one does not heal: the
379            // chunk is replayed to the next subscription and breaks it the
380            // same way. Counting it here, and only when the connection
381            // delivered nothing, keeps a restarting engine reconnecting for
382            // as long as it takes while a poisoned stream still reaches the
383            // consumer as an error instead of silence. The classification
384            // table stays as it is: the other two streams in this SDK have
385            // their own give-up rules.
386            StreamErrorClass::Transient
387                if status.as_ref().map(Status::code) == Some(Code::Internal) =>
388            {
389                consecutive_other += 1;
390                consecutive_other >= STREAM_MAX_CONSECUTIVE_OTHER_FAILURES
391            }
392            StreamErrorClass::Transient => false,
393            StreamErrorClass::Fatal => true,
394            StreamErrorClass::Other => {
395                consecutive_other += 1;
396                consecutive_other >= STREAM_MAX_CONSECUTIVE_OTHER_FAILURES
397            }
398        };
399        if give_up {
400            // What was held goes out first: giving up is no reason to throw
401            // away chunks that did arrive.
402            let mut held = Vec::new();
403            ordering.flush(&mut held);
404            for item in held {
405                if chunks.send(item).await.is_err() {
406                    return;
407                }
408            }
409            let status = status.unwrap_or_else(|| Status::unknown("the output stream ended"));
410            tracing::warn!(
411                code = ?status.code(),
412                message = status.message(),
413                consecutive_other,
414                "giving up on a run's output stream",
415            );
416            // The last thing the consumer is told, so the failure is not
417            // indistinguishable from a run that produced nothing.
418            let _ = chunks
419                .send(Err(Error::ResultStream(Box::new(status))))
420                .await;
421            return;
422        }
423
424        let delay = backoff(attempt);
425        attempt = attempt.saturating_add(1);
426        log_reconnect(attempt, consecutive_other, delay, status.as_ref());
427        tokio::select! {
428            () = cancel.cancelled() => return,
429            () = tokio::time::sleep(delay) => {}
430        }
431    }
432}
433
434/// How one connection ended.
435enum Outcome {
436    /// Nothing more will be delivered: the engine hung up, or the consumer
437    /// went away.
438    Done,
439    /// The stream ended; `status` is `None` when it ended cleanly.
440    Ended {
441        status: Option<Status>,
442        /// Whether this connection delivered at least one chunk.
443        received: bool,
444    },
445}
446
447/// Hands every chunk on `events` to the consumer until the stream ends, the
448/// engine hangs up, or the consumer goes away.
449async fn serve<S>(
450    mut events: S,
451    chunks: &mpsc::Sender<Result<StreamChunk>>,
452    cancel: &CancellationToken,
453    ordering: &mut Reorder,
454) -> Outcome
455where
456    S: Stream<Item = std::result::Result<WorkflowEvent, Status>> + Unpin,
457{
458    use tokio_stream::StreamExt;
459
460    let mut received = false;
461    // Reused rather than allocated per event: the ordinary chunk releases
462    // exactly one item.
463    let mut ready: Vec<Result<StreamChunk>> = Vec::new();
464    loop {
465        let item = tokio::select! {
466            () = cancel.cancelled() => return Outcome::Done,
467            item = events.next() => item,
468        };
469        match item {
470            Some(Ok(event)) => {
471                // The hang-up flag is read before the event is consumed, and
472                // acted on after it is delivered: the engine is allowed to put
473                // it on a chunk, and dropping that chunk would lose the last
474                // thing the run said.
475                let hangup = event.hangup;
476                ready.clear();
477                if let Some(chunk) = chunk_of(event) {
478                    ordering.accept(chunk, &mut ready);
479                }
480                if hangup {
481                    // Everything held goes out before the stream ends — the
482                    // hang-up is the last chance to release it.
483                    ordering.flush(&mut ready);
484                }
485                for item in ready.drain(..) {
486                    // What resets the backoff is a chunk *delivered*; a
487                    // reported gap is not evidence that the stream is healthy.
488                    received |= item.is_ok();
489                    if chunks.send(item).await.is_err() {
490                        return Outcome::Done;
491                    }
492                }
493                if hangup {
494                    tracing::debug!("the engine hung up on a run's output stream");
495                    return Outcome::Done;
496                }
497            }
498            Some(Err(status)) => {
499                return Outcome::Ended {
500                    status: Some(status),
501                    received,
502                };
503            }
504            None => {
505                return Outcome::Ended {
506                    status: None,
507                    received,
508                };
509            }
510        }
511    }
512}
513
514/// The chunk in `event`, or `None` for an event that is not one.
515///
516/// The filter is the event *type* alone, as the official Go SDK's is
517/// (`pkg/client/listener.go:422`): the same subscription carries a run's
518/// lifecycle events (STARTED, COMPLETED, …) and this is what tells them apart.
519/// `resource_type` is deliberately not looked at — the engine decides what it
520/// puts there, and a second condition could only ever make chunks disappear.
521fn chunk_of(event: WorkflowEvent) -> Option<StreamChunk> {
522    if event.event_type != ResourceEventType::Stream as i32 {
523        return None;
524    }
525    Some(StreamChunk {
526        run_id: event.workflow_run_id,
527        task_run_id: event.resource_id,
528        index: event.event_index,
529        payload: event.event_payload,
530    })
531}
532
533/// How many chunks of one task run are held back while an earlier one is
534/// missing.
535///
536/// Measured rather than guessed: against both supported engines, a task run's
537/// chunks arrived in index order every time, with no duplicates, including a
538/// hundred pushed back to back. Holding chunks costs latency and memory, so
539/// the buffer is only as deep as the engine was observed to need — and the gap
540/// report below is what makes a shallow buffer safe, because a hole is then
541/// *said* rather than waited out.
542const REORDER_BUFFER: usize = 1;
543
544/// One task run's chunk sequence.
545struct Sequence {
546    /// The index to deliver next.
547    next: i64,
548    /// Chunks that arrived before their turn, by index.
549    held: BTreeMap<i64, StreamChunk>,
550}
551
552/// Puts the chunks of one subscription back into `event_index` order, per task
553/// run.
554///
555/// It lives across reconnects on purpose: a new subscription may replay chunks
556/// this one has already delivered, and remembering where each task run got to
557/// is what turns those into duplicates to drop rather than output to hand over
558/// twice.
559///
560/// Three rules, and the reasons they are not the obvious ones:
561///
562/// * **The first index seen starts the sequence.** A subscription opened after
563///   the run started has not "lost" the chunks before it — it was never there
564///   for them — and reporting them as a gap would make the ordinary case look
565///   broken.
566/// * **An index at or below the last one delivered is dropped.** Handing a
567///   consumer the same chunk twice is worse than telling it one is missing.
568/// * **A hole is waited out only so far.** Past [`REORDER_BUFFER`] chunks
569///   behind it, what was held is released in index order and every hole that
570///   release steps over — there can be more than one — is reported as its own
571///   [`Error::StreamGap`](crate::Error::StreamGap). Nothing waits for ever.
572#[derive(Default)]
573struct Reorder {
574    tasks: HashMap<String, Sequence>,
575}
576
577impl Reorder {
578    /// Takes one chunk and appends whatever is now ready, in order, to `out`.
579    fn accept(&mut self, chunk: StreamChunk, out: &mut Vec<Result<StreamChunk>>) {
580        let Some(index) = chunk.index else {
581            // Nothing to order it by, and nothing it can be ordered against:
582            // an engine that does not fill `event_index` gets arrival order,
583            // which is what it gave us.
584            out.push(Ok(chunk));
585            return;
586        };
587        let task_run_id = chunk.task_run_id.clone();
588        let sequence = self.tasks.entry(task_run_id).or_insert(Sequence {
589            next: index,
590            held: BTreeMap::new(),
591        });
592        if index < sequence.next {
593            tracing::debug!(
594                task_run_id = %chunk.task_run_id,
595                index,
596                expected = sequence.next,
597                "dropping a chunk of a run's output stream that was already delivered",
598            );
599            return;
600        }
601        if index == sequence.next {
602            out.push(Ok(chunk));
603            sequence.next += 1;
604            // Whatever was waiting behind it can go now.
605            while let Some(next) = sequence.held.remove(&sequence.next) {
606                out.push(Ok(next));
607                sequence.next += 1;
608            }
609            return;
610        }
611        let task_run_id = chunk.task_run_id.clone();
612        sequence.held.insert(index, chunk);
613        if sequence.held.len() > REORDER_BUFFER {
614            release(&task_run_id, sequence, out);
615        }
616    }
617
618    /// Appends everything still held, in index order, to `out` — what the end
619    /// of a subscription releases.
620    fn flush(&mut self, out: &mut Vec<Result<StreamChunk>>) {
621        for (task_run_id, sequence) in &mut self.tasks {
622            release(task_run_id, sequence, out);
623        }
624    }
625}
626
627/// Releases everything `sequence` holds, in index order, reporting every hole
628/// it steps over.
629///
630/// One report per hole rather than one per batch: a batch can have more than
631/// one, and a second hole passed over in silence is exactly the loss
632/// [`Error::StreamGap`](crate::Error::StreamGap) exists to prevent.
633fn release(task_run_id: &str, sequence: &mut Sequence, out: &mut Vec<Result<StreamChunk>>) {
634    // `BTreeMap` iterates in key order, which is index order.
635    for (index, chunk) in std::mem::take(&mut sequence.held) {
636        if index > sequence.next {
637            tracing::warn!(
638                task_run_id,
639                expected = sequence.next,
640                got = index,
641                "a run's output stream skipped chunks",
642            );
643            out.push(Err(Error::StreamGap {
644                task_run_id: task_run_id.to_string(),
645                expected: sequence.next,
646                got: index,
647            }));
648        }
649        sequence.next = index + 1;
650        out.push(Ok(chunk));
651    }
652}
653
654/// Reconnecting is normal, so only the first attempt and every fifth one are
655/// worth a warning; the rest would drown a long outage in log lines.
656fn log_reconnect(attempt: u32, consecutive_other: u32, delay: Duration, status: Option<&Status>) {
657    let code = status.map(Status::code);
658    let message = status.map(Status::message).unwrap_or_default();
659    if attempt == 1 || attempt.is_multiple_of(5) {
660        tracing::warn!(
661            attempt,
662            consecutive_other,
663            ?delay,
664            ?code,
665            message,
666            "reconnecting to a run's output stream",
667        );
668    } else {
669        tracing::debug!(
670            attempt,
671            consecutive_other,
672            ?delay,
673            ?code,
674            message,
675            "reconnecting to a run's output stream",
676        );
677    }
678}
679
680#[cfg(test)]
681mod tests {
682    use std::collections::VecDeque;
683    use std::sync::atomic::{AtomicUsize, Ordering};
684    use std::sync::{Arc, Mutex};
685
686    use tokio::sync::mpsc;
687    use tokio::time::Instant;
688    use tokio_stream::wrappers::ReceiverStream;
689
690    use super::*;
691
692    /// Nothing here may block for ever: a regression should fail the test
693    /// rather than hang the suite. The clock is paused, so a wait that is
694    /// going to be answered costs no real time.
695    const NEVER: Duration = Duration::from_secs(10 * 60);
696
697    type ScriptedStream = ReceiverStream<std::result::Result<WorkflowEvent, Status>>;
698    type ScriptedSender = mpsc::Sender<std::result::Result<WorkflowEvent, Status>>;
699
700    /// One scripted connection: a stream the test drives, or a failure to
701    /// connect at all.
702    enum Step {
703        Stream(mpsc::Receiver<std::result::Result<WorkflowEvent, Status>>),
704        Fail(Status),
705    }
706
707    /// A connection the test drives: the script's step, and the sender that
708    /// feeds it. Dropping the sender ends that connection cleanly.
709    fn connection() -> (ScriptedSender, Step) {
710        let (tx, rx) = mpsc::channel(8);
711        (tx, Step::Stream(rx))
712    }
713
714    /// A `connect` function driven by a script, which also records when each
715    /// connection was opened. Once the script runs out it blocks for ever, so
716    /// the loop sits waiting rather than spinning.
717    #[derive(Clone)]
718    struct Script {
719        steps: Arc<Mutex<VecDeque<Step>>>,
720        calls: Arc<AtomicUsize>,
721        opened: mpsc::UnboundedSender<Instant>,
722    }
723
724    impl Script {
725        fn new(steps: Vec<Step>) -> (Self, mpsc::UnboundedReceiver<Instant>) {
726            let (opened, opens) = mpsc::unbounded_channel();
727            (
728                Self {
729                    steps: Arc::new(Mutex::new(steps.into())),
730                    calls: Arc::new(AtomicUsize::new(0)),
731                    opened,
732                },
733                opens,
734            )
735        }
736
737        async fn connect(self) -> std::result::Result<ScriptedStream, Status> {
738            self.calls.fetch_add(1, Ordering::SeqCst);
739            let _ = self.opened.send(Instant::now());
740            let step = self
741                .steps
742                .lock()
743                .unwrap_or_else(|poisoned| poisoned.into_inner())
744                .pop_front();
745            match step {
746                Some(Step::Stream(events)) => Ok(ReceiverStream::new(events)),
747                Some(Step::Fail(status)) => Err(status),
748                None => std::future::pending().await,
749            }
750        }
751
752        fn calls(&self) -> usize {
753            self.calls.load(Ordering::SeqCst)
754        }
755    }
756
757    /// An output stream over `steps`, whose reconnect delay is
758    /// `backoff(attempt)` instead of the policy's jittered one.
759    async fn opened(
760        steps: Vec<Step>,
761        backoff: impl FnMut(u32) -> Duration + Send + 'static,
762    ) -> (OutputStream, Script, mpsc::UnboundedReceiver<Instant>) {
763        let (script, opens) = Script::new(steps);
764        let stream = open_with(
765            {
766                let script = script.clone();
767                move || script.clone().connect()
768            },
769            backoff,
770        )
771        .await
772        .expect("a scripted first connection is never fatal");
773        (stream, script, opens)
774    }
775
776    fn chunk(index: i64, payload: &str) -> WorkflowEvent {
777        WorkflowEvent {
778            workflow_run_id: "run-1".to_string(),
779            resource_id: "task-1".to_string(),
780            event_type: ResourceEventType::Stream as i32,
781            event_payload: payload.to_string(),
782            event_index: Some(index),
783            ..WorkflowEvent::default()
784        }
785    }
786
787    /// When the next connection was opened, or a failed test rather than a
788    /// hung one.
789    async fn next_open(opens: &mut mpsc::UnboundedReceiver<Instant>) -> Instant {
790        tokio::time::timeout(NEVER, opens.recv())
791            .await
792            .expect("timed out waiting for a connection")
793            .expect("the loop dropped the recording channel")
794    }
795
796    /// The next item, or a failed test rather than a hung one.
797    async fn next_item(stream: &mut OutputStream) -> Option<Result<StreamChunk>> {
798        tokio::time::timeout(NEVER, stream.next())
799            .await
800            .expect("timed out waiting for the stream")
801    }
802
803    /// The ordering state, and what one call to `accept` released.
804    fn accepted(ordering: &mut Reorder, chunk: StreamChunk) -> Vec<Result<StreamChunk>> {
805        let mut out = Vec::new();
806        ordering.accept(chunk, &mut out);
807        out
808    }
809
810    /// The payloads of the chunks in `items`, and a `"gap(expected..got)"`
811    /// marker for each gap — so a test can assert on one flat sequence.
812    fn shape(items: Vec<Result<StreamChunk>>) -> Vec<String> {
813        items
814            .into_iter()
815            .map(|item| match item {
816                Ok(chunk) => chunk.payload,
817                Err(Error::StreamGap { expected, got, .. }) => format!("gap({expected}..{got})"),
818                Err(other) => format!("error({other})"),
819            })
820            .collect()
821    }
822
823    fn numbered(task: &str, index: i64) -> StreamChunk {
824        StreamChunk {
825            run_id: "run-1".to_string(),
826            task_run_id: task.to_string(),
827            index: Some(index),
828            payload: format!("chunk-{index}"),
829        }
830    }
831
832    #[test]
833    fn chunks_that_arrive_in_order_pass_straight_through() {
834        let mut ordering = Reorder::default();
835        for n in 0..3 {
836            assert_eq!(
837                shape(accepted(&mut ordering, numbered("task-1", n))),
838                [format!("chunk-{n}")],
839            );
840        }
841    }
842
843    /// Criterion 2, without a socket: 2, 0, 1 comes out 0, 1, 2.
844    #[test]
845    fn an_early_chunk_waits_for_the_ones_before_it() {
846        let mut ordering = Reorder::default();
847        // The first index seen is where the sequence starts, so this test
848        // begins at 0 on purpose: 2 arriving first would simply *be* the start
849        // (see `a_late_subscription_starts_where_it_joined`).
850        assert_eq!(
851            shape(accepted(&mut ordering, numbered("task-1", 0))),
852            ["chunk-0"],
853        );
854        assert!(shape(accepted(&mut ordering, numbered("task-1", 2))).is_empty());
855        assert_eq!(
856            shape(accepted(&mut ordering, numbered("task-1", 1))),
857            ["chunk-1", "chunk-2"],
858        );
859    }
860
861    /// Criterion 3: a chunk that never comes does not block the stream for
862    /// ever. Once more than `REORDER_BUFFER` chunks are waiting behind the
863    /// hole, the hole is reported and what was held is released in order.
864    #[test]
865    fn a_hole_is_reported_once_the_buffer_is_over_full_and_the_rest_follows() {
866        let mut ordering = Reorder::default();
867        accepted(&mut ordering, numbered("task-1", 0));
868        // Index 1 never arrives. Everything after it waits…
869        let mut released = Vec::new();
870        for n in 2..=(2 + REORDER_BUFFER as i64) {
871            released.extend(shape(accepted(&mut ordering, numbered("task-1", n))));
872        }
873        // …until the buffer is over full, and then the gap is reported once
874        // and the held chunks follow it in index order.
875        let expected: Vec<String> = std::iter::once("gap(1..2)".to_string())
876            .chain((2..=(2 + REORDER_BUFFER as i64)).map(|n| format!("chunk-{n}")))
877            .collect();
878        assert_eq!(released, expected);
879        // And the sequence carries on from where the engine is.
880        assert_eq!(
881            shape(accepted(
882                &mut ordering,
883                numbered("task-1", 3 + REORDER_BUFFER as i64),
884            )),
885            [format!("chunk-{}", 3 + REORDER_BUFFER as i64)],
886        );
887    }
888
889    /// Two holes in one batch are two reports: the consumer is told about
890    /// every stretch that is missing, not only the first one. With
891    /// `REORDER_BUFFER = 1` the release happens with two chunks held, so
892    /// `{3, 7}` behind `next = 1` is exactly the case that reaches it.
893    #[test]
894    fn every_hole_in_a_released_batch_is_reported() {
895        let mut ordering = Reorder::default();
896        // The sequence starts at 0, so 1 is what is expected next.
897        accepted(&mut ordering, numbered("task-1", 0));
898        assert!(shape(accepted(&mut ordering, numbered("task-1", 3))).is_empty());
899        assert_eq!(
900            shape(accepted(&mut ordering, numbered("task-1", 7))),
901            ["gap(1..3)", "chunk-3", "gap(4..7)", "chunk-7"],
902        );
903        // And the sequence carries on from the last index released.
904        assert_eq!(
905            shape(accepted(&mut ordering, numbered("task-1", 8))),
906            ["chunk-8"],
907        );
908    }
909
910    /// A chunk that was already delivered is dropped rather than handed over
911    /// twice — reconnecting can replay, and a duplicate is worse for a
912    /// consumer than a reported gap.
913    #[test]
914    fn a_chunk_that_was_already_delivered_is_dropped() {
915        let mut ordering = Reorder::default();
916        accepted(&mut ordering, numbered("task-1", 0));
917        accepted(&mut ordering, numbered("task-1", 1));
918        assert!(shape(accepted(&mut ordering, numbered("task-1", 0))).is_empty());
919        assert!(shape(accepted(&mut ordering, numbered("task-1", 1))).is_empty());
920        assert_eq!(
921            shape(accepted(&mut ordering, numbered("task-1", 2))),
922            ["chunk-2"],
923        );
924    }
925
926    /// Indexes are per task run, so two tasks streaming at once do not hold
927    /// each other up.
928    #[test]
929    fn each_task_run_has_its_own_sequence() {
930        let mut ordering = Reorder::default();
931        assert_eq!(
932            shape(accepted(&mut ordering, numbered("task-1", 0))),
933            ["chunk-0"],
934        );
935        // Task two's 0 is not task one's 1, and does not wait for anything.
936        assert_eq!(
937            shape(accepted(&mut ordering, numbered("task-2", 0))),
938            ["chunk-0"],
939        );
940        assert!(shape(accepted(&mut ordering, numbered("task-1", 2))).is_empty());
941        // Task two is unaffected by task one's hole.
942        assert_eq!(
943            shape(accepted(&mut ordering, numbered("task-2", 1))),
944            ["chunk-1"],
945        );
946    }
947
948    /// An engine that does not fill `event_index` still delivers: those chunks
949    /// go straight through, and they neither wait for nor disturb a sequence.
950    #[test]
951    fn a_chunk_without_an_index_goes_straight_through() {
952        let mut ordering = Reorder::default();
953        accepted(&mut ordering, numbered("task-1", 0));
954        assert!(shape(accepted(&mut ordering, numbered("task-1", 2))).is_empty());
955        let unnumbered = StreamChunk {
956            index: None,
957            payload: "no index".to_string(),
958            ..numbered("task-1", 0)
959        };
960        assert_eq!(shape(accepted(&mut ordering, unnumbered)), ["no index"]);
961        // The held chunk is still held, and 1 still completes it.
962        assert_eq!(
963            shape(accepted(&mut ordering, numbered("task-1", 1))),
964            ["chunk-1", "chunk-2"],
965        );
966    }
967
968    /// A subscription opened after the run started joins at whatever index it
969    /// first sees. Reporting everything before that as a gap would make the
970    /// ordinary case — subscribe after trigger — look broken.
971    #[test]
972    fn a_late_subscription_starts_where_it_joined() {
973        let mut ordering = Reorder::default();
974        assert_eq!(
975            shape(accepted(&mut ordering, numbered("task-1", 40))),
976            ["chunk-40"],
977        );
978        assert_eq!(
979            shape(accepted(&mut ordering, numbered("task-1", 41))),
980            ["chunk-41"],
981        );
982    }
983
984    /// The end of the stream releases what is still held, gap and all: chunks
985    /// that did arrive are not thrown away because one did not.
986    #[test]
987    fn flushing_releases_what_is_held_and_reports_the_hole() {
988        let mut ordering = Reorder::default();
989        accepted(&mut ordering, numbered("task-1", 0));
990        accepted(&mut ordering, numbered("task-1", 2));
991        let mut out = Vec::new();
992        ordering.flush(&mut out);
993        assert_eq!(shape(out), ["gap(1..2)", "chunk-2"]);
994        // Flushing twice releases nothing more.
995        let mut out = Vec::new();
996        ordering.flush(&mut out);
997        assert!(out.is_empty());
998    }
999
1000    /// Under full jitter the real delay may be any length including zero, so
1001    /// only a fixed backoff can tell a reconnect delay that is there from one
1002    /// that was deleted.
1003    #[tokio::test(start_paused = true)]
1004    async fn a_reconnect_waits_for_the_backoff() {
1005        let (first, one) = connection();
1006        let (second, two) = connection();
1007        let (mut chunks, _script, mut opens) =
1008            opened(vec![one, two], |_| Duration::from_secs(7)).await;
1009        let opened_at = next_open(&mut opens).await;
1010
1011        // A clean end of stream: worth reconnecting, after the backoff.
1012        drop(first);
1013
1014        let reopened_at = next_open(&mut opens).await;
1015        assert!(
1016            reopened_at.duration_since(opened_at) >= Duration::from_secs(7),
1017            "reconnected after {:?}",
1018            reopened_at.duration_since(opened_at),
1019        );
1020        second
1021            .send(Ok(chunk(0, "after")))
1022            .await
1023            .expect("the second connection is open");
1024        let chunk = next_item(&mut chunks)
1025            .await
1026            .expect("a chunk")
1027            .expect("not an error");
1028        assert_eq!(chunk.payload, "after");
1029    }
1030
1031    /// Ten unclassified failures in a row is the SDK's give-up rule
1032    /// (`STREAM_MAX_CONSECUTIVE_OTHER_FAILURES`), and giving up is told to the
1033    /// consumer rather than looking like a run with no output.
1034    #[tokio::test(start_paused = true)]
1035    async fn ten_unclassified_failures_in_a_row_give_up() {
1036        let (first, one) = connection();
1037        let mut steps = vec![one];
1038        steps.extend((0..10).map(|_| Step::Fail(Status::unknown("what?"))));
1039        let (mut chunks, script, _opens) = opened(steps, |_| Duration::ZERO).await;
1040
1041        // A clean end is transient, so it does not count towards the ten.
1042        drop(first);
1043
1044        let error = next_item(&mut chunks)
1045            .await
1046            .expect("the give-up is the last item")
1047            .expect_err("giving up is an error");
1048        assert!(matches!(error, Error::ResultStream(_)), "{error:?}");
1049        assert!(next_item(&mut chunks).await.is_none(), "and then it ends");
1050        assert_eq!(script.calls(), 11, "one good connection and ten failures");
1051    }
1052
1053    /// A connection that delivered a chunk was healthy, whatever it did
1054    /// afterwards: the give-up counter starts again from it.
1055    #[tokio::test(start_paused = true)]
1056    async fn a_delivered_chunk_resets_the_give_up_counter() {
1057        let (first, one) = connection();
1058        let (middle, two) = connection();
1059        let (last, three) = connection();
1060        let mut steps = vec![one];
1061        steps.extend((0..9).map(|_| Step::Fail(Status::unknown("what?"))));
1062        steps.push(two);
1063        steps.extend((0..9).map(|_| Step::Fail(Status::unknown("what?"))));
1064        steps.push(three);
1065        let (mut chunks, script, _opens) = opened(steps, |_| Duration::ZERO).await;
1066
1067        drop(first);
1068        middle
1069            .send(Ok(chunk(0, "healthy")))
1070            .await
1071            .expect("the middle connection is open");
1072        assert_eq!(
1073            next_item(&mut chunks)
1074                .await
1075                .expect("a chunk")
1076                .expect("not an error")
1077                .payload,
1078            "healthy",
1079        );
1080        drop(middle);
1081
1082        // Nine more failures after that: still under the limit, so the stream
1083        // is alive and the last connection delivers.
1084        last.send(Ok(chunk(1, "still here")))
1085            .await
1086            .expect("the last connection is open");
1087        assert_eq!(
1088            next_item(&mut chunks)
1089                .await
1090                .expect("a chunk")
1091                .expect("not an error")
1092                .payload,
1093            "still here",
1094        );
1095        assert_eq!(script.calls(), 21);
1096    }
1097}