apalis-rtui 0.1.0-rc.1

A drop-in replacement for apalis-board: HTTP surface, live updates, and the apalis-tui dashboard
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
//! The `task-status` ring buffer behind `GET /events/status` and `GET /events/activity`
//! (`docs/PLAN.md` §6.2), and the tracing-derived fallback for
//! `GET /queues/{queue}/tasks/{task_id}/events` (`docs/PLAN.md` §6.3).
//!
//! [`EventRing`] is fed from the exact same pipeline [`TracingSubscriber`](super::TracingSubscriber)
//! already writes [`LogEntry`] frames into: [`EventRing::ensure_feeding`] opens its own
//! [`TracingBroadcaster`] client (the same call `GET /events`'s handler makes for its own client)
//! and derives a [`TaskStatusEvent`] from every frame whose span names a `task.start`/`task.done`/
//! `task.failed` message, exactly like [`GET /events`](super) already filters to `span.is_some()`
//! frames. This keeps `/events` and `/events/status` two independent *readers* of one upstream
//! signal, never two separate signals that could drift apart.

use std::{
    collections::VecDeque,
    fmt,
    sync::{Arc, Mutex},
    time::{SystemTime, UNIX_EPOCH},
};

use apalis_rtui_types::{ActivityBucket, EntryType, LogEntry, TaskStatusEvent};
use futures::{
    StreamExt,
    channel::mpsc::{Receiver, Sender, channel},
};

use crate::api::{
    capability::{STATE_POISON, SharedState},
    sse::{Client, TracingBroadcaster},
};

/// How many recent [`TaskStatusEvent`]s a late `/events/status` (or per-task fallback) client
/// replays before joining the live stream. Bounded so this buffer's memory footprint stays flat
/// on a long-running server regardless of how many task transitions have happened since boot,
/// trading a full history for a generous recent window (about the last 1000 lifecycle
/// transitions across every queue).
const REPLAY_CAPACITY: usize = 1000;

/// One-minute activity buckets kept for `GET /events/activity`: an hour of history
/// (`docs/PLAN.md` §6.2).
const ACTIVITY_MINUTES: usize = 60;

/// Per-subscriber outgoing channel capacity, matching [`TracingBroadcaster::new_client`]'s own.
const CLIENT_CHANNEL_CAPACITY: usize = 100;

/// The `.expect()` message every [`EventRing`] lock site uses.
pub(crate) const RING_POISON: &str = "apalis-rtui: EventRing mutex poisoned";

/// A `/events/status` (or per-task fallback) subscription filter, checked before `try_send`
/// (`docs/PLAN.md` §6.4) so a client only ever receives, and only ever replays, what it asked
/// for. `None` on either field means "no filter on this axis".
#[derive(Debug, Clone, Default)]
pub(crate) struct StatusFilter {
    pub(crate) queue: Option<String>,
    pub(crate) task_id: Option<String>,
}

impl StatusFilter {
    fn matches(&self, event: &TaskStatusEvent) -> bool {
        self.queue.as_deref().is_none_or(|q| q == event.queue)
            && self.task_id.as_deref().is_none_or(|id| id == event.task_id)
    }
}

/// One minute's running counts, plus which minute (Unix epoch, truncated) it currently
/// represents. `minute_epoch: None` means this slot has never been written.
#[derive(Debug, Clone, Copy, Default)]
struct ActivitySlot {
    minute_epoch: Option<i64>,
    start: u64,
    done: u64,
    failed: u64,
}

/// A live `/events/status` (or per-task fallback) subscriber: an outgoing channel plus the
/// filter installed for it at subscribe time.
struct StatusClient {
    filter: StatusFilter,
    sender: Sender<TaskStatusEvent>,
}

impl fmt::Debug for StatusClient {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_struct("StatusClient")
            .field("filter", &self.filter)
            .finish_non_exhaustive()
    }
}

/// The bounded ring buffer plus task-status fan-out behind `/events/status`, `/events/activity`,
/// and the per-task fallback route. See the module docs for how it is fed.
#[derive(Debug)]
pub(crate) struct EventRing {
    activity: [ActivitySlot; ACTIVITY_MINUTES],
    replay: VecDeque<TaskStatusEvent>,
    clients: Vec<StatusClient>,
    feeding: bool,
}

impl Default for EventRing {
    fn default() -> Self {
        Self {
            activity: [ActivitySlot::default(); ACTIVITY_MINUTES],
            replay: VecDeque::with_capacity(REPLAY_CAPACITY),
            clients: Vec::new(),
            feeding: false,
        }
    }
}

impl EventRing {
    /// Create a new `EventRing` wrapped in an `Arc<Mutex<>>`, matching
    /// [`TracingBroadcaster::create`]'s own shape.
    #[must_use]
    pub(crate) fn create() -> Arc<Mutex<Self>> {
        Arc::new(Mutex::new(Self::default()))
    }

    /// Ensure the background task draining `broadcaster`'s tracing pipeline into `ring` is
    /// running. Idempotent, and safe to call on every `/events/status`/`/events/activity`/
    /// per-task-fallback request: the actual [`tokio::spawn`] happens at most once per `EventRing`
    /// (the same "first caller wins" shape `TracingSubscriber`'s own process-wide global-default
    /// install already has, see `tests/support/harness.rs`'s module docs).
    ///
    /// **Limit worth stating plainly**: the replay buffer and activity buckets only ever hold
    /// what this feeder captured *after* it started, which is after the *first* request to any
    /// of those three routes. Neither `EventRing` nor `TracingBroadcaster` keeps history from
    /// before that point, so "replay for a late client" means late relative to whenever this
    /// process first got asked for live status, not late relative to server boot. In practice a
    /// long-running server's first status/activity/per-task request happens moments after boot
    /// (the TUI subscribes as soon as it loads), so this is a startup-only gap.
    pub(crate) fn ensure_feeding(
        ring: &Arc<Mutex<Self>>,
        broadcaster: &Arc<Mutex<TracingBroadcaster>>,
        state: Arc<Mutex<SharedState>>,
    ) {
        let mut guard = ring.lock().expect(RING_POISON);
        if guard.feeding {
            return;
        }
        guard.feeding = true;
        drop(guard);

        let client = broadcaster
            .lock()
            .expect("invariant: broadcaster mutex is never held across a panic")
            .new_client();
        tokio::spawn(feed(Arc::clone(ring), client, state));
    }

    /// Record `event`: bump its minute's activity bucket, push it onto the replay buffer
    /// (evicting the oldest entry once over [`REPLAY_CAPACITY`]), and fan it out to every live
    /// subscriber whose filter matches, dropping any subscriber whose channel has disconnected.
    ///
    /// A subscriber whose channel is full keeps its place and misses this event. Its stream then
    /// carries on with a hole in it and no marker, and the client has no `Last-Event-ID` to
    /// resume from, so the gap is invisible on the wire. The alternative, cutting a slow reader
    /// off so it reconnects into the replay buffer, would turn a brief stall into a dropped
    /// connection. Never short-circuiting matters more: one stalled subscriber must not stop the
    /// others from getting the event.
    fn record(&mut self, event: &TaskStatusEvent) {
        self.bump_activity(&event.status);
        if self.replay.len() >= REPLAY_CAPACITY {
            self.replay.pop_front();
        }
        self.replay.push_back(event.clone());
        self.clients.retain_mut(|client| {
            if !client.filter.matches(event) {
                return !client.sender.is_closed();
            }
            match client.sender.try_send(event.clone()) {
                Ok(()) => true,
                Err(error) => !error.is_disconnected(),
            }
        });
    }

    fn bump_activity(&mut self, status: &str) {
        let minute = unix_now_minute();
        let idx = bucket_index(minute);
        let slot = &mut self.activity[idx];
        if slot.minute_epoch != Some(minute) {
            *slot = ActivitySlot {
                minute_epoch: Some(minute),
                ..ActivitySlot::default()
            };
        }
        match status {
            "Running" => slot.start += 1,
            "Done" => slot.done += 1,
            "Failed" | "Killed" => slot.failed += 1,
            _ => {}
        }
    }

    /// The last [`ACTIVITY_MINUTES`] minutes, oldest first, zero-filled for any minute that saw
    /// no events. Always exactly [`ACTIVITY_MINUTES`] entries, so a client can render it as a
    /// fixed-width sparkline without gap handling.
    ///
    /// The slots are keyed by minute counter (Unix seconds over 60), which is what
    /// [`bucket_index`] needs, but
    /// [`ActivityBucket::minute_epoch`](apalis_rtui_types::ActivityBucket::minute_epoch) is Unix
    /// epoch **seconds** truncated to the minute, so each one is multiplied back up on the way
    /// out.
    pub(crate) fn activity(&self) -> Vec<ActivityBucket> {
        let now = unix_now_minute();
        (0..ACTIVITY_MINUTES)
            .map(|offset_from_oldest| {
                let minute = now
                    - i64::try_from(ACTIVITY_MINUTES - 1 - offset_from_oldest).unwrap_or(i64::MAX);
                let minute_epoch = minute.saturating_mul(60);
                let slot = &self.activity[bucket_index(minute)];
                if slot.minute_epoch == Some(minute) {
                    ActivityBucket {
                        minute_epoch,
                        start: slot.start,
                        done: slot.done,
                        failed: slot.failed,
                    }
                } else {
                    ActivityBucket {
                        minute_epoch,
                        ..ActivityBucket::default()
                    }
                }
            })
            .collect()
    }

    /// Subscribe to live [`TaskStatusEvent`]s matching `filter`, returning a snapshot of the
    /// (already `filter`-matched) replay buffer to send first, plus the live receiver to chain
    /// after it (`docs/PLAN.md` §6.2's reconnect-replay contract).
    pub(crate) fn subscribe(
        &mut self,
        filter: StatusFilter,
    ) -> (Vec<TaskStatusEvent>, Receiver<TaskStatusEvent>) {
        let replay: Vec<_> = self
            .replay
            .iter()
            .filter(|event| filter.matches(event))
            .cloned()
            .collect();
        let (sender, receiver) = channel(CLIENT_CHANNEL_CAPACITY);
        self.clients.push(StatusClient { filter, sender });
        (replay, receiver)
    }
}

fn bucket_index(minute: i64) -> usize {
    usize::try_from(minute.rem_euclid(i64::try_from(ACTIVITY_MINUTES).unwrap_or(i64::MAX)))
        .unwrap_or(0)
}

fn unix_now_minute() -> i64 {
    let secs = SystemTime::now()
        .duration_since(UNIX_EPOCH)
        .unwrap_or_default()
        .as_secs();
    i64::try_from(secs / 60).unwrap_or(i64::MAX)
}

/// The background loop [`EventRing::ensure_feeding`] spawns: read every [`LogEntry`] the
/// broadcaster hands this client, derive a [`TaskStatusEvent`] from the ones that describe a
/// task lifecycle transition, and [`EventRing::record`] it. Exits once `client`'s channel closes
/// (the broadcaster itself was dropped), which only happens at process shutdown in practice.
async fn feed(ring: Arc<Mutex<EventRing>>, mut client: Client, state: Arc<Mutex<SharedState>>) {
    while let Some(Ok(entry)) = client.next().await {
        if let Some(event) = derive_event(&entry, &state) {
            ring.lock().expect(RING_POISON).record(&event);
        }
    }
}

/// Derive a [`TaskStatusEvent`] from `entry`, or `None` when it is not one of the three task
/// lifecycle frames `apalis`'s `TraceLayer` emits inside its `"task"` span
/// (`apalis-1.0.0-rc.9/src/layers/tracing/`, confirmed against
/// `tests/goldens/events-task-*.json`): `task.start` (no `done_in`/`result` yet), `task.done`,
/// `task.failed`. `queue` comes from [`SharedState::queue_for_task`], since the span itself never
/// carries a queue name. See that method's docs for the mechanism and its limits.
fn derive_event(entry: &LogEntry, state: &Mutex<SharedState>) -> Option<TaskStatusEvent> {
    let span = entry.span.as_ref()?;
    let message = entry.entry.message.as_deref()?;
    let status = match message {
        "task.start" => "Running",
        "task.done" => "Done",
        "task.failed" => "Failed",
        _ => return None,
    };
    let (done_in_ms, result) = match &entry.entry.entry_type {
        Some(EntryType::Result(result_entry)) => (
            parse_done_in_ms(&result_entry.done_in),
            Some(result_entry.result.clone()),
        ),
        _ => (None, None),
    };
    let attempt = u32::try_from(span.attempt).unwrap_or(0);
    let queue = state
        .lock()
        .expect(STATE_POISON)
        .queue_for_task(&span.task_id)
        .map(str::to_owned)
        .unwrap_or_default();
    Some(TaskStatusEvent {
        task_id: span.task_id.clone(),
        queue,
        status: status.to_owned(),
        attempt,
        done_in_ms,
        result,
    })
}

/// Parse a `done_in` string in whatever unit `apalis`'s `DefaultOnResponse`/`LatencyUnit` chose
/// into whole milliseconds. The four units are `ns`, `μs`, `ms`, and `s`
/// (`apalis-1.0.0-rc.9/src/layers/tracing/on_response.rs`). `None` for any other shape.
fn parse_done_in_ms(raw: &str) -> Option<u64> {
    let trimmed = raw.trim();
    if let Some(value) = trimmed.strip_suffix("ms") {
        value.trim().parse().ok()
    } else if let Some(value) = trimmed.strip_suffix("μs") {
        value.trim().parse::<u64>().ok().map(|us| us / 1_000)
    } else if let Some(value) = trimmed.strip_suffix("ns") {
        value.trim().parse::<u64>().ok().map(|ns| ns / 1_000_000)
    } else if let Some(value) = trimmed.strip_suffix('s') {
        value.trim().parse::<f64>().ok().and_then(|secs| {
            let millis = secs * 1000.0;
            (millis.is_finite() && millis >= 0.0).then_some(millis.round() as u64)
        })
    } else {
        None
    }
}

#[cfg(test)]
mod tests {
    use apalis_rtui_types::{Entry, LineEntry, Span};

    use super::*;

    fn task_frame(message: &str, task_id: &str, attempt: i64) -> LogEntry {
        LogEntry {
            span: Some(Span {
                attempt,
                task_id: task_id.to_owned(),
                name: "task".to_owned(),
            }),
            level: apalis_rtui_types::LogLevel::Debug,
            timestamp: "2024-01-01T00:00:00Z".to_owned(),
            target: "apalis::layers::tracing::on_request".to_owned(),
            entry: Entry {
                message: Some(message.to_owned()),
                entry_type: None,
            },
        }
    }

    fn terminal_frame(
        message: &str,
        task_id: &str,
        attempt: i64,
        done_in: &str,
        result: &str,
    ) -> LogEntry {
        let mut entry = task_frame(message, task_id, attempt);
        entry.entry.entry_type = Some(EntryType::Result(apalis_rtui_types::ResultEntry {
            done_in: done_in.to_owned(),
            result: result.to_owned(),
        }));
        entry
    }

    #[test]
    fn parse_done_in_ms_handles_every_apalis_unit() {
        assert_eq!(parse_done_in_ms("120ms"), Some(120));
        assert_eq!(parse_done_in_ms("1500μs"), Some(1));
        assert_eq!(parse_done_in_ms("2500000ns"), Some(2));
        assert_eq!(parse_done_in_ms("1.5s"), Some(1500));
        assert_eq!(parse_done_in_ms("garbage"), None);
    }

    #[test]
    fn derive_event_skips_frames_with_no_span_or_unrecognized_message() {
        let state = Arc::new(Mutex::new(SharedState::default()));
        let mut no_span = task_frame("task.start", "t1", 0);
        no_span.span = None;
        assert!(derive_event(&no_span, &state).is_none());

        let other_message = task_frame("something else", "t1", 0);
        assert!(derive_event(&other_message, &state).is_none());
    }

    #[test]
    fn derive_event_reports_running_for_task_start_with_no_terminal_fields() {
        let state = Arc::new(Mutex::new(SharedState::default()));
        let event = derive_event(&task_frame("task.start", "t1", 2), &state)
            .expect("task.start derives an event");
        assert_eq!(event.status, "Running");
        assert_eq!(event.attempt, 2);
        assert_eq!(event.done_in_ms, None);
        assert_eq!(event.result, None);
        assert_eq!(
            event.queue, "",
            "unrecorded task_id resolves to an empty queue"
        );
    }

    #[test]
    fn derive_event_resolves_queue_once_recorded() {
        let state = Arc::new(Mutex::new(SharedState::default()));
        state.lock().unwrap().record_task_queue("t1", "emails");
        let event = derive_event(&task_frame("task.done", "t1", 1), &state)
            .expect("task.done derives an event");
        assert_eq!(event.queue, "emails");
    }

    #[test]
    fn derive_event_carries_done_in_and_result_on_terminal_frames() {
        let state = Arc::new(Mutex::new(SharedState::default()));
        let frame = terminal_frame("task.failed", "t1", 1, "12ms", "boom");
        let event = derive_event(&frame, &state).expect("task.failed derives an event");
        assert_eq!(event.status, "Failed");
        assert_eq!(event.done_in_ms, Some(12));
        assert_eq!(event.result.as_deref(), Some("boom"));
    }

    #[test]
    fn derive_event_ignores_a_line_entry_result_type() {
        let state = Arc::new(Mutex::new(SharedState::default()));
        let mut frame = task_frame("some log line", "t1", 0);
        frame.entry.entry_type = Some(EntryType::Line(LineEntry::default()));
        assert!(derive_event(&frame, &state).is_none());
    }

    #[test]
    fn status_filter_matches_on_every_axis_independently() {
        let event = TaskStatusEvent {
            task_id: "t1".to_owned(),
            queue: "emails".to_owned(),
            status: "Done".to_owned(),
            attempt: 0,
            done_in_ms: None,
            result: None,
        };
        assert!(StatusFilter::default().matches(&event));
        assert!(
            StatusFilter {
                queue: Some("emails".to_owned()),
                task_id: None,
            }
            .matches(&event)
        );
        assert!(
            !StatusFilter {
                queue: Some("notifications".to_owned()),
                task_id: None,
            }
            .matches(&event)
        );
        assert!(
            StatusFilter {
                queue: None,
                task_id: Some("t1".to_owned()),
            }
            .matches(&event)
        );
        assert!(
            !StatusFilter {
                queue: None,
                task_id: Some("t2".to_owned()),
            }
            .matches(&event)
        );
    }

    #[test]
    fn record_bumps_the_matching_activity_bucket_and_replays_it() {
        let mut ring = EventRing::default();
        ring.record(&TaskStatusEvent {
            task_id: "t1".to_owned(),
            queue: "emails".to_owned(),
            status: "Running".to_owned(),
            attempt: 0,
            done_in_ms: None,
            result: None,
        });
        ring.record(&TaskStatusEvent {
            task_id: "t1".to_owned(),
            queue: "emails".to_owned(),
            status: "Done".to_owned(),
            attempt: 0,
            done_in_ms: Some(5),
            result: Some("ok".to_owned()),
        });
        let current = ring.activity().pop().expect("60 buckets, at least one");
        assert_eq!(current.start, 1);
        assert_eq!(current.done, 1);
        assert_eq!(current.failed, 0);
        assert_eq!(
            current.minute_epoch,
            unix_now_minute() * 60,
            "minute_epoch is Unix epoch seconds truncated to the minute, not the minute counter"
        );

        let (replay, _live) = ring.subscribe(StatusFilter::default());
        assert_eq!(replay.len(), 2);
    }

    #[test]
    fn replay_buffer_is_bounded_at_capacity() {
        let mut ring = EventRing::default();
        for i in 0..REPLAY_CAPACITY + 10 {
            ring.record(&TaskStatusEvent {
                task_id: format!("t{i}"),
                queue: "emails".to_owned(),
                status: "Done".to_owned(),
                attempt: 0,
                done_in_ms: None,
                result: None,
            });
        }
        let (replay, _live) = ring.subscribe(StatusFilter::default());
        assert_eq!(replay.len(), REPLAY_CAPACITY);
        assert_eq!(replay[0].task_id, "t10", "the oldest 10 should be evicted");
    }

    #[test]
    fn activity_is_always_exactly_60_zero_filled_buckets_when_nothing_recorded() {
        let ring = EventRing::default();
        let buckets = ring.activity();
        assert_eq!(buckets.len(), ACTIVITY_MINUTES);
        assert!(
            buckets
                .iter()
                .all(|b| b.start == 0 && b.done == 0 && b.failed == 0)
        );
    }

    /// The wire contract `apalis_rtui_types::ActivityBucket` documents: Unix epoch **seconds**
    /// truncated to the minute, one bucket every 60 seconds, oldest first, ending on the current
    /// minute. The ring keys its own slots by minute counter, so this is the conversion that
    /// used to be missing.
    #[test]
    fn activity_buckets_are_consecutive_minute_truncated_epoch_seconds() {
        let ring = EventRing::default();
        let buckets = ring.activity();
        let last = buckets.last().expect("60 buckets").minute_epoch;
        assert_eq!(last, unix_now_minute() * 60);
        for (index, bucket) in buckets.iter().enumerate() {
            assert_eq!(
                bucket.minute_epoch % 60,
                0,
                "bucket {index} is not truncated to the minute"
            );
            let steps_before_last = i64::try_from(ACTIVITY_MINUTES - 1 - index).expect("fits i64");
            assert_eq!(bucket.minute_epoch, last - steps_before_last * 60);
        }
    }

    #[tokio::test]
    async fn subscribe_delivers_live_events_matching_the_filter_only() {
        let mut ring = EventRing::default();
        let (_replay, mut live) = ring.subscribe(StatusFilter {
            queue: None,
            task_id: Some("t1".to_owned()),
        });
        ring.record(&TaskStatusEvent {
            task_id: "t2".to_owned(),
            queue: "emails".to_owned(),
            status: "Done".to_owned(),
            attempt: 0,
            done_in_ms: None,
            result: None,
        });
        ring.record(&TaskStatusEvent {
            task_id: "t1".to_owned(),
            queue: "emails".to_owned(),
            status: "Done".to_owned(),
            attempt: 0,
            done_in_ms: None,
            result: None,
        });
        let event = live.next().await.expect("t1's event was delivered");
        assert_eq!(event.task_id, "t1");
        assert!(
            futures::poll!(live.next()).is_pending(),
            "t2's event must not have been delivered"
        );
    }
}