zakura 1.8.8

Zakura, an independent, consensus-compatible implementation of a Zcash node
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
//! A task that gossips any [`zakura_chain::transaction::UnminedTxId`] that enters the mempool to peers.
//!
//! This module is just a function [`run_mempool_transaction_id_gossip`] that
//! treats mempool insertion events received in a channel as wakeup signals,
//! drains the transaction IDs that still need gossip from the mempool service,
//! and advertises them to peers. Draining from the mempool service lets the task
//! recover the IDs behind any wakeups dropped by a lagged channel.
//!
//! Each advertisement completes in its own task, so a slow peer delays only its
//! own broadcast, never the next batch of transaction IDs.

use std::time::Duration;

use tokio::{
    sync::broadcast::{
        self,
        error::{RecvError, TryRecvError},
    },
    task::JoinSet,
};
use tower::{timeout::Timeout, Service, ServiceExt};

use zakura_network::MAX_TX_INV_IN_SENT_MESSAGE;

use zakura_network as zn;
use zakura_node_services::mempool::{MempoolChange, Request, Response};

use crate::{components::sync::TIPS_RESPONSE_TIMEOUT, BoxError};

/// Controls how long we wait between starting successive batches of transaction ID gossip.
///
/// Block gossip has no delay, so this delay only applies to transactions. It is a third of
/// Zcash's historical 7 second gossip delay, matching the 25 second block spacing in ZIP 218.
///
/// ## Correctness
///
/// If this delay is set too high, transactions won't propagate through the network efficiently.
///
/// If this delay is set too low, the peer set and remote peers can get overloaded.
pub const TRANSACTION_GOSSIP_DELAY: Duration = Duration::from_secs(2);

/// How long the gossip task waits for the network to accept a broadcast.
///
/// The task waits for the network before it takes any pending transaction IDs,
/// so a timeout leaves them in the mempool for the next attempt.
pub const BROADCAST_READY_TIMEOUT: Duration = TIPS_RESPONSE_TIMEOUT;

/// The maximum number of channel messages we will combine into a single peer broadcast.
pub const MAX_CHANGES_BEFORE_SEND: usize = 10;

// Safe because the protocol limit of 25,000 fits in usize on all targets.
const MAX_TX_INV_IN_SENT_MESSAGE_USIZE: usize = MAX_TX_INV_IN_SENT_MESSAGE as usize;

/// The number of mempool change notifications buffered for gossip subscribers.
///
/// Keep this close to the number of changes the gossip task can drain in one
/// broadcast, so sustained overload triggers lag recovery instead of building
/// up a large backlog of stale notifications.
pub(super) const MEMPOOL_CHANGE_CHANNEL_CAPACITY: usize = MAX_CHANGES_BEFORE_SEND * 4;

/// Runs continuously, gossiping new [`UnminedTxId`](zakura_chain::transaction::UnminedTxId)s to peers.
///
/// Broadcasts any new transaction IDs that are stored in the mempool to
/// multiple ready peers. Broadcasts start at most once per
/// [`TRANSACTION_GOSSIP_DELAY`], and each one finishes in the background within
/// [`TIPS_RESPONSE_TIMEOUT`], so only a few broadcasts are ever in flight.
pub(crate) async fn run_mempool_transaction_id_gossip<ZN, ZM>(
    mut receiver: broadcast::Receiver<MempoolChange>,
    broadcast_network: ZN,
    mut mempool: ZM,
) -> Result<(), BoxError>
where
    ZN: Service<zn::Request, Response = zn::Response, Error = BoxError> + Send + Clone + 'static,
    ZN::Future: Send + 'static,
    ZM: Service<Request, Response = Response, Error = BoxError> + Send + Clone + 'static,
    ZM::Future: Send + 'static,
{
    info!("initializing transaction gossip task");

    // use the same timeout as tips requests,
    // so broadcasts don't occupy peers the syncer needs for too long
    let mut broadcast_network = Timeout::new(broadcast_network, TIPS_RESPONSE_TIMEOUT);
    let mut in_flight_broadcasts = JoinSet::new();
    let mut drain_pending_without_wakeup = false;

    loop {
        // This count is only used in logs. It is zero for a pending-set drain
        // that runs without consuming a channel notification.
        let mut combined_changes = 0;

        if !drain_pending_without_wakeup {
            combined_changes = 1;

            // once we get new data in the channel, drain pending transaction IDs
            // from the mempool service and broadcast them to peers.
            //
            // The channel is a wakeup signal. The mempool service keeps the
            // authoritative pending gossip set, so lagged wakeups can recover
            // without re-advertising the entire mempool.
            loop {
                match receiver.recv().await {
                    Ok(mempool_change) if mempool_change.is_added() => break,
                    Ok(_) => {
                        // ignore other changes, we only want to gossip added transactions
                        continue;
                    }
                    Err(RecvError::Lagged(skip_count)) => {
                        info!(
                            ?skip_count,
                            "dropped mempool changes before gossiping, draining pending transaction IDs"
                        );
                        metrics::counter!("mempool.gossip.lagged.events.total")
                            .increment(skip_count);
                        // Exit the wait loop to re-advertise the pending transaction IDs.
                        break;
                    }
                    Err(closed @ RecvError::Closed) => Err(closed)?,
                }
            }

            // also consume wakeups that arrived shortly after this one,
            // but limit the number of changes so the loop terminates.
            while combined_changes <= MAX_CHANGES_BEFORE_SEND {
                match receiver.try_recv() {
                    Ok(mempool_change) if mempool_change.is_added() => {}
                    Ok(_) => {
                        // ignore other changes, we only want to gossip added transactions
                        continue;
                    }
                    Err(TryRecvError::Empty) => break,
                    Err(TryRecvError::Lagged(skip_count)) => {
                        info!(
                            ?skip_count,
                            "dropped mempool changes before gossiping, draining pending transaction IDs"
                        );
                        metrics::counter!("mempool.gossip.lagged.events.total")
                            .increment(skip_count);
                    }
                    Err(closed @ TryRecvError::Closed) => Err(closed)?,
                }

                combined_changes += 1;
            }
        } else {
            drain_pending_without_wakeup = false;
        }

        reap_finished_broadcasts(&mut in_flight_broadcasts);

        let advertised_count = match start_pending_mempool_transaction_id_broadcast(
            &mut mempool,
            &mut broadcast_network,
            &mut in_flight_broadcasts,
            combined_changes,
        )
        .await?
        {
            BroadcastStart::Started { tx_count } => tx_count,
            BroadcastStart::NothingPending => continue,
            BroadcastStart::NetworkNotReady => {
                // The pending IDs are still in the mempool, so retry without
                // waiting for another wakeup.
                drain_pending_without_wakeup = true;
                continue;
            }
        };

        // A full batch means more transaction IDs may still be pending, so
        // drain again after the delay instead of waiting for another wakeup.
        drain_pending_without_wakeup = advertised_count == MAX_TX_INV_IN_SENT_MESSAGE;

        // Waiting between batches can delay transaction propagation, in order to reduce peer load.
        // The delay runs from the start of the broadcast, not from its completion.
        tokio::time::sleep(TRANSACTION_GOSSIP_DELAY).await;
    }
}

/// The outcome of one attempt to start a pending transaction ID broadcast.
#[derive(Copy, Clone, Debug, Eq, PartialEq)]
enum BroadcastStart {
    /// A broadcast of `tx_count` transaction IDs is running in the background.
    Started { tx_count: u64 },

    /// The mempool had no transaction IDs waiting for gossip.
    NothingPending,

    /// The network was not ready within [`BROADCAST_READY_TIMEOUT`],
    /// so no pending transaction IDs were taken from the mempool.
    NetworkNotReady,
}

/// Takes up to one inv worth of transaction IDs from the mempool's pending
/// gossip set, and starts advertising them in a task in `in_flight_broadcasts`.
///
/// Waits for the network to be ready before taking any IDs, so that a network
/// that stays unready leaves them pending.
async fn start_pending_mempool_transaction_id_broadcast<ZN, ZM>(
    mempool: &mut ZM,
    broadcast_network: &mut Timeout<ZN>,
    in_flight_broadcasts: &mut JoinSet<()>,
    combined_changes: usize,
) -> Result<BroadcastStart, BoxError>
where
    ZN: Service<zn::Request, Response = zn::Response, Error = BoxError> + Send + Clone + 'static,
    ZN::Future: Send + 'static,
    ZM: Service<Request, Response = Response, Error = BoxError> + Send + Clone + 'static,
    ZM::Future: Send + 'static,
{
    // `Timeout` only bounds the response future, so readiness needs its own timeout.
    let Ok(broadcast_network) =
        tokio::time::timeout(BROADCAST_READY_TIMEOUT, broadcast_network.ready()).await
    else {
        info!(
            timeout = ?BROADCAST_READY_TIMEOUT,
            "network was not ready for a transaction broadcast, retrying"
        );
        metrics::counter!("mempool.gossip.network.unready.total").increment(1);

        return Ok(BroadcastStart::NetworkNotReady);
    };
    let broadcast_network = broadcast_network?;

    let Response::TransactionIds(tx_ids) = mempool
        .ready()
        .await?
        .call(Request::TakePendingGossipTransactionIds {
            limit: MAX_TX_INV_IN_SENT_MESSAGE_USIZE,
        })
        .await?
    else {
        return Err(std::io::Error::other(
            "mempool pending gossip request returned a different response variant",
        )
        .into());
    };

    if tx_ids.is_empty() {
        return Ok(BroadcastStart::NothingPending);
    }

    let tx_count: u64 = tx_ids
        .len()
        .try_into()
        .expect("the mempool returns at most one inv of transaction IDs, which fits in u64");
    let request = zn::Request::AdvertiseTransactionIds(tx_ids, None);

    info!(%request, changes = %combined_changes, "sending pending mempool transaction broadcast");
    debug!(
        ?request,
        changes = ?combined_changes,
        "full list of pending mempool transactions in broadcast"
    );

    // The response only reports that the selected peers finished, so waiting
    // for it here would let one slow peer hold back the next batch. The task
    // must still poll the response: dropping it could cancel a buffered
    // request before it reaches any peers.
    let response = broadcast_network.call(request);
    in_flight_broadcasts.spawn(async move {
        if let Err(error) = response.await {
            debug!(%error, "transaction broadcast did not finish");
        }
    });

    metrics::counter!("mempool.gossip.pending.transactions.total").increment(tx_count);
    metrics::counter!("mempool.gossiped.transactions.total").increment(tx_count);

    Ok(BroadcastStart::Started { tx_count })
}

/// Removes finished broadcast tasks, and propagates any panic from them.
fn reap_finished_broadcasts(in_flight_broadcasts: &mut JoinSet<()>) {
    while let Some(result) = in_flight_broadcasts.try_join_next() {
        if let Err(error) = result {
            // Only the gossip task's own exit aborts these tasks, so any other
            // join error is a panic.
            if let Ok(panic) = error.try_into_panic() {
                std::panic::resume_unwind(panic);
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use std::{
        collections::{HashSet, VecDeque},
        sync::{Arc, Mutex},
        time::Duration,
    };

    use futures::future;
    use tokio::{
        sync::{broadcast, mpsc},
        time::Instant,
    };
    use tower::service_fn;

    use zakura_chain::transaction::{self, UnminedTxId};

    use super::*;

    fn test_tx_ids(count: usize, seed: u8) -> HashSet<UnminedTxId> {
        (0..count)
            .map(|index| {
                let index: u64 = index
                    .try_into()
                    .expect("test transaction ID index fits in u64");
                let mut bytes = [seed; 32];
                bytes[..8].copy_from_slice(&index.to_le_bytes());

                UnminedTxId::Legacy(transaction::Hash(bytes))
            })
            .collect()
    }

    fn mempool_service(
        pending_batches: Vec<HashSet<UnminedTxId>>,
    ) -> (
        impl Service<Request, Response = Response, Error = BoxError, Future: Send> + Clone,
        mpsc::Receiver<usize>,
    ) {
        let pending_batches = Arc::new(Mutex::new(VecDeque::from(pending_batches)));
        let (limit_sender, limit_receiver) = mpsc::channel(16);
        let service = service_fn(move |request| {
            let pending_batches = pending_batches.clone();
            let limit_sender = limit_sender.clone();

            async move {
                match request {
                    Request::TakePendingGossipTransactionIds { limit } => {
                        assert_eq!(
                            limit, MAX_TX_INV_IN_SENT_MESSAGE_USIZE,
                            "gossip task should bound each pending drain to one inv"
                        );
                        limit_sender
                            .send(limit)
                            .await
                            .expect("limit receiver should be open");

                        let tx_ids = pending_batches
                            .lock()
                            .expect("pending batch mutex should not be poisoned")
                            .pop_front()
                            .unwrap_or_default();

                        Ok(Response::TransactionIds(tx_ids))
                    }
                    unexpected_request => {
                        panic!("unexpected mempool request: {unexpected_request:?}")
                    }
                }
            }
        });

        (service, limit_receiver)
    }

    fn peer_set_service() -> (
        impl Service<zn::Request, Response = zn::Response, Error = BoxError, Future: Send + 'static>
            + Clone,
        mpsc::Receiver<zn::Request>,
    ) {
        let (advertised_sender, advertised_receiver) = mpsc::channel(16);

        let service = service_fn(move |request| {
            let advertised_sender = advertised_sender.clone();

            async move {
                advertised_sender
                    .send(request)
                    .await
                    .expect("advertised request receiver should be open");

                Ok(zn::Response::Nil)
            }
        });

        (service, advertised_receiver)
    }

    async fn expect_advertised_transaction_ids(
        advertised_receiver: &mut mpsc::Receiver<zn::Request>,
    ) -> HashSet<UnminedTxId> {
        let advertised_request =
            tokio::time::timeout(Duration::from_secs(1), advertised_receiver.recv())
                .await
                .expect("gossip task should advertise pending mempool txids")
                .expect("peer set should advertise a request before the task exits");

        let zn::Request::AdvertiseTransactionIds(advertised_tx_ids, None) = advertised_request
        else {
            panic!("unexpected advertised request: {advertised_request:?}");
        };

        advertised_tx_ids
    }

    #[tokio::test]
    async fn added_mempool_gossip_drains_pending_transaction_ids() {
        let _init_guard = zakura_test::init();

        let pending_tx_ids = test_tx_ids(2, 1);
        let (mempool, mut limit_receiver) = mempool_service(vec![pending_tx_ids.clone()]);
        let (peer_set, mut advertised_receiver) = peer_set_service();
        let (sender, receiver) = broadcast::channel(MEMPOOL_CHANGE_CHANNEL_CAPACITY);

        sender
            .send(MempoolChange::added(test_tx_ids(1, 2)))
            .expect("receiver should be subscribed");

        let gossip_task = tokio::spawn(run_mempool_transaction_id_gossip(
            receiver, peer_set, mempool,
        ));

        assert_eq!(
            limit_receiver
                .recv()
                .await
                .expect("gossip task should request pending txids"),
            MAX_TX_INV_IN_SENT_MESSAGE_USIZE
        );
        assert_eq!(
            expect_advertised_transaction_ids(&mut advertised_receiver).await,
            pending_tx_ids,
            "happy path should advertise the pending mempool txids",
        );

        gossip_task.abort();
    }

    #[tokio::test]
    async fn lagged_mempool_gossip_drains_pending_transaction_ids() {
        let _init_guard = zakura_test::init();

        let pending_tx_ids = test_tx_ids(2, 1);
        let dropped_tx_ids = test_tx_ids(2, 2);
        let (mempool, mut limit_receiver) = mempool_service(vec![pending_tx_ids.clone()]);
        let (peer_set, mut advertised_receiver) = peer_set_service();
        let (sender, receiver) = broadcast::channel(1);

        let mut lagged_events = dropped_tx_ids
            .into_iter()
            .map(|tx_id| MempoolChange::added([tx_id].into_iter().collect()));

        sender
            .send(
                lagged_events
                    .next()
                    .expect("first lagged mempool change should exist"),
            )
            .expect("receiver should be subscribed");
        sender
            .send(
                lagged_events
                    .next()
                    .expect("second lagged mempool change should exist"),
            )
            .expect("receiver should be subscribed");

        let gossip_task = tokio::spawn(run_mempool_transaction_id_gossip(
            receiver, peer_set, mempool,
        ));

        assert_eq!(
            limit_receiver
                .recv()
                .await
                .expect("gossip task should request pending txids"),
            MAX_TX_INV_IN_SENT_MESSAGE_USIZE
        );
        assert_eq!(
            expect_advertised_transaction_ids(&mut advertised_receiver).await,
            pending_tx_ids,
            "lag recovery should advertise pending txids, not dropped channel payloads",
        );

        gossip_task.abort();
    }

    #[tokio::test(start_paused = true)]
    async fn lagged_mempool_gossip_recovers_pending_transaction_ids_in_bounded_cycles() {
        let _init_guard = zakura_test::init();

        let first_batch = test_tx_ids(MAX_TX_INV_IN_SENT_MESSAGE_USIZE, 1);
        let second_batch = test_tx_ids(2, 2);
        let (mempool, mut limit_receiver) =
            mempool_service(vec![first_batch.clone(), second_batch.clone()]);
        let (peer_set, mut advertised_receiver) = peer_set_service();
        let (sender, receiver) = broadcast::channel(1);

        sender
            .send(MempoolChange::added(
                [UnminedTxId::Legacy(transaction::Hash([42; 32]))]
                    .into_iter()
                    .collect(),
            ))
            .expect("receiver should be subscribed");
        sender
            .send(MempoolChange::added(
                [UnminedTxId::Legacy(transaction::Hash([43; 32]))]
                    .into_iter()
                    .collect(),
            ))
            .expect("receiver should be subscribed");

        let gossip_task = tokio::spawn(run_mempool_transaction_id_gossip(
            receiver, peer_set, mempool,
        ));

        assert_eq!(
            limit_receiver
                .recv()
                .await
                .expect("first drain should request pending txids"),
            MAX_TX_INV_IN_SENT_MESSAGE_USIZE
        );
        let advertised_tx_ids = expect_advertised_transaction_ids(&mut advertised_receiver).await;
        assert_eq!(advertised_tx_ids, first_batch);
        assert_eq!(
            advertised_tx_ids.len(),
            MAX_TX_INV_IN_SENT_MESSAGE_USIZE,
            "first recovery cycle should be bounded to one inv-sized batch",
        );

        tokio::time::advance(TRANSACTION_GOSSIP_DELAY).await;

        assert_eq!(
            limit_receiver
                .recv()
                .await
                .expect("second drain should happen without another wakeup"),
            MAX_TX_INV_IN_SENT_MESSAGE_USIZE
        );
        let advertised_tx_ids = expect_advertised_transaction_ids(&mut advertised_receiver).await;
        assert_eq!(advertised_tx_ids, second_batch);
        assert!(
            advertised_tx_ids.len() < MAX_TX_INV_IN_SENT_MESSAGE_USIZE,
            "second recovery cycle should only advertise the remaining txids",
        );

        gossip_task.abort();
    }

    #[tokio::test]
    async fn empty_pending_mempool_gossip_wakeup_does_not_advertise() {
        let _init_guard = zakura_test::init();

        let (mempool, mut limit_receiver) = mempool_service(vec![HashSet::new()]);
        let (peer_set, mut advertised_receiver) = peer_set_service();
        let (sender, receiver) = broadcast::channel(MEMPOOL_CHANGE_CHANNEL_CAPACITY);

        sender
            .send(MempoolChange::added(test_tx_ids(1, 1)))
            .expect("receiver should be subscribed");

        let gossip_task = tokio::spawn(run_mempool_transaction_id_gossip(
            receiver, peer_set, mempool,
        ));

        assert_eq!(
            limit_receiver
                .recv()
                .await
                .expect("gossip task should request pending txids"),
            MAX_TX_INV_IN_SENT_MESSAGE_USIZE
        );
        assert!(
            tokio::time::timeout(Duration::from_millis(50), advertised_receiver.recv())
                .await
                .is_err(),
            "empty pending gossip wakeups should not advertise to peers",
        );
        assert!(
            !gossip_task.is_finished(),
            "gossip task should remain alive after an empty pending drain",
        );

        gossip_task.abort();
    }

    /// A peer set that records each request and its start time, then never responds,
    /// like a broadcast that is stuck on a slow peer.
    fn stalled_peer_set_service() -> (
        impl Service<zn::Request, Response = zn::Response, Error = BoxError, Future: Send + 'static>
            + Clone,
        mpsc::Receiver<(zn::Request, Instant)>,
    ) {
        let (advertised_sender, advertised_receiver) = mpsc::channel(16);

        let service = service_fn(move |request| {
            let advertised_sender = advertised_sender.clone();

            async move {
                advertised_sender
                    .send((request, Instant::now()))
                    .await
                    .expect("advertised request receiver should be open");

                future::pending::<Result<zn::Response, BoxError>>().await
            }
        });

        (service, advertised_receiver)
    }

    /// Returns the next transaction IDs sent to a stalled peer set, and when they were sent.
    async fn expect_stalled_advertised_transaction_ids(
        advertised_receiver: &mut mpsc::Receiver<(zn::Request, Instant)>,
    ) -> (HashSet<UnminedTxId>, Instant) {
        // The paused clock auto-advances, so this only needs to exceed the gossip delay.
        let (request, started_at) =
            tokio::time::timeout(Duration::from_secs(30), advertised_receiver.recv())
                .await
                .expect("gossip task should advertise pending mempool txids")
                .expect("peer set should advertise a request before the task exits");
        let zn::Request::AdvertiseTransactionIds(tx_ids, None) = request else {
            panic!("unexpected advertised request: {request:?}");
        };

        (tx_ids, started_at)
    }

    /// A peer set that never becomes ready.
    #[derive(Clone)]
    struct NeverReadyPeerSet;

    impl Service<zn::Request> for NeverReadyPeerSet {
        type Response = zn::Response;
        type Error = BoxError;
        type Future = future::Ready<Result<zn::Response, BoxError>>;

        fn poll_ready(
            &mut self,
            _cx: &mut std::task::Context<'_>,
        ) -> std::task::Poll<Result<(), BoxError>> {
            std::task::Poll::Pending
        }

        fn call(&mut self, request: zn::Request) -> Self::Future {
            panic!("a peer set that is never ready must not be called: {request:?}")
        }
    }

    #[tokio::test(start_paused = true)]
    async fn stalled_broadcast_does_not_delay_next_batch() {
        let _init_guard = zakura_test::init();

        // A full first batch makes the task drain again without another wakeup.
        let first_batch = test_tx_ids(MAX_TX_INV_IN_SENT_MESSAGE_USIZE, 1);
        let second_batch = test_tx_ids(2, 2);
        let (mempool, _limit_receiver) =
            mempool_service(vec![first_batch.clone(), second_batch.clone()]);
        let (peer_set, mut advertised_receiver) = stalled_peer_set_service();
        let (sender, receiver) = broadcast::channel(MEMPOOL_CHANGE_CHANNEL_CAPACITY);

        sender
            .send(MempoolChange::added(test_tx_ids(1, 3)))
            .expect("receiver should be subscribed");

        let gossip_task = tokio::spawn(run_mempool_transaction_id_gossip(
            receiver, peer_set, mempool,
        ));

        let (advertised_tx_ids, first_started_at) =
            expect_stalled_advertised_transaction_ids(&mut advertised_receiver).await;
        assert_eq!(advertised_tx_ids, first_batch);

        let (advertised_tx_ids, second_started_at) =
            expect_stalled_advertised_transaction_ids(&mut advertised_receiver).await;
        assert_eq!(advertised_tx_ids, second_batch);

        let gap = second_started_at.duration_since(first_started_at);
        assert!(
            gap >= TRANSACTION_GOSSIP_DELAY,
            "batches should still be paced by the gossip delay, gap: {gap:?}",
        );
        assert!(
            gap < TIPS_RESPONSE_TIMEOUT,
            "a stalled broadcast should not hold back the next batch, gap: {gap:?}",
        );

        gossip_task.abort();
    }

    #[tokio::test(start_paused = true)]
    async fn unready_network_leaves_pending_ids_in_mempool() {
        let _init_guard = zakura_test::init();

        let (mempool, mut limit_receiver) = mempool_service(vec![test_tx_ids(2, 1)]);
        let (sender, receiver) = broadcast::channel(MEMPOOL_CHANGE_CHANNEL_CAPACITY);

        sender
            .send(MempoolChange::added(test_tx_ids(1, 2)))
            .expect("receiver should be subscribed");

        let gossip_task = tokio::spawn(run_mempool_transaction_id_gossip(
            receiver,
            NeverReadyPeerSet,
            mempool,
        ));

        // Long enough for several readiness timeouts and retries.
        tokio::time::sleep(BROADCAST_READY_TIMEOUT * 3).await;

        assert!(
            matches!(
                limit_receiver.try_recv(),
                Err(mpsc::error::TryRecvError::Empty)
            ),
            "the task should not take pending txids while the network is unready",
        );
        assert!(
            !gossip_task.is_finished(),
            "gossip task should keep retrying while the network is unready",
        );

        gossip_task.abort();
    }
}