Skip to main content

pb_mapper_server/
manager.rs

1//! The routing manager's work queue, and its registry of connection IDs and
2//! the mailboxes they are reached through.
3
4use std::collections::VecDeque;
5use std::fmt;
6use std::sync::{
7    Arc,
8    atomic::{AtomicBool, Ordering},
9};
10use std::time::{Duration, Instant};
11
12use pb_mapper_core::conn_id::RemoteConnId;
13use tracing::instrument;
14
15use crate::{ConnTask, ManagerTask};
16
17pub type SenderChan<T> = kanal::AsyncSender<T>;
18pub type ReceiverChan<T> = kanal::AsyncReceiver<T>;
19
20/// The routing manager's queue is closed: the relay has stopped routing.
21#[derive(Debug, Clone, Copy, PartialEq, Eq)]
22pub struct ManagerClosed;
23
24impl fmt::Display for ManagerClosed {
25    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
26        f.write_str("routing manager queue closed")
27    }
28}
29
30impl std::error::Error for ManagerClosed {}
31
32/// Why [`TaskSender::try_send`] did not queue a task, with the task handed back.
33pub enum TrySendError {
34    /// The queue is at capacity; the task can still be sent by waiting.
35    Full(ManagerTask),
36    /// The manager has stopped and will never serve the task.
37    Closed(ManagerTask),
38}
39
40impl fmt::Debug for TrySendError {
41    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
42        match self {
43            Self::Full(_) => f.write_str("Full(..)"),
44            Self::Closed(_) => f.write_str("Closed(..)"),
45        }
46    }
47}
48
49struct Envelope {
50    task: ManagerTask,
51    scope: Option<Arc<AtomicBool>>,
52}
53
54/// Bounded manager mailbox with an unambiguous, cancellation-safe send boundary.
55#[derive(Clone)]
56pub struct TaskSender {
57    sender: tokio::sync::mpsc::Sender<Envelope>,
58    scope: Option<Arc<AtomicBool>>,
59}
60
61impl fmt::Debug for TaskSender {
62    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
63        f.debug_struct("TaskSender")
64            .field("connection_scoped", &self.scope.is_some())
65            .finish()
66    }
67}
68
69impl TaskSender {
70    /// A fresh connection incarnation; clones share its one-time cleanup fence.
71    pub(crate) fn for_connection(&self) -> Self {
72        Self {
73            sender: self.sender.clone(),
74            scope: Some(Arc::new(AtomicBool::new(true))),
75        }
76    }
77
78    pub async fn send(&self, task: ManagerTask) -> Result<(), ManagerClosed> {
79        self.sender
80            .send(Envelope {
81                task,
82                scope: self.scope.clone(),
83            })
84            .await
85            .map_err(|_| ManagerClosed)
86    }
87
88    #[expect(
89        clippy::result_large_err,
90        reason = "the task is handed back so the caller can send it again by waiting"
91    )]
92    pub fn try_send(&self, task: ManagerTask) -> Result<(), TrySendError> {
93        self.sender
94            .try_send(Envelope {
95                task,
96                scope: self.scope.clone(),
97            })
98            .map_err(|error| match error {
99                tokio::sync::mpsc::error::TrySendError::Full(message) => {
100                    TrySendError::Full(message.task)
101                }
102                tokio::sync::mpsc::error::TrySendError::Closed(message) => {
103                    TrySendError::Closed(message.task)
104                }
105            })
106    }
107}
108
109pub(crate) struct TaskReceiver(tokio::sync::mpsc::Receiver<Envelope>);
110
111impl TaskReceiver {
112    #[cfg(test)]
113    pub(crate) fn is_empty(&self) -> bool {
114        self.0.is_empty()
115    }
116
117    /// The next task from a live connection incarnation, or `None` once every
118    /// sender is gone.
119    pub(crate) async fn recv(&mut self) -> Option<ManagerTask> {
120        while let Some(message) = self.0.recv().await {
121            if let Some(scope) = message.scope {
122                let valid = if message.task.ends_connection() {
123                    scope.swap(false, Ordering::AcqRel)
124                } else {
125                    scope.load(Ordering::Acquire)
126                };
127                if !valid {
128                    tracing::debug!(
129                        event = "stale_connection_task_ignored",
130                        "ignored a task from a closed connection incarnation"
131                    );
132                    continue;
133                }
134            }
135            return Some(message.task);
136        }
137        None
138    }
139}
140
141pub(crate) fn task_channel(capacity: usize) -> (TaskSender, TaskReceiver) {
142    let (sender, receiver) = tokio::sync::mpsc::channel(capacity);
143    (
144        TaskSender {
145            sender,
146            scope: None,
147        },
148        TaskReceiver(receiver),
149    )
150}
151
152const MANAGER_QUEUE_CAPACITY: usize = 1024;
153
154/// How long a registration's ID rests, once the registration has ended, before
155/// another connection may take it.
156///
157/// Past the two seconds a publisher waits at most before it reconnects, so the
158/// registration that replaces one does not take its ID: an administrator who
159/// retires the old one by the ID a listing showed then retires nothing, rather
160/// than its healthy replacement.
161pub(crate) const REGISTRATION_ID_REST: Duration = Duration::from_secs(10);
162
163/// Hands out connection IDs, and routes each registered ID to its mailbox.
164pub struct TaskManager {
165    /// The next ID never handed out; every ID below it has been. It stops at
166    /// `u32::MAX`, which is never handed out, rather than wrap onto IDs that
167    /// live connections may hold.
168    next_conn_id: RemoteConnId,
169    /// Held for the manager's whole life, so its receiver never reports closed.
170    sender: TaskSender,
171    receiver: TaskReceiver,
172    /// IDs returned by connections that ended, the next accept's first.
173    ///
174    /// Each was handed to a connection and comes back once, when it ends, and
175    /// an accept takes one back out, so this pool and `resting_conn_ids`
176    /// together hold at most as many IDs as were ever open at once,
177    /// connections that never authenticated included, plus the registrations
178    /// that ended within one [`REGISTRATION_ID_REST`], and keep that size
179    /// until the relay stops.
180    idle_conn_ids: Vec<RemoteConnId>,
181    /// IDs of ended registrations, oldest first, each with when its rest ends.
182    resting_conn_ids: VecDeque<(RemoteConnId, Instant)>,
183    /// The IDs in `idle_conn_ids` and `resting_conn_ids`, so that a return is
184    /// checked against them in constant time on the routing loop, however
185    /// large a flood made the pool.
186    idle_conn_set: hashbrown::HashSet<RemoteConnId>,
187    active_conn_map: hashbrown::HashMap<RemoteConnId, SenderChan<ConnTask>>,
188}
189
190impl Default for TaskManager {
191    fn default() -> Self {
192        Self::new()
193    }
194}
195
196impl TaskManager {
197    pub fn new() -> Self {
198        let (sender, receiver) = task_channel(MANAGER_QUEUE_CAPACITY);
199        Self {
200            next_conn_id: RemoteConnId::default(),
201            sender,
202            receiver,
203            idle_conn_ids: Vec::new(),
204            resting_conn_ids: VecDeque::new(),
205            idle_conn_set: hashbrown::HashSet::new(),
206            active_conn_map: hashbrown::HashMap::new(),
207        }
208    }
209
210    /// The next task to route.
211    ///
212    /// `None` would mean every sender is gone, which cannot happen while the
213    /// manager holds one of its own.
214    #[inline]
215    pub async fn wait_for_task(&mut self) -> Option<ManagerTask> {
216        self.receiver.recv().await
217    }
218
219    pub fn get_task_sender(&self) -> TaskSender {
220        self.sender.clone()
221    }
222
223    #[inline]
224    pub fn get_conn_sender_chan(&self, conn_id: &RemoteConnId) -> Option<SenderChan<ConnTask>> {
225        self.active_conn_map.get(conn_id).map(|v| v.to_owned())
226    }
227
228    #[inline]
229    pub fn has_conn_sender(&self, conn_id: &RemoteConnId) -> bool {
230        self.active_conn_map.contains_key(conn_id)
231    }
232
233    #[inline]
234    pub fn active_conn_count(&self) -> usize {
235        self.active_conn_map.len()
236    }
237
238    /// IDs returned and not yet handed out again, resting ones included.
239    #[inline]
240    pub fn idle_conn_count(&self) -> usize {
241        self.idle_conn_ids.len() + self.resting_conn_ids.len()
242    }
243
244    /// Stop routing to a connection, without returning its ID to the idle pool.
245    ///
246    /// Dropping the sender and recycling the ID are separate steps because they
247    /// become safe at different moments. The sender must go the instant the
248    /// connection stops being usable — a retirement, a subscribe that found it
249    /// unreachable — but the socket task is still running then, and still owns a
250    /// guard that will deregister when it unwinds. Recycling at that point would
251    /// let a freshly accepted socket take the ID and register under it, and the
252    /// old task's deregistration, which matches on ID alone, would unwind the
253    /// replacement instead.
254    ///
255    /// So the ID stays out of circulation until [`Self::recycle_conn_id`] or
256    /// `rest_conn_id`, which the manager calls when the guard's deregistration
257    /// finally arrives — the one point at which nothing will speak for that ID
258    /// again. Every accepted connection produces exactly one such task, however
259    /// it ends, so no ID is stranded.
260    #[inline]
261    pub fn drop_conn_sender(&mut self, conn_id: RemoteConnId) -> bool {
262        self.active_conn_map.remove(&conn_id).is_some()
263    }
264
265    /// Return an ID to the idle pool, if it is not already there: an ID queued
266    /// twice could go to two connections before either had signed up.
267    #[inline]
268    pub fn recycle_conn_id(&mut self, conn_id: RemoteConnId) {
269        if self.idle_conn_set.insert(conn_id) {
270            self.idle_conn_ids.push(conn_id);
271        }
272    }
273
274    /// Return an ended registration's ID, to be handed out again only once it
275    /// has rested [`REGISTRATION_ID_REST`] from `now`, if it is not already
276    /// returned.
277    pub(crate) fn rest_conn_id(&mut self, conn_id: RemoteConnId, now: Instant) {
278        if self.idle_conn_set.insert(conn_id) {
279            self.resting_conn_ids
280                .push_back((conn_id, now + REGISTRATION_ID_REST));
281        }
282    }
283
284    /// An ID for a newly accepted connection: an idle one when one is free,
285    /// otherwise one never handed out. `registered` names the IDs the service
286    /// map still holds, which are never reused. An ended registration's ID is
287    /// idle only once it has rested ten seconds, so that the registration
288    /// replacing it does not take it.
289    ///
290    /// `None` once every ID is held, which takes four billion connections
291    /// open at once: the caller refuses the connection, and an ID that comes
292    /// back is handed out again.
293    pub fn get_conn_id(
294        &mut self,
295        registered: impl Iterator<Item = RemoteConnId> + Clone,
296    ) -> Option<RemoteConnId> {
297        self.take_conn_id(registered, Instant::now())
298    }
299
300    /// [`Self::get_conn_id`] at `now`.
301    pub(crate) fn take_conn_id(
302        &mut self,
303        registered: impl Iterator<Item = RemoteConnId> + Clone,
304        now: Instant,
305    ) -> Option<RemoteConnId> {
306        while let Some(&(conn_id, rested)) = self.resting_conn_ids.front()
307            && rested <= now
308        {
309            self.resting_conn_ids.pop_front();
310            self.idle_conn_ids.push(conn_id);
311        }
312        while let Some(conn_id) = self.idle_conn_ids.pop() {
313            if let Some(conn_id) = self.reusable(conn_id, &registered) {
314                return Some(conn_id);
315            }
316        }
317        if let Some(next) = u32::from(self.next_conn_id).checked_add(1) {
318            let conn_id = self.next_conn_id;
319            self.next_conn_id = next.into();
320            return Some(conn_id);
321        }
322        // Every ID has been handed out: one still resting serves rather than
323        // the connection being refused.
324        while let Some((conn_id, _)) = self.resting_conn_ids.pop_front() {
325            if let Some(conn_id) = self.reusable(conn_id, &registered) {
326                return Some(conn_id);
327            }
328        }
329        None
330    }
331
332    /// `conn_id`, just taken from the pool, if nothing holds it.
333    fn reusable(
334        &mut self,
335        conn_id: RemoteConnId,
336        registered: &(impl Iterator<Item = RemoteConnId> + Clone),
337    ) -> Option<RemoteConnId> {
338        self.idle_conn_set.remove(&conn_id);
339        if self.is_issued(&conn_id)
340            && !self.active_conn_map.contains_key(&conn_id)
341            && !registered.clone().any(|id| id == conn_id)
342        {
343            return Some(conn_id);
344        }
345        tracing::warn!(
346            "You got invalid or active `conn_id` from `idle_list`!
347                That may nerver happen!
348                 We will generate a new connId by default"
349        );
350        None
351    }
352
353    #[instrument(skip(self, sender))]
354    pub fn sign_up_conn_sender(&mut self, conn_id: RemoteConnId, sender: SenderChan<ConnTask>) {
355        if !self.is_issued(&conn_id) {
356            tracing::warn!("invalid conn id,may be index out of bound,we do nothing");
357            return;
358        }
359        if self.active_conn_map.insert(conn_id, sender).is_some() {
360            tracing::error!("conn id {conn_id} is already active; replacing sender");
361        }
362    }
363
364    fn is_issued(&self, conn_id: &RemoteConnId) -> bool {
365        conn_id < &self.next_conn_id
366    }
367}
368
369#[cfg(test)]
370mod tests {
371    use super::*;
372
373    fn sender() -> SenderChan<ConnTask> {
374        kanal::bounded_async(1).0
375    }
376
377    #[test]
378    fn recycling_an_id_twice_queues_it_once() {
379        let mut manager = TaskManager::new();
380        let first = manager.get_conn_id(std::iter::empty()).unwrap();
381        manager.sign_up_conn_sender(first, sender());
382        assert!(manager.drop_conn_sender(first));
383
384        manager.recycle_conn_id(first);
385        manager.recycle_conn_id(first);
386
387        assert_eq!(manager.idle_conn_count(), 1);
388        assert_eq!(manager.get_conn_id(std::iter::empty()), Some(first));
389        assert_eq!(
390            manager.get_conn_id(std::iter::empty()),
391            Some(RemoteConnId::from(1))
392        );
393    }
394
395    /// Past the last fresh ID a new connection is refused, not handed an ID
396    /// that wrapped round to one a live connection holds.
397    #[test]
398    fn the_end_of_the_id_space_refuses_rather_than_wraps() {
399        let mut manager = TaskManager::new();
400        let live = manager.get_conn_id(std::iter::empty()).unwrap();
401        manager.sign_up_conn_sender(live, sender());
402        manager.next_conn_id = RemoteConnId::from(u32::MAX - 1);
403
404        let last = manager.get_conn_id(std::iter::empty()).unwrap();
405        assert_eq!(last, RemoteConnId::from(u32::MAX - 1));
406        assert_eq!(manager.get_conn_id(std::iter::empty()), None);
407        assert_eq!(manager.get_conn_id(std::iter::empty()), None);
408
409        // An ID that comes back serves the next connection, even one that
410        // would otherwise rest first.
411        manager.recycle_conn_id(last);
412        assert_eq!(manager.get_conn_id(std::iter::empty()), Some(last));
413        manager.rest_conn_id(last, Instant::now());
414        assert_eq!(manager.get_conn_id(std::iter::empty()), Some(last));
415        assert!(manager.get_conn_sender_chan(&live).is_some());
416    }
417
418    /// The ends of a flood of connections, as many as the shipped descriptor
419    /// limit lets a peer hold open before it authenticates, each return their
420    /// ID without a scan of the IDs already returned: the routing loop doing it
421    /// serves every other connection meanwhile. The bound is far above what
422    /// that takes (milliseconds) and far below what a scan per return took
423    /// (seconds in a debug build).
424    #[test]
425    fn a_flood_of_connection_ends_returns_its_ids_cheaply() {
426        const FLOOD: u32 = 65_535;
427        let mut manager = TaskManager::new();
428        let ids = (0..FLOOD)
429            .map(|_| manager.get_conn_id(std::iter::empty()).unwrap())
430            .collect::<Vec<_>>();
431        let started = std::time::Instant::now();
432        for id in ids {
433            manager.recycle_conn_id(id);
434        }
435        let elapsed = started.elapsed();
436        assert!(
437            elapsed < std::time::Duration::from_secs(1),
438            "returning {FLOOD} ids took {elapsed:?}"
439        );
440        assert_eq!(manager.idle_conn_count(), FLOOD as usize);
441    }
442
443    #[test]
444    fn idle_id_is_not_reused_while_active() {
445        let mut manager = TaskManager::new();
446        let active = manager.get_conn_id(std::iter::empty()).unwrap();
447        manager.sign_up_conn_sender(active, sender());
448        manager.recycle_conn_id(active);
449
450        assert_eq!(
451            manager.get_conn_id(std::iter::empty()),
452            Some(RemoteConnId::from(1))
453        );
454        assert!(manager.get_conn_sender_chan(&active).is_some());
455    }
456
457    #[tokio::test]
458    async fn old_connection_cannot_clean_up_or_update_a_reused_id() {
459        let (sender, mut receiver) = task_channel(8);
460        let old = sender.for_connection();
461        let replacement = sender.for_connection();
462        let cleanup = |key: &str| ManagerTask::DeRegisterServerConn {
463            key: key.into(),
464            conn_id: 7_u32.into(),
465        };
466        old.send(cleanup("old")).await.unwrap();
467        assert!(matches!(
468            receiver.recv().await.unwrap(),
469            ManagerTask::DeRegisterServerConn { .. }
470        ));
471        // The routing ID may now be recycled. Its new scope must remain usable.
472        old.send(cleanup("old")).await.unwrap();
473        old.send(ManagerTask::ServerConnActivity {
474            key: "old".into(),
475            conn_id: 7_u32.into(),
476        })
477        .await
478        .unwrap();
479        replacement.send(cleanup("new")).await.unwrap();
480        assert!(
481            matches!(receiver.recv().await.unwrap(), ManagerTask::DeRegisterServerConn { key, .. } if key.as_ref() == "new")
482        );
483        assert!(receiver.is_empty());
484    }
485}