1use std::collections::VecDeque;
5use std::fmt;
6use std::sync::{
7 Arc,
8 atomic::{AtomicBool, Ordering},
9};
10use std::time::{Duration, Instant};
11
12use pb_mapper_core::conn_id::RemoteConnId;
13use tracing::instrument;
14
15use crate::{ConnTask, ManagerTask};
16
17pub type SenderChan<T> = kanal::AsyncSender<T>;
18pub type ReceiverChan<T> = kanal::AsyncReceiver<T>;
19
20#[derive(Debug, Clone, Copy, PartialEq, Eq)]
22pub struct ManagerClosed;
23
24impl fmt::Display for ManagerClosed {
25 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
26 f.write_str("routing manager queue closed")
27 }
28}
29
30impl std::error::Error for ManagerClosed {}
31
32pub enum TrySendError {
34 Full(ManagerTask),
36 Closed(ManagerTask),
38}
39
40impl fmt::Debug for TrySendError {
41 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
42 match self {
43 Self::Full(_) => f.write_str("Full(..)"),
44 Self::Closed(_) => f.write_str("Closed(..)"),
45 }
46 }
47}
48
49struct Envelope {
50 task: ManagerTask,
51 scope: Option<Arc<AtomicBool>>,
52}
53
54#[derive(Clone)]
56pub struct TaskSender {
57 sender: tokio::sync::mpsc::Sender<Envelope>,
58 scope: Option<Arc<AtomicBool>>,
59}
60
61impl fmt::Debug for TaskSender {
62 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
63 f.debug_struct("TaskSender")
64 .field("connection_scoped", &self.scope.is_some())
65 .finish()
66 }
67}
68
69impl TaskSender {
70 pub(crate) fn for_connection(&self) -> Self {
72 Self {
73 sender: self.sender.clone(),
74 scope: Some(Arc::new(AtomicBool::new(true))),
75 }
76 }
77
78 pub async fn send(&self, task: ManagerTask) -> Result<(), ManagerClosed> {
79 self.sender
80 .send(Envelope {
81 task,
82 scope: self.scope.clone(),
83 })
84 .await
85 .map_err(|_| ManagerClosed)
86 }
87
88 #[expect(
89 clippy::result_large_err,
90 reason = "the task is handed back so the caller can send it again by waiting"
91 )]
92 pub fn try_send(&self, task: ManagerTask) -> Result<(), TrySendError> {
93 self.sender
94 .try_send(Envelope {
95 task,
96 scope: self.scope.clone(),
97 })
98 .map_err(|error| match error {
99 tokio::sync::mpsc::error::TrySendError::Full(message) => {
100 TrySendError::Full(message.task)
101 }
102 tokio::sync::mpsc::error::TrySendError::Closed(message) => {
103 TrySendError::Closed(message.task)
104 }
105 })
106 }
107}
108
109pub(crate) struct TaskReceiver(tokio::sync::mpsc::Receiver<Envelope>);
110
111impl TaskReceiver {
112 #[cfg(test)]
113 pub(crate) fn is_empty(&self) -> bool {
114 self.0.is_empty()
115 }
116
117 pub(crate) async fn recv(&mut self) -> Option<ManagerTask> {
120 while let Some(message) = self.0.recv().await {
121 if let Some(scope) = message.scope {
122 let valid = if message.task.ends_connection() {
123 scope.swap(false, Ordering::AcqRel)
124 } else {
125 scope.load(Ordering::Acquire)
126 };
127 if !valid {
128 tracing::debug!(
129 event = "stale_connection_task_ignored",
130 "ignored a task from a closed connection incarnation"
131 );
132 continue;
133 }
134 }
135 return Some(message.task);
136 }
137 None
138 }
139}
140
141pub(crate) fn task_channel(capacity: usize) -> (TaskSender, TaskReceiver) {
142 let (sender, receiver) = tokio::sync::mpsc::channel(capacity);
143 (
144 TaskSender {
145 sender,
146 scope: None,
147 },
148 TaskReceiver(receiver),
149 )
150}
151
152const MANAGER_QUEUE_CAPACITY: usize = 1024;
153
154pub(crate) const REGISTRATION_ID_REST: Duration = Duration::from_secs(10);
162
163pub struct TaskManager {
165 next_conn_id: RemoteConnId,
169 sender: TaskSender,
171 receiver: TaskReceiver,
172 idle_conn_ids: Vec<RemoteConnId>,
181 resting_conn_ids: VecDeque<(RemoteConnId, Instant)>,
183 idle_conn_set: hashbrown::HashSet<RemoteConnId>,
187 active_conn_map: hashbrown::HashMap<RemoteConnId, SenderChan<ConnTask>>,
188}
189
190impl Default for TaskManager {
191 fn default() -> Self {
192 Self::new()
193 }
194}
195
196impl TaskManager {
197 pub fn new() -> Self {
198 let (sender, receiver) = task_channel(MANAGER_QUEUE_CAPACITY);
199 Self {
200 next_conn_id: RemoteConnId::default(),
201 sender,
202 receiver,
203 idle_conn_ids: Vec::new(),
204 resting_conn_ids: VecDeque::new(),
205 idle_conn_set: hashbrown::HashSet::new(),
206 active_conn_map: hashbrown::HashMap::new(),
207 }
208 }
209
210 #[inline]
215 pub async fn wait_for_task(&mut self) -> Option<ManagerTask> {
216 self.receiver.recv().await
217 }
218
219 pub fn get_task_sender(&self) -> TaskSender {
220 self.sender.clone()
221 }
222
223 #[inline]
224 pub fn get_conn_sender_chan(&self, conn_id: &RemoteConnId) -> Option<SenderChan<ConnTask>> {
225 self.active_conn_map.get(conn_id).map(|v| v.to_owned())
226 }
227
228 #[inline]
229 pub fn has_conn_sender(&self, conn_id: &RemoteConnId) -> bool {
230 self.active_conn_map.contains_key(conn_id)
231 }
232
233 #[inline]
234 pub fn active_conn_count(&self) -> usize {
235 self.active_conn_map.len()
236 }
237
238 #[inline]
240 pub fn idle_conn_count(&self) -> usize {
241 self.idle_conn_ids.len() + self.resting_conn_ids.len()
242 }
243
244 #[inline]
261 pub fn drop_conn_sender(&mut self, conn_id: RemoteConnId) -> bool {
262 self.active_conn_map.remove(&conn_id).is_some()
263 }
264
265 #[inline]
268 pub fn recycle_conn_id(&mut self, conn_id: RemoteConnId) {
269 if self.idle_conn_set.insert(conn_id) {
270 self.idle_conn_ids.push(conn_id);
271 }
272 }
273
274 pub(crate) fn rest_conn_id(&mut self, conn_id: RemoteConnId, now: Instant) {
278 if self.idle_conn_set.insert(conn_id) {
279 self.resting_conn_ids
280 .push_back((conn_id, now + REGISTRATION_ID_REST));
281 }
282 }
283
284 pub fn get_conn_id(
294 &mut self,
295 registered: impl Iterator<Item = RemoteConnId> + Clone,
296 ) -> Option<RemoteConnId> {
297 self.take_conn_id(registered, Instant::now())
298 }
299
300 pub(crate) fn take_conn_id(
302 &mut self,
303 registered: impl Iterator<Item = RemoteConnId> + Clone,
304 now: Instant,
305 ) -> Option<RemoteConnId> {
306 while let Some(&(conn_id, rested)) = self.resting_conn_ids.front()
307 && rested <= now
308 {
309 self.resting_conn_ids.pop_front();
310 self.idle_conn_ids.push(conn_id);
311 }
312 while let Some(conn_id) = self.idle_conn_ids.pop() {
313 if let Some(conn_id) = self.reusable(conn_id, ®istered) {
314 return Some(conn_id);
315 }
316 }
317 if let Some(next) = u32::from(self.next_conn_id).checked_add(1) {
318 let conn_id = self.next_conn_id;
319 self.next_conn_id = next.into();
320 return Some(conn_id);
321 }
322 while let Some((conn_id, _)) = self.resting_conn_ids.pop_front() {
325 if let Some(conn_id) = self.reusable(conn_id, ®istered) {
326 return Some(conn_id);
327 }
328 }
329 None
330 }
331
332 fn reusable(
334 &mut self,
335 conn_id: RemoteConnId,
336 registered: &(impl Iterator<Item = RemoteConnId> + Clone),
337 ) -> Option<RemoteConnId> {
338 self.idle_conn_set.remove(&conn_id);
339 if self.is_issued(&conn_id)
340 && !self.active_conn_map.contains_key(&conn_id)
341 && !registered.clone().any(|id| id == conn_id)
342 {
343 return Some(conn_id);
344 }
345 tracing::warn!(
346 "You got invalid or active `conn_id` from `idle_list`!
347 That may nerver happen!
348 We will generate a new connId by default"
349 );
350 None
351 }
352
353 #[instrument(skip(self, sender))]
354 pub fn sign_up_conn_sender(&mut self, conn_id: RemoteConnId, sender: SenderChan<ConnTask>) {
355 if !self.is_issued(&conn_id) {
356 tracing::warn!("invalid conn id,may be index out of bound,we do nothing");
357 return;
358 }
359 if self.active_conn_map.insert(conn_id, sender).is_some() {
360 tracing::error!("conn id {conn_id} is already active; replacing sender");
361 }
362 }
363
364 fn is_issued(&self, conn_id: &RemoteConnId) -> bool {
365 conn_id < &self.next_conn_id
366 }
367}
368
369#[cfg(test)]
370mod tests {
371 use super::*;
372
373 fn sender() -> SenderChan<ConnTask> {
374 kanal::bounded_async(1).0
375 }
376
377 #[test]
378 fn recycling_an_id_twice_queues_it_once() {
379 let mut manager = TaskManager::new();
380 let first = manager.get_conn_id(std::iter::empty()).unwrap();
381 manager.sign_up_conn_sender(first, sender());
382 assert!(manager.drop_conn_sender(first));
383
384 manager.recycle_conn_id(first);
385 manager.recycle_conn_id(first);
386
387 assert_eq!(manager.idle_conn_count(), 1);
388 assert_eq!(manager.get_conn_id(std::iter::empty()), Some(first));
389 assert_eq!(
390 manager.get_conn_id(std::iter::empty()),
391 Some(RemoteConnId::from(1))
392 );
393 }
394
395 #[test]
398 fn the_end_of_the_id_space_refuses_rather_than_wraps() {
399 let mut manager = TaskManager::new();
400 let live = manager.get_conn_id(std::iter::empty()).unwrap();
401 manager.sign_up_conn_sender(live, sender());
402 manager.next_conn_id = RemoteConnId::from(u32::MAX - 1);
403
404 let last = manager.get_conn_id(std::iter::empty()).unwrap();
405 assert_eq!(last, RemoteConnId::from(u32::MAX - 1));
406 assert_eq!(manager.get_conn_id(std::iter::empty()), None);
407 assert_eq!(manager.get_conn_id(std::iter::empty()), None);
408
409 manager.recycle_conn_id(last);
412 assert_eq!(manager.get_conn_id(std::iter::empty()), Some(last));
413 manager.rest_conn_id(last, Instant::now());
414 assert_eq!(manager.get_conn_id(std::iter::empty()), Some(last));
415 assert!(manager.get_conn_sender_chan(&live).is_some());
416 }
417
418 #[test]
425 fn a_flood_of_connection_ends_returns_its_ids_cheaply() {
426 const FLOOD: u32 = 65_535;
427 let mut manager = TaskManager::new();
428 let ids = (0..FLOOD)
429 .map(|_| manager.get_conn_id(std::iter::empty()).unwrap())
430 .collect::<Vec<_>>();
431 let started = std::time::Instant::now();
432 for id in ids {
433 manager.recycle_conn_id(id);
434 }
435 let elapsed = started.elapsed();
436 assert!(
437 elapsed < std::time::Duration::from_secs(1),
438 "returning {FLOOD} ids took {elapsed:?}"
439 );
440 assert_eq!(manager.idle_conn_count(), FLOOD as usize);
441 }
442
443 #[test]
444 fn idle_id_is_not_reused_while_active() {
445 let mut manager = TaskManager::new();
446 let active = manager.get_conn_id(std::iter::empty()).unwrap();
447 manager.sign_up_conn_sender(active, sender());
448 manager.recycle_conn_id(active);
449
450 assert_eq!(
451 manager.get_conn_id(std::iter::empty()),
452 Some(RemoteConnId::from(1))
453 );
454 assert!(manager.get_conn_sender_chan(&active).is_some());
455 }
456
457 #[tokio::test]
458 async fn old_connection_cannot_clean_up_or_update_a_reused_id() {
459 let (sender, mut receiver) = task_channel(8);
460 let old = sender.for_connection();
461 let replacement = sender.for_connection();
462 let cleanup = |key: &str| ManagerTask::DeRegisterServerConn {
463 key: key.into(),
464 conn_id: 7_u32.into(),
465 };
466 old.send(cleanup("old")).await.unwrap();
467 assert!(matches!(
468 receiver.recv().await.unwrap(),
469 ManagerTask::DeRegisterServerConn { .. }
470 ));
471 old.send(cleanup("old")).await.unwrap();
473 old.send(ManagerTask::ServerConnActivity {
474 key: "old".into(),
475 conn_id: 7_u32.into(),
476 })
477 .await
478 .unwrap();
479 replacement.send(cleanup("new")).await.unwrap();
480 assert!(
481 matches!(receiver.recv().await.unwrap(), ManagerTask::DeRegisterServerConn { key, .. } if key.as_ref() == "new")
482 );
483 assert!(receiver.is_empty());
484 }
485}