Skip to main content

turnloop_mongodb/
topology.rs

1//! server-discovery-and-monitoring/server-discovery-and-monitoring.md §§ Parsing
2//! hello, TopologyType table, updateRSFromPrimary, Error Handling; server-selection/
3//! server-selection.md §§ Read Preference, Latency Window, max-staleness.
4use crate::Instant;
5use crate::{
6    Error, ErrorKind, Result,
7    uri::{Options, ReadPreference},
8};
9use bson::{Document, oid::ObjectId};
10use std::{
11    collections::{BTreeMap, VecDeque},
12    time::Duration,
13};
14#[derive(Clone, Copy, Debug, PartialEq, Eq)]
15pub enum ServerType {
16    Unknown,
17    PossiblePrimary,
18    Standalone,
19    Mongos,
20    RSPrimary,
21    RSSecondary,
22    RSArbiter,
23    RSOther,
24    RSGhost,
25}
26impl ServerType {
27    pub fn readable(self) -> bool {
28        matches!(
29            self,
30            Self::Standalone | Self::Mongos | Self::RSPrimary | Self::RSSecondary
31        )
32    }
33}
34#[derive(Clone, Copy, Debug, PartialEq, Eq)]
35pub enum TopologyType {
36    Unknown,
37    Single,
38    ReplicaSetNoPrimary,
39    ReplicaSetWithPrimary,
40    Sharded,
41}
42#[derive(Clone, Debug)]
43pub struct Server {
44    pub kind: ServerType,
45    pub set_name: Option<String>,
46    pub hosts: Vec<String>,
47    pub me: Option<String>,
48    pub primary: Option<String>,
49    pub set_version: Option<i32>,
50    pub election_id: Option<ObjectId>,
51    pub min_wire_version: i32,
52    pub max_wire_version: i32,
53    pub session_timeout: Option<i64>,
54    pub topology_version: Option<Document>,
55    pub error: Option<String>,
56    pub generation: u64,
57    pub rtt: Option<Duration>,
58    pub last_update: Instant,
59    pub last_write_ms: Option<i64>,
60    pub tags: BTreeMap<String, String>,
61    pub next_check: Option<Instant>,
62    pub hello_ok: bool,
63    pub operation_count: u32,
64}
65impl Server {
66    fn unknown(now: Instant, generation: u64) -> Self {
67        Self {
68            kind: ServerType::Unknown,
69            set_name: None,
70            hosts: Vec::new(),
71            me: None,
72            primary: None,
73            set_version: None,
74            election_id: None,
75            min_wire_version: 0,
76            max_wire_version: 0,
77            session_timeout: None,
78            topology_version: None,
79            error: None,
80            generation,
81            rtt: None,
82            last_update: now,
83            last_write_ms: None,
84            tags: BTreeMap::new(),
85            next_check: Some(now),
86            hello_ok: false,
87            operation_count: 0,
88        }
89    }
90    fn parse(d: &Document, now: Instant, old: &Server, rtt: Duration, heartbeat: Duration) -> Self {
91        let mut s = Self::unknown(now, old.generation);
92        s.next_check = Some(now + heartbeat);
93        s.hello_ok = d.get_bool("helloOk").unwrap_or(false);
94        let ok = d
95            .get_f64("ok")
96            .ok()
97            .or_else(|| d.get_i32("ok").ok().map(f64::from))
98            .unwrap_or(0.0)
99            != 0.0;
100        if !ok {
101            s.error = Some(d.get_str("errmsg").unwrap_or("hello failed").to_owned());
102            return s;
103        }
104        s.kind = if d.get_str("msg").ok() == Some("isdbgrid") {
105            ServerType::Mongos
106        } else if d.get_bool("isreplicaset").ok() == Some(true) {
107            ServerType::RSGhost
108        } else if d.contains_key("setName") {
109            if d.get_bool("hidden").ok() == Some(true) {
110                ServerType::RSOther
111            } else if d
112                .get_bool("isWritablePrimary")
113                .or_else(|_| d.get_bool("ismaster"))
114                .unwrap_or(false)
115            {
116                ServerType::RSPrimary
117            } else if d.get_bool("secondary").ok() == Some(true) {
118                ServerType::RSSecondary
119            } else if d.get_bool("arbiterOnly").ok() == Some(true) {
120                ServerType::RSArbiter
121            } else {
122                ServerType::RSOther
123            }
124        } else {
125            ServerType::Standalone
126        };
127        s.set_name = d.get_str("setName").ok().map(str::to_owned);
128        s.me = d.get_str("me").ok().map(str::to_lowercase);
129        s.primary = d.get_str("primary").ok().map(str::to_lowercase);
130        for key in ["hosts", "passives", "arbiters"] {
131            if let Ok(a) = d.get_array(key) {
132                s.hosts
133                    .extend(a.iter().filter_map(|b| b.as_str().map(str::to_lowercase)));
134            }
135        }
136        s.set_version = d.get_i32("setVersion").ok();
137        s.election_id = d.get_object_id("electionId").ok();
138        s.min_wire_version = d.get_i32("minWireVersion").unwrap_or(0);
139        s.max_wire_version = d.get_i32("maxWireVersion").unwrap_or(0);
140        s.session_timeout = integer(d, "logicalSessionTimeoutMinutes");
141        s.topology_version = d.get_document("topologyVersion").ok().cloned();
142        s.last_write_ms = d
143            .get_document("lastWrite")
144            .ok()
145            .and_then(|w| w.get_datetime("lastWriteDate").ok())
146            .map(|v| v.timestamp_millis());
147        if let Ok(tags) = d.get_document("tags") {
148            s.tags = tags
149                .iter()
150                .filter_map(|(k, v)| v.as_str().map(|s| (k.clone(), s.to_owned())))
151                .collect();
152        }
153        s.rtt = Some(
154            old.rtt
155                .map_or(rtt, |old| old.mul_f64(0.8) + rtt.mul_f64(0.2)),
156        );
157        s.operation_count = old.operation_count;
158        s
159    }
160}
161#[derive(Clone, Debug)]
162pub enum TopologyEvent {
163    Check { address: String, hello: bool },
164    ServerAdded(String),
165    ServerRemoved(String),
166    ServerChanged(String),
167    TopologyChanged(TopologyType),
168    ClearPool { address: String, generation: u64 },
169}
170pub struct Topology {
171    pub kind: TopologyType,
172    pub set_name: Option<String>,
173    pub servers: BTreeMap<String, Server>,
174    pub max_set_version: Option<i32>,
175    pub max_election_id: Option<ObjectId>,
176    pub heartbeat: Duration,
177    pub local_threshold: Duration,
178    seed_count: usize,
179    events: VecDeque<TopologyEvent>,
180}
181/// Retained candidate names for async callers which cannot keep references to a
182/// mutating topology between polls. Capacities survive selection and filtering.
183#[derive(Default)]
184pub struct SelectionBuffer {
185    names: Vec<String>,
186    len: usize,
187}
188trait CandidateList<'a> {
189    fn clear(&mut self);
190    fn push(&mut self, name: &'a str);
191    fn names(&self) -> impl Iterator<Item = &str>;
192    fn retain(&mut self, keep: impl FnMut(&str) -> bool);
193    fn is_empty(&self) -> bool;
194}
195impl<'a> CandidateList<'a> for Vec<&'a str> {
196    fn clear(&mut self) {
197        Vec::clear(self);
198    }
199    fn push(&mut self, name: &'a str) {
200        Vec::push(self, name);
201    }
202    fn names(&self) -> impl Iterator<Item = &str> {
203        self.iter().copied()
204    }
205    fn retain(&mut self, mut keep: impl FnMut(&str) -> bool) {
206        Vec::retain(self, |s| keep(s));
207    }
208    fn is_empty(&self) -> bool {
209        Vec::is_empty(self)
210    }
211}
212impl<'a> CandidateList<'a> for SelectionBuffer {
213    fn clear(&mut self) {
214        self.len = 0;
215    }
216    fn push(&mut self, name: &'a str) {
217        if self.len == self.names.len() {
218            self.names.push(String::with_capacity(name.len()));
219        }
220        self.names[self.len].clear();
221        self.names[self.len].push_str(name);
222        self.len += 1;
223    }
224    fn names(&self) -> impl Iterator<Item = &str> {
225        self.names[..self.len].iter().map(String::as_str)
226    }
227    fn retain(&mut self, mut keep: impl FnMut(&str) -> bool) {
228        let mut to = 0;
229        for from in 0..self.len {
230            if keep(&self.names[from]) {
231                self.names.swap(to, from);
232                to += 1;
233            }
234        }
235        self.len = to;
236    }
237    fn is_empty(&self) -> bool {
238        self.len == 0
239    }
240}
241impl Topology {
242    pub fn new(options: &Options, now: Instant) -> Self {
243        let kind = if options.direct {
244            TopologyType::Single
245        } else if options.replica_set.is_some() {
246            TopologyType::ReplicaSetNoPrimary
247        } else {
248            TopologyType::Unknown
249        };
250        let servers = options
251            .seeds
252            .iter()
253            .map(|a| (a.authority(), Server::unknown(now, 0)))
254            .collect();
255        Self {
256            kind,
257            set_name: options.replica_set.clone(),
258            servers,
259            max_set_version: None,
260            max_election_id: None,
261            heartbeat: options.heartbeat,
262            local_threshold: options.local_threshold,
263            seed_count: options.seeds.len(),
264            events: VecDeque::with_capacity(16),
265        }
266    }
267    pub fn next_timeout(&self) -> Option<Instant> {
268        self.servers.values().filter_map(|s| s.next_check).min()
269    }
270    pub fn handle_timeout(&mut self, now: Instant) {
271        for (a, s) in &mut self.servers {
272            if s.next_check.is_some_and(|d| d <= now) {
273                s.next_check = None;
274                self.events.push_back(TopologyEvent::Check {
275                    address: a.clone(),
276                    hello: s.hello_ok,
277                });
278            }
279        }
280    }
281    pub fn request_check(&mut self, now: Instant) {
282        for s in self.servers.values_mut() {
283            let at = (s.last_update + Duration::from_millis(500)).max(now);
284            s.next_check = Some(s.next_check.map_or(at, |d| d.min(at)));
285        }
286    }
287    pub fn poll_event(&mut self) -> Option<TopologyEvent> {
288        self.events.pop_front()
289    }
290    pub fn update(&mut self, address: &str, hello: &Document, now: Instant, rtt: Duration) {
291        let address = address.to_ascii_lowercase();
292        let Some(old) = self.servers.get(&address) else {
293            return;
294        };
295        if older(
296            old.topology_version.as_ref(),
297            hello.get_document("topologyVersion").ok(),
298            false,
299        ) {
300            return;
301        }
302        let mut s = Server::parse(hello, now, old, rtt, self.heartbeat);
303        let before = self.kind;
304        if s.kind == ServerType::Unknown {
305            s.generation += 1;
306            self.events.push_back(TopologyEvent::ClearPool {
307                address: address.clone(),
308                generation: s.generation,
309            });
310        }
311        self.servers.insert(address.clone(), s.clone());
312        if self.kind == TopologyType::Single {
313            if self.set_name.is_some() && self.set_name != s.set_name {
314                let mut unknown = Server::unknown(now, s.generation);
315                unknown.next_check = Some(now + self.heartbeat);
316                self.servers.insert(address.clone(), unknown);
317            }
318        } else if self.kind == TopologyType::Sharded {
319            if !matches!(s.kind, ServerType::Unknown | ServerType::Mongos) {
320                self.remove(&address);
321            }
322        } else {
323            match s.kind {
324                ServerType::Standalone => {
325                    if self.kind == TopologyType::Unknown && self.seed_count == 1 {
326                        self.kind = TopologyType::Single;
327                    } else {
328                        self.remove(&address);
329                    }
330                }
331                ServerType::Mongos => {
332                    if self.kind == TopologyType::Unknown {
333                        self.kind = TopologyType::Sharded;
334                    } else {
335                        self.remove(&address);
336                    }
337                }
338                ServerType::RSPrimary
339                | ServerType::RSSecondary
340                | ServerType::RSArbiter
341                | ServerType::RSOther => {
342                    if self.set_name.is_none() {
343                        self.set_name = s.set_name.clone();
344                    }
345                    if self.set_name != s.set_name {
346                        self.remove(&address);
347                    } else if s.kind == ServerType::RSPrimary {
348                        let stale = if s.max_wire_version >= 17 {
349                            (s.election_id, s.set_version)
350                                < (self.max_election_id, self.max_set_version)
351                        } else {
352                            s.set_version.is_some()
353                                && s.election_id.is_some()
354                                && self.max_set_version.is_some()
355                                && self.max_election_id.is_some()
356                                && (s.set_version, s.election_id)
357                                    < (self.max_set_version, self.max_election_id)
358                        };
359                        if stale {
360                            let mut unknown = Server::unknown(now, s.generation);
361                            unknown.next_check = Some(now + self.heartbeat);
362                            unknown.error = Some(
363                                "primary marked stale due to electionId/setVersion mismatch".into(),
364                            );
365                            self.servers.insert(address.clone(), unknown);
366                        } else {
367                            if s.max_wire_version >= 17 {
368                                self.max_election_id = s.election_id;
369                                self.max_set_version = s.set_version;
370                            } else {
371                                if s.election_id.is_some() && s.set_version.is_some() {
372                                    self.max_election_id = s.election_id;
373                                }
374                                self.max_set_version = self.max_set_version.max(s.set_version);
375                            }
376                            for (a, other) in &mut self.servers {
377                                if a != &address && other.kind == ServerType::RSPrimary {
378                                    let generation = other.generation;
379                                    *other = Server::unknown(now, generation);
380                                    other.error = Some(
381                                        "primary marked stale due to discovery of newer primary"
382                                            .into(),
383                                    );
384                                }
385                            }
386                            self.add_hosts(&s.hosts, now);
387                            let remove: Vec<_> = self
388                                .servers
389                                .keys()
390                                .filter(|a| !s.hosts.contains(a))
391                                .cloned()
392                                .collect();
393                            for a in remove {
394                                self.remove(&a);
395                            }
396                        }
397                    } else {
398                        if self.kind != TopologyType::ReplicaSetWithPrimary {
399                            self.add_hosts(&s.hosts, now);
400                            if let Some(primary) = &s.primary
401                                && let Some(p) = self.servers.get_mut(primary)
402                                && p.kind == ServerType::Unknown
403                            {
404                                p.kind = ServerType::PossiblePrimary;
405                            }
406                        }
407                        if s.me.as_ref().is_some_and(|me| me != &address) {
408                            self.remove(&address);
409                        }
410                    }
411                    if self.kind == TopologyType::Unknown {
412                        self.kind = TopologyType::ReplicaSetNoPrimary;
413                    }
414                }
415                _ => {}
416            }
417            if matches!(
418                self.kind,
419                TopologyType::ReplicaSetNoPrimary | TopologyType::ReplicaSetWithPrimary
420            ) {
421                self.check_primary();
422            }
423        }
424        if self.servers.contains_key(&address) {
425            self.events.push_back(TopologyEvent::ServerChanged(address));
426        }
427        if before != self.kind {
428            self.events
429                .push_back(TopologyEvent::TopologyChanged(self.kind));
430        }
431    }
432    fn add_hosts(&mut self, hosts: &[String], now: Instant) {
433        for host in hosts {
434            if !self.servers.contains_key(host) {
435                self.servers.insert(host.clone(), Server::unknown(now, 0));
436                self.events
437                    .push_back(TopologyEvent::ServerAdded(host.clone()));
438            }
439        }
440    }
441    fn remove(&mut self, address: &str) {
442        if self.servers.remove(address).is_some() {
443            self.events
444                .push_back(TopologyEvent::ServerRemoved(address.to_owned()));
445        }
446    }
447    fn check_primary(&mut self) {
448        self.kind = if self
449            .servers
450            .values()
451            .any(|s| s.kind == ServerType::RSPrimary)
452        {
453            TopologyType::ReplicaSetWithPrimary
454        } else {
455            TopologyType::ReplicaSetNoPrimary
456        };
457    }
458    pub fn session_timeout(&self) -> Option<i64> {
459        let mut timeout = None;
460        for s in self.servers.values().filter(|s| s.kind.readable()) {
461            let n = s.session_timeout?;
462            timeout = Some(timeout.map_or(n, |v: i64| v.min(n)));
463        }
464        timeout
465    }
466    /// Wire 6 through 27: MongoDB 3.6 through 8.2. Unknown servers do not constrain.
467    pub fn compatible(&self) -> bool {
468        self.servers.values().all(|s| {
469            matches!(s.kind, ServerType::Unknown | ServerType::PossiblePrimary)
470                || (s.min_wire_version <= 27 && s.max_wire_version >= 6)
471        })
472    }
473    pub fn application_error(&mut self, address: &str, error: ApplicationError<'_>, now: Instant) {
474        let Some(s) = self.servers.get_mut(address) else {
475            return;
476        };
477        if error.generation < s.generation
478            || older(
479                s.topology_version.as_ref(),
480                error
481                    .response
482                    .and_then(|d| d.get_document("topologyVersion").ok()),
483                true,
484            )
485        {
486            return;
487        }
488        let d = error.response;
489        let detail = d
490            .and_then(|d| d.get_document("writeConcernError").ok())
491            .or(d);
492        let code = detail.and_then(|d| d.get_i32("code").ok());
493        let message = detail.and_then(|d| d.get_str("errmsg").ok()).unwrap_or("");
494        let state_change = code.map_or_else(
495            || {
496                message.contains("not master")
497                    || message.contains("not primary")
498                    || message.contains("node is recovering")
499            },
500            |c| matches!(c, 11600 | 11602 | 13436 | 189 | 91 | 10107 | 13435 | 10058),
501        );
502        if error.kind == ApplicationErrorKind::Timeout && !state_change {
503            return;
504        }
505        if state_change || error.kind == ApplicationErrorKind::Network || !error.handshake_complete
506        {
507            let clear = error.kind != ApplicationErrorKind::Command
508                || !error.handshake_complete
509                || matches!(code, Some(11600 | 91))
510                || error.max_wire_version < 8;
511            let generation = s.generation + u64::from(clear);
512            let tv = if state_change {
513                d.and_then(|d| d.get_document("topologyVersion").ok())
514                    .cloned()
515            } else {
516                None
517            };
518            *s = Server::unknown(now, generation);
519            s.error = Some(message.to_owned());
520            s.topology_version = tv;
521            s.next_check = Some(now + Duration::from_millis(500));
522            if clear {
523                self.events.push_back(TopologyEvent::ClearPool {
524                    address: address.into(),
525                    generation,
526                });
527            }
528            if matches!(
529                self.kind,
530                TopologyType::ReplicaSetWithPrimary | TopologyType::ReplicaSetNoPrimary
531            ) {
532                self.check_primary();
533            }
534        }
535    }
536    /// Writes use Primary; host supplies random entropy for unbiased selection from
537    /// the latency window. `candidates` reuses a caller-owned vector.
538    pub fn candidates<'a>(
539        &'a self,
540        pref: ReadPreference,
541        tags: &[BTreeMap<String, String>],
542        max_staleness: Option<Duration>,
543        out: &mut Vec<&'a str>,
544    ) -> Result<()> {
545        self.candidates_deprioritized(pref, tags, max_staleness, &[], out)
546    }
547    pub fn candidates_deprioritized<'a>(
548        &'a self,
549        pref: ReadPreference,
550        tags: &[BTreeMap<String, String>],
551        max_staleness: Option<Duration>,
552        excluded: &[&str],
553        out: &mut Vec<&'a str>,
554    ) -> Result<()> {
555        self.candidates_inner(pref, tags, max_staleness, excluded, out)?;
556        if out.is_empty() && !excluded.is_empty() {
557            self.candidates_inner(pref, tags, max_staleness, &[], out)?;
558        }
559        Ok(())
560    }
561    /// Select with retained owned scratch space, using the same filtering policy
562    /// as candidates_deprioritized. No allocations after topology warm-up.
563    pub fn select_reusing<'a>(
564        &'a self,
565        pref: ReadPreference,
566        tags: &[BTreeMap<String, String>],
567        max_staleness: Option<Duration>,
568        excluded: &[&str],
569        entropy: [u64; 2],
570        scratch: &mut SelectionBuffer,
571    ) -> Result<Option<&'a str>> {
572        self.candidates_inner(pref, tags, max_staleness, excluded, scratch)?;
573        if scratch.is_empty() && !excluded.is_empty() {
574            self.candidates_inner(pref, tags, max_staleness, &[], scratch)?;
575        }
576        if scratch.len == 0 {
577            return Ok(None);
578        }
579        let a = ((u128::from(entropy[0]) * scratch.len as u128) >> 64) as usize;
580        let mut b = if scratch.len == 1 {
581            a
582        } else {
583            ((u128::from(entropy[1]) * (scratch.len - 1) as u128) >> 64) as usize
584        };
585        if scratch.len > 1 && b >= a {
586            b += 1;
587        }
588        let two = [scratch.names[a].as_str(), scratch.names[b].as_str()];
589        Ok(self
590            .choose(&two, [0, 0])
591            .and_then(|s| self.servers.get_key_value(s).map(|(a, _)| a.as_str())))
592    }
593    /// Power-of-two choice; supply independent uniformly distributed host entropy.
594    pub fn choose<'a>(&self, candidates: &[&'a str], entropy: [u64; 2]) -> Option<&'a str> {
595        if candidates.is_empty() {
596            return None;
597        }
598        let a = ((entropy[0] as u128 * candidates.len() as u128) >> 64) as usize;
599        if candidates.len() == 1 {
600            return Some(candidates[0]);
601        }
602        let mut b = ((entropy[1] as u128 * (candidates.len() - 1) as u128) >> 64) as usize;
603        if b >= a {
604            b += 1;
605        }
606        Some(
607            if self.servers.get(candidates[a])?.operation_count
608                <= self.servers.get(candidates[b])?.operation_count
609            {
610                candidates[a]
611            } else {
612                candidates[b]
613            },
614        )
615    }
616    pub fn operation_started(&mut self, address: &str) -> Result<()> {
617        let s = self
618            .servers
619            .get_mut(address)
620            .ok_or_else(|| Error::protocol("Selected server was removed"))?;
621        s.operation_count = s
622            .operation_count
623            .checked_add(1)
624            .ok_or_else(|| Error::protocol("Server operation count overflow"))?;
625        Ok(())
626    }
627    pub fn operation_finished(&mut self, address: &str) {
628        if let Some(s) = self.servers.get_mut(address) {
629            s.operation_count = s.operation_count.saturating_sub(1);
630        }
631    }
632    fn candidates_inner<'a>(
633        &'a self,
634        pref: ReadPreference,
635        tags: &[BTreeMap<String, String>],
636        max_staleness: Option<Duration>,
637        excluded: &[&str],
638        out: &mut impl CandidateList<'a>,
639    ) -> Result<()> {
640        out.clear();
641        if !self.compatible() {
642            return Err(Error::new(
643                ErrorKind::InvalidArgument,
644                "Incompatible MongoDB wire version",
645            ));
646        }
647        if pref == ReadPreference::Primary
648            && (tags.iter().any(|t| !t.is_empty()) || max_staleness.is_some())
649        {
650            return Err(Error::new(
651                ErrorKind::InvalidArgument,
652                "Primary read preference does not allow tags or staleness",
653            ));
654        }
655        if matches!(
656            self.kind,
657            TopologyType::ReplicaSetWithPrimary | TopologyType::ReplicaSetNoPrimary
658        ) && max_staleness.is_some_and(|n| {
659            n < Duration::from_secs(90) || n < self.heartbeat + Duration::from_secs(10)
660        }) {
661            return Err(Error::new(
662                ErrorKind::InvalidArgument,
663                "maxStalenessSeconds is too small",
664            ));
665        }
666        let primary = self
667            .servers
668            .iter()
669            .filter(|(a, _)| !excluded.contains(&a.as_str()))
670            .map(|(_, s)| s)
671            .find(|s| s.kind == ServerType::RSPrimary);
672        let max_write = self
673            .servers
674            .values()
675            .filter(|s| s.kind == ServerType::RSSecondary)
676            .filter_map(|s| s.last_write_ms)
677            .max();
678        let filter = |s: &Server| -> bool {
679            if matches!(self.kind, TopologyType::Single | TopologyType::Sharded) {
680                return !matches!(
681                    s.kind,
682                    ServerType::Unknown | ServerType::PossiblePrimary | ServerType::RSGhost
683                );
684            }
685            let candidate = match pref {
686                ReadPreference::Primary => s.kind == ServerType::RSPrimary,
687                ReadPreference::PrimaryPreferred if primary.is_some() => {
688                    s.kind == ServerType::RSPrimary
689                }
690                ReadPreference::Secondary
691                | ReadPreference::SecondaryPreferred
692                | ReadPreference::PrimaryPreferred => s.kind == ServerType::RSSecondary,
693                ReadPreference::Nearest => {
694                    matches!(s.kind, ServerType::RSPrimary | ServerType::RSSecondary)
695                }
696            };
697            if !candidate {
698                return false;
699            }
700            if s.kind == ServerType::RSSecondary
701                && let Some(max) = max_staleness
702            {
703                let Some(write) = s.last_write_ms else {
704                    return false;
705                };
706                let stale = if let Some(p) = primary {
707                    let Some(pw) = p.last_write_ms else {
708                        return false;
709                    };
710                    let update_delta = if s.last_update >= p.last_update {
711                        s.last_update.duration_since(p.last_update).as_millis() as i128
712                    } else {
713                        -(p.last_update.duration_since(s.last_update).as_millis() as i128)
714                    };
715                    update_delta + pw as i128 - write as i128 + self.heartbeat.as_millis() as i128
716                } else {
717                    max_write.unwrap_or(write) as i128 - write as i128
718                        + self.heartbeat.as_millis() as i128
719                };
720                if stale > max.as_millis() as i128 {
721                    return false;
722                }
723            }
724            true
725        };
726        for (a, s) in &self.servers {
727            if !excluded.contains(&a.as_str()) && filter(s) {
728                out.push(a);
729            }
730        }
731        if !matches!(self.kind, TopologyType::Single | TopologyType::Sharded)
732            && !(pref == ReadPreference::PrimaryPreferred && primary.is_some())
733            && !tags.is_empty()
734        {
735            let matching = tags.iter().find(|t| {
736                out.names().any(|a| {
737                    t.iter()
738                        .all(|(k, v)| self.servers[a].tags.get(k) == Some(v))
739                })
740            });
741            if let Some(t) = matching {
742                out.retain(|a| {
743                    t.iter()
744                        .all(|(k, v)| self.servers[a].tags.get(k) == Some(v))
745                });
746            } else {
747                out.clear();
748            }
749        }
750        if out.is_empty() && pref == ReadPreference::SecondaryPreferred {
751            for (a, s) in &self.servers {
752                if !excluded.contains(&a.as_str()) && s.kind == ServerType::RSPrimary {
753                    out.push(a);
754                }
755            }
756        }
757        if let Some(min) = out.names().filter_map(|a| self.servers[a].rtt).min() {
758            out.retain(|a| {
759                self.servers[a]
760                    .rtt
761                    .is_some_and(|r| r <= min + self.local_threshold)
762            });
763        }
764        Ok(())
765    }
766}
767#[derive(Clone, Copy, Debug, PartialEq, Eq)]
768pub enum ApplicationErrorKind {
769    Command,
770    Network,
771    Timeout,
772}
773pub struct ApplicationError<'a> {
774    pub generation: u64,
775    pub max_wire_version: i32,
776    pub handshake_complete: bool,
777    pub kind: ApplicationErrorKind,
778    pub response: Option<&'a Document>,
779}
780fn integer(d: &Document, k: &str) -> Option<i64> {
781    d.get_i64(k)
782        .ok()
783        .or_else(|| d.get_i32(k).ok().map(i64::from))
784}
785fn older(current: Option<&Document>, incoming: Option<&Document>, equal: bool) -> bool {
786    let Some((c, i)) = current.zip(incoming) else {
787        return false;
788    };
789    if c.get_object_id("processId").ok() != i.get_object_id("processId").ok() {
790        return false;
791    }
792    match (integer(c, "counter"), integer(i, "counter")) {
793        (Some(c), Some(i)) => {
794            if equal {
795                c >= i
796            } else {
797                c > i
798            }
799        }
800        _ => false,
801    }
802}