openlogi_device/inventory.rs
1//! Enumerate connected HID++ receivers and their paired devices.
2
3use std::{
4 collections::{HashMap, HashSet},
5 hash::Hash,
6 sync::Arc,
7 time::Duration,
8};
9
10use futures_concurrency::future::Join as _;
11use hidpp::channel::HidppChannel;
12use openlogi_core::device::DeviceInventory;
13use thiserror::Error;
14use tokio::time::timeout;
15use tracing::{debug, warn};
16
17use crate::ChannelRegistry;
18use crate::backend::{BackendError, HidBackend, NodeId, NodeInfo};
19use crate::channel::route::{DeviceRoute, is_receiver_pid};
20use ledger::NodeLedger;
21
22mod cache;
23mod features;
24pub mod hotplug;
25mod ledger;
26mod mappings;
27pub mod persist;
28mod probe;
29pub mod standalone;
30
31use cache::{CACHE_MISS_GRACE, CacheKey, CacheOutcome, Cached};
32use persist::{ProbeCacheSnapshot, ProbeCacheStore};
33use probe::{NodeProbe, probe_one};
34
35/// How long to wait for device-arrival event bursts before assuming the
36/// receiver has finished reporting. MX Master 4 (and other devices that may
37/// be asleep) need a generous window to wake and respond to the arrival
38/// ping; we err on the side of waiting.
39const ARRIVAL_DRAIN: Duration = Duration::from_millis(1500);
40
41/// Maximum number of pairing slots a Bolt receiver supports. We iterate this
42/// range to surface paired-but-offline devices that won't fire arrival events.
43const MAX_BOLT_SLOTS: u8 = 6;
44
45/// Upper bound on probing one HID node. `hidpp`'s request/response has no
46/// timeout of its own, so without this a single unresponsive (e.g. asleep)
47/// device wedges the whole enumeration — and the GUI runs `enumerate` on a
48/// polling watcher, so a permanent hang would stall every later refresh.
49///
50/// A timed-out node is skipped and re-probed on the next watcher tick (~2 s),
51/// and the first probe usually wakes the device so the retry succeeds fast.
52/// Slots are probed concurrently on both receiver paths, so a receiver's worst
53/// case is the 1.5 s arrival drain plus a single slot's [`BOLT_SLOT_PROBE`] /
54/// [`UNIFYING_SLOT_PROBE`] — not their sum — plus, on Bolt only, the
55/// sequential pairing-register pass that precedes the slot walk. This stays
56/// comfortably above that, so awake devices never trip it.
57///
58/// Sized for the Bluetooth-direct feature walk, the long pole: a ~35-entry
59/// table over a link that drops individual reports, which `hidpp::device`
60/// re-asks for per entry. At 6 s one lost report consumed the whole budget and
61/// the walk was abandoned mid-table, surfacing as a mouse that never appeared.
62const PROBE_BUDGET: Duration = Duration::from_secs(25);
63
64/// Probe budget for receiver nodes (Bolt/Unifying/Lightspeed dongles).
65///
66/// The 25 s [`PROBE_BUDGET`] is sized for Bluetooth-direct feature walks that
67/// receivers never perform. Keeping the receiver budget tighter matters
68/// because a full-budget timeout is also the detection path for a channel
69/// whose input-report delivery died (observed on macOS with concurrent opens
70/// of the same node: requests keep being written and answered, but the
71/// replies are delivered only to the other open handle). Until the channel is
72/// replaced every write on it stalls — DPI, SmartShift, ring haptics — so
73/// this budget bounds that outage.
74///
75/// It must still fit a receiver probe's real worst case, which is NOT the
76/// millisecond register reads but a paired device's full HID++ 2.0 feature
77/// walk: 1.5 s arrival drain + the sequential pairing-register pass + one
78/// slot's [`BOLT_SLOT_PROBE`] (10 s). 6 s proved too tight — a legitimate
79/// deep walk tripped the dead-delivery eviction, the surfaced-empty inventory
80/// tore down capture plans, and a pinned stale channel Arc then deadlocked
81/// recovery (dead buttons until restart). 13 s clears the honest worst case.
82const RECEIVER_PROBE_BUDGET: Duration = Duration::from_secs(13);
83
84/// Per-slot budget for the HID++ 2.0 feature walk on a Unifying paired device.
85///
86/// Unifying wireless round-trips are slower than Bolt BTLE: some devices (e.g.
87/// K540) take ~3 s for the version ping to return. Running multiple slow slots
88/// concurrently can still consume the full PROBE_BUDGET and get cancelled
89/// mid-walk — the probe returns nothing rather than partial features. A
90/// per-slot cap ensures each slot's feature walk is bounded independently of
91/// how many other slots are being probed at the same time. A timed-out slot
92/// still surfaces in the inventory (kind + wpid from the arrival event) — it
93/// just lacks capabilities / battery until the next tick.
94const UNIFYING_SLOT_PROBE: Duration = Duration::from_millis(3500);
95
96/// Per-slot budget when a Unifying device already has a fresh immutable probe.
97///
98/// This path normally performs just one battery read. Some Lightspeed devices
99/// occasionally omit that reply even though their receiver has just emitted a
100/// live device-arrival event. Do not let that optional refresh consume the
101/// full first-sight feature-walk budget or delay publication of a known-online
102/// mouse on every watcher tick.
103const UNIFYING_CACHED_SLOT_PROBE: Duration = Duration::from_millis(750);
104
105/// Per-slot budget for the HID++ 2.0 feature walk on a Bolt paired device.
106///
107/// Bounds a single device that stops answering its feature-walk reads (seen on
108/// a recent macOS IOHID stack with a new MX Master 4) so it falls back to its
109/// cached / identity-only data instead of pinning its slot future forever
110/// (#218). Slots walk *concurrently* (mirroring the Unifying path), so this
111/// budget covers the slowest single slot rather than dividing [`PROBE_BUDGET`]
112/// across the slot count. A healthy walk is not always fast either: a
113/// feature-rich device enumerates a large table one round-trip per feature
114/// (the MX Master 4's 45 features take ~1–1.6 s over Bolt even awake), and on
115/// high-latency USB paths (a Bolt receiver behind a KVM's USB emulation) it
116/// takes several seconds — the previous 3 s cap starved every slot there, so a
117/// newly paired device could never acquire model info at all. 10 s is generous
118/// headroom for degraded-but-alive paths while still fitting [`PROBE_BUDGET`]
119/// after the 1.5 s arrival drain and Bolt's sequential pairing-register pass.
120const BOLT_SLOT_PROBE: Duration = Duration::from_secs(10);
121
122/// Errors raised while enumerating HID++ devices.
123#[derive(Debug, Error)]
124pub enum InventoryError {
125 /// Underlying HID backend error.
126 #[error("HID transport error")]
127 Hid(#[from] BackendError),
128 /// More than one indistinguishable standalone raw-HID node was found.
129 #[error("multiple indistinguishable standalone raw HID devices found")]
130 AmbiguousRawDevice,
131}
132
133/// Stateful device enumerator: holds the per-device probe cache so the polling
134/// watcher reuses immutable data across ticks instead of re-handshaking every
135/// device every ~2s. One-shot callers use the [`enumerate`] free function, which
136/// runs against a fresh (empty) cache.
137pub struct Enumerator {
138 /// The HID stack this enumerator walks. `openlogi-hid` supplies this
139 /// host's; tests and other hosts supply their own.
140 backend: Arc<dyn HidBackend>,
141 cache: HashMap<CacheKey, Cached>,
142 /// Consecutive ticks each cached device has been missing, for grace-period
143 /// eviction.
144 misses: HashMap<CacheKey, u8>,
145 /// Open HID++ channels reused across ticks, keyed by OS node id. Opening (and
146 /// tearing down) a device every ~2s tick is the churn issue #99 is about —
147 /// each open also leaks an `io_service_t` in async-hid's macOS backend — so a
148 /// steadily-connected node is opened once here and reused until it
149 /// disconnects.
150 channels: ChannelCache<NodeId, CachedChannel>,
151 /// Per-node last-good inventory + consecutive-failure counts: replays a
152 /// node's snapshot through transient probe failures and decides when its
153 /// cached channel must be dropped and reopened (see [`crate::inventory::ledger`]).
154 ledger: NodeLedger<NodeId>,
155 /// Optional publication sink used by the persistent Agent watcher. One-shot
156 /// callers keep this `None` and retain the route-opening library behavior.
157 registry: Option<ChannelRegistry>,
158 tick: u64,
159 /// Where the immutable probe cache is kept across restarts, `None` for a
160 /// memory-only enumerator (one-shot CLI calls, tests).
161 store: Option<Arc<dyn ProbeCacheStore>>,
162 /// Whether the persistable cache content changed since the last save —
163 /// fresh full probes and evictions, not per-tick battery refreshes.
164 cache_dirty: bool,
165 /// Whether the most recent tick failed to open at least one HID++ node.
166 open_failures_last_tick: bool,
167}
168
169/// An open channel to a receiver / direct-device HID node, held across
170/// `enumerate` ticks. Evicting it (on disconnect, or when the `Enumerator`
171/// drops) closes the device and joins the channel's read thread via
172/// [`HidppChannel`]'s `Drop`.
173struct CachedChannel {
174 info: NodeInfo,
175 channel: Arc<HidppChannel>,
176}
177
178struct PreparedNodes {
179 active: Vec<(NodeInfo, Arc<HidppChannel>)>,
180 open_failures: Vec<NodeId>,
181 retiring: Vec<NodeId>,
182}
183
184/// Disjoint active and retiring channels, generic so ownership transitions can
185/// be tested without constructing a platform HID node.
186struct ChannelCache<Node, Channel> {
187 active: HashMap<Node, Channel>,
188 retiring: HashMap<Node, Channel>,
189}
190
191impl<Node, Channel> Default for ChannelCache<Node, Channel> {
192 fn default() -> Self {
193 Self {
194 active: HashMap::new(),
195 retiring: HashMap::new(),
196 }
197 }
198}
199
200impl<Node: Eq + Hash + Clone, Channel> ChannelCache<Node, Channel> {
201 fn get(&self, node: &Node) -> Option<&Channel> {
202 self.active.get(node)
203 }
204
205 fn insert(&mut self, node: Node, channel: Channel) {
206 debug_assert!(!self.retiring.contains_key(&node));
207 self.active.insert(node, channel);
208 }
209
210 fn retire_node(&mut self, node: &Node) -> Option<&Channel> {
211 let channel = self.active.remove(node)?;
212 // Overwrite rather than keep an older retirement. Holding a node in
213 // both maps is a bug `insert` only debug-asserts against, and the
214 // caller uses what comes back to release *this* channel's cache pin —
215 // handed the stale one, it would clear the wrong pointer and leave the
216 // real pin in place, which is what blocks a node from reopening.
217 self.retiring.insert(node.clone(), channel);
218 self.retiring.get(node)
219 }
220
221 /// Whether this node may be opened during the current tick. A quiescent
222 /// retirement is dropped here, but opening remains deferred to a later tick.
223 fn prepare_open(&mut self, node: &Node, is_quiescent: impl FnOnce(&Channel) -> bool) -> bool {
224 let Some(channel) = self.retiring.get(node) else {
225 return true;
226 };
227 if is_quiescent(channel) {
228 self.retiring.remove(node);
229 }
230 false
231 }
232
233 fn retire_absent(&mut self, seen: &HashSet<Node>, mut on_retire: impl FnMut(&Channel)) {
234 let absent = self
235 .active
236 .keys()
237 .filter(|node| !seen.contains(*node))
238 .cloned()
239 .collect::<Vec<_>>();
240 for node in absent {
241 if let Some(channel) = self.retire_node(&node) {
242 on_retire(channel);
243 }
244 }
245 }
246
247 fn reap_absent(&mut self, seen: &HashSet<Node>, is_quiescent: impl Fn(&Channel) -> bool) {
248 self.retiring
249 .retain(|node, channel| seen.contains(node) || !is_quiescent(channel));
250 }
251
252 #[cfg(test)]
253 fn is_retiring(&self, node: &Node) -> bool {
254 self.retiring.contains_key(node)
255 }
256}
257
258fn routes_for_inventories(inventories: &[DeviceInventory]) -> Vec<DeviceRoute> {
259 inventories
260 .iter()
261 .flat_map(|inventory| {
262 inventory
263 .paired
264 .iter()
265 .filter_map(|paired| DeviceRoute::device_route_for(inventory, paired.slot))
266 })
267 .collect()
268}
269
270fn settle_unhealthy_node<Node: Eq + Hash + Clone>(
271 ledger: &mut NodeLedger<Node>,
272 node: &Node,
273 all_complete: &mut bool,
274 all_healthy: &mut bool,
275) -> Option<DeviceInventory> {
276 *all_complete = false;
277 *all_healthy = false;
278 ledger.settle(node, false, None).inventory
279}
280
281/// Enumerate all Logitech HID++ receivers visible to the current process and
282/// the devices paired to each.
283///
284/// Combines two data sources per receiver:
285///
286/// - `trigger_device_arrival` events — the only path to a device's wireless
287/// PID in hidpp 0.2 (the `wpid` field on `BoltDevicePairingInformation` is
288/// private). Only online, responsive devices show up here.
289/// - `get_device_pairing_information` polled per slot — covers paired-but-
290/// offline devices (sleeping mice, devices on a different host) that the
291/// arrival ping doesn't wake. No wpid for these.
292///
293/// We merge the two so an MX Master that's been asleep still shows up with
294/// its codename and kind even before you click it.
295pub async fn enumerate(
296 backend: Arc<dyn HidBackend>,
297) -> Result<Vec<DeviceInventory>, InventoryError> {
298 // The polling [`Enumerator`] keeps a per-node ledger across ticks, so a
299 // transient probe miss replays the node's last good inventory. A one-shot
300 // caller (CLI `list` / `diag`) builds a fresh `Enumerator` whose ledger is
301 // empty, so a miss has nothing to replay and would surface as an empty or
302 // partial list — the two isolated runs in #218 read 3 devices and 0. Retry a
303 // few times instead, reusing the same enumerator so its ledger accumulates a
304 // snapshot a later attempt can replay and the opened channel stays warm.
305 // #226's 5 s request timeout inside `HidppChannel::send` makes a dead probe
306 // fail fast, so a short bounded retry is cheap. Some transports can answer
307 // while still yielding a short device set (for example, a Unifying arrival
308 // event landing just after the drain window). When every node answered this
309 // cycle but that healthy pass is still short, two identical inventories mean
310 // the expected stable Unifying offline drain has settled. A failed/timed-out
311 // probe must keep using the full retry budget so the next attempt can reopen
312 // the channel and recover.
313 let mut enumerator = Enumerator::with_backend(backend);
314 let mut previous_inventories: Option<Vec<DeviceInventory>> = None;
315 let mut attempt = 1u8;
316 loop {
317 let (inventories, all_complete, all_healthy) =
318 enumerator.enumerate_reporting_completeness().await?;
319 if one_shot_should_stop(
320 previous_inventories.as_deref(),
321 &inventories,
322 all_complete,
323 all_healthy,
324 attempt,
325 ) {
326 return Ok(inventories);
327 }
328 debug!(
329 attempt,
330 all_complete,
331 all_healthy,
332 "one-shot enumerate inventory incomplete or still changing — retrying"
333 );
334 // Only a healthy pass is valid evidence for the unchanged-inventory
335 // stop, so the equality check below only ever compares two consecutive
336 // healthy snapshots. A failed/timed-out probe (replayed last-good or
337 // partial live result) is cleared so it can't count as one of the two
338 // "stable" reads and short-circuit a later healthy-but-short pass.
339 previous_inventories = if all_healthy { Some(inventories) } else { None };
340 tokio::time::sleep(ONESHOT_RETRY_DELAY).await;
341 attempt += 1;
342 }
343}
344
345/// Stop the one-shot retry loop when the snapshot is complete, when a healthy
346/// but short pass has stabilized (the expected Unifying offline-drain case), or
347/// when the explicit attempt cap is reached. An unchanged inventory from a
348/// failed probe is not stable evidence; it must keep retrying until the cap.
349fn one_shot_should_stop(
350 previous: Option<&[DeviceInventory]>,
351 current: &[DeviceInventory],
352 all_complete: bool,
353 all_healthy: bool,
354 attempt: u8,
355) -> bool {
356 all_complete
357 || (all_healthy && previous.is_some_and(|previous| previous == current))
358 || attempt >= ONESHOT_ATTEMPTS
359}
360
361/// Attempts a one-shot [`enumerate`] makes before returning whatever it last
362/// read, when an inventory keeps coming back incomplete or changing.
363const ONESHOT_ATTEMPTS: u8 = 4;
364
365/// Delay between one-shot [`enumerate`] retries. A first probe usually wakes an
366/// asleep device, so a short pause lets the next attempt read it cleanly.
367const ONESHOT_RETRY_DELAY: Duration = Duration::from_millis(300);
368
369/// Nodes that remain valid for this tick: everything the OS enumerated plus
370/// cached channels whose open transport still reports a live connection.
371fn retained_nodes<K>(
372 enumerated: &HashSet<K>,
373 cached_channels: impl IntoIterator<Item = (K, bool)>,
374) -> HashSet<K>
375where
376 K: Clone + Eq + Hash,
377{
378 let mut retained = enumerated.clone();
379 retained.extend(
380 cached_channels
381 .into_iter()
382 .filter_map(|(node, connected)| connected.then_some(node)),
383 );
384 retained
385}
386
387/// Add cached channels omitted by this OS enumeration while their open
388/// transport still reports a live connection.
389fn append_live_cached_channels(
390 nodes: &mut HashSet<NodeId>,
391 channels: &ChannelCache<NodeId, CachedChannel>,
392 active: &mut Vec<(NodeInfo, Arc<HidppChannel>)>,
393) {
394 let retained = retained_nodes(
395 nodes,
396 channels
397 .active
398 .iter()
399 .map(|(node, open)| (node.clone(), open.channel.is_connected())),
400 );
401 for node in retained.difference(nodes) {
402 if let Some(open) = channels.get(node) {
403 debug!(
404 ?node,
405 name = %open.info.name,
406 "OS enumeration omitted a live HID node; probing cached channel"
407 );
408 active.push((open.info.clone(), Arc::clone(&open.channel)));
409 }
410 }
411 *nodes = retained;
412}
413
414impl Enumerator {
415 /// Whether the most recent [`enumerate`](Self::enumerate) tick failed to
416 /// open at least one HID++ node. `false` before the first tick.
417 ///
418 /// On macOS a run of ticks with this set is the observable signature of a
419 /// denied Input Monitoring grant or a stale permission session — paired
420 /// with the grant state it separates "grant it" from "log out", which the
421 /// bare open error cannot (the denial is silent).
422 #[must_use]
423 pub fn open_failures_last_tick(&self) -> bool {
424 self.open_failures_last_tick
425 }
426
427 /// An enumerator that walks `backend` — this host's HID stack, a scripted
428 /// device tree in tests, or another host's.
429 #[must_use]
430 pub fn with_backend(backend: Arc<dyn HidBackend>) -> Self {
431 Self {
432 backend,
433 cache: HashMap::new(),
434 misses: HashMap::new(),
435 channels: ChannelCache::default(),
436 ledger: NodeLedger::default(),
437 registry: None,
438 tick: 0,
439 store: None,
440 cache_dirty: false,
441 open_failures_last_tick: false,
442 }
443 }
444
445 /// Publish this enumerator's already-open channels into `registry` after
446 /// each settled inventory tick.
447 #[must_use]
448 pub fn with_registry(mut self, registry: ChannelRegistry) -> Self {
449 self.registry = Some(registry);
450 self
451 }
452
453 /// Warm-start this enumerator's immutable probe cache from `store`, and
454 /// write it back there whenever its persistable content changes.
455 ///
456 /// A modifier rather than a constructor: persistence is orthogonal to the
457 /// channel registry and the backend, so an enumerator can carry all three.
458 #[must_use]
459 pub fn with_probe_cache(mut self, store: Arc<dyn ProbeCacheStore>) -> Self {
460 let cache = store.load().into_entries();
461 if !cache.is_empty() {
462 debug!(entries = cache.len(), "probe cache warm-started");
463 }
464 self.cache.extend(cache);
465 self.store = Some(store);
466 self
467 }
468
469 async fn prepare_nodes(
470 &mut self,
471 backend: &dyn HidBackend,
472 candidates: Vec<NodeInfo>,
473 ) -> PreparedNodes {
474 let mut active = Vec::new();
475 let mut seen_nodes = HashSet::new();
476 let mut open_failures = Vec::new();
477 let mut retiring = Vec::new();
478 for info in candidates {
479 let node = info.id.clone();
480 seen_nodes.insert(node.clone());
481 if !self
482 .channels
483 .prepare_open(&node, |cached| Arc::strong_count(&cached.channel) == 1)
484 {
485 debug!("node still retiring — waiting for its channel's remaining users to drop");
486 retiring.push(node);
487 continue;
488 }
489 if let Some(open) = self.channels.get(&node) {
490 active.push((open.info.clone(), Arc::clone(&open.channel)));
491 continue;
492 }
493 match backend.open_hidpp(&info).await {
494 Ok(Some(channel)) => {
495 self.channels.insert(
496 node,
497 CachedChannel {
498 info: info.clone(),
499 channel: Arc::clone(&channel),
500 },
501 );
502 active.push((info, channel));
503 }
504 Ok(None) => {}
505 Err(e) => {
506 warn!(error = ?e, "failed to open HID++ channel — retrying next tick");
507 open_failures.push(node);
508 }
509 }
510 }
511
512 // IOHIDManager can temporarily omit a Bluetooth device's vendor HID++
513 // collection while its already-open handle and ordinary mouse link are
514 // still live. Keep probing that cached channel instead of turning one
515 // incomplete OS snapshot into an offline device and stopping capture.
516 append_live_cached_channels(&mut seen_nodes, &self.channels, &mut active);
517
518 if let Some(registry) = &self.registry {
519 registry.retain_nodes(&seen_nodes);
520 }
521 self.channels.retire_absent(&seen_nodes, |cached| {
522 crate::write::clear_haptic_feature_cache_for(&cached.channel);
523 });
524 self.channels.reap_absent(&seen_nodes, |cached| {
525 Arc::strong_count(&cached.channel) == 1
526 });
527 self.ledger.retain_nodes(&seen_nodes);
528
529 PreparedNodes {
530 active,
531 open_failures,
532 retiring,
533 }
534 }
535
536 /// Write the cache through to its store when the persistable content
537 /// changed this tick. Best-effort: a failed write is logged and retried on
538 /// the next dirty tick.
539 fn flush_cache(&mut self) {
540 if !self.cache_dirty {
541 return;
542 }
543 let Some(store) = &self.store else {
544 return;
545 };
546 match store.save(&ProbeCacheSnapshot::of(&self.cache)) {
547 Ok(()) => self.cache_dirty = false,
548 Err(e) => warn!(error = %e, "failed to persist probe cache"),
549 }
550 }
551
552 /// One enumeration pass, reusing the cache from prior passes. Probes every
553 /// HID candidate concurrently (so one asleep node that burns the whole
554 /// `PROBE_BUDGET` can't stall the others), reusing each device's cached
555 /// immutable data when it's present and fresh.
556 ///
557 /// A node the OS still lists but whose probe fails (receiver registers
558 /// unanswered, probe timeout, open failure) is **not** reported as absent:
559 /// its last completed inventory is replayed for a bounded grace and its
560 /// channel is reopened, so a transient HID++ glitch can't masquerade as
561 /// "no devices" (#218) — see the node ledger.
562 pub async fn enumerate(&mut self) -> Result<Vec<DeviceInventory>, InventoryError> {
563 self.enumerate_reporting_completeness()
564 .await
565 .map(|(inv, _, _)| inv)
566 }
567
568 /// [`Self::enumerate`] plus whether every probed node produced a complete
569 /// enough snapshot for the one-shot caller to stop early, and whether every
570 /// probed node answered this cycle. Completeness is separate from per-node
571 /// health: a node can answer cleanly enough for the ledger to accept its
572 /// live inventory while still reporting a known count/list shortfall that
573 /// the one-shot retry should give one more chance to settle. Only healthy
574 /// shortfalls can use the unchanged-inventory early stop; failed probes must
575 /// run through the retry budget so a later attempt can recover.
576 async fn enumerate_reporting_completeness(
577 &mut self,
578 ) -> Result<(Vec<DeviceInventory>, bool, bool), InventoryError> {
579 self.tick = self.tick.wrapping_add(1);
580 let tick = self.tick;
581 let backend = Arc::clone(&self.backend);
582 let candidates = backend.enumerate_hidpp().await?;
583 debug!(count = candidates.len(), "HID++ candidate interfaces");
584
585 // Reuse an open channel per node, opening only when no active or
586 // retiring connection owns that OS node.
587 let PreparedNodes {
588 active,
589 open_failures,
590 retiring: retiring_nodes,
591 } = self.prepare_nodes(&*backend, candidates).await;
592 self.open_failures_last_tick = !open_failures.is_empty();
593
594 // Probe each open channel concurrently, sharing `&cache` read-only;
595 // updates are collected and applied afterwards (no `RefCell`).
596 let results = {
597 let cache = &self.cache;
598 active
599 .into_iter()
600 .map(|(info, channel)| async move {
601 let node = info.id.clone();
602 // Receivers answer register reads over local USB in
603 // milliseconds; only direct (esp. Bluetooth) devices need
604 // the long feature-walk budget. A tight receiver budget
605 // bounds the outage when its channel's input-report
606 // delivery dies (writes accepted, replies never seen —
607 // observed on macOS with concurrent opens of one node).
608 let receiver = is_receiver_pid(info.product_id);
609 let budget = if receiver {
610 RECEIVER_PROBE_BUDGET
611 } else {
612 PROBE_BUDGET
613 };
614 let probe =
615 timeout(budget, probe_one(info, Arc::clone(&channel), cache, tick)).await;
616 (node, channel, probe, budget, receiver)
617 })
618 .collect::<Vec<_>>()
619 .join()
620 .await
621 };
622
623 let mut inventories = Vec::new();
624 let mut outcomes = Vec::new();
625 // Aggregates for the one-shot retry. `all_complete` can stop
626 // immediately; `all_healthy` gates the unchanged-inventory shortcut so
627 // failed probes keep retrying. The ledger's own per-node replay is
628 // governed by `probe.healthy`.
629 let mut all_complete = true;
630 let mut all_healthy = true;
631 for (node, channel, result, budget, receiver) in results {
632 let probe = if let Ok(probe) = result {
633 probe
634 } else {
635 // The probe burned the whole budget — an asleep direct device,
636 // or a channel whose input-report delivery died (writes
637 // accepted, replies never seen). Either way: "couldn't
638 // check", not "nothing there".
639 warn!(
640 ?budget,
641 receiver, "device probe timed out — treating as a failed probe"
642 );
643 NodeProbe::failed()
644 };
645 all_complete &= probe.complete;
646 all_healthy &= probe.healthy;
647 outcomes.extend(probe.outcomes);
648 let settled = self.ledger.settle(&node, probe.healthy, probe.inventory);
649 // Every node waits for the ledger's consecutive-failure threshold,
650 // receivers included. One full-budget timeout is not evidence of
651 // dead delivery: [`RECEIVER_PROBE_BUDGET`] leaves barely a second
652 // over its own documented worst case, so a legitimate deep walk
653 // plus a single lost reply (5 s `SEND_RESPONSE_TIMEOUT`) already
654 // exceeds it. Evicting on that unpublishes *every* device behind
655 // the receiver — a Bolt publishes all six slots under one node —
656 // and tears down each one's capture plan. A channel whose delivery
657 // really is dead times out again on the next tick and is replaced
658 // then, with the ledger replaying its last-good inventory
659 // meanwhile, so nothing disappears from the GUI in between.
660 if settled.evict_channel {
661 if let Some(registry) = &self.registry {
662 registry.remove_node(&node);
663 }
664 if let Some(cached) = self.channels.retire_node(&node) {
665 // Release the haptic cache's pin on this channel NOW —
666 // waiting for the next haptic route-miss deadlocks when
667 // capture dies with it (see clear_haptic_feature_cache_for).
668 crate::write::clear_haptic_feature_cache_for(&cached.channel);
669 warn!("node probe keeps failing — retiring its channel before reopen");
670 }
671 } else if let Some(registry) = &self.registry {
672 let routes = settled
673 .inventory
674 .as_ref()
675 .map_or_else(Vec::new, |inventory| {
676 routes_for_inventories(std::slice::from_ref(inventory))
677 });
678 if routes.is_empty() {
679 registry.remove_node(&node);
680 } else {
681 registry.replace_node(node.clone(), routes, channel);
682 }
683 }
684 inventories.extend(settled.inventory);
685 }
686 // A listed node whose old connection is still retiring is an unhealthy
687 // probe, not a disconnect: preserve the ledger's normal replay grace.
688 for node in retiring_nodes {
689 inventories.extend(settle_unhealthy_node(
690 &mut self.ledger,
691 &node,
692 &mut all_complete,
693 &mut all_healthy,
694 ));
695 }
696 // Nodes that wouldn't open this tick still replay their last snapshot
697 // (they have no cached channel to evict).
698 for node in open_failures {
699 inventories.extend(settle_unhealthy_node(
700 &mut self.ledger,
701 &node,
702 &mut all_complete,
703 &mut all_healthy,
704 ));
705 }
706
707 let seen_keys = self.apply_outcomes(outcomes);
708 self.evict_unseen(&seen_keys);
709 self.flush_cache();
710 Ok((inventories, all_complete, all_healthy))
711 }
712
713 /// Fold this tick's probe outcomes into the cache, returning the keys seen
714 /// so [`Self::evict_unseen`] can age out the rest.
715 fn apply_outcomes(&mut self, outcomes: Vec<CacheOutcome>) -> HashSet<CacheKey> {
716 let mut seen_keys = HashSet::new();
717 for outcome in outcomes {
718 match outcome {
719 CacheOutcome::Fresh(key, cached) => {
720 seen_keys.insert(key.clone());
721 // A completed full probe of a persistable device is worth
722 // writing through; battery `Update`s are not (they would
723 // rewrite the file every tick for a value that is re-read
724 // live anyway), and neither are keys `persist::save`
725 // filters out — dirtying on those would rewrite an
726 // unchanged file on every refresh of a direct-only system.
727 self.cache_dirty |= persist::is_persistable(&key);
728 self.cache.insert(key, cached);
729 }
730 CacheOutcome::Update(key, cached) => {
731 seen_keys.insert(key.clone());
732 self.cache.insert(key, cached);
733 }
734 CacheOutcome::Seen(key) => {
735 seen_keys.insert(key);
736 }
737 CacheOutcome::Unkeyed => {}
738 }
739 }
740 seen_keys
741 }
742
743 /// Drop cache entries for devices not seen this tick, after a short grace so
744 /// a transient receiver timeout doesn't discard a still-present device.
745 fn evict_unseen(&mut self, seen_keys: &HashSet<CacheKey>) {
746 for key in seen_keys {
747 self.misses.remove(key);
748 }
749 let missing: Vec<CacheKey> = self
750 .cache
751 .keys()
752 .filter(|k| !seen_keys.contains(*k))
753 .cloned()
754 .collect();
755 for key in missing {
756 let misses = self.misses.entry(key.clone()).or_insert(0);
757 *misses += 1;
758 if *misses > CACHE_MISS_GRACE {
759 self.cache.remove(&key);
760 self.misses.remove(&key);
761 self.cache_dirty |= persist::is_persistable(&key);
762 }
763 }
764 }
765}
766
767#[cfg(test)]
768mod tests;