Skip to main content

subc_daemon/
registry.rs

1use std::{
2    collections::{HashMap, HashSet},
3    error::Error,
4    fmt,
5    sync::{Mutex, MutexGuard},
6};
7
8use subc_protocol::manifest::{CapabilityDeclarations, ModuleManifest, ProviderRole};
9use tokio::sync::watch;
10use tracing::info;
11
12/// Per-connection identity assigned by [`crate::Router`] while serving a socket.
13#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
14pub struct ConnectionId(u64);
15
16impl ConnectionId {
17    /// Synthetic connection id used by unit tests. Router-issued connection ids
18    /// start at 1, so this 0 value never collides with a real socket owner.
19    #[cfg(test)]
20    pub const LOCAL: Self = Self(0);
21
22    pub const fn new(raw: u64) -> Self {
23        Self(raw)
24    }
25
26    pub fn get(self) -> u64 {
27        self.0
28    }
29}
30
31/// Lifecycle state for a module's channel allocation.
32#[derive(Debug, Clone, Copy, PartialEq, Eq)]
33pub enum ChannelState {
34    Active,
35    Closed,
36}
37
38/// Why a module registration stopped belonging to its connection.
39#[derive(Debug, Clone, Copy, PartialEq, Eq)]
40pub(crate) enum RegistrationEndReason {
41    ConnectionClosed,
42    ExplicitGoodbye,
43    Replaced,
44    SupervisorStop,
45    SupervisorRestart,
46    RegistrationFailed,
47}
48
49impl RegistrationEndReason {
50    fn as_str(self) -> &'static str {
51        match self {
52            Self::ConnectionClosed => "connection_closed",
53            Self::ExplicitGoodbye => "explicit_goodbye",
54            Self::Replaced => "replaced",
55            Self::SupervisorStop => "supervisor_stop",
56            Self::SupervisorRestart => "supervisor_restart",
57            Self::RegistrationFailed => "registration_failed",
58        }
59    }
60}
61
62/// Registry record for one active module registration.
63#[derive(Debug, Clone, PartialEq)]
64pub struct ModuleRegistration {
65    pub manifest: ModuleManifest,
66    pub ready: bool,
67    pub negotiated_ver: u8,
68    pub state: ChannelState,
69    pub connection_id: ConnectionId,
70    pub control_ops: Vec<String>,
71}
72
73/// Which registration a lookup or a lifecycle wait is about.
74///
75/// A module id alone stops naming one process once a blue/green swap runs two
76/// processes under the same id: the incumbent in the active slot, the
77/// replacement in the candidate slot, and, after cutover, the old incumbent
78/// demoted until its connection goes away. A wait keyed on the bare id would
79/// confuse them (a successful swap never empties the id's active slot, and a
80/// candidate's "has it registered yet" would be answered by the incumbent).
81#[derive(Debug, Clone, Copy, PartialEq, Eq)]
82pub enum RegistrationSlot<'a> {
83    /// The routable registration for this module id. This is what every plain
84    /// start, stop and restart path means by "the module is registered".
85    Active(&'a str),
86    /// A swap candidate registered under this module id and not yet promoted.
87    Candidate(&'a str),
88    /// Whatever registration this module connection holds, in any slot.
89    Connection(ConnectionId),
90}
91
92/// The result of promoting a module's candidate registration to active.
93#[derive(Debug, Clone, PartialEq)]
94pub struct RegistryCutover {
95    /// The registration that is now active (the former candidate).
96    pub promoted: ModuleRegistration,
97    /// The former active registration, now demoted. `None` when the incumbent
98    /// had already gone away before the promotion.
99    pub superseded: Option<ModuleRegistration>,
100}
101
102/// Control-plane registry for module manifests and supervision ownership.
103///
104/// Duplicate active `module_id`s are rejected rather than replaced. Rejection is
105/// the safer v1 behavior because replacing a still-connected module could hijack
106/// in-flight routes. Stale registrations are removed by connection cleanup; a
107/// reconnect after the old connection drops can then register the same id again.
108///
109/// A blue/green swap is the one sanctioned way two processes hold the same id.
110/// The replacement registers into a separate candidate slot, which every by-id
111/// lookup ignores, so it stays unroutable until [`Registry::promote_candidate`]
112/// swaps it in. The old incumbent is then kept as "superseded" until its
113/// connection deregisters. Lookups keyed by connection id search all three
114/// slots, because a connection must be able to update and remove its own
115/// registration whichever slot it currently sits in.
116#[derive(Debug, Default)]
117pub struct Registry {
118    inner: Mutex<RegistryInner>,
119}
120
121#[derive(Debug, Default)]
122struct RegistryInner {
123    /// Only subscribed module ids have a sender. Changes coalesce, and writers
124    /// never wait for a supervisor to receive them.
125    // The value counts accepted active registrations. Keep that evidence even
126    // when register and disconnect coalesce before a supervisor receives them.
127    module_changes: HashMap<String, watch::Sender<u64>>,
128    modules: HashMap<String, ModuleRegistration>,
129    /// Swap candidates by module id: registered, never routable, never listed.
130    candidates: HashMap<String, ModuleRegistration>,
131    /// Former incumbents demoted by a promotion, kept only so their own
132    /// connection can still find and remove them. Never routable, never listed.
133    superseded: Vec<ModuleRegistration>,
134    /// Connections whose HELLO passed the launch-nonce check required for a
135    /// reserved module or swap candidate. Kept outside ModuleRegistration so
136    /// adding this catalog fact does not break callers that construct the public
137    /// registration struct.
138    nonce_verified: HashSet<ConnectionId>,
139    /// The active connection that displaced each superseded registration.
140    superseded_by: HashMap<ConnectionId, ConnectionId>,
141    /// Connections whose registration is reserved but not yet published.
142    ///
143    /// A HELLO registers in the registry before the daemon installs the
144    /// module's forwarding endpoint, and until that endpoint exists a
145    /// `route.open` to the module cannot be relayed. A registration in this set
146    /// already holds its module id (a duplicate HELLO is refused) and is found
147    /// by every lookup except two: [`Registry::list_modules`], which feeds
148    /// `catalog.list`, and [`Registry::published_registration`], which the swap
149    /// supervisor reads to decide that a candidate has registered. Adding a
150    /// registration to this set does not bump the catalog generation;
151    /// [`Registry::publish_registration`] removes it and bumps the generation
152    /// in the same critical section, so a catalog reader sees the module and
153    /// the new generation together.
154    unpublished: HashSet<ConnectionId>,
155    generation: u64,
156}
157
158impl Registry {
159    /// Subscribe before reading a registration so a change between the read
160    /// and the supervisor's wait remains pending on this receiver.
161    pub(crate) fn subscribe_module_changes(
162        &self,
163        module_id: &str,
164    ) -> Result<watch::Receiver<u64>, RegistryError> {
165        let mut inner = self.lock_inner()?;
166        // Retired supervisors must not leave an ever-growing set of ids.
167        inner
168            .module_changes
169            .retain(|_, sender| sender.receiver_count() > 0);
170        Ok(inner
171            .module_changes
172            .entry(module_id.to_string())
173            .or_insert_with(|| watch::channel(0).0)
174            .subscribe())
175    }
176
177    /// Register a module manifest with the module's effective granted control op set.
178    ///
179    /// The registration is published at once: it is listed by
180    /// [`Self::list_modules`] and bumps the catalog generation. The HELLO path
181    /// uses [`Self::register_unpublished_with_control_ops`] instead, because it
182    /// still has to install the module's forwarding endpoint.
183    pub fn register_with_control_ops(
184        &self,
185        manifest: ModuleManifest,
186        negotiated_ver: u8,
187        connection_id: ConnectionId,
188        control_ops: Vec<String>,
189    ) -> Result<ModuleRegistration, RegistryError> {
190        self.register_active(
191            manifest,
192            negotiated_ver,
193            connection_id,
194            control_ops,
195            true,
196            false,
197        )
198    }
199
200    /// Register a module manifest without listing it yet.
201    ///
202    /// The registration holds its module id and is visible to every lookup
203    /// except [`Self::list_modules`] (and so `catalog.list`) and
204    /// [`Self::published_registration`]. The catalog generation does not move.
205    /// Call [`Self::publish_registration`] once the module can actually be
206    /// routed to; if that never becomes possible, deregistering the connection
207    /// removes the entry without the catalog ever having listed it.
208    pub fn register_unpublished_with_control_ops(
209        &self,
210        manifest: ModuleManifest,
211        negotiated_ver: u8,
212        connection_id: ConnectionId,
213        control_ops: Vec<String>,
214    ) -> Result<ModuleRegistration, RegistryError> {
215        self.register_active(
216            manifest,
217            negotiated_ver,
218            connection_id,
219            control_ops,
220            false,
221            false,
222        )
223    }
224
225    /// Register a module after its HELLO is accepted, without listing it before forwarding is installed.
226    pub(crate) fn register_unpublished_verified_with_control_ops(
227        &self,
228        manifest: ModuleManifest,
229        negotiated_ver: u8,
230        connection_id: ConnectionId,
231        control_ops: Vec<String>,
232        nonce_verified: bool,
233    ) -> Result<ModuleRegistration, RegistryError> {
234        self.register_active(
235            manifest,
236            negotiated_ver,
237            connection_id,
238            control_ops,
239            false,
240            nonce_verified,
241        )
242    }
243
244    fn register_active(
245        &self,
246        manifest: ModuleManifest,
247        negotiated_ver: u8,
248        connection_id: ConnectionId,
249        control_ops: Vec<String>,
250        publish: bool,
251        nonce_verified: bool,
252    ) -> Result<ModuleRegistration, RegistryError> {
253        let module_id = manifest.module_id.clone();
254        if let Err(reason) = module_id_path_hazard(&module_id) {
255            return Err(RegistryError::PathHazardModuleId { module_id, reason });
256        }
257        let mut inner = self.lock_inner()?;
258        if inner.modules.contains_key(&module_id) {
259            return Err(RegistryError::DuplicateModuleId { module_id });
260        }
261
262        let ready = manifest.ready.unwrap_or(true);
263        let registration = ModuleRegistration {
264            manifest,
265            ready,
266            negotiated_ver,
267            state: ChannelState::Active,
268            connection_id,
269            control_ops,
270        };
271
272        inner.modules.insert(module_id, registration.clone());
273        if nonce_verified {
274            inner.nonce_verified.insert(connection_id);
275        } else {
276            inner.nonce_verified.remove(&connection_id);
277        }
278        if publish {
279            inner.bump_generation();
280        } else {
281            inner.unpublished.insert(connection_id);
282        }
283        if let Some(sender) = inner.module_changes.get(&registration.manifest.module_id) {
284            sender.send_modify(|accepted| *accepted = accepted.wrapping_add(1));
285        }
286        Ok(registration)
287    }
288
289    /// Register a swap candidate for `manifest.module_id` into the candidate slot.
290    ///
291    /// The candidate is invisible to [`Self::get_module`], [`Self::list_modules`]
292    /// and every other by-id lookup until [`Self::promote_candidate`]. An active
293    /// registration for the id is not required, because the incumbent may die
294    /// while the swap is open; deciding whether a candidate may register at all
295    /// belongs to the caller that admits it. A second candidate for the same id
296    /// is refused.
297    ///
298    /// The candidate is published at once, meaning candidate-slot lookups
299    /// through [`Self::published_registration`] return it immediately. The
300    /// HELLO path uses
301    /// [`Self::register_unpublished_candidate_with_control_ops`] instead.
302    pub fn register_candidate_with_control_ops(
303        &self,
304        manifest: ModuleManifest,
305        negotiated_ver: u8,
306        connection_id: ConnectionId,
307        control_ops: Vec<String>,
308    ) -> Result<ModuleRegistration, RegistryError> {
309        self.register_candidate(
310            manifest,
311            negotiated_ver,
312            connection_id,
313            control_ops,
314            true,
315            false,
316        )
317    }
318
319    /// Register a swap candidate that [`Self::published_registration`] does not
320    /// return until [`Self::publish_registration`] is called for its
321    /// connection. The swap supervisor reads that lookup to decide the
322    /// candidate has registered, and it promotes a ready candidate straight
323    /// away; promoting one whose forwarding endpoint is not installed yet would
324    /// fail the cutover.
325    pub fn register_unpublished_candidate_with_control_ops(
326        &self,
327        manifest: ModuleManifest,
328        negotiated_ver: u8,
329        connection_id: ConnectionId,
330        control_ops: Vec<String>,
331    ) -> Result<ModuleRegistration, RegistryError> {
332        self.register_candidate(
333            manifest,
334            negotiated_ver,
335            connection_id,
336            control_ops,
337            false,
338            false,
339        )
340    }
341
342    /// Register a blue/green swap candidate after its required launch nonce has been verified.
343    pub(crate) fn register_unpublished_verified_candidate_with_control_ops(
344        &self,
345        manifest: ModuleManifest,
346        negotiated_ver: u8,
347        connection_id: ConnectionId,
348        control_ops: Vec<String>,
349    ) -> Result<ModuleRegistration, RegistryError> {
350        self.register_candidate(
351            manifest,
352            negotiated_ver,
353            connection_id,
354            control_ops,
355            false,
356            true,
357        )
358    }
359
360    fn register_candidate(
361        &self,
362        manifest: ModuleManifest,
363        negotiated_ver: u8,
364        connection_id: ConnectionId,
365        control_ops: Vec<String>,
366        publish: bool,
367        nonce_verified: bool,
368    ) -> Result<ModuleRegistration, RegistryError> {
369        let module_id = manifest.module_id.clone();
370        if let Err(reason) = module_id_path_hazard(&module_id) {
371            return Err(RegistryError::PathHazardModuleId { module_id, reason });
372        }
373        let mut inner = self.lock_inner()?;
374        if inner.candidates.contains_key(&module_id) {
375            return Err(RegistryError::DuplicateModuleId { module_id });
376        }
377        let ready = manifest.ready.unwrap_or(true);
378        let registration = ModuleRegistration {
379            manifest,
380            ready,
381            negotiated_ver,
382            state: ChannelState::Active,
383            connection_id,
384            control_ops,
385        };
386        inner.candidates.insert(module_id, registration.clone());
387        if nonce_verified {
388            inner.nonce_verified.insert(connection_id);
389        } else {
390            inner.nonce_verified.remove(&connection_id);
391        }
392        if !publish {
393            inner.unpublished.insert(connection_id);
394        }
395        inner.notify_module_changed(&registration.manifest.module_id);
396        Ok(registration)
397    }
398
399    /// Publish the registration `connection_id` holds, in whichever slot.
400    ///
401    /// For an active registration this is the moment `catalog.list` starts
402    /// listing the module, and the catalog generation advances under the same
403    /// lock, so no reader can see the new generation without the module or the
404    /// module without the new generation. A candidate is not listed either way;
405    /// publishing it only makes it visible to
406    /// [`Self::published_registration`].
407    ///
408    /// Returns `false` when there is nothing to publish: the connection holds
409    /// no registration any more, or it was already published.
410    pub fn publish_registration(&self, connection_id: ConnectionId) -> Result<bool, RegistryError> {
411        let mut inner = self.lock_inner()?;
412        if !inner.unpublished.remove(&connection_id) {
413            return Ok(false);
414        }
415        let Some((slot, registration)) = inner.find_by_connection(connection_id) else {
416            return Ok(false);
417        };
418        let module_id = registration.manifest.module_id.clone();
419        if matches!(slot, SlotKind::Active) {
420            inner.bump_generation();
421        }
422        inner.notify_module_changed(&module_id);
423        Ok(true)
424    }
425
426    /// Move the candidate for `module_id` into the active slot and demote the
427    /// previous active registration, in one registry critical section.
428    ///
429    /// Returns `Ok(None)` when there is no candidate to promote. Bumps the
430    /// catalog generation, because the listed registration for the id changed.
431    pub fn promote_candidate(
432        &self,
433        module_id: &str,
434    ) -> Result<Option<RegistryCutover>, RegistryError> {
435        let mut inner = self.lock_inner()?;
436        let Some(promoted) = inner.candidates.remove(module_id) else {
437            return Ok(None);
438        };
439        let superseded = inner
440            .modules
441            .insert(module_id.to_string(), promoted.clone());
442        if let Some(superseded) = superseded.clone() {
443            let replaced_connection_id = superseded.connection_id.get();
444            inner
445                .superseded_by
446                .insert(superseded.connection_id, promoted.connection_id);
447            inner.superseded.push(superseded);
448            info!(
449                target: "subc.registry",
450                module_id = %module_id,
451                connection_id = promoted.connection_id.get(),
452                replaced = true,
453                replaced_connection_id,
454                "module registration promoted"
455            );
456        }
457        inner.bump_generation();
458        inner.notify_module_changed(module_id);
459        Ok(Some(RegistryCutover {
460            promoted,
461            superseded,
462        }))
463    }
464
465    /// The ACTIVE registration for `module_id`. Candidates and superseded
466    /// incumbents are never returned: this is the lookup routing decisions use.
467    pub fn get_module(&self, module_id: &str) -> Result<Option<ModuleRegistration>, RegistryError> {
468        Ok(self.lock_inner()?.modules.get(module_id).cloned())
469    }
470
471    /// The swap candidate registered for `module_id`, if any.
472    pub fn get_candidate(
473        &self,
474        module_id: &str,
475    ) -> Result<Option<ModuleRegistration>, RegistryError> {
476        Ok(self.lock_inner()?.candidates.get(module_id).cloned())
477    }
478
479    /// The registration held in one specific slot. See [`RegistrationSlot`].
480    pub fn registration(
481        &self,
482        slot: RegistrationSlot<'_>,
483    ) -> Result<Option<ModuleRegistration>, RegistryError> {
484        let inner = self.lock_inner()?;
485        Ok(match slot {
486            RegistrationSlot::Active(module_id) => inner.modules.get(module_id).cloned(),
487            RegistrationSlot::Candidate(module_id) => inner.candidates.get(module_id).cloned(),
488            RegistrationSlot::Connection(connection_id) => inner
489                .find_by_connection(connection_id)
490                .map(|(_, registration)| registration.clone()),
491        })
492    }
493
494    pub fn active_registration_count(&self) -> Result<usize, RegistryError> {
495        Ok(self.lock_inner()?.modules.len())
496    }
497
498    /// The catalog generation and the active registrations `catalog.list`
499    /// shows. Unpublished registrations (see
500    /// [`Self::register_unpublished_with_control_ops`]) are left out.
501    pub fn list_modules(&self) -> Result<(u64, Vec<ModuleRegistration>), RegistryError> {
502        let inner = self.lock_inner()?;
503        let mut modules = inner
504            .modules
505            .values()
506            .filter(|registration| !inner.unpublished.contains(&registration.connection_id))
507            .cloned()
508            .collect::<Vec<_>>();
509        modules.sort_by(|left, right| left.manifest.module_id.cmp(&right.manifest.module_id));
510        Ok((inner.generation, modules))
511    }
512
513    /// List published active registrations together with their HELLO nonce-verification status.
514    /// Both values are captured under the registry lock so a concurrent swap or
515    /// disconnect cannot separate a registration from its verification fact.
516    pub fn list_modules_with_nonce_verification(
517        &self,
518    ) -> Result<(u64, Vec<(ModuleRegistration, bool)>), RegistryError> {
519        let inner = self.lock_inner()?;
520        let mut registrations = inner
521            .modules
522            .values()
523            .filter(|registration| !inner.unpublished.contains(&registration.connection_id))
524            .collect::<Vec<_>>();
525        registrations.sort_by(|left, right| left.manifest.module_id.cmp(&right.manifest.module_id));
526        let modules = registrations
527            .into_iter()
528            .map(|registration| {
529                (
530                    registration.clone(),
531                    inner.nonce_verified.contains(&registration.connection_id),
532                )
533            })
534            .collect();
535        Ok((inner.generation, modules))
536    }
537
538    /// Every active registration, including unpublished ones.
539    ///
540    /// For the capability evaluator and the capability deny census, which
541    /// count a module as soon as it has registered. Whether `catalog.list`
542    /// shows the module yet does not change their verdicts.
543    pub fn list_registered_modules(&self) -> Result<Vec<ModuleRegistration>, RegistryError> {
544        let inner = self.lock_inner()?;
545        let mut modules = inner.modules.values().cloned().collect::<Vec<_>>();
546        modules.sort_by(|left, right| left.manifest.module_id.cmp(&right.manifest.module_id));
547        Ok(modules)
548    }
549
550    /// Like [`Self::registration`], but a registration that is not published
551    /// yet is reported as absent.
552    pub fn published_registration(
553        &self,
554        slot: RegistrationSlot<'_>,
555    ) -> Result<Option<ModuleRegistration>, RegistryError> {
556        let inner = self.lock_inner()?;
557        let registration = match slot {
558            RegistrationSlot::Active(module_id) => inner.modules.get(module_id),
559            RegistrationSlot::Candidate(module_id) => inner.candidates.get(module_id),
560            RegistrationSlot::Connection(connection_id) => inner
561                .find_by_connection(connection_id)
562                .map(|(_, registration)| registration),
563        };
564        Ok(registration
565            .filter(|registration| !inner.unpublished.contains(&registration.connection_id))
566            .cloned())
567    }
568
569    pub fn generation(&self) -> Result<u64, RegistryError> {
570        Ok(self.lock_inner()?.generation)
571    }
572
573    #[cfg(test)]
574    pub(crate) fn set_module_state_for_test(
575        &self,
576        module_id: &str,
577        state: ChannelState,
578    ) -> Result<bool, RegistryError> {
579        let mut inner = self.lock_inner()?;
580        let Some(registration) = inner.modules.get_mut(module_id) else {
581            return Ok(false);
582        };
583        registration.state = state;
584        inner.notify_module_changed(module_id);
585        Ok(true)
586    }
587
588    /// The registration owned by `connection_id`, searching the active,
589    /// candidate and superseded slots in that order.
590    pub fn get_module_by_connection(
591        &self,
592        connection_id: ConnectionId,
593    ) -> Result<Option<ModuleRegistration>, RegistryError> {
594        Ok(self
595            .lock_inner()?
596            .find_by_connection(connection_id)
597            .map(|(_, registration)| registration.clone()))
598    }
599
600    /// Replace the provider role list and, when supplied, the attested capability
601    /// declaration for the module owned by `connection_id`.
602    ///
603    /// Searches every slot: a swap candidate declares itself ready through this
604    /// call, and if only the active slot were searched its update would find
605    /// nothing and the candidate would never become ready. Only a change to the
606    /// active slot bumps the catalog generation, because only the active slot is
607    /// listed, and only once that registration is published.
608    pub fn replace_catalog_for_connection(
609        &self,
610        connection_id: ConnectionId,
611        provides: Vec<ProviderRole>,
612        capabilities: Option<CapabilityDeclarations>,
613        ready: Option<bool>,
614    ) -> Result<Option<ModuleRegistration>, RegistryError> {
615        let mut inner = self.lock_inner()?;
616        let Some((slot, _)) = inner.find_by_connection(connection_id) else {
617            return Ok(None);
618        };
619        let registration = inner
620            .registration_mut(slot, connection_id)
621            .expect("registration discovered under the same registry lock must still exist");
622        if let Some(capabilities) = capabilities.as_ref() {
623            let declares_federated = |caps: Option<&CapabilityDeclarations>| {
624                caps.is_some_and(|caps| {
625                    caps.provides
626                        .iter()
627                        .any(|cap| cap == subc_protocol::scope::SCOPE_FEDERATED_CAPABILITY)
628                })
629            };
630            // Pending binds rely on this declaration while the target module's outbound queue is full.
631            if declares_federated(registration.manifest.capabilities.as_ref())
632                != declares_federated(Some(capabilities))
633            {
634                return Err(RegistryError::FederatedCapabilityChanged);
635            }
636        }
637        registration.manifest.provides = provides;
638        if let Some(capabilities) = capabilities {
639            registration.manifest.capabilities = Some(capabilities);
640        }
641        if let Some(ready) = ready {
642            registration.ready = ready;
643            registration.manifest.ready = Some(ready);
644        }
645        let updated = registration.clone();
646        if matches!(slot, SlotKind::Active) && !inner.unpublished.contains(&connection_id) {
647            inner.bump_generation();
648        }
649        inner.notify_module_changed(&updated.manifest.module_id);
650        Ok(Some(updated))
651    }
652
653    /// Deregister every module owned by a dropped connection, in any slot.
654    pub fn deregister_connection(
655        &self,
656        connection_id: ConnectionId,
657    ) -> Result<Vec<ModuleRegistration>, RegistryError> {
658        self.deregister_connection_with_reason(
659            connection_id,
660            RegistrationEndReason::ConnectionClosed,
661        )
662    }
663
664    pub(crate) fn deregister_connection_with_reason(
665        &self,
666        connection_id: ConnectionId,
667        reason: RegistrationEndReason,
668    ) -> Result<Vec<ModuleRegistration>, RegistryError> {
669        let mut inner = self.lock_inner()?;
670        // A registration that was never published was never listed, so
671        // removing it leaves the catalog, and its generation, as they were.
672        let was_published = !inner.unpublished.remove(&connection_id);
673        let module_ids: Vec<String> = inner
674            .modules
675            .iter()
676            .filter(|(_, registration)| registration.connection_id == connection_id)
677            .map(|(module_id, _)| module_id.clone())
678            .collect();
679
680        let mut closed: Vec<ModuleRegistration> = module_ids
681            .into_iter()
682            .filter_map(|module_id| inner.close_module(&module_id, was_published))
683            .collect();
684
685        let candidate_ids: Vec<String> = inner
686            .candidates
687            .iter()
688            .filter(|(_, registration)| registration.connection_id == connection_id)
689            .map(|(module_id, _)| module_id.clone())
690            .collect();
691        for module_id in candidate_ids {
692            if let Some(mut registration) = inner.candidates.remove(&module_id) {
693                registration.state = ChannelState::Closed;
694                closed.push(registration);
695            }
696        }
697
698        let (removed, kept): (Vec<_>, Vec<_>) = std::mem::take(&mut inner.superseded)
699            .into_iter()
700            .partition(|registration| registration.connection_id == connection_id);
701        let superseded_connections: std::collections::HashSet<_> = removed
702            .iter()
703            .map(|registration| registration.connection_id)
704            .collect();
705        inner.superseded = kept;
706        closed.extend(removed.into_iter().map(|mut registration| {
707            registration.state = ChannelState::Closed;
708            registration
709        }));
710        for registration in &closed {
711            inner.nonce_verified.remove(&registration.connection_id);
712            inner.notify_module_changed(&registration.manifest.module_id);
713            let replaced = superseded_connections.contains(&registration.connection_id);
714            let end_reason = if replaced {
715                RegistrationEndReason::Replaced
716            } else {
717                reason
718            };
719            let replaced_by_connection_id = if replaced {
720                inner
721                    .superseded_by
722                    .remove(&registration.connection_id)
723                    .map(ConnectionId::get)
724            } else {
725                None
726            };
727            if let Some(replaced_by_connection_id) = replaced_by_connection_id {
728                info!(
729                    target: "subc.registry",
730                    module_id = %registration.manifest.module_id,
731                    connection_id = registration.connection_id.get(),
732                    reason = %end_reason.as_str(),
733                    replaced_by_connection_id,
734                    "module registration ended"
735                );
736                continue;
737            }
738            info!(
739                target: "subc.registry",
740                module_id = %registration.manifest.module_id,
741                connection_id = registration.connection_id.get(),
742                reason = %end_reason.as_str(),
743                "module registration ended"
744            );
745        }
746        Ok(closed)
747    }
748
749    fn lock_inner(&self) -> Result<MutexGuard<'_, RegistryInner>, RegistryError> {
750        self.inner.lock().map_err(|_| RegistryError::Poisoned)
751    }
752}
753
754/// Which internal slot a connection-keyed lookup found its registration in.
755#[derive(Debug, Clone, Copy, PartialEq, Eq)]
756enum SlotKind {
757    Active,
758    Candidate,
759    Superseded,
760}
761
762impl RegistryInner {
763    fn notify_module_changed(&self, module_id: &str) {
764        if let Some(sender) = self.module_changes.get(module_id) {
765            sender.send_modify(|_| {});
766        }
767    }
768
769    fn find_by_connection(
770        &self,
771        connection_id: ConnectionId,
772    ) -> Option<(SlotKind, &ModuleRegistration)> {
773        let owned_by =
774            |registration: &&ModuleRegistration| registration.connection_id == connection_id;
775        self.modules
776            .values()
777            .find(owned_by)
778            .map(|registration| (SlotKind::Active, registration))
779            .or_else(|| {
780                self.candidates
781                    .values()
782                    .find(owned_by)
783                    .map(|registration| (SlotKind::Candidate, registration))
784            })
785            .or_else(|| {
786                self.superseded
787                    .iter()
788                    .find(owned_by)
789                    .map(|registration| (SlotKind::Superseded, registration))
790            })
791    }
792
793    fn registration_mut(
794        &mut self,
795        slot: SlotKind,
796        connection_id: ConnectionId,
797    ) -> Option<&mut ModuleRegistration> {
798        let owned_by =
799            |registration: &&mut ModuleRegistration| registration.connection_id == connection_id;
800        match slot {
801            SlotKind::Active => self.modules.values_mut().find(owned_by),
802            SlotKind::Candidate => self.candidates.values_mut().find(owned_by),
803            SlotKind::Superseded => self.superseded.iter_mut().find(owned_by),
804        }
805    }
806
807    fn close_module(&mut self, module_id: &str, listed: bool) -> Option<ModuleRegistration> {
808        let mut registration = self.modules.remove(module_id)?;
809        registration.state = ChannelState::Closed;
810        if listed {
811            self.bump_generation();
812        }
813        Some(registration)
814    }
815
816    fn bump_generation(&mut self) {
817        self.generation = self.generation.wrapping_add(1);
818    }
819}
820
821#[derive(Debug, Clone, PartialEq, Eq)]
822pub enum RegistryError {
823    FederatedCapabilityChanged,
824    DuplicateModuleId {
825        module_id: String,
826    },
827    /// The id is unusable as a single path component. Enforced at
828    /// registration because the daemon MINTS A STORAGE DESCRIPTOR from the
829    /// self-claimed id verbatim (`<data_home>/cortexkit/<module_id>/store.db`),
830    /// so an id carrying separators or dot components is a path-traversal or
831    /// store-collision primitive handed to whoever claims it (issue #32). The
832    /// derivations deliberately do NOT sanitize instead: sanitizing here would
833    /// silently re-path every deployed store and desynchronize the Rust and TS
834    /// derivations, while refusal changes nothing for any id that ever worked.
835    PathHazardModuleId {
836        module_id: String,
837        reason: String,
838    },
839    Poisoned,
840}
841
842impl fmt::Display for RegistryError {
843    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
844        match self {
845            Self::FederatedCapabilityChanged => write!(f, "scope-federated/v1 is fixed at HELLO; re-register on a new connection to change it"),
846            Self::DuplicateModuleId { module_id } => {
847                write!(f, "module_id '{module_id}' is already registered")
848            }
849            Self::PathHazardModuleId { module_id, reason } => {
850                write!(
851                    f,
852                    "module_id '{}' is not usable as a path component: {reason}",
853                    module_id.escape_debug()
854                )
855            }
856            Self::Poisoned => write!(f, "registry lock was poisoned"),
857        }
858    }
859}
860
861impl Error for RegistryError {}
862
863/// Why `module_id` cannot serve as a single path component, or `Ok(())`.
864///
865/// This is a REFUSAL predicate, not a sanitizer: every currently-working fleet
866/// id passes untouched, and anything refused here never worked meaningfully --
867/// it either escaped `<data_home>/cortexkit/` (separators, dot components) or
868/// aliased another module's store (`a/b` vs `a//b` collapsing on POSIX).
869/// Colons are allowed: reserved-namespace children (`mcp:...`) register today
870/// and a colon cannot traverse. Windows path legality is the store library's
871/// concern, not an identity rule.
872pub fn module_id_path_hazard(module_id: &str) -> Result<(), String> {
873    subc_protocol::manifest::validate_module_id(module_id)
874}
875
876#[cfg(test)]
877mod change_notification_tests {
878    use super::*;
879
880    fn manifest() -> ModuleManifest {
881        ModuleManifest::builder("watched", "1.0.0")
882            .protocol_ver(1)
883            .build()
884    }
885
886    fn changed(receiver: &mut watch::Receiver<u64>) {
887        assert!(
888            receiver.has_changed().unwrap(),
889            "registry write must notify"
890        );
891        receiver.borrow_and_update();
892    }
893
894    #[test]
895    fn every_registration_write_notifies_only_its_module() {
896        let registry = Registry::default();
897        let mut events = registry.subscribe_module_changes("watched").unwrap();
898        let other = registry.subscribe_module_changes("other").unwrap();
899        let active = ConnectionId::new(1);
900        let candidate = ConnectionId::new(2);
901        registry
902            .register_with_control_ops(manifest(), 1, active, vec![])
903            .unwrap();
904        changed(&mut events);
905        registry
906            .set_module_state_for_test("watched", ChannelState::Active)
907            .unwrap();
908        changed(&mut events);
909        registry
910            .replace_catalog_for_connection(active, vec![], None, Some(true))
911            .unwrap();
912        changed(&mut events);
913        registry
914            .register_candidate_with_control_ops(manifest(), 1, candidate, vec![])
915            .unwrap();
916        changed(&mut events);
917        registry
918            .replace_catalog_for_connection(candidate, vec![], None, Some(true))
919            .unwrap();
920        changed(&mut events);
921        registry.promote_candidate("watched").unwrap().unwrap();
922        changed(&mut events);
923        // Superseded connections can still update and deregister their slot.
924        registry
925            .replace_catalog_for_connection(active, vec![], None, Some(false))
926            .unwrap();
927        changed(&mut events);
928        registry.deregister_connection(active).unwrap();
929        changed(&mut events);
930        registry.deregister_connection(candidate).unwrap();
931        changed(&mut events);
932        registry
933            .register_candidate_with_control_ops(manifest(), 1, candidate, vec![])
934            .unwrap();
935        changed(&mut events);
936        registry.deregister_connection(candidate).unwrap();
937        changed(&mut events);
938        assert!(
939            !other.has_changed().unwrap(),
940            "other module must stay parked"
941        );
942    }
943
944    #[test]
945    fn subscription_before_read_keeps_a_change_until_waited_on() {
946        let registry = Registry::default();
947        let mut events = registry.subscribe_module_changes("watched").unwrap();
948        assert!(registry.get_module("watched").unwrap().is_none());
949        registry
950            .register_with_control_ops(manifest(), 1, ConnectionId::new(1), vec![])
951            .unwrap();
952        // Neither a later lookup nor a coalesced catalog update consumes it.
953        registry.get_module("watched").unwrap().unwrap();
954        registry
955            .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
956            .unwrap();
957        changed(&mut events);
958        assert!(!events.has_changed().unwrap());
959    }
960}
961
962#[cfg(test)]
963mod publication_tests {
964    use super::*;
965
966    fn manifest() -> ModuleManifest {
967        ModuleManifest::builder("m", "1.0.0")
968            .protocol_ver(1)
969            .build()
970    }
971
972    /// An unpublished registration holds its id and answers routing lookups,
973    /// but `list_modules` leaves it out and the generation stays put until
974    /// it is published; publishing lists it and bumps the generation once.
975    #[test]
976    fn unpublished_registration_is_unlisted_until_published() {
977        let registry = Registry::default();
978        let connection = ConnectionId::new(1);
979        registry
980            .register_unpublished_with_control_ops(manifest(), 1, connection, vec![])
981            .unwrap();
982        assert_eq!(registry.list_modules().unwrap(), (0, Vec::new()));
983        assert!(registry.get_module("m").unwrap().is_some());
984        assert_eq!(registry.list_registered_modules().unwrap().len(), 1);
985        assert!(registry
986            .published_registration(RegistrationSlot::Active("m"))
987            .unwrap()
988            .is_none());
989        assert!(matches!(
990            registry.register_with_control_ops(manifest(), 1, ConnectionId::new(2), vec![]),
991            Err(RegistryError::DuplicateModuleId { .. })
992        ));
993        // A catalog update before publication does not move the generation.
994        registry
995            .replace_catalog_for_connection(connection, vec![], None, Some(true))
996            .unwrap();
997        assert_eq!(registry.generation().unwrap(), 0);
998
999        assert!(registry.publish_registration(connection).unwrap());
1000        let (generation, listed) = registry.list_modules().unwrap();
1001        assert_eq!(generation, 1);
1002        assert_eq!(listed.len(), 1);
1003        assert!(!registry.publish_registration(connection).unwrap());
1004        assert_eq!(registry.generation().unwrap(), 1);
1005    }
1006
1007    /// Removing a registration that was never published leaves no catalog
1008    /// trace: the generation does not move, and a later publish is a no-op.
1009    #[test]
1010    fn deregistering_an_unpublished_registration_leaves_the_generation() {
1011        let registry = Registry::default();
1012        let connection = ConnectionId::new(1);
1013        registry
1014            .register_unpublished_with_control_ops(manifest(), 1, connection, vec![])
1015            .unwrap();
1016        assert_eq!(registry.deregister_connection(connection).unwrap().len(), 1);
1017        assert_eq!(registry.list_modules().unwrap(), (0, Vec::new()));
1018        assert!(!registry.publish_registration(connection).unwrap());
1019        assert_eq!(registry.generation().unwrap(), 0);
1020    }
1021
1022    /// An unpublished candidate holds the candidate slot but
1023    /// `published_registration` does not return it until it is published.
1024    /// Publishing a candidate does not bump the generation: candidates are
1025    /// never listed.
1026    #[test]
1027    fn unpublished_candidate_is_hidden_from_published_lookups() {
1028        let registry = Registry::default();
1029        let candidate = ConnectionId::new(2);
1030        registry
1031            .register_unpublished_candidate_with_control_ops(manifest(), 1, candidate, vec![])
1032            .unwrap();
1033        assert!(registry
1034            .registration(RegistrationSlot::Candidate("m"))
1035            .unwrap()
1036            .is_some());
1037        assert!(registry
1038            .published_registration(RegistrationSlot::Candidate("m"))
1039            .unwrap()
1040            .is_none());
1041        assert!(registry
1042            .published_registration(RegistrationSlot::Connection(candidate))
1043            .unwrap()
1044            .is_none());
1045        assert!(registry.publish_registration(candidate).unwrap());
1046        assert!(registry
1047            .published_registration(RegistrationSlot::Candidate("m"))
1048            .unwrap()
1049            .is_some());
1050        assert_eq!(registry.generation().unwrap(), 0);
1051    }
1052}
1053
1054#[cfg(test)]
1055mod path_hazard_tests {
1056    use super::*;
1057    use crate::ConnectionId;
1058    use subc_protocol::manifest::ModuleManifest;
1059
1060    fn manifest(module_id: &str) -> ModuleManifest {
1061        ModuleManifest::builder(module_id, "0.1.0")
1062            .protocol_ver(1)
1063            .build()
1064    }
1065
1066    #[test]
1067    fn path_hazard_ids_are_refused_and_nothing_registers() {
1068        let registry = Registry::default();
1069        for (bad, reason_fragment) in [
1070            ("../escape", "path separator"),
1071            ("a/b", "path separator"),
1072            ("a\\b", "path separator"),
1073            ("..", "dot path component"),
1074            (".", "dot path component"),
1075            ("", "empty"),
1076            ("evil\u{0}id", "control character"),
1077        ] {
1078            let err = registry
1079                .register_with_control_ops(manifest(bad), 1, ConnectionId::new(7), Vec::new())
1080                .expect_err("path-hazard id must refuse");
1081            // Reason asserted so a predicate throwing the WRONG refusal fails.
1082            assert!(
1083                err.to_string().contains(reason_fragment),
1084                "id {bad:?}: expected {reason_fragment:?} in {err}"
1085            );
1086        }
1087        // THE EFFECT, not just the verdicts: no refusal left a registration
1088        // behind, and the generation never moved.
1089        assert_eq!(registry.active_registration_count().unwrap(), 0);
1090        assert_eq!(registry.generation().unwrap(), 0);
1091    }
1092
1093    #[test]
1094    fn module_id_path_component_length_matches_shared_refusal_vectors() {
1095        let doc: serde_json::Value = serde_json::from_str(include_str!(
1096            "../tests/golden/module_id_path_component_refusals.json"
1097        ))
1098        .expect("refusal fixture parses");
1099
1100        for case in doc["vectors"].as_array().expect("vectors array") {
1101            let name = case["name"].as_str().expect("name");
1102            let module_id = case["module_id"]["unit"]
1103                .as_str()
1104                .expect("module_id unit")
1105                .repeat(
1106                    case["module_id"]["repeat"]
1107                        .as_u64()
1108                        .expect("module_id repeat") as usize,
1109                );
1110            assert_eq!(
1111                module_id.len(),
1112                case["utf8_bytes"].as_u64().expect("utf8 bytes") as usize
1113            );
1114
1115            let expected = case["expect_reason"].as_str().map(str::to_owned);
1116            assert_eq!(
1117                module_id_path_hazard(&module_id).err(),
1118                expected,
1119                "shared refusal vector {name:?} diverged"
1120            );
1121        }
1122    }
1123
1124    #[test]
1125    fn working_id_shapes_register_including_namespace_colons() {
1126        let registry = Registry::default();
1127        for (i, good) in ["magic-context", "mcp:everything", "v1.2-module"]
1128            .iter()
1129            .enumerate()
1130        {
1131            registry
1132                .register_with_control_ops(
1133                    manifest(good),
1134                    1,
1135                    ConnectionId::new(10 + i as u64),
1136                    Vec::new(),
1137                )
1138                .unwrap_or_else(|err| panic!("id {good:?} must register: {err}"));
1139        }
1140        assert_eq!(registry.active_registration_count().unwrap(), 3);
1141    }
1142}
1143
1144#[cfg(test)]
1145mod swap_slot_tests {
1146    use super::*;
1147
1148    fn manifest(module_id: &str, ready: Option<bool>) -> ModuleManifest {
1149        let mut manifest = ModuleManifest::builder(module_id, "0.1.0").build();
1150        manifest.ready = ready;
1151        manifest
1152    }
1153
1154    const INCUMBENT: ConnectionId = ConnectionId(1);
1155    const CANDIDATE: ConnectionId = ConnectionId(2);
1156
1157    fn registry_with_candidate() -> Registry {
1158        let registry = Registry::default();
1159        registry
1160            .register_with_control_ops(manifest("m", None), 1, INCUMBENT, Vec::new())
1161            .unwrap();
1162        registry
1163            .register_candidate_with_control_ops(
1164                manifest("m", Some(false)),
1165                1,
1166                CANDIDATE,
1167                Vec::new(),
1168            )
1169            .unwrap();
1170        registry
1171    }
1172
1173    #[test]
1174    fn candidate_is_invisible_to_by_id_lookups_and_listing() {
1175        let registry = registry_with_candidate();
1176        let generation = registry.generation().unwrap();
1177        assert_eq!(
1178            registry.get_module("m").unwrap().unwrap().connection_id,
1179            INCUMBENT
1180        );
1181        let (listed_generation, listed) = registry.list_modules().unwrap();
1182        assert_eq!(listed.len(), 1);
1183        assert_eq!(listed[0].connection_id, INCUMBENT);
1184        assert_eq!(listed_generation, generation);
1185        assert_eq!(registry.active_registration_count().unwrap(), 1);
1186        assert_eq!(
1187            registry.get_candidate("m").unwrap().unwrap().connection_id,
1188            CANDIDATE
1189        );
1190        assert_eq!(
1191            registry
1192                .register_candidate_with_control_ops(
1193                    manifest("m", None),
1194                    1,
1195                    ConnectionId(3),
1196                    Vec::new()
1197                )
1198                .unwrap_err(),
1199            RegistryError::DuplicateModuleId {
1200                module_id: "m".to_string()
1201            }
1202        );
1203    }
1204
1205    #[test]
1206    fn promoted_candidate_keeps_its_verified_nonce_fact() {
1207        let registry = Registry::default();
1208        registry
1209            .register_unpublished_verified_candidate_with_control_ops(
1210                manifest("m", Some(true)),
1211                1,
1212                CANDIDATE,
1213                Vec::new(),
1214            )
1215            .unwrap();
1216        assert!(registry.publish_registration(CANDIDATE).unwrap());
1217        assert!(registry
1218            .list_modules_with_nonce_verification()
1219            .unwrap()
1220            .1
1221            .is_empty());
1222
1223        registry.promote_candidate("m").unwrap().unwrap();
1224        let (_, modules) = registry.list_modules_with_nonce_verification().unwrap();
1225        assert_eq!(modules.len(), 1);
1226        assert!(modules[0].1);
1227    }
1228
1229    /// Without the candidate slot in the connection-keyed search, this update
1230    /// returns `Ok(None)` and the candidate never becomes ready.
1231    #[test]
1232    fn candidate_catalog_update_reaches_the_candidate_registration() {
1233        let registry = registry_with_candidate();
1234        assert!(!registry.get_candidate("m").unwrap().unwrap().ready);
1235
1236        let updated = registry
1237            .replace_catalog_for_connection(CANDIDATE, Vec::new(), None, Some(true))
1238            .unwrap()
1239            .expect("the candidate's own connection finds its registration");
1240
1241        assert_eq!(updated.connection_id, CANDIDATE);
1242        assert!(registry.get_candidate("m").unwrap().unwrap().ready);
1243        assert_eq!(
1244            registry.get_module_by_connection(CANDIDATE).unwrap(),
1245            Some(updated)
1246        );
1247        assert_eq!(
1248            registry.get_module("m").unwrap().unwrap().connection_id,
1249            INCUMBENT,
1250            "a candidate's update must not touch the active registration"
1251        );
1252    }
1253
1254    #[test]
1255    fn promotion_swaps_slots_and_each_connection_still_deregisters_its_own() {
1256        let registry = registry_with_candidate();
1257        let before = registry.generation().unwrap();
1258        let cutover = registry.promote_candidate("m").unwrap().unwrap();
1259        assert_eq!(cutover.promoted.connection_id, CANDIDATE);
1260        assert_eq!(cutover.superseded.unwrap().connection_id, INCUMBENT);
1261        assert_ne!(registry.generation().unwrap(), before);
1262        assert_eq!(registry.promote_candidate("m").unwrap(), None);
1263
1264        assert_eq!(
1265            registry
1266                .registration(RegistrationSlot::Active("m"))
1267                .unwrap()
1268                .unwrap()
1269                .connection_id,
1270            CANDIDATE
1271        );
1272        assert!(registry
1273            .registration(RegistrationSlot::Candidate("m"))
1274            .unwrap()
1275            .is_none());
1276        assert!(registry
1277            .registration(RegistrationSlot::Connection(INCUMBENT))
1278            .unwrap()
1279            .is_some());
1280
1281        let closed = registry.deregister_connection(INCUMBENT).unwrap();
1282        assert_eq!(closed.len(), 1);
1283        assert_eq!(closed[0].connection_id, INCUMBENT);
1284        assert_eq!(closed[0].state, ChannelState::Closed);
1285        assert!(registry
1286            .registration(RegistrationSlot::Connection(INCUMBENT))
1287            .unwrap()
1288            .is_none());
1289        assert_eq!(
1290            registry.get_module("m").unwrap().unwrap().connection_id,
1291            CANDIDATE
1292        );
1293    }
1294
1295    #[test]
1296    fn a_dropped_candidate_deregisters_from_the_candidate_slot_only() {
1297        let registry = registry_with_candidate();
1298        let closed = registry.deregister_connection(CANDIDATE).unwrap();
1299        assert_eq!(closed.len(), 1);
1300        assert_eq!(closed[0].connection_id, CANDIDATE);
1301        assert!(registry.get_candidate("m").unwrap().is_none());
1302        assert_eq!(
1303            registry.get_module("m").unwrap().unwrap().connection_id,
1304            INCUMBENT
1305        );
1306    }
1307}