Skip to main content

subc_daemon/config_gate/
delivery.rs

1//! Connection-bound delivery attempts; durable vectors remain in ConfigStore.
2
3use std::collections::HashSet;
4use subc_protocol::config::{ConfigAppliedRequest, ConfigGetReply, ConfigGetRequest};
5
6use super::*;
7use crate::config_gate::apply::Transition;
8use crate::config_store::Tuple;
9
10pub(crate) const CONFIG_LIVE_ACK_TIMEOUT: Duration = Duration::from_secs(30);
11pub(crate) const CONFIG_WARMUP_SLACK: Duration = Duration::from_secs(10);
12
13fn allocate_spawn(
14    state: &mut OwnerState,
15    module: &str,
16    generation: u64,
17    at: tokio::time::Instant,
18    register: Duration,
19    health: Duration,
20) -> Result<DeliverySpawn> {
21    #[cfg(test)]
22    if state.delivery.failed_spawn_module.as_deref() == Some(module) {
23        return Err(GateError::new(
24            "delivery_spawn_exhausted",
25            "delivery spawn identifiers exhausted",
26        ));
27    }
28    state.delivery.next_spawn = state.delivery.next_spawn.checked_add(1).ok_or_else(|| {
29        GateError::new(
30            "delivery_spawn_exhausted",
31            "delivery spawn identifiers exhausted",
32        )
33    })?;
34    Ok(DeliverySpawn {
35        id: state.delivery.next_spawn,
36        module: module.into(),
37        generation,
38        deadline: at + register + health + CONFIG_WARMUP_SLACK,
39    })
40}
41
42#[derive(Clone, Debug)]
43struct Attempt {
44    module: String,
45    generation: u64,
46    id: u64,
47    transition: Transition,
48    ready: bool,
49    acknowledged: bool,
50    delivered: bool,
51    deadline: Option<tokio::time::Instant>,
52    // Catch-up belongs to one promoted child, not to the durable generation.
53    // A daemon restart drops it: every fresh launch receives the active vector.
54    catchup: bool,
55    write_deadline: Option<tokio::time::Instant>,
56    policy_probe_requested: bool,
57}
58
59#[derive(Clone, Debug)]
60pub struct DeliverySpawn {
61    id: u64,
62    module: String,
63    generation: u64,
64    deadline: tokio::time::Instant,
65}
66
67impl DeliverySpawn {
68    pub(crate) fn module(&self) -> &str {
69        &self.module
70    }
71    pub(crate) fn generation(&self) -> u64 {
72        self.generation
73    }
74}
75
76#[derive(Default)]
77pub(super) struct DeliveryState {
78    #[cfg(test)]
79    failed_spawn_module: Option<String>,
80    next_spawn: u64,
81    next_attempt: u64,
82    policy_epochs: HashMap<String, u64>,
83    attempts: HashMap<ConnectionId, Attempt>,
84    served: HashMap<ConnectionId, (Option<u64>, Option<u64>)>,
85    stamp_running: HashSet<ConnectionId>,
86    initial_generation: u64,
87    children: HashMap<ConnectionId, DeliverySpawn>,
88    healthy_children: HashSet<(ConnectionId, u64)>,
89    readiness_observed: HashSet<(ConnectionId, u64)>,
90    unregistered_children: HashMap<(String, bool), DeliverySpawn>,
91    recoveries: Vec<(ConnectionId, String, AppliedDisposition)>,
92    exited: HashMap<String, ExitedDelivery>,
93    // Transport cleanup may precede the process-exit observation. Retain the
94    // last active child's assigned generation, not an unattempted deferred one.
95    last_target_attempt: HashMap<String, (u64, u64)>,
96}
97
98impl DeliveryState {
99    pub(super) fn reset_for_authority_exit(&mut self, generation: u64) {
100        *self = Self {
101            next_spawn: self.next_spawn,
102            next_attempt: self.next_attempt,
103            ..Self::new(generation)
104        };
105    }
106    pub(super) fn started_at_generation(&self) -> u64 {
107        self.initial_generation
108    }
109    pub(super) fn policy_authorization(
110        &self,
111        module: &str,
112        generation: u64,
113        connection: ConnectionId,
114    ) -> Option<(u64, tokio::time::Instant)> {
115        let attempt = self.attempts.get(&connection)?;
116        let deadline = attempt.deadline?;
117        (attempt.module == module
118            && attempt.generation == generation
119            && attempt.delivered
120            && attempt.acknowledged
121            && tokio::time::Instant::now() < deadline)
122            .then_some((attempt.id, deadline))
123    }
124    pub(super) fn fenced_modules(&self) -> impl Iterator<Item = String> + '_ {
125        self.exited.keys().cloned()
126    }
127}
128
129/// Captured before a health RPC; only the same child and installed policy may
130/// consume its result. The fields are private so a completion cannot retarget it.
131#[derive(Clone, Debug)]
132pub struct DeliveryProbe {
133    connection: ConnectionId,
134    module: String,
135    generation: u64,
136    attempt: Option<u64>,
137    policy_epoch: u64,
138    route_readiness: bool,
139}
140
141impl DeliveryProbe {
142    pub(crate) fn connection(&self) -> ConnectionId {
143        self.connection
144    }
145    pub(crate) fn owns_failure(&self) -> bool {
146        self.route_readiness
147    }
148}
149
150#[derive(Clone)]
151pub(crate) struct ExitedDelivery {
152    pub module: String,
153    spawn: u64,
154    generation: u64,
155    changed: tokio::sync::watch::Sender<bool>,
156}
157
158#[derive(Clone, Debug, PartialEq, Eq)]
159pub enum AppliedDisposition {
160    Ignored,
161    WaitingForReadiness,
162    Active,
163    CatchupComplete,
164    Failed {
165        generation: u64,
166        restore: Option<u64>,
167        keep_running: bool,
168    },
169    CatchupFailed {
170        active: u64,
171    },
172}
173
174#[derive(Clone, Copy)]
175pub(crate) struct DeliveryWindow {
176    pub generation: u64,
177    pub attempt: u64,
178    pub delivered: bool,
179    pub deadline: Option<tokio::time::Instant>,
180}
181
182impl DeliveryState {
183    pub(super) fn invalidate_policy(&mut self, module: &str, generation: u64) {
184        for attempt in self
185            .attempts
186            .values_mut()
187            .filter(|attempt| attempt.module == module && attempt.generation == generation)
188        {
189            attempt.ready = false;
190            attempt.policy_probe_requested = false;
191        }
192    }
193    pub(super) fn policy_installed(
194        &mut self,
195        module: &str,
196        generation: u64,
197        target: Option<ConnectionId>,
198    ) -> Result<()> {
199        // The generation may be unchanged after a restore and retry. Use a
200        // separate epoch so a probe begun before this installation cannot count.
201        let epoch = self.policy_epochs.entry(module.into()).or_default();
202        *epoch = epoch.checked_add(1).ok_or_else(|| {
203            GateError::new(
204                "delivery_probe_exhausted",
205                "policy readiness epochs exhausted",
206            )
207        })?;
208        self.invalidate_policy(module, generation);
209        let connections: HashSet<_> = self
210            .children
211            .iter()
212            .filter(|(_, spawn)| spawn.module == module)
213            .map(|(connection, _)| *connection)
214            .chain(
215                self.attempts
216                    .iter()
217                    .filter(|(_, attempt)| attempt.module == module)
218                    .map(|(connection, _)| *connection),
219            )
220            .chain(target)
221            .collect();
222        self.readiness_observed.retain(|(connection, observed)| {
223            *observed != generation || !connections.contains(connection)
224        });
225        self.healthy_children.retain(|(connection, observed)| {
226            *observed != generation || !connections.contains(connection)
227        });
228        Ok(())
229    }
230    pub(super) fn declared(&mut self, connection: ConnectionId, module: &str, active: bool) {
231        if let Some(spawn) = self.unregistered_children.remove(&(module.into(), active)) {
232            self.children.insert(connection, spawn);
233        }
234    }
235    pub(super) fn new(initial_generation: u64) -> Self {
236        Self {
237            initial_generation,
238            ..Self::default()
239        }
240    }
241
242    pub(super) fn disconnected(&mut self, connection: ConnectionId) {
243        self.attempts.remove(&connection);
244        self.served.remove(&connection);
245        self.stamp_running.remove(&connection);
246        self.children.remove(&connection);
247        self.healthy_children
248            .retain(|(child, _)| *child != connection);
249        self.readiness_observed
250            .retain(|(child, _)| *child != connection);
251    }
252
253    pub(super) fn remember_stamp_connections(&mut self, connections: Vec<ConnectionId>) {
254        self.stamp_running.extend(connections);
255    }
256
257    fn boundary_open(
258        &self,
259        connection: ConnectionId,
260        generation: u64,
261        transition: Transition,
262    ) -> bool {
263        if transition == Transition::DaemonRestart && generation > self.initial_generation {
264            return false;
265        }
266        !transition.requires_spawn()
267            || self
268                .children
269                .get(&connection)
270                .is_some_and(|spawn| spawn.generation == generation)
271    }
272
273    fn allocate(
274        &mut self,
275        module: &str,
276        generation: u64,
277        transition: Transition,
278    ) -> Result<Attempt> {
279        self.next_attempt = self.next_attempt.checked_add(1).ok_or_else(|| {
280            GateError::new(
281                "delivery_attempt_exhausted",
282                "delivery attempt ids exhausted",
283            )
284        })?;
285        Ok(Attempt {
286            module: module.into(),
287            generation,
288            id: self.next_attempt,
289            transition,
290            ready: transition == Transition::Live,
291            acknowledged: false,
292            delivered: false,
293            deadline: None,
294            catchup: false,
295            write_deadline: None,
296            policy_probe_requested: false,
297        })
298    }
299}
300
301pub(super) fn visible_reply(
302    state: &OwnerState,
303    connection: ConnectionId,
304    mut reply: ConfigGetReply,
305    transition: Transition,
306) -> Result<ConfigGetReply> {
307    if reply.pending.as_ref().is_some_and(|view| {
308        !state
309            .delivery
310            .boundary_open(connection, view.generation, transition)
311    }) {
312        reply.pending = None;
313    }
314    if reply.active.is_none() && reply.pending.is_none() {
315        return Err(GateError::new(
316            "config_gated",
317            "generation waits for its required supervision or daemon boundary",
318        ));
319    }
320    Ok(reply)
321}
322
323pub(super) fn bind_get(
324    state: &mut OwnerState,
325    connection: ConnectionId,
326    module: &str,
327    request: &ConfigGetRequest,
328    reply: ConfigGetReply,
329    transition: Transition,
330) -> Result<ConfigGetReply> {
331    let mut reply = visible_reply(state, connection, reply, transition)?;
332    match (request.generation, request.attempt) {
333        (None, None) => {}
334        (Some(generation), Some(attempt))
335            if state
336                .delivery
337                .attempts
338                .get(&connection)
339                .is_some_and(|bound| {
340                    bound.module == module && bound.generation == generation && bound.id == attempt
341                }) => {}
342        _ => {
343            return Err(GateError::new(
344                "config_gated",
345                "requested generation and attempt are not assigned to this connection",
346            ))
347        }
348    }
349    if reply.pending.is_none()
350        && state
351            .delivery
352            .attempts
353            .get(&connection)
354            .is_some_and(|bound| {
355                bound.catchup
356                    && reply
357                        .active
358                        .as_ref()
359                        .is_some_and(|view| view.generation == bound.generation)
360            })
361    {
362        reply.pending = reply.active.clone();
363        if let Some(view) = &mut reply.pending {
364            view.status = ConfigStatus::Pending;
365        }
366    }
367    if let Some(pending) = &mut reply.pending {
368        let matches = state
369            .delivery
370            .attempts
371            .get(&connection)
372            .is_some_and(|bound| bound.module == module && bound.generation == pending.generation);
373        if !matches {
374            let mut attempt = state
375                .delivery
376                .allocate(module, pending.generation, transition)?;
377            if transition.requires_spawn() {
378                attempt.ready = state
379                    .delivery
380                    .healthy_children
381                    .contains(&(connection, pending.generation));
382                attempt.deadline = state
383                    .delivery
384                    .children
385                    .get(&connection)
386                    .filter(|warmup| warmup.generation == pending.generation)
387                    .map(|warmup| warmup.deadline);
388            }
389            attempt.ready &= super::route_policy::ready(state, module, pending.generation)?;
390            state.delivery.attempts.insert(connection, attempt);
391        }
392        pending.attempt = state
393            .delivery
394            .attempts
395            .get(&connection)
396            .map(|bound| bound.id);
397        if state.active_declarations.get(module) == Some(&connection) {
398            if let Some(spawn) = state.delivery.children.get(&connection) {
399                let id = spawn.id;
400                state
401                    .delivery
402                    .last_target_attempt
403                    .insert(module.into(), (id, pending.generation));
404            }
405        }
406    } else {
407        state.delivery.attempts.remove(&connection);
408    }
409    if request
410        .generation
411        .is_some_and(|id| reply.pending.as_ref().map(|view| view.generation) != Some(id))
412    {
413        return Err(GateError::new(
414            "config_gated",
415            "selected delivery has been superseded",
416        ));
417    }
418    Ok(reply)
419}
420
421impl RuntimeOwner {
422    pub(crate) fn policy_probe_due(&self, connection: ConnectionId) -> Result<Option<(u64, u64)>> {
423        let mut state = self.lock()?;
424        let Some(bound) = state.delivery.attempts.get(&connection) else {
425            return Ok(None);
426        };
427        let (module, generation, attempt) = (bound.module.clone(), bound.generation, bound.id);
428        if bound.ready
429            || bound.policy_probe_requested
430            || state.active_declarations.get(&module) != Some(&connection)
431            || !state
432                .store
433                .body()?
434                .generations
435                .get(&generation)
436                .is_some_and(|entry| {
437                    super::route_policy::required(&entry.generation.changeset, &module)
438                })
439            || !super::route_policy::ready(&mut state, &module, generation)?
440        {
441            return Ok(None);
442        }
443        state
444            .delivery
445            .attempts
446            .get_mut(&connection)
447            .unwrap()
448            .policy_probe_requested = true;
449        Ok(Some((generation, attempt)))
450    }
451
452    pub(crate) fn policy_probe_pending(
453        &self,
454        connection: ConnectionId,
455        generation: u64,
456        attempt: u64,
457    ) -> Result<bool> {
458        let mut state = self.lock()?;
459        let Some(bound) = state.delivery.attempts.get(&connection) else {
460            return Ok(false);
461        };
462        let module = bound.module.clone();
463        Ok(bound.generation == generation
464            && bound.id == attempt
465            && !bound.ready
466            && state.active_declarations.get(&module) == Some(&connection)
467            && super::route_policy::ready(&mut state, &module, generation)?)
468    }
469    pub(crate) fn delivery_health_probe(
470        &self,
471        connection: ConnectionId,
472        module: &str,
473    ) -> Result<Option<DeliveryProbe>> {
474        let mut state = self.lock()?;
475        if state.active_declarations.get(module) != Some(&connection) {
476            return Ok(None);
477        }
478        let attempt = state
479            .delivery
480            .attempts
481            .get(&connection)
482            .map(|attempt| (attempt.generation, attempt.id));
483        let generation = attempt.map(|attempt| attempt.0).or_else(|| {
484            state
485                .delivery
486                .children
487                .get(&connection)
488                .map(|child| child.generation)
489        });
490        let route_readiness = match generation {
491            Some(generation) if attempt.is_some() => state
492                .store
493                .body()?
494                .generations
495                .get(&generation)
496                .is_some_and(|entry| {
497                    super::route_policy::required(&entry.generation.changeset, module)
498                }),
499            _ => false,
500        };
501        Ok(generation.map(|generation| DeliveryProbe {
502            connection,
503            module: module.into(),
504            generation,
505            attempt: attempt.map(|attempt| attempt.1),
506            route_readiness,
507            policy_epoch: state
508                .delivery
509                .policy_epochs
510                .get(module)
511                .copied()
512                .unwrap_or_default(),
513        }))
514    }
515
516    pub(crate) fn delivery_health_result(
517        &self,
518        probe: DeliveryProbe,
519        healthy: bool,
520    ) -> Result<Option<AppliedDisposition>> {
521        let attempt = {
522            let mut state = self.lock()?;
523            if state.active_declarations.get(&probe.module) != Some(&probe.connection)
524                || state
525                    .delivery
526                    .policy_epochs
527                    .get(&probe.module)
528                    .copied()
529                    .unwrap_or_default()
530                    != probe.policy_epoch
531            {
532                return Ok(None);
533            }
534            let current = state.delivery.attempts.get(&probe.connection);
535            if probe
536                .attempt
537                .is_some_and(|id| current.is_none_or(|bound| bound.id != id))
538                || current.is_some_and(|bound| bound.generation != probe.generation)
539            {
540                return Ok(None);
541            }
542            let attempt = current.map(|bound| bound.id);
543            if !healthy
544                && (!super::route_policy::ready(&mut state, &probe.module, probe.generation)?
545                    || !state
546                        .store
547                        .body()?
548                        .generations
549                        .get(&probe.generation)
550                        .is_some_and(|entry| {
551                            super::route_policy::required(
552                                &entry.generation.changeset,
553                                &probe.module,
554                            )
555                        }))
556            {
557                return Ok(None);
558            }
559            attempt
560        };
561        if healthy {
562            self.delivery_readiness_complete(probe.connection, &probe.module, probe.generation)
563        } else if let Some(attempt) = attempt {
564            self.module_outcome(
565                probe.connection,
566                &probe.module,
567                ConfigAppliedRequest::new(
568                    probe.generation,
569                    attempt,
570                    subc_protocol::config::ConfigApplyResult::Nack,
571                    vec![],
572                ),
573                true,
574            )
575            .map(Some)
576        } else {
577            Ok(None)
578        }
579    }
580
581    #[cfg(test)]
582    pub(super) fn ready(
583        &self,
584        connection: ConnectionId,
585        module: &str,
586        generation: u64,
587    ) -> Result<Option<AppliedDisposition>> {
588        self.delivery_readiness_complete(connection, module, generation)
589    }
590
591    pub(crate) fn pending_config_restarts(&self) -> Result<Vec<(String, u64)>> {
592        let mut state = self.lock()?;
593        state.store.delivery_ready()?;
594        let body = state.store.body()?;
595        body.versions
596            .iter()
597            .filter(|(tuple, _)| tuple.project.is_none())
598            .filter_map(|(tuple, versions)| {
599                let generation = versions.pending?;
600                let entry = body.generations.get(&generation)?;
601                Some(
602                    Transition::for_module(&entry.generation.changeset, &tuple.module).map(
603                        |transition| {
604                            (transition == Transition::ModuleRestart)
605                                .then(|| (tuple.module.clone(), generation))
606                        },
607                    ),
608                )
609            })
610            .collect::<Result<Vec<_>>>()
611            .map(|entries| entries.into_iter().flatten().collect())
612    }
613
614    pub(crate) fn requires_fresher_spawn(
615        &self,
616        module: &str,
617        spawn: Option<&DeliverySpawn>,
618    ) -> Result<Option<u64>> {
619        let mut state = self.lock()?;
620        state.store.delivery_ready()?;
621        Self::requires_fresher_spawn_loaded(&state, module, spawn)
622    }
623
624    pub(super) fn requires_fresher_spawn_loaded(
625        state: &OwnerState,
626        module: &str,
627        spawn: Option<&DeliverySpawn>,
628    ) -> Result<Option<u64>> {
629        let body = state.store.loaded_admission_state().1;
630        let pending = body
631            .versions
632            .iter()
633            .find(|(tuple, _)| tuple.module == module && tuple.project.is_none())
634            .and_then(|(_, versions)| versions.pending);
635        let Some(generation) = pending else {
636            return Ok(None);
637        };
638        let entry = body
639            .generations
640            .get(&generation)
641            .ok_or_else(|| GateError::new("config_gated", "pending generation is absent"))?;
642        let transition = Transition::for_module(&entry.generation.changeset, module)?;
643        let deferred_daemon = transition == Transition::DaemonRestart
644            && generation > state.delivery.initial_generation;
645        Ok((transition.requires_spawn()
646            && !deferred_daemon
647            && spawn
648                .is_none_or(|spawn| spawn.module != module || spawn.generation() != generation))
649        .then_some(generation))
650    }
651
652    /// A supervised process exit, not a transport disconnect, fails warm-up.
653    /// The synchronous fence closes the gap before the async failure commit.
654    pub(crate) fn delivery_child_exited(&self, spawn: &DeliverySpawn) -> Result<()> {
655        let mut state = self.lock()?;
656        let pending = state
657            .store
658            .body()?
659            .versions
660            .iter()
661            .find(|(tuple, _)| tuple.module == spawn.module && tuple.project.is_none())
662            .and_then(|(_, versions)| versions.pending);
663        let pending = pending.filter(|generation| {
664            state
665                .store
666                .loaded_admission_state()
667                .1
668                .generations
669                .get(generation)
670                .and_then(|entry| entry.delivery.as_ref())
671                .is_some_and(|delivery| delivery.declarations.contains_key(&spawn.module))
672        });
673        if let Some(generation) = pending.filter(|generation| {
674            *generation == spawn.generation
675                || state.delivery.last_target_attempt.get(&spawn.module)
676                    == Some(&(spawn.id, *generation))
677        }) {
678            state
679                .delivery
680                .exited
681                .entry(spawn.module.clone())
682                .or_insert_with(|| ExitedDelivery {
683                    module: spawn.module.clone(),
684                    spawn: spawn.id,
685                    generation,
686                    changed: tokio::sync::watch::channel(true).0,
687                });
688        }
689        self.notify_delivery_changed();
690        Ok(())
691    }
692
693    pub(crate) fn delivery_exit_fence(
694        &self,
695        module: &str,
696    ) -> Result<Option<tokio::sync::watch::Receiver<bool>>> {
697        Ok(self
698            .lock()?
699            .delivery
700            .exited
701            .get(module)
702            .map(|entry| entry.changed.subscribe()))
703    }
704
705    pub(crate) fn delivery_exits(&self) -> Result<Vec<ExitedDelivery>> {
706        Ok(self.lock()?.delivery.exited.values().cloned().collect())
707    }
708
709    /// The delivery worker holds this module's asynchronous configuration lock
710    /// while recording the failure. Respawn stays blocked until the failure has
711    /// restored the previous active generation or, if none exists, the supervisor
712    /// has canceled respawn and stopped the module.
713    /// `AnchorPending` still restores the previous active generation in memory;
714    /// its durable failure record is retried before delivery can resume.
715    pub(crate) fn delivery_record_exit(&self, exited: &ExitedDelivery) -> Result<bool> {
716        #[cfg(test)]
717        self.exit_record_attempts
718            .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
719        let mut state = self.lock()?;
720        if !state
721            .delivery
722            .exited
723            .get(&exited.module)
724            .is_some_and(|entry| entry.spawn == exited.spawn)
725        {
726            return Ok(false);
727        }
728        let tuple = Tuple {
729            module: exited.module.clone(),
730            project: None,
731        };
732        let versions = state
733            .store
734            .body()?
735            .versions
736            .iter()
737            .find(|(key, _)| key == &tuple)
738            .map(|(_, versions)| versions.clone())
739            .unwrap_or_default();
740        if versions
741            .pending
742            .is_some_and(|generation| generation != exited.generation)
743        {
744            return Ok(false);
745        }
746        if versions.pending == Some(exited.generation) {
747            state.restore_route_policy(&exited.module, exited.generation);
748            let result = state
749                .store
750                .record_delivery(&tuple, exited.generation, false);
751            if let Err(error) = result {
752                if error.code != crate::config_store::StoreErrorCode::AnchorPending {
753                    return Err(error.into());
754                }
755            }
756            state.delivery.attempts.retain(|_, attempt| {
757                attempt.module != exited.module || attempt.generation != exited.generation
758            });
759        }
760        Ok(versions.active.is_none())
761    }
762
763    #[cfg(all(test, unix))]
764    pub(crate) fn exit_record_attempts_for_test(&self) -> usize {
765        self.exit_record_attempts
766            .load(std::sync::atomic::Ordering::SeqCst)
767    }
768
769    pub(crate) fn delivery_release_exit(&self, exited: &ExitedDelivery) -> Result<()> {
770        let mut state = self.lock()?;
771        if state
772            .delivery
773            .exited
774            .get(&exited.module)
775            .is_some_and(|entry| entry.spawn == exited.spawn)
776        {
777            if let Some(entry) = state.delivery.exited.remove(&exited.module) {
778                entry.changed.send_replace(false);
779                // A prior release may have reached the worker while this fence
780                // still blocked admission. Recheck it after the fence is gone;
781                // the worker starts only enabled modules with no remaining hold.
782                state.admission_releases.push(exited.module.clone());
783            }
784        }
785        self.notify_delivery_changed();
786        Ok(())
787    }
788    pub(crate) fn delivery_abandon_spawn(&self, spawn: &DeliverySpawn, active: bool) {
789        if let Ok(mut state) = self.lock() {
790            let key = (spawn.module.clone(), active);
791            if state
792                .delivery
793                .unregistered_children
794                .get(&key)
795                .is_some_and(|current| current.id == spawn.id)
796            {
797                state.delivery.unregistered_children.remove(&key);
798            }
799        }
800    }
801    pub(crate) fn delivery_queue_abandoned_spawn(&self, spawn: DeliverySpawn, active: bool) {
802        if let Ok(mut abandoned) = self.abandoned_spawns.lock() {
803            abandoned.push((spawn, active));
804        }
805        self.notify_delivery_changed();
806    }
807    pub(super) fn drain_abandoned_spawns(&self, state: &mut OwnerState) {
808        if let Ok(mut abandoned) = self.abandoned_spawns.lock() {
809            for (spawn, active) in abandoned.drain(..) {
810                let key = (spawn.module.clone(), active);
811                if state
812                    .delivery
813                    .unregistered_children
814                    .get(&key)
815                    .is_some_and(|current| current.id == spawn.id)
816                {
817                    state.delivery.unregistered_children.remove(&key);
818                }
819            }
820        }
821    }
822
823    /// The full authority check and reservation ran on a blocking worker while
824    /// the caller owned the module gate. Recheck only that exact witness here;
825    /// contention asks the async caller to wait off the runtime thread.
826    pub(crate) fn delivery_launch_current(
827        &self,
828        spawn: &DeliverySpawn,
829        active: bool,
830    ) -> Result<Option<bool>> {
831        let state = match self.inner.try_lock() {
832            Ok(state) => state,
833            Err(std::sync::TryLockError::WouldBlock) => return Ok(None),
834            Err(_) => {
835                return Err(GateError::new(
836                    "config_unavailable",
837                    "configuration owner lock is poisoned",
838                ))
839            }
840        };
841        Ok(Some(
842            super::admission::AdmissionCache::from_state(&state)
843                .spawn_generation(&spawn.module)
844                .is_ok_and(|generation| generation == spawn.generation)
845                && state
846                    .delivery
847                    .unregistered_children
848                    .get(&(spawn.module.clone(), active))
849                    .is_some_and(|current| current.id == spawn.id)
850                && !state.delivery.exited.contains_key(&spawn.module),
851        ))
852    }
853    #[cfg(test)]
854    pub(crate) fn pause_spawn_preflight_for_test(&self) {
855        let hook = self.spawn_preflight_hook.lock().unwrap().take();
856        if let Some(hook) = hook {
857            hook();
858        }
859    }
860    #[cfg(all(test, unix))]
861    pub(crate) fn exhaust_delivery_spawns_for_test(&self) {
862        self.lock().unwrap().delivery.next_spawn = u64::MAX;
863    }
864    #[cfg(all(test, unix))]
865    pub(crate) fn restore_unused_delivery_spawns_for_test(&self) {
866        self.lock().unwrap().delivery.next_spawn = 1000;
867    }
868    #[cfg(all(test, unix))]
869    pub(crate) fn fail_delivery_spawns_for_module_for_test(&self, module: &str) {
870        self.lock().unwrap().delivery.failed_spawn_module = Some(module.into());
871    }
872    #[cfg(all(test, unix))]
873    pub(crate) fn clear_delivery_spawn_failure_for_test(&self) {
874        self.lock().unwrap().delivery.failed_spawn_module = None;
875    }
876    #[cfg(all(test, unix))]
877    pub(crate) fn set_spawn_preflight_hook_for_test(&self, hook: impl FnOnce() + Send + 'static) {
878        *self.spawn_preflight_hook.lock().unwrap() = Some(Box::new(hook));
879    }
880    #[cfg(all(test, unix))]
881    pub(crate) fn unregistered_spawn_count_for_test(&self) -> usize {
882        self.lock().unwrap().delivery.unregistered_children.len()
883    }
884    pub(crate) fn delivery_spawned_for_admission_in_slot(
885        &self,
886        module: &str,
887        admission: &super::admission::SpawnAdmission,
888        at: tokio::time::Instant,
889        register: Duration,
890        health: Duration,
891        active: bool,
892    ) -> Result<Option<DeliverySpawn>> {
893        let mut state = OwnerGuard(
894            self.inner.try_lock().map_err(|error| {
895                if matches!(error, std::sync::TryLockError::Poisoned(_)) {
896                    return GateError::new("config_unavailable", "configuration owner is poisoned");
897                }
898                GateError::new(
899                    "config_delivery_deferred",
900                    "configuration publication is busy",
901                )
902            })?,
903            &self.published,
904            &self.delivery_changed,
905            &self.module_gates,
906            self,
907        );
908        state.store.check()?;
909        let spawn = allocate_spawn(
910            &mut state,
911            module,
912            admission.generation(),
913            at,
914            register,
915            health,
916        )?;
917        state
918            .delivery
919            .unregistered_children
920            .insert((module.into(), active), spawn.clone());
921        Ok(Some(spawn))
922    }
923
924    #[cfg(test)]
925    pub(crate) fn delivery_spawned_in_slot(
926        &self,
927        module: &str,
928        at: tokio::time::Instant,
929        register: Duration,
930        health: Duration,
931        active: bool,
932    ) -> Result<Option<DeliverySpawn>> {
933        let spawn = self.delivery_spawned(module, at, register, health)?;
934        if let Some(spawn) = &spawn {
935            self.lock()?
936                .delivery
937                .unregistered_children
938                .insert((module.into(), active), spawn.clone());
939        }
940        Ok(spawn)
941    }
942    pub(crate) fn delivery_take_recoveries(
943        &self,
944    ) -> Result<Vec<(ConnectionId, String, AppliedDisposition)>> {
945        Ok(std::mem::take(&mut self.lock()?.delivery.recoveries))
946    }
947
948    pub(crate) fn delivery_window(
949        &self,
950        connection: ConnectionId,
951    ) -> Result<Option<DeliveryWindow>> {
952        Ok(self
953            .lock()?
954            .delivery
955            .attempts
956            .get(&connection)
957            .map(|bound| DeliveryWindow {
958                generation: bound.generation,
959                attempt: bound.id,
960                delivered: bound.delivered,
961                deadline: bound.deadline,
962            }))
963    }
964
965    pub(crate) fn delivery_write_started(
966        &self,
967        connection: ConnectionId,
968        generation: u64,
969        attempt: u64,
970        now: tokio::time::Instant,
971    ) -> Result<()> {
972        if let Some(bound) = self
973            .lock()?
974            .delivery
975            .attempts
976            .get_mut(&connection)
977            .filter(|bound| {
978                bound.generation == generation && bound.id == attempt && !bound.delivered
979            })
980        {
981            bound
982                .write_deadline
983                .get_or_insert(now + CONFIG_LIVE_ACK_TIMEOUT);
984        }
985        Ok(())
986    }
987
988    pub(crate) fn delivery_timeout(
989        &self,
990        connection: ConnectionId,
991        module: &str,
992        generation: u64,
993        attempt: u64,
994    ) -> Result<AppliedDisposition> {
995        self.module_outcome(
996            connection,
997            module,
998            ConfigAppliedRequest::new(
999                generation,
1000                attempt,
1001                subc_protocol::config::ConfigApplyResult::Nack,
1002                vec![],
1003            ),
1004            true,
1005        )
1006    }
1007
1008    /// Capture the common spawn instant, not the later HELLO instant, so a slow
1009    /// registration cannot extend the approved generation's warm-up deadline.
1010    pub(crate) fn delivery_spawned(
1011        &self,
1012        module: &str,
1013        spawned_at: tokio::time::Instant,
1014        register_timeout: Duration,
1015        first_health_budget: Duration,
1016    ) -> Result<Option<DeliverySpawn>> {
1017        let mut state = self.lock()?;
1018        state.store.check()?;
1019        let generation =
1020            super::admission::AdmissionCache::from_state(&state).spawn_generation(module)?;
1021        allocate_spawn(
1022            &mut state,
1023            module,
1024            generation,
1025            spawned_at,
1026            register_timeout,
1027            first_health_budget,
1028        )
1029        .map(Some)
1030    }
1031
1032    pub(crate) fn delivery_child_registered(
1033        &self,
1034        connection: ConnectionId,
1035        spawn: &DeliverySpawn,
1036    ) -> Result<()> {
1037        let mut state = self.lock()?;
1038        state.delivery.children.insert(connection, spawn.clone());
1039        if let Some(bound) = state
1040            .delivery
1041            .attempts
1042            .get_mut(&connection)
1043            .filter(|bound| bound.module == spawn.module && bound.generation == spawn.generation)
1044        {
1045            if bound.transition.requires_spawn() {
1046                bound.deadline = Some(spawn.deadline);
1047            }
1048        }
1049        Ok(())
1050    }
1051
1052    pub(crate) fn module_view_delivered(
1053        &self,
1054        connection: ConnectionId,
1055        module: &str,
1056        reply: &ConfigGetReply,
1057        written_at: tokio::time::Instant,
1058        active: bool,
1059    ) -> Result<()> {
1060        {
1061            let mut state = self.lock()?;
1062            if !state
1063                .declarations
1064                .get(&connection)
1065                .is_some_and(|(id, _)| id == module)
1066            {
1067                return Err(GateError::new(
1068                    "not_delivery_target",
1069                    "configuration connection has ended",
1070                ));
1071            }
1072            let body = state.store.body()?;
1073            if body.stamped {
1074                let versions = body
1075                    .versions
1076                    .iter()
1077                    .find(|(tuple, _)| tuple.module == module && tuple.project.is_none())
1078                    .map(|(_, versions)| versions.clone())
1079                    .unwrap_or_default();
1080                if reply
1081                    .active
1082                    .as_ref()
1083                    .is_some_and(|view| versions.active != Some(view.generation))
1084                    || reply.pending.as_ref().is_some_and(|view| {
1085                        // A promoted child catches up to the durable active
1086                        // vector through its own pending, child-local attempt.
1087                        let catchup = versions.active == Some(view.generation)
1088                            && state
1089                                .delivery
1090                                .attempts
1091                                .get(&connection)
1092                                .is_some_and(|bound| {
1093                                    bound.catchup
1094                                        && bound.generation == view.generation
1095                                        && Some(bound.id) == view.attempt
1096                                });
1097                        versions.pending != Some(view.generation) && !catchup
1098                    })
1099                {
1100                    return Err(GateError::new(
1101                        "delivery_superseded",
1102                        "configuration changed while its view was being written",
1103                    ));
1104                }
1105            }
1106        }
1107        if active {
1108            if let Some(view) = &reply.pending {
1109                if let Some(attempt) = view.attempt {
1110                    self.delivery_written(connection, view.generation, attempt, written_at)?;
1111                }
1112            }
1113        }
1114        let mut state = self.lock()?;
1115        if !state
1116            .declarations
1117            .get(&connection)
1118            .is_some_and(|(id, _)| id == module)
1119        {
1120            return Err(GateError::new(
1121                "not_delivery_target",
1122                "configuration connection has ended",
1123            ));
1124        }
1125        state.delivery.served.insert(
1126            connection,
1127            (
1128                reply.active.as_ref().map(|view| view.generation),
1129                reply.pending.as_ref().map(|view| view.generation),
1130            ),
1131        );
1132        Ok(())
1133    }
1134    /// Completion is supplied by the supervisor after registration, the first
1135    /// healthy probe and any required route teardown, never by a module body.
1136    pub(crate) fn delivery_readiness_complete(
1137        &self,
1138        connection: ConnectionId,
1139        module: &str,
1140        generation: u64,
1141    ) -> Result<Option<AppliedDisposition>> {
1142        let attempt = {
1143            let mut state = self.lock()?;
1144            if state.active_declarations.get(module) != Some(&connection) {
1145                return Err(GateError::new(
1146                    "not_delivery_target",
1147                    "readiness belongs to another connection",
1148                ));
1149            }
1150            let started_for_generation = state
1151                .delivery
1152                .children
1153                .get(&connection)
1154                .is_some_and(|warmup| warmup.generation == generation);
1155            if started_for_generation {
1156                state
1157                    .delivery
1158                    .healthy_children
1159                    .insert((connection, generation));
1160            }
1161            let policy_ready = super::route_policy::ready(&mut state, module, generation)?;
1162            let Some(bound) = state
1163                .delivery
1164                .attempts
1165                .get_mut(&connection)
1166                .filter(|bound| bound.module == module && bound.generation == generation)
1167            else {
1168                return Ok(None);
1169            };
1170            if bound.transition.requires_spawn() && !started_for_generation {
1171                return Ok(None);
1172            }
1173            bound.ready = policy_ready;
1174            let acknowledged = bound.ready && bound.acknowledged;
1175            let id = bound.id;
1176            state
1177                .delivery
1178                .readiness_observed
1179                .insert((connection, generation));
1180            acknowledged.then_some(id)
1181        };
1182        attempt
1183            .map(|attempt| {
1184                self.module_applied(
1185                    connection,
1186                    module,
1187                    ConfigAppliedRequest::new(
1188                        generation,
1189                        attempt,
1190                        subc_protocol::config::ConfigApplyResult::Ack,
1191                        vec![],
1192                    ),
1193                )
1194            })
1195            .transpose()
1196    }
1197
1198    /// Both forwarding and registry promotion happen under the caller's module
1199    /// gate before this connection becomes an acknowledgement target.
1200    pub(crate) fn delivery_promoted(
1201        &self,
1202        connection: ConnectionId,
1203        module: &str,
1204        cutover_at: tokio::time::Instant,
1205    ) -> Result<Option<(u64, u64)>> {
1206        let mut state = self.lock()?;
1207        state.store.delivery_ready()?;
1208        Self::promote_loaded(&mut state, connection, module, cutover_at)
1209    }
1210
1211    pub(super) fn promote_loaded(
1212        state: &mut OwnerState,
1213        connection: ConnectionId,
1214        module: &str,
1215        cutover_at: tokio::time::Instant,
1216    ) -> Result<Option<(u64, u64)>> {
1217        if state.store.loaded_admission_state().0 == crate::config_store::StoreState::Unstamped {
1218            if state
1219                .declarations
1220                .get(&connection)
1221                .is_some_and(|(id, _)| id == module)
1222            {
1223                state.phone_target_changed(module, connection);
1224                state.active_declarations.insert(module.into(), connection);
1225            }
1226            return Ok(None);
1227        }
1228        let Some((id, declaration)) = state
1229            .declarations
1230            .get(&connection)
1231            .cloned()
1232            .filter(|(id, _)| id == module)
1233        else {
1234            return Err(GateError::new(
1235                "config_gated",
1236                "candidate has not declared its approved configuration",
1237            ));
1238        };
1239        let tuple = Tuple {
1240            module: module.into(),
1241            project: None,
1242        };
1243        let body = state.store.loaded_admission_state().1;
1244        let versions = body
1245            .versions
1246            .iter()
1247            .find(|(key, _)| key == &tuple)
1248            .map(|(_, versions)| versions.clone())
1249            .unwrap_or_default();
1250        let transition = versions
1251            .pending
1252            .and_then(|id| body.generations.get(&id))
1253            .map(|entry| Transition::for_module(&entry.generation.changeset, module))
1254            .transpose()?
1255            .unwrap_or(Transition::Live);
1256        let pending_declaration = versions
1257            .pending
1258            .and_then(|id| body.generations.get(&id))
1259            .and_then(|entry| entry.delivery.as_ref())
1260            .and_then(|snapshot| snapshot.declarations.get(module))
1261            .cloned();
1262        let active_declaration = versions
1263            .active
1264            .and_then(|id| body.generations.get(&id))
1265            .and_then(|entry| entry.delivery.as_ref())
1266            .and_then(|snapshot| snapshot.declarations.get(module))
1267            .cloned();
1268        let pending = versions.pending.filter(|generation| {
1269            state
1270                .delivery
1271                .boundary_open(connection, *generation, transition)
1272        });
1273        if pending != versions.pending && transition != Transition::DaemonRestart {
1274            return Err(GateError::new(
1275                "config_gated",
1276                "candidate launch was superseded by a configuration requiring a fresh start",
1277            ));
1278        }
1279        let expected = if pending.is_some() {
1280            pending_declaration
1281        } else {
1282            active_declaration
1283        };
1284        if expected.as_ref() != Some(&declaration) {
1285            return Err(GateError::new(
1286                "config_gated",
1287                "candidate declaration differs from the approved declaration",
1288            ));
1289        }
1290        let transition = if pending.is_some() {
1291            transition
1292        } else {
1293            Transition::Live
1294        };
1295        let vector = (versions.active, pending);
1296        let catchup = pending.is_none()
1297            && versions.active.is_some()
1298            && state.delivery.served.get(&connection) != Some(&vector);
1299        state.phone_target_changed(module, connection);
1300        state.active_declarations.insert(id, connection);
1301        state.delivery.attempts.remove(&connection);
1302        let Some(generation) = pending.or_else(|| catchup.then_some(versions.active).flatten())
1303        else {
1304            return Ok(None);
1305        };
1306        let mut attempt = state.delivery.allocate(module, generation, transition)?;
1307        attempt.catchup = catchup;
1308        // Swap readiness completed before promotion. Its new target now gets
1309        // a full live ACK window, independently of the finished spawn budget.
1310        // A route obligation still needs its own post-install probe: the
1311        // candidate's earlier HEALTHY did not bind the installed policy epoch.
1312        attempt.ready = super::route_policy::ready_loaded(state, module, generation)
1313            && !state
1314                .store
1315                .loaded_admission_state()
1316                .1
1317                .generations
1318                .get(&generation)
1319                .is_some_and(|entry| {
1320                    super::route_policy::required(&entry.generation.changeset, module)
1321                });
1322        attempt.deadline = Some(cutover_at + CONFIG_LIVE_ACK_TIMEOUT);
1323        let assignment = (generation, attempt.id);
1324        state.delivery.attempts.insert(connection, attempt);
1325        state
1326            .delivery
1327            .readiness_observed
1328            .insert((connection, generation));
1329        Ok(Some(assignment))
1330    }
1331
1332    /// The control adapter calls this only for the active target while holding
1333    /// its module gate, after the delivery frame's writer flush succeeds.
1334    pub(crate) fn delivery_written(
1335        &self,
1336        connection: ConnectionId,
1337        generation: u64,
1338        attempt: u64,
1339        written_at: tokio::time::Instant,
1340    ) -> Result<()> {
1341        let mut state = self.lock()?;
1342        let bound = state
1343            .delivery
1344            .attempts
1345            .get_mut(&connection)
1346            .filter(|bound| bound.generation == generation && bound.id == attempt)
1347            .ok_or_else(|| {
1348                GateError::new(
1349                    "delivery_superseded",
1350                    "writer receipt no longer names the current delivery attempt",
1351                )
1352            })?;
1353        if bound
1354            .write_deadline
1355            .is_some_and(|deadline| written_at >= deadline)
1356            || bound
1357                .deadline
1358                .is_some_and(|deadline| written_at >= deadline)
1359        {
1360            return Err(GateError::new(
1361                "delivery_expired",
1362                "configuration delivery timed out",
1363            ));
1364        }
1365        bound.delivered = true;
1366        if !bound.transition.requires_spawn() && bound.deadline.is_none() {
1367            bound.deadline = Some(written_at + CONFIG_LIVE_ACK_TIMEOUT);
1368        }
1369        Ok(())
1370    }
1371
1372    pub(crate) fn module_applied(
1373        &self,
1374        connection: ConnectionId,
1375        module: &str,
1376        request: ConfigAppliedRequest,
1377    ) -> Result<AppliedDisposition> {
1378        self.module_outcome(connection, module, request, false)
1379    }
1380
1381    fn module_outcome(
1382        &self,
1383        connection: ConnectionId,
1384        module: &str,
1385        request: ConfigAppliedRequest,
1386        timed_out: bool,
1387    ) -> Result<AppliedDisposition> {
1388        let mut state = self.lock()?;
1389        if !state
1390            .declarations
1391            .get(&connection)
1392            .is_some_and(|(id, _)| id == module)
1393        {
1394            return Err(GateError::new(
1395                "declaration_order",
1396                "config.declare must precede get or applied on this connection",
1397            ));
1398        }
1399        if request.project_id.is_some() {
1400            return Err(GateError::new(
1401                "project_registry_unavailable",
1402                "project delivery requires the authoritative project registry",
1403            ));
1404        }
1405        if state.active_declarations.get(module) != Some(&connection) {
1406            return Err(GateError::new(
1407                "not_delivery_target",
1408                "connection is not the active delivery target",
1409            ));
1410        }
1411        state.store.check()?;
1412        let Some(bound) = state
1413            .delivery
1414            .attempts
1415            .get(&connection)
1416            .cloned()
1417            .filter(|bound| {
1418                bound.module == module
1419                    && bound.generation == request.generation
1420                    && bound.id == request.attempt
1421            })
1422        else {
1423            return Ok(AppliedDisposition::Ignored);
1424        };
1425        if !bound.delivered && !timed_out {
1426            return Ok(AppliedDisposition::Ignored);
1427        }
1428        let tuple = Tuple {
1429            module: module.into(),
1430            project: None,
1431        };
1432        let versions = state
1433            .store
1434            .body()?
1435            .versions
1436            .iter()
1437            .find(|(key, _)| key == &tuple)
1438            .map(|(_, versions)| versions.clone())
1439            .unwrap_or_default();
1440        let ack = request.result.is_ack()
1441            && !bound
1442                .deadline
1443                .is_some_and(|deadline| tokio::time::Instant::now() >= deadline);
1444        if bound.catchup {
1445            // A later approval waiting for a new daemon cannot replace the
1446            // active configuration this child is catching up to now.
1447            let pending_transition = versions
1448                .pending
1449                .map(|generation| {
1450                    let entry = state
1451                        .store
1452                        .body()?
1453                        .generations
1454                        .get(&generation)
1455                        .ok_or_else(|| {
1456                            GateError::new("config_gated", "pending generation is absent")
1457                        })?;
1458                    Transition::for_module(&entry.generation.changeset, module)
1459                })
1460                .transpose()?;
1461            let pending_is_deliverable =
1462                versions
1463                    .pending
1464                    .zip(pending_transition)
1465                    .is_some_and(|(generation, transition)| {
1466                        state
1467                            .delivery
1468                            .boundary_open(connection, generation, transition)
1469                    });
1470            if pending_is_deliverable || versions.active != Some(bound.generation) {
1471                return Ok(AppliedDisposition::Ignored);
1472            }
1473            state.delivery.attempts.remove(&connection);
1474            if !ack {
1475                state.delivery.recoveries.push((
1476                    connection,
1477                    module.into(),
1478                    AppliedDisposition::CatchupFailed {
1479                        active: bound.generation,
1480                    },
1481                ));
1482            }
1483            return Ok(if ack {
1484                AppliedDisposition::CatchupComplete
1485            } else {
1486                AppliedDisposition::CatchupFailed {
1487                    active: bound.generation,
1488                }
1489            });
1490        }
1491        if versions.pending != Some(bound.generation) {
1492            return Ok(AppliedDisposition::Ignored);
1493        }
1494        if ack
1495            && (!bound.ready
1496                || (bound.transition == Transition::DaemonRestart
1497                    && bound.generation > state.delivery.initial_generation))
1498        {
1499            if let Some(bound) = state.delivery.attempts.get_mut(&connection) {
1500                bound.acknowledged = true;
1501            }
1502            return Ok(AppliedDisposition::WaitingForReadiness);
1503        }
1504        if !ack {
1505            state.restore_route_policy(module, bound.generation);
1506        }
1507        let result = state.store.record_delivery(&tuple, bound.generation, ack);
1508        if result.is_ok() && ack {
1509            state.commit_route_policy(module, bound.generation);
1510        }
1511        // A pending-anchor NACK has already restored memory. Discard its attempt
1512        // even though the durable failure must wait for the anchor retry.
1513        if result.is_ok() || !ack {
1514            state.delivery.attempts.remove(&connection);
1515        }
1516        if !ack {
1517            let keep_running =
1518                versions.active.is_none() && state.delivery.stamp_running.contains(&connection);
1519            state.delivery.recoveries.push((
1520                connection,
1521                module.into(),
1522                AppliedDisposition::Failed {
1523                    generation: bound.generation,
1524                    restore: versions.active,
1525                    keep_running,
1526                },
1527            ));
1528        }
1529        result?;
1530        Ok(if ack {
1531            AppliedDisposition::Active
1532        } else {
1533            AppliedDisposition::Failed {
1534                generation: bound.generation,
1535                restore: versions.active,
1536                keep_running: versions.active.is_none()
1537                    && state.delivery.stamp_running.contains(&connection),
1538            }
1539        })
1540    }
1541}