Skip to main content

subc_daemon/config_gate/
delivery.rs

1//! Connection-bound delivery attempts; durable vectors remain in ConfigStore.
2
3use std::collections::HashSet;
4use subc_protocol::config::{ConfigAppliedRequest, ConfigGetReply, ConfigGetRequest};
5
6use super::*;
7use crate::config_gate::apply::Transition;
8use crate::config_store::Tuple;
9
10pub(crate) const CONFIG_LIVE_ACK_TIMEOUT: Duration = Duration::from_secs(30);
11pub(crate) const CONFIG_WARMUP_SLACK: Duration = Duration::from_secs(10);
12
13#[allow(clippy::too_many_arguments)]
14fn allocate_spawn(
15    state: &mut OwnerState,
16    module: &str,
17    generation: u64,
18    at: tokio::time::Instant,
19    register: Duration,
20    health: Duration,
21    authority: Option<(
22        std::sync::Arc<crate::spawn_authority::SpawnAuthorityRecord>,
23        bool,
24    )>,
25    program: Option<super::admission::AdmittedProgram>,
26) -> Result<DeliverySpawn> {
27    #[cfg(test)]
28    if state.delivery.failed_spawn_module.as_deref() == Some(module) {
29        return Err(GateError::new(
30            "delivery_spawn_exhausted",
31            "delivery spawn identifiers exhausted",
32        ));
33    }
34    state.delivery.next_spawn = state.delivery.next_spawn.checked_add(1).ok_or_else(|| {
35        GateError::new(
36            "delivery_spawn_exhausted",
37            "delivery spawn identifiers exhausted",
38        )
39    })?;
40    let (authority, stamped) =
41        authority.map_or((None, false), |(record, stamped)| (Some(record), stamped));
42    Ok(DeliverySpawn {
43        id: state.delivery.next_spawn,
44        module: module.into(),
45        generation,
46        deadline: at + register + health + CONFIG_WARMUP_SLACK,
47        authority,
48        stamped,
49        program,
50    })
51}
52
53#[derive(Clone, Debug)]
54struct Attempt {
55    module: String,
56    generation: u64,
57    id: u64,
58    transition: Transition,
59    ready: bool,
60    acknowledged: bool,
61    delivered: bool,
62    deadline: Option<tokio::time::Instant>,
63    // Catch-up belongs to one promoted child, not to the durable generation.
64    // A daemon restart drops it: every fresh launch receives the active vector.
65    catchup: bool,
66    write_deadline: Option<tokio::time::Instant>,
67    policy_probe_requested: bool,
68    /// The spawn this attempt was assigned to, when it carries an authority
69    /// record. Its acknowledgement counts toward that spawn's first-party
70    /// mark only, never another spawn's.
71    spawn: Option<DeliverySpawn>,
72    /// A catch-up assigned only so a fresh spawn acknowledges the active
73    /// generation for its first-party mark. Its refusal or timeout leaves the
74    /// spawn unmarked and starts no recovery.
75    mark_only: bool,
76}
77
78#[derive(Clone, Debug)]
79pub struct DeliverySpawn {
80    id: u64,
81    module: String,
82    generation: u64,
83    deadline: tokio::time::Instant,
84    /// The launch's spawn authority record, when the supervisor launched it.
85    authority: Option<std::sync::Arc<crate::spawn_authority::SpawnAuthorityRecord>>,
86    /// The stamped store gated this module when the spawn was admitted. Only
87    /// such a spawn can ever carry the first-party mark.
88    stamped: bool,
89    /// The program the spawn was admitted under, when the stamped store
90    /// gated its module. A placement or approval that supersedes it rules
91    /// the spawn out of the mark.
92    program: Option<super::admission::AdmittedProgram>,
93}
94
95impl DeliverySpawn {
96    pub(crate) fn module(&self) -> &str {
97        &self.module
98    }
99    pub(crate) fn generation(&self) -> u64 {
100        self.generation
101    }
102    pub(crate) fn authority(
103        &self,
104    ) -> Option<&std::sync::Arc<crate::spawn_authority::SpawnAuthorityRecord>> {
105        self.authority.as_ref()
106    }
107    pub(crate) fn stamped(&self) -> bool {
108        self.stamped
109    }
110    pub(super) fn admitted_program(&self) -> Option<&super::admission::AdmittedProgram> {
111        self.program.as_ref()
112    }
113    /// Attach a launch's authority record, as the supervisor's admission path
114    /// does, to a spawn a test allocated directly.
115    #[cfg(test)]
116    pub(crate) fn with_authority_for_test(
117        mut self,
118        record: std::sync::Arc<crate::spawn_authority::SpawnAuthorityRecord>,
119        stamped: bool,
120    ) -> Self {
121        self.authority = Some(record);
122        self.stamped = stamped;
123        self
124    }
125}
126
127#[derive(Default)]
128pub(super) struct DeliveryState {
129    #[cfg(test)]
130    failed_spawn_module: Option<String>,
131    next_spawn: u64,
132    next_attempt: u64,
133    policy_epochs: HashMap<String, u64>,
134    attempts: HashMap<ConnectionId, Attempt>,
135    served: HashMap<ConnectionId, (Option<u64>, Option<u64>)>,
136    pub(super) unstamped_views: HashMap<ConnectionId, std::sync::Arc<ConfigView>>,
137    stamp_running: HashSet<ConnectionId>,
138    initial_generation: u64,
139    children: HashMap<ConnectionId, DeliverySpawn>,
140    healthy_children: HashSet<(ConnectionId, u64)>,
141    readiness_observed: HashSet<(ConnectionId, u64)>,
142    unregistered_children: HashMap<(String, bool), DeliverySpawn>,
143    recoveries: Vec<(ConnectionId, String, AppliedDisposition)>,
144    exited: HashMap<String, ExitedDelivery>,
145    // Transport cleanup may precede the process-exit observation. Retain the
146    // last active child's assigned generation, not an unattempted deferred one.
147    last_target_attempt: HashMap<String, (u64, u64)>,
148    /// Every spawn admitted under the stamp, which carries or may still earn
149    /// the first-party mark. Unlike the rest of this state it survives
150    /// `reset_for_authority_exit`, so a committed unstamp still finds the
151    /// spawns whose marks must be lowered. Each is kept with the program it
152    /// was admitted under.
153    mark_records: Vec<(
154        std::sync::Weak<crate::spawn_authority::SpawnAuthorityRecord>,
155        Option<super::admission::AdmittedProgram>,
156    )>,
157}
158
159impl DeliveryState {
160    pub(super) fn reset_for_authority_exit(&mut self, generation: u64) {
161        *self = Self {
162            next_spawn: self.next_spawn,
163            next_attempt: self.next_attempt,
164            mark_records: std::mem::take(&mut self.mark_records),
165            ..Self::new(generation)
166        };
167    }
168    /// Remember `spawn` for the mark sweep when it was admitted under the
169    /// stamp with an authority record.
170    pub(super) fn track_mark(&mut self, spawn: &DeliverySpawn) {
171        let Some(record) = spawn.authority.as_ref().filter(|_| spawn.stamped) else {
172            return;
173        };
174        if !self
175            .mark_records
176            .iter()
177            .any(|(known, _)| std::ptr::eq(known.as_ptr(), std::sync::Arc::as_ptr(record)))
178        {
179            self.mark_records
180                .push((std::sync::Arc::downgrade(record), spawn.program.clone()));
181        }
182    }
183    /// The live spawns remembered for the mark sweep, each with the program
184    /// it was admitted under; forgets dead ones.
185    pub(super) fn mark_records(
186        &mut self,
187    ) -> Vec<(
188        std::sync::Arc<crate::spawn_authority::SpawnAuthorityRecord>,
189        Option<super::admission::AdmittedProgram>,
190    )> {
191        self.mark_records
192            .retain(|(record, _)| record.upgrade().is_some_and(|record| record.alive()));
193        self.mark_records
194            .iter()
195            .filter_map(|(record, program)| Some((record.upgrade()?, program.clone())))
196            .collect()
197    }
198    pub(super) fn started_at_generation(&self) -> u64 {
199        self.initial_generation
200    }
201    pub(super) fn policy_authorization(
202        &self,
203        module: &str,
204        generation: u64,
205        connection: ConnectionId,
206    ) -> Option<(u64, tokio::time::Instant)> {
207        let attempt = self.attempts.get(&connection)?;
208        let deadline = attempt.deadline?;
209        (attempt.module == module
210            && attempt.generation == generation
211            && attempt.delivered
212            && attempt.acknowledged
213            && tokio::time::Instant::now() < deadline)
214            .then_some((attempt.id, deadline))
215    }
216    pub(super) fn fenced_modules(&self) -> impl Iterator<Item = String> + '_ {
217        self.exited.keys().cloned()
218    }
219}
220
221/// Captured before a health RPC; only the same child and installed policy may
222/// consume its result. The fields are private so a completion cannot retarget it.
223#[derive(Clone, Debug)]
224pub struct DeliveryProbe {
225    connection: ConnectionId,
226    module: String,
227    generation: u64,
228    attempt: Option<u64>,
229    policy_epoch: u64,
230    route_readiness: bool,
231}
232
233impl DeliveryProbe {
234    pub(crate) fn connection(&self) -> ConnectionId {
235        self.connection
236    }
237    pub(crate) fn owns_failure(&self) -> bool {
238        self.route_readiness
239    }
240}
241
242#[derive(Clone)]
243pub(crate) struct ExitedDelivery {
244    pub module: String,
245    spawn: u64,
246    generation: u64,
247    changed: tokio::sync::watch::Sender<bool>,
248}
249
250#[derive(Clone, Debug, PartialEq, Eq)]
251pub enum AppliedDisposition {
252    Ignored,
253    WaitingForReadiness,
254    Active,
255    CatchupComplete,
256    Failed {
257        generation: u64,
258        restore: Option<u64>,
259        keep_running: bool,
260    },
261    CatchupFailed {
262        active: u64,
263    },
264}
265
266#[derive(Clone, Copy)]
267pub(crate) struct DeliveryWindow {
268    pub generation: u64,
269    pub attempt: u64,
270    pub delivered: bool,
271    pub deadline: Option<tokio::time::Instant>,
272}
273
274impl DeliveryState {
275    pub(super) fn invalidate_policy(&mut self, module: &str, generation: u64) {
276        for attempt in self
277            .attempts
278            .values_mut()
279            .filter(|attempt| attempt.module == module && attempt.generation == generation)
280        {
281            attempt.ready = false;
282            attempt.policy_probe_requested = false;
283        }
284    }
285    pub(super) fn policy_installed(
286        &mut self,
287        module: &str,
288        generation: u64,
289        target: Option<ConnectionId>,
290    ) -> Result<()> {
291        // The generation may be unchanged after a restore and retry. Use a
292        // separate epoch so a probe begun before this installation cannot count.
293        let epoch = self.policy_epochs.entry(module.into()).or_default();
294        *epoch = epoch.checked_add(1).ok_or_else(|| {
295            GateError::new(
296                "delivery_probe_exhausted",
297                "policy readiness epochs exhausted",
298            )
299        })?;
300        self.invalidate_policy(module, generation);
301        let connections: HashSet<_> = self
302            .children
303            .iter()
304            .filter(|(_, spawn)| spawn.module == module)
305            .map(|(connection, _)| *connection)
306            .chain(
307                self.attempts
308                    .iter()
309                    .filter(|(_, attempt)| attempt.module == module)
310                    .map(|(connection, _)| *connection),
311            )
312            .chain(target)
313            .collect();
314        self.readiness_observed.retain(|(connection, observed)| {
315            *observed != generation || !connections.contains(connection)
316        });
317        self.healthy_children.retain(|(connection, observed)| {
318            *observed != generation || !connections.contains(connection)
319        });
320        Ok(())
321    }
322    pub(super) fn declared(&mut self, connection: ConnectionId, module: &str, active: bool) {
323        if let Some(spawn) = self.unregistered_children.remove(&(module.into(), active)) {
324            self.children.insert(connection, spawn);
325        }
326    }
327    pub(super) fn new(initial_generation: u64) -> Self {
328        Self {
329            initial_generation,
330            ..Self::default()
331        }
332    }
333
334    pub(super) fn disconnected(&mut self, connection: ConnectionId) {
335        self.attempts.remove(&connection);
336        self.served.remove(&connection);
337        self.unstamped_views.remove(&connection);
338        self.stamp_running.remove(&connection);
339        self.children.remove(&connection);
340        self.healthy_children
341            .retain(|(child, _)| *child != connection);
342        self.readiness_observed
343            .retain(|(child, _)| *child != connection);
344    }
345
346    pub(super) fn remember_stamp_connections(&mut self, connections: Vec<ConnectionId>) {
347        self.stamp_running.extend(connections);
348    }
349
350    fn boundary_open(
351        &self,
352        connection: ConnectionId,
353        generation: u64,
354        transition: Transition,
355    ) -> bool {
356        if transition == Transition::DaemonRestart && generation > self.initial_generation {
357            return false;
358        }
359        !transition.requires_spawn()
360            || self
361                .children
362                .get(&connection)
363                .is_some_and(|spawn| spawn.generation == generation)
364    }
365
366    fn allocate(
367        &mut self,
368        module: &str,
369        generation: u64,
370        transition: Transition,
371    ) -> Result<Attempt> {
372        self.next_attempt = self.next_attempt.checked_add(1).ok_or_else(|| {
373            GateError::new(
374                "delivery_attempt_exhausted",
375                "delivery attempt ids exhausted",
376            )
377        })?;
378        Ok(Attempt {
379            module: module.into(),
380            generation,
381            id: self.next_attempt,
382            transition,
383            ready: transition == Transition::Live,
384            acknowledged: false,
385            delivered: false,
386            deadline: None,
387            catchup: false,
388            write_deadline: None,
389            policy_probe_requested: false,
390            spawn: None,
391            mark_only: false,
392        })
393    }
394}
395
396pub(super) fn visible_reply(
397    state: &OwnerState,
398    connection: ConnectionId,
399    mut reply: ConfigGetReply,
400    transition: Transition,
401) -> Result<ConfigGetReply> {
402    if reply.pending.as_ref().is_some_and(|view| {
403        !state
404            .delivery
405            .boundary_open(connection, view.generation, transition)
406    }) {
407        reply.pending = None;
408    }
409    if reply.active.is_none() && reply.pending.is_none() {
410        return Err(GateError::new(
411            "config_gated",
412            "generation waits for its required supervision or daemon boundary",
413        ));
414    }
415    Ok(reply)
416}
417
418pub(super) fn bind_get(
419    state: &mut OwnerState,
420    connection: ConnectionId,
421    module: &str,
422    request: &ConfigGetRequest,
423    reply: ConfigGetReply,
424    transition: Transition,
425) -> Result<ConfigGetReply> {
426    let mut reply = visible_reply(state, connection, reply, transition)?;
427    match (request.generation, request.attempt) {
428        (None, None) => {}
429        (Some(generation), Some(attempt))
430            if state
431                .delivery
432                .attempts
433                .get(&connection)
434                .is_some_and(|bound| {
435                    bound.module == module && bound.generation == generation && bound.id == attempt
436                }) => {}
437        _ => {
438            return Err(GateError::new(
439                "config_gated",
440                "requested generation and attempt are not assigned to this connection",
441            ))
442        }
443    }
444    if reply.pending.is_none()
445        && state
446            .delivery
447            .attempts
448            .get(&connection)
449            .is_some_and(|bound| {
450                bound.catchup
451                    && reply
452                        .active
453                        .as_ref()
454                        .is_some_and(|view| view.generation == bound.generation)
455            })
456    {
457        reply.pending = reply.active.clone();
458        if let Some(view) = &mut reply.pending {
459            view.status = ConfigStatus::Pending;
460        }
461    }
462    if reply.pending.is_none() && request.generation.is_none() {
463        let generation = reply.active.as_ref().map(|view| view.generation);
464        if let Some(attempt) = generation
465            .map(|generation| mark_acknowledgement(state, connection, module, generation))
466            .transpose()?
467            .flatten()
468        {
469            state.delivery.attempts.insert(connection, attempt);
470            reply.pending = reply.active.clone();
471            if let Some(view) = &mut reply.pending {
472                view.status = ConfigStatus::Pending;
473            }
474        }
475    }
476    if let Some(pending) = &mut reply.pending {
477        let matches = state
478            .delivery
479            .attempts
480            .get(&connection)
481            .is_some_and(|bound| bound.module == module && bound.generation == pending.generation);
482        if !matches {
483            let mut attempt = state
484                .delivery
485                .allocate(module, pending.generation, transition)?;
486            if transition.requires_spawn() {
487                attempt.ready = state
488                    .delivery
489                    .healthy_children
490                    .contains(&(connection, pending.generation));
491                attempt.deadline = state
492                    .delivery
493                    .children
494                    .get(&connection)
495                    .filter(|warmup| warmup.generation == pending.generation)
496                    .map(|warmup| warmup.deadline);
497            }
498            attempt.ready &= super::route_policy::ready(state, module, pending.generation)?;
499            attempt.spawn = state.delivery.children.get(&connection).cloned();
500            state.delivery.attempts.insert(connection, attempt);
501        }
502        pending.attempt = state
503            .delivery
504            .attempts
505            .get(&connection)
506            .map(|bound| bound.id);
507        let mark_only = state
508            .delivery
509            .attempts
510            .get(&connection)
511            .is_some_and(|bound| bound.mark_only);
512        if state.active_declarations.get(module) == Some(&connection) && !mark_only {
513            if let Some(spawn) = state.delivery.children.get(&connection) {
514                let id = spawn.id;
515                state
516                    .delivery
517                    .last_target_attempt
518                    .insert(module.into(), (id, pending.generation));
519            }
520        }
521    } else {
522        state.delivery.attempts.remove(&connection);
523    }
524    if request
525        .generation
526        .is_some_and(|id| reply.pending.as_ref().map(|view| view.generation) != Some(id))
527    {
528        return Err(GateError::new(
529            "config_gated",
530            "selected delivery has been superseded",
531        ));
532    }
533    Ok(reply)
534}
535
536/// The catch-up attempt that asks a fresh spawn to acknowledge its module's
537/// active generation, so the spawn can earn the first-party mark: a spawn is
538/// marked only after it acknowledged the active generation and reported
539/// ready, on every start path. Delivering
540/// the active view at launch is not proof the spawn applied it, and without a
541/// pending change a crash, health, manual or boot restart is otherwise never
542/// asked for an acknowledgement.
543///
544/// Assigned only to the active target of a stamped, gated module whose spawn
545/// carries an authority record that could still be marked, at most once per
546/// active generation; never when a pending change already has an attempt.
547/// Unstamped and ungated modules are never asked, so their delivery is
548/// unchanged. A refusal or timeout leaves the spawn unmarked and starts no
549/// recovery.
550fn mark_acknowledgement(
551    state: &mut OwnerState,
552    connection: ConnectionId,
553    module: &str,
554    generation: u64,
555) -> Result<Option<Attempt>> {
556    if state.store.loaded_admission_state().0 != crate::config_store::StoreState::Stamped
557        || state.active_declarations.get(module) != Some(&connection)
558        || state.delivery.attempts.contains_key(&connection)
559    {
560        return Ok(None);
561    }
562    let Some(spawn) = state
563        .delivery
564        .children
565        .get(&connection)
566        .filter(|spawn| spawn.stamped && spawn.module == module)
567        .cloned()
568    else {
569        return Ok(None);
570    };
571    let Some(record) = spawn.authority.as_ref() else {
572        return Ok(None);
573    };
574    let facts = record.mark_facts();
575    if !facts.unmarked || !facts.origin_qualifies || !record.request_acknowledgement(generation) {
576        return Ok(None);
577    }
578    let mut attempt = state
579        .delivery
580        .allocate(module, generation, Transition::Live)?;
581    attempt.catchup = true;
582    attempt.mark_only = true;
583    attempt.spawn = Some(spawn);
584    Ok(Some(attempt))
585}
586
587impl RuntimeOwner {
588    pub(crate) fn policy_probe_due(&self, connection: ConnectionId) -> Result<Option<(u64, u64)>> {
589        let mut state = self.lock()?;
590        let Some(bound) = state.delivery.attempts.get(&connection) else {
591            return Ok(None);
592        };
593        let (module, generation, attempt) = (bound.module.clone(), bound.generation, bound.id);
594        // A mark-only attempt asks for an acknowledgement and nothing else;
595        // readiness and route policy belong to the spawn's ordinary delivery.
596        if bound.ready
597            || bound.mark_only
598            || bound.policy_probe_requested
599            || state.active_declarations.get(&module) != Some(&connection)
600            || !state
601                .store
602                .body()?
603                .generations
604                .get(&generation)
605                .is_some_and(|entry| {
606                    super::route_policy::required(&entry.generation.changeset, &module)
607                })
608            || !super::route_policy::ready(&mut state, &module, generation)?
609        {
610            return Ok(None);
611        }
612        state
613            .delivery
614            .attempts
615            .get_mut(&connection)
616            .unwrap()
617            .policy_probe_requested = true;
618        Ok(Some((generation, attempt)))
619    }
620
621    pub(crate) fn policy_probe_pending(
622        &self,
623        connection: ConnectionId,
624        generation: u64,
625        attempt: u64,
626    ) -> Result<bool> {
627        let mut state = self.lock()?;
628        let Some(bound) = state.delivery.attempts.get(&connection) else {
629            return Ok(false);
630        };
631        let module = bound.module.clone();
632        Ok(bound.generation == generation
633            && bound.id == attempt
634            && !bound.ready
635            && state.active_declarations.get(&module) == Some(&connection)
636            && super::route_policy::ready(&mut state, &module, generation)?)
637    }
638    pub(crate) fn delivery_health_probe(
639        &self,
640        connection: ConnectionId,
641        module: &str,
642    ) -> Result<Option<DeliveryProbe>> {
643        let mut state = self.lock()?;
644        if state.active_declarations.get(module) != Some(&connection) {
645            return Ok(None);
646        }
647        let attempt = state
648            .delivery
649            .attempts
650            .get(&connection)
651            .map(|attempt| (attempt.generation, attempt.id));
652        let generation = attempt.map(|attempt| attempt.0).or_else(|| {
653            state
654                .delivery
655                .children
656                .get(&connection)
657                .map(|child| child.generation)
658        });
659        let route_readiness = match generation {
660            Some(generation) if attempt.is_some() => state
661                .store
662                .body()?
663                .generations
664                .get(&generation)
665                .is_some_and(|entry| {
666                    super::route_policy::required(&entry.generation.changeset, module)
667                }),
668            _ => false,
669        };
670        Ok(generation.map(|generation| DeliveryProbe {
671            connection,
672            module: module.into(),
673            generation,
674            attempt: attempt.map(|attempt| attempt.1),
675            route_readiness,
676            policy_epoch: state
677                .delivery
678                .policy_epochs
679                .get(module)
680                .copied()
681                .unwrap_or_default(),
682        }))
683    }
684
685    pub(crate) fn delivery_health_result(
686        &self,
687        probe: DeliveryProbe,
688        healthy: bool,
689    ) -> Result<Option<AppliedDisposition>> {
690        let attempt = {
691            let mut state = self.lock()?;
692            if state.active_declarations.get(&probe.module) != Some(&probe.connection)
693                || state
694                    .delivery
695                    .policy_epochs
696                    .get(&probe.module)
697                    .copied()
698                    .unwrap_or_default()
699                    != probe.policy_epoch
700            {
701                return Ok(None);
702            }
703            let current = state.delivery.attempts.get(&probe.connection);
704            if probe
705                .attempt
706                .is_some_and(|id| current.is_none_or(|bound| bound.id != id))
707                || current.is_some_and(|bound| bound.generation != probe.generation)
708            {
709                return Ok(None);
710            }
711            // A warm-up probe answering not ready is no answer to a mark-only
712            // attempt: only the spawn's own acknowledgement settles it, so a
713            // spawn still warming up is never refused the mark by a probe.
714            let attempt = current
715                .filter(|bound| !bound.mark_only)
716                .map(|bound| bound.id);
717            if !healthy
718                && (!super::route_policy::ready(&mut state, &probe.module, probe.generation)?
719                    || !state
720                        .store
721                        .body()?
722                        .generations
723                        .get(&probe.generation)
724                        .is_some_and(|entry| {
725                            super::route_policy::required(
726                                &entry.generation.changeset,
727                                &probe.module,
728                            )
729                        }))
730            {
731                return Ok(None);
732            }
733            attempt
734        };
735        if healthy {
736            self.delivery_readiness_complete(probe.connection, &probe.module, probe.generation)
737        } else if let Some(attempt) = attempt {
738            self.module_outcome(
739                probe.connection,
740                &probe.module,
741                ConfigAppliedRequest::new(
742                    probe.generation,
743                    attempt,
744                    subc_protocol::config::ConfigApplyResult::Nack,
745                    vec![],
746                ),
747                true,
748            )
749            .map(Some)
750        } else {
751            Ok(None)
752        }
753    }
754
755    #[cfg(test)]
756    pub(super) fn ready(
757        &self,
758        connection: ConnectionId,
759        module: &str,
760        generation: u64,
761    ) -> Result<Option<AppliedDisposition>> {
762        self.delivery_readiness_complete(connection, module, generation)
763    }
764
765    pub(crate) fn pending_config_restarts(&self) -> Result<Vec<(String, u64)>> {
766        let mut state = self.lock()?;
767        state.store.delivery_ready()?;
768        let body = state.store.body()?;
769        body.versions
770            .iter()
771            .filter(|(tuple, _)| tuple.project.is_none())
772            .filter_map(|(tuple, versions)| {
773                let generation = versions.pending?;
774                let entry = body.generations.get(&generation)?;
775                Some(
776                    Transition::for_module(&entry.generation.changeset, &tuple.module).map(
777                        |transition| {
778                            (transition == Transition::ModuleRestart)
779                                .then(|| (tuple.module.clone(), generation))
780                        },
781                    ),
782                )
783            })
784            .collect::<Result<Vec<_>>>()
785            .map(|entries| entries.into_iter().flatten().collect())
786    }
787
788    pub(crate) fn requires_fresher_spawn(
789        &self,
790        module: &str,
791        spawn: Option<&DeliverySpawn>,
792    ) -> Result<Option<u64>> {
793        let mut state = self.lock()?;
794        state.store.delivery_ready()?;
795        Self::requires_fresher_spawn_loaded(&state, module, spawn)
796    }
797
798    pub(super) fn requires_fresher_spawn_loaded(
799        state: &OwnerState,
800        module: &str,
801        spawn: Option<&DeliverySpawn>,
802    ) -> Result<Option<u64>> {
803        let body = state.store.loaded_admission_state().1;
804        let pending = body
805            .versions
806            .iter()
807            .find(|(tuple, _)| tuple.module == module && tuple.project.is_none())
808            .and_then(|(_, versions)| versions.pending);
809        let Some(generation) = pending else {
810            return Ok(None);
811        };
812        let entry = body
813            .generations
814            .get(&generation)
815            .ok_or_else(|| GateError::new("config_gated", "pending generation is absent"))?;
816        let transition = Transition::for_module(&entry.generation.changeset, module)?;
817        let deferred_daemon = transition == Transition::DaemonRestart
818            && generation > state.delivery.initial_generation;
819        Ok((transition.requires_spawn()
820            && !deferred_daemon
821            && spawn
822                .is_none_or(|spawn| spawn.module != module || spawn.generation() != generation))
823        .then_some(generation))
824    }
825
826    /// A supervised process exit, not a transport disconnect, fails warm-up.
827    /// The synchronous fence closes the gap before the async failure commit.
828    pub(crate) fn delivery_child_exited(&self, spawn: &DeliverySpawn) -> Result<()> {
829        let mut state = self.lock()?;
830        let pending = state
831            .store
832            .body()?
833            .versions
834            .iter()
835            .find(|(tuple, _)| tuple.module == spawn.module && tuple.project.is_none())
836            .and_then(|(_, versions)| versions.pending);
837        let pending = pending.filter(|generation| {
838            state
839                .store
840                .loaded_admission_state()
841                .1
842                .generations
843                .get(generation)
844                .and_then(|entry| entry.delivery.as_ref())
845                .is_some_and(|delivery| delivery.declarations.contains_key(&spawn.module))
846        });
847        if let Some(generation) = pending.filter(|generation| {
848            *generation == spawn.generation
849                || state.delivery.last_target_attempt.get(&spawn.module)
850                    == Some(&(spawn.id, *generation))
851        }) {
852            state
853                .delivery
854                .exited
855                .entry(spawn.module.clone())
856                .or_insert_with(|| ExitedDelivery {
857                    module: spawn.module.clone(),
858                    spawn: spawn.id,
859                    generation,
860                    changed: tokio::sync::watch::channel(true).0,
861                });
862        }
863        self.notify_delivery_changed();
864        Ok(())
865    }
866
867    pub(crate) fn delivery_exit_fence(
868        &self,
869        module: &str,
870    ) -> Result<Option<tokio::sync::watch::Receiver<bool>>> {
871        Ok(self
872            .lock()?
873            .delivery
874            .exited
875            .get(module)
876            .map(|entry| entry.changed.subscribe()))
877    }
878
879    pub(crate) fn delivery_exits(&self) -> Result<Vec<ExitedDelivery>> {
880        Ok(self.lock()?.delivery.exited.values().cloned().collect())
881    }
882
883    /// The delivery worker holds this module's asynchronous configuration lock
884    /// while recording the failure. Respawn stays blocked until the failure has
885    /// restored the previous active generation or, if none exists, the supervisor
886    /// has canceled respawn and stopped the module.
887    /// `AnchorPending` still restores the previous active generation in memory;
888    /// its durable failure record is retried before delivery can resume.
889    pub(crate) fn delivery_record_exit(&self, exited: &ExitedDelivery) -> Result<bool> {
890        #[cfg(test)]
891        self.exit_record_attempts
892            .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
893        let mut state = self.lock()?;
894        if !state
895            .delivery
896            .exited
897            .get(&exited.module)
898            .is_some_and(|entry| entry.spawn == exited.spawn)
899        {
900            return Ok(false);
901        }
902        let tuple = Tuple {
903            module: exited.module.clone(),
904            project: None,
905        };
906        let versions = state
907            .store
908            .body()?
909            .versions
910            .iter()
911            .find(|(key, _)| key == &tuple)
912            .map(|(_, versions)| versions.clone())
913            .unwrap_or_default();
914        if versions
915            .pending
916            .is_some_and(|generation| generation != exited.generation)
917        {
918            return Ok(false);
919        }
920        if versions.pending == Some(exited.generation) {
921            state.restore_route_policy(&exited.module, exited.generation);
922            let result = state
923                .store
924                .record_delivery(&tuple, exited.generation, false);
925            if let Err(error) = result {
926                if error.code != crate::config_store::StoreErrorCode::AnchorPending {
927                    return Err(error.into());
928                }
929            }
930            state.delivery.attempts.retain(|_, attempt| {
931                attempt.module != exited.module || attempt.generation != exited.generation
932            });
933        }
934        Ok(versions.active.is_none())
935    }
936
937    #[cfg(all(test, unix))]
938    pub(crate) fn exit_record_attempts_for_test(&self) -> usize {
939        self.exit_record_attempts
940            .load(std::sync::atomic::Ordering::SeqCst)
941    }
942
943    pub(crate) fn delivery_release_exit(&self, exited: &ExitedDelivery) -> Result<()> {
944        let mut state = self.lock()?;
945        if state
946            .delivery
947            .exited
948            .get(&exited.module)
949            .is_some_and(|entry| entry.spawn == exited.spawn)
950        {
951            if let Some(entry) = state.delivery.exited.remove(&exited.module) {
952                entry.changed.send_replace(false);
953                // A prior release may have reached the worker while this fence
954                // still blocked admission. Recheck it after the fence is gone;
955                // the worker starts only enabled modules with no remaining hold.
956                state.admission_releases.push(exited.module.clone());
957            }
958        }
959        self.notify_delivery_changed();
960        Ok(())
961    }
962    pub(crate) fn delivery_abandon_spawn(&self, spawn: &DeliverySpawn, active: bool) {
963        if let Ok(mut state) = self.lock() {
964            let key = (spawn.module.clone(), active);
965            if state
966                .delivery
967                .unregistered_children
968                .get(&key)
969                .is_some_and(|current| current.id == spawn.id)
970            {
971                state.delivery.unregistered_children.remove(&key);
972            }
973        }
974    }
975    pub(crate) fn delivery_queue_abandoned_spawn(&self, spawn: DeliverySpawn, active: bool) {
976        if let Ok(mut abandoned) = self.abandoned_spawns.lock() {
977            abandoned.push((spawn, active));
978        }
979        self.notify_delivery_changed();
980    }
981    pub(super) fn drain_abandoned_spawns(&self, state: &mut OwnerState) {
982        if let Ok(mut abandoned) = self.abandoned_spawns.lock() {
983            for (spawn, active) in abandoned.drain(..) {
984                let key = (spawn.module.clone(), active);
985                if state
986                    .delivery
987                    .unregistered_children
988                    .get(&key)
989                    .is_some_and(|current| current.id == spawn.id)
990                {
991                    state.delivery.unregistered_children.remove(&key);
992                }
993            }
994        }
995    }
996
997    /// The full authority check and reservation ran on a blocking worker while
998    /// the caller owned the module gate. Recheck only that exact witness here;
999    /// contention asks the async caller to wait off the runtime thread.
1000    pub(crate) fn delivery_launch_current(
1001        &self,
1002        spawn: &DeliverySpawn,
1003        active: bool,
1004    ) -> Result<Option<bool>> {
1005        let state = match self.inner.try_lock() {
1006            Ok(state) => state,
1007            Err(std::sync::TryLockError::WouldBlock) => return Ok(None),
1008            Err(_) => {
1009                return Err(GateError::new(
1010                    "config_unavailable",
1011                    "configuration owner lock is poisoned",
1012                ))
1013            }
1014        };
1015        Ok(Some(
1016            super::admission::AdmissionCache::from_state(&state)
1017                .spawn_generation(&spawn.module)
1018                .is_ok_and(|generation| generation == spawn.generation)
1019                && state
1020                    .delivery
1021                    .unregistered_children
1022                    .get(&(spawn.module.clone(), active))
1023                    .is_some_and(|current| current.id == spawn.id)
1024                && !state.delivery.exited.contains_key(&spawn.module),
1025        ))
1026    }
1027    #[cfg(test)]
1028    pub(crate) fn pause_spawn_preflight_for_test(&self) {
1029        let hook = self.spawn_preflight_hook.lock().unwrap().take();
1030        if let Some(hook) = hook {
1031            hook();
1032        }
1033    }
1034    #[cfg(all(test, unix))]
1035    pub(crate) fn exhaust_delivery_spawns_for_test(&self) {
1036        self.lock().unwrap().delivery.next_spawn = u64::MAX;
1037    }
1038    #[cfg(all(test, unix))]
1039    pub(crate) fn restore_unused_delivery_spawns_for_test(&self) {
1040        self.lock().unwrap().delivery.next_spawn = 1000;
1041    }
1042    #[cfg(all(test, unix))]
1043    pub(crate) fn fail_delivery_spawns_for_module_for_test(&self, module: &str) {
1044        self.lock().unwrap().delivery.failed_spawn_module = Some(module.into());
1045    }
1046    #[cfg(all(test, unix))]
1047    pub(crate) fn clear_delivery_spawn_failure_for_test(&self) {
1048        self.lock().unwrap().delivery.failed_spawn_module = None;
1049    }
1050    #[cfg(all(test, unix))]
1051    pub(crate) fn set_spawn_preflight_hook_for_test(&self, hook: impl FnOnce() + Send + 'static) {
1052        *self.spawn_preflight_hook.lock().unwrap() = Some(Box::new(hook));
1053    }
1054    #[cfg(all(test, unix))]
1055    pub(crate) fn unregistered_spawn_count_for_test(&self) -> usize {
1056        self.lock().unwrap().delivery.unregistered_children.len()
1057    }
1058    #[allow(clippy::too_many_arguments)]
1059    pub(crate) fn delivery_spawned_for_admission_in_slot(
1060        &self,
1061        module: &str,
1062        admission: &super::admission::SpawnAdmission,
1063        at: tokio::time::Instant,
1064        register: Duration,
1065        health: Duration,
1066        active: bool,
1067        authority: Option<std::sync::Arc<crate::spawn_authority::SpawnAuthorityRecord>>,
1068    ) -> Result<Option<DeliverySpawn>> {
1069        let mut state = OwnerGuard(
1070            self.inner.try_lock().map_err(|error| {
1071                if matches!(error, std::sync::TryLockError::Poisoned(_)) {
1072                    return GateError::new("config_unavailable", "configuration owner is poisoned");
1073                }
1074                GateError::new(
1075                    "config_delivery_deferred",
1076                    "configuration publication is busy",
1077                )
1078            })?,
1079            &self.published,
1080            &self.delivery_changed,
1081            &self.module_gates,
1082            self,
1083        );
1084        state.store.check()?;
1085        // Admission under the stamped gate is the only way to the mark: a
1086        // stamp never marks a child admitted before it.
1087        let stamped = admission.gated()
1088            && state.store.loaded_admission_state().0 == crate::config_store::StoreState::Stamped;
1089        let spawn = allocate_spawn(
1090            &mut state,
1091            module,
1092            admission.generation(),
1093            at,
1094            register,
1095            health,
1096            authority.map(|record| (record, stamped)),
1097            admission.admitted_program(),
1098        )?;
1099        state.delivery.track_mark(&spawn);
1100        state
1101            .delivery
1102            .unregistered_children
1103            .insert((module.into(), active), spawn.clone());
1104        Ok(Some(spawn))
1105    }
1106
1107    #[cfg(test)]
1108    pub(crate) fn delivery_spawned_in_slot(
1109        &self,
1110        module: &str,
1111        at: tokio::time::Instant,
1112        register: Duration,
1113        health: Duration,
1114        active: bool,
1115    ) -> Result<Option<DeliverySpawn>> {
1116        let spawn = self.delivery_spawned(module, at, register, health)?;
1117        if let Some(spawn) = &spawn {
1118            self.lock()?
1119                .delivery
1120                .unregistered_children
1121                .insert((module.into(), active), spawn.clone());
1122        }
1123        Ok(spawn)
1124    }
1125    pub(crate) fn delivery_take_recoveries(
1126        &self,
1127    ) -> Result<Vec<(ConnectionId, String, AppliedDisposition)>> {
1128        Ok(std::mem::take(&mut self.lock()?.delivery.recoveries))
1129    }
1130
1131    pub(crate) fn delivery_window(
1132        &self,
1133        connection: ConnectionId,
1134    ) -> Result<Option<DeliveryWindow>> {
1135        Ok(self
1136            .lock()?
1137            .delivery
1138            .attempts
1139            .get(&connection)
1140            .map(|bound| DeliveryWindow {
1141                generation: bound.generation,
1142                attempt: bound.id,
1143                delivered: bound.delivered,
1144                deadline: bound.deadline,
1145            }))
1146    }
1147
1148    pub(crate) fn delivery_write_started(
1149        &self,
1150        connection: ConnectionId,
1151        generation: u64,
1152        attempt: u64,
1153        now: tokio::time::Instant,
1154    ) -> Result<()> {
1155        if let Some(bound) = self
1156            .lock()?
1157            .delivery
1158            .attempts
1159            .get_mut(&connection)
1160            .filter(|bound| {
1161                bound.generation == generation && bound.id == attempt && !bound.delivered
1162            })
1163        {
1164            bound
1165                .write_deadline
1166                .get_or_insert(now + CONFIG_LIVE_ACK_TIMEOUT);
1167        }
1168        Ok(())
1169    }
1170
1171    pub(crate) fn delivery_timeout(
1172        &self,
1173        connection: ConnectionId,
1174        module: &str,
1175        generation: u64,
1176        attempt: u64,
1177    ) -> Result<AppliedDisposition> {
1178        self.module_outcome(
1179            connection,
1180            module,
1181            ConfigAppliedRequest::new(
1182                generation,
1183                attempt,
1184                subc_protocol::config::ConfigApplyResult::Nack,
1185                vec![],
1186            ),
1187            true,
1188        )
1189    }
1190
1191    /// Capture the common spawn instant, not the later HELLO instant, so a slow
1192    /// registration cannot extend the approved generation's warm-up deadline.
1193    pub(crate) fn delivery_spawned(
1194        &self,
1195        module: &str,
1196        spawned_at: tokio::time::Instant,
1197        register_timeout: Duration,
1198        first_health_budget: Duration,
1199    ) -> Result<Option<DeliverySpawn>> {
1200        let mut state = self.lock()?;
1201        state.store.check()?;
1202        let admission = super::admission::AdmissionCache::from_state(&state);
1203        let generation = admission.spawn_generation(module)?;
1204        allocate_spawn(
1205            &mut state,
1206            module,
1207            generation,
1208            spawned_at,
1209            register_timeout,
1210            first_health_budget,
1211            None,
1212            admission.admitted_program(module),
1213        )
1214        .map(Some)
1215    }
1216
1217    pub(crate) fn delivery_child_registered(
1218        &self,
1219        connection: ConnectionId,
1220        spawn: &DeliverySpawn,
1221    ) -> Result<()> {
1222        let mut state = self.lock()?;
1223        state.delivery.track_mark(spawn);
1224        state.delivery.children.insert(connection, spawn.clone());
1225        if let Some(bound) = state
1226            .delivery
1227            .attempts
1228            .get_mut(&connection)
1229            .filter(|bound| bound.module == spawn.module && bound.generation == spawn.generation)
1230        {
1231            if bound.transition.requires_spawn() {
1232                bound.deadline = Some(spawn.deadline);
1233            }
1234        }
1235        Ok(())
1236    }
1237
1238    pub(crate) fn module_view_delivered(
1239        &self,
1240        connection: ConnectionId,
1241        module: &str,
1242        reply: &ConfigGetReply,
1243        written_at: tokio::time::Instant,
1244        active: bool,
1245    ) -> Result<()> {
1246        {
1247            let mut state = self.lock()?;
1248            if !state
1249                .declarations
1250                .get(&connection)
1251                .is_some_and(|(id, _)| id == module)
1252            {
1253                return Err(GateError::new(
1254                    "not_delivery_target",
1255                    "configuration connection has ended",
1256                ));
1257            }
1258            let body = state.store.body()?;
1259            if body.stamped {
1260                let versions = body
1261                    .versions
1262                    .iter()
1263                    .find(|(tuple, _)| tuple.module == module && tuple.project.is_none())
1264                    .map(|(_, versions)| versions.clone())
1265                    .unwrap_or_default();
1266                if reply
1267                    .active
1268                    .as_ref()
1269                    .is_some_and(|view| versions.active != Some(view.generation))
1270                    || reply.pending.as_ref().is_some_and(|view| {
1271                        // A promoted child catches up to the durable active
1272                        // vector through its own pending, child-local attempt.
1273                        let catchup = versions.active == Some(view.generation)
1274                            && state
1275                                .delivery
1276                                .attempts
1277                                .get(&connection)
1278                                .is_some_and(|bound| {
1279                                    bound.catchup
1280                                        && bound.generation == view.generation
1281                                        && Some(bound.id) == view.attempt
1282                                });
1283                        versions.pending != Some(view.generation) && !catchup
1284                    })
1285                {
1286                    return Err(GateError::new(
1287                        "delivery_superseded",
1288                        "configuration changed while its view was being written",
1289                    ));
1290                }
1291            }
1292        }
1293        if active {
1294            if let Some(view) = &reply.pending {
1295                if let Some(attempt) = view.attempt {
1296                    self.delivery_written(connection, view.generation, attempt, written_at)?;
1297                }
1298            }
1299        }
1300        let mut state = self.lock()?;
1301        if !state
1302            .declarations
1303            .get(&connection)
1304            .is_some_and(|(id, _)| id == module)
1305        {
1306            return Err(GateError::new(
1307                "not_delivery_target",
1308                "configuration connection has ended",
1309            ));
1310        }
1311        state.delivery.served.insert(
1312            connection,
1313            (
1314                reply.active.as_ref().map(|view| view.generation),
1315                reply.pending.as_ref().map(|view| view.generation),
1316            ),
1317        );
1318        if let Some(view) = reply
1319            .active
1320            .as_ref()
1321            .filter(|view| view.status == ConfigStatus::Unstamped)
1322        {
1323            state
1324                .delivery
1325                .unstamped_views
1326                .insert(connection, std::sync::Arc::new(view.clone()));
1327        } else {
1328            state.delivery.unstamped_views.remove(&connection);
1329        }
1330        Ok(())
1331    }
1332    /// Completion is supplied by the supervisor after registration, the first
1333    /// healthy probe and any required route teardown, never by a module body.
1334    pub(crate) fn delivery_readiness_complete(
1335        &self,
1336        connection: ConnectionId,
1337        module: &str,
1338        generation: u64,
1339    ) -> Result<Option<AppliedDisposition>> {
1340        let attempt = {
1341            let mut state = self.lock()?;
1342            if state.active_declarations.get(module) != Some(&connection) {
1343                return Err(GateError::new(
1344                    "not_delivery_target",
1345                    "readiness belongs to another connection",
1346                ));
1347            }
1348            let started_for_generation = state
1349                .delivery
1350                .children
1351                .get(&connection)
1352                .is_some_and(|warmup| warmup.generation == generation);
1353            if started_for_generation {
1354                state
1355                    .delivery
1356                    .healthy_children
1357                    .insert((connection, generation));
1358            }
1359            let policy_ready = super::route_policy::ready(&mut state, module, generation)?;
1360            let Some(bound) = state
1361                .delivery
1362                .attempts
1363                .get_mut(&connection)
1364                .filter(|bound| bound.module == module && bound.generation == generation)
1365            else {
1366                return Ok(None);
1367            };
1368            if bound.transition.requires_spawn() && !started_for_generation {
1369                return Ok(None);
1370            }
1371            bound.ready = policy_ready;
1372            let acknowledged = bound.ready && bound.acknowledged;
1373            let id = bound.id;
1374            state
1375                .delivery
1376                .readiness_observed
1377                .insert((connection, generation));
1378            acknowledged.then_some(id)
1379        };
1380        attempt
1381            .map(|attempt| {
1382                self.module_applied(
1383                    connection,
1384                    module,
1385                    ConfigAppliedRequest::new(
1386                        generation,
1387                        attempt,
1388                        subc_protocol::config::ConfigApplyResult::Ack,
1389                        vec![],
1390                    ),
1391                )
1392            })
1393            .transpose()
1394    }
1395
1396    /// Both forwarding and registry promotion happen under the caller's module
1397    /// gate before this connection becomes an acknowledgement target.
1398    pub(crate) fn delivery_promoted(
1399        &self,
1400        connection: ConnectionId,
1401        module: &str,
1402        cutover_at: tokio::time::Instant,
1403    ) -> Result<Option<(u64, u64)>> {
1404        let mut state = self.lock()?;
1405        state.store.delivery_ready()?;
1406        Self::promote_loaded(&mut state, connection, module, cutover_at)
1407    }
1408
1409    pub(super) fn promote_loaded(
1410        state: &mut OwnerState,
1411        connection: ConnectionId,
1412        module: &str,
1413        cutover_at: tokio::time::Instant,
1414    ) -> Result<Option<(u64, u64)>> {
1415        if state.store.loaded_admission_state().0 == crate::config_store::StoreState::Unstamped {
1416            if state
1417                .declarations
1418                .get(&connection)
1419                .is_some_and(|(id, _)| id == module)
1420            {
1421                state.phone_target_changed(module, connection);
1422                state.active_declarations.insert(module.into(), connection);
1423            }
1424            return Ok(None);
1425        }
1426        let Some((id, declaration)) = state
1427            .declarations
1428            .get(&connection)
1429            .cloned()
1430            .filter(|(id, _)| id == module)
1431        else {
1432            return Err(GateError::new(
1433                "config_gated",
1434                "candidate has not declared its approved configuration",
1435            ));
1436        };
1437        let tuple = Tuple {
1438            module: module.into(),
1439            project: None,
1440        };
1441        let body = state.store.loaded_admission_state().1;
1442        let versions = body
1443            .versions
1444            .iter()
1445            .find(|(key, _)| key == &tuple)
1446            .map(|(_, versions)| versions.clone())
1447            .unwrap_or_default();
1448        let transition = versions
1449            .pending
1450            .and_then(|id| body.generations.get(&id))
1451            .map(|entry| Transition::for_module(&entry.generation.changeset, module))
1452            .transpose()?
1453            .unwrap_or(Transition::Live);
1454        let pending_declaration = versions
1455            .pending
1456            .and_then(|id| body.generations.get(&id))
1457            .and_then(|entry| entry.delivery.as_ref())
1458            .and_then(|snapshot| snapshot.declarations.get(module))
1459            .cloned();
1460        let active_declaration = versions
1461            .active
1462            .and_then(|id| body.generations.get(&id))
1463            .and_then(|entry| entry.delivery.as_ref())
1464            .and_then(|snapshot| snapshot.declarations.get(module))
1465            .cloned();
1466        let pending = versions.pending.filter(|generation| {
1467            state
1468                .delivery
1469                .boundary_open(connection, *generation, transition)
1470        });
1471        if pending != versions.pending && transition != Transition::DaemonRestart {
1472            return Err(GateError::new(
1473                "config_gated",
1474                "candidate launch was superseded by a configuration requiring a fresh start",
1475            ));
1476        }
1477        let expected = if pending.is_some() {
1478            pending_declaration
1479        } else {
1480            active_declaration
1481        };
1482        if expected.as_ref() != Some(&declaration) {
1483            return Err(GateError::new(
1484                "config_gated",
1485                "candidate declaration differs from the approved declaration",
1486            ));
1487        }
1488        let transition = if pending.is_some() {
1489            transition
1490        } else {
1491            Transition::Live
1492        };
1493        let vector = (versions.active, pending);
1494        let catchup = pending.is_none()
1495            && versions.active.is_some()
1496            && state.delivery.served.get(&connection) != Some(&vector);
1497        state.phone_target_changed(module, connection);
1498        state.active_declarations.insert(id, connection);
1499        state.delivery.attempts.remove(&connection);
1500        let Some(generation) = pending.or_else(|| catchup.then_some(versions.active).flatten())
1501        else {
1502            return Ok(None);
1503        };
1504        let mut attempt = state.delivery.allocate(module, generation, transition)?;
1505        attempt.catchup = catchup;
1506        attempt.spawn = state.delivery.children.get(&connection).cloned();
1507        // Swap readiness completed before promotion. Its new target now gets
1508        // a full live ACK window, independently of the finished spawn budget.
1509        // A route obligation still needs its own post-install probe: the
1510        // candidate's earlier HEALTHY did not bind the installed policy epoch.
1511        attempt.ready = super::route_policy::ready_loaded(state, module, generation)
1512            && !state
1513                .store
1514                .loaded_admission_state()
1515                .1
1516                .generations
1517                .get(&generation)
1518                .is_some_and(|entry| {
1519                    super::route_policy::required(&entry.generation.changeset, module)
1520                });
1521        attempt.deadline = Some(cutover_at + CONFIG_LIVE_ACK_TIMEOUT);
1522        let assignment = (generation, attempt.id);
1523        state.delivery.attempts.insert(connection, attempt);
1524        state
1525            .delivery
1526            .readiness_observed
1527            .insert((connection, generation));
1528        Ok(Some(assignment))
1529    }
1530
1531    /// The control adapter calls this only for the active target while holding
1532    /// its module gate, after the delivery frame's writer flush succeeds.
1533    pub(crate) fn delivery_written(
1534        &self,
1535        connection: ConnectionId,
1536        generation: u64,
1537        attempt: u64,
1538        written_at: tokio::time::Instant,
1539    ) -> Result<()> {
1540        let mut state = self.lock()?;
1541        let bound = state
1542            .delivery
1543            .attempts
1544            .get_mut(&connection)
1545            .filter(|bound| bound.generation == generation && bound.id == attempt)
1546            .ok_or_else(|| {
1547                GateError::new(
1548                    "delivery_superseded",
1549                    "writer receipt no longer names the current delivery attempt",
1550                )
1551            })?;
1552        if bound
1553            .write_deadline
1554            .is_some_and(|deadline| written_at >= deadline)
1555            || bound
1556                .deadline
1557                .is_some_and(|deadline| written_at >= deadline)
1558        {
1559            return Err(GateError::new(
1560                "delivery_expired",
1561                "configuration delivery timed out",
1562            ));
1563        }
1564        bound.delivered = true;
1565        if !bound.transition.requires_spawn() && bound.deadline.is_none() {
1566            bound.deadline = Some(written_at + CONFIG_LIVE_ACK_TIMEOUT);
1567        }
1568        Ok(())
1569    }
1570
1571    pub(crate) fn module_applied(
1572        &self,
1573        connection: ConnectionId,
1574        module: &str,
1575        request: ConfigAppliedRequest,
1576    ) -> Result<AppliedDisposition> {
1577        self.module_outcome(connection, module, request, false)
1578    }
1579
1580    fn module_outcome(
1581        &self,
1582        connection: ConnectionId,
1583        module: &str,
1584        request: ConfigAppliedRequest,
1585        timed_out: bool,
1586    ) -> Result<AppliedDisposition> {
1587        let mut state = self.lock()?;
1588        if !state
1589            .declarations
1590            .get(&connection)
1591            .is_some_and(|(id, _)| id == module)
1592        {
1593            return Err(GateError::new(
1594                "declaration_order",
1595                "config.declare must precede get or applied on this connection",
1596            ));
1597        }
1598        if request.project_id.is_some() {
1599            return Err(GateError::new(
1600                "project_registry_unavailable",
1601                "project delivery requires the authoritative project registry",
1602            ));
1603        }
1604        if state.active_declarations.get(module) != Some(&connection) {
1605            return Err(GateError::new(
1606                "not_delivery_target",
1607                "connection is not the active delivery target",
1608            ));
1609        }
1610        state.store.check()?;
1611        let Some(bound) = state
1612            .delivery
1613            .attempts
1614            .get(&connection)
1615            .cloned()
1616            .filter(|bound| {
1617                bound.module == module
1618                    && bound.generation == request.generation
1619                    && bound.id == request.attempt
1620            })
1621        else {
1622            return Ok(AppliedDisposition::Ignored);
1623        };
1624        if !bound.delivered && !timed_out {
1625            return Ok(AppliedDisposition::Ignored);
1626        }
1627        let tuple = Tuple {
1628            module: module.into(),
1629            project: None,
1630        };
1631        let versions = state
1632            .store
1633            .body()?
1634            .versions
1635            .iter()
1636            .find(|(key, _)| key == &tuple)
1637            .map(|(_, versions)| versions.clone())
1638            .unwrap_or_default();
1639        let ack = request.result.is_ack()
1640            && !bound
1641                .deadline
1642                .is_some_and(|deadline| tokio::time::Instant::now() >= deadline);
1643        if bound.catchup {
1644            // A later approval waiting for a new daemon cannot replace the
1645            // active configuration this child is catching up to now.
1646            let pending_transition = versions
1647                .pending
1648                .map(|generation| {
1649                    let entry = state
1650                        .store
1651                        .body()?
1652                        .generations
1653                        .get(&generation)
1654                        .ok_or_else(|| {
1655                            GateError::new("config_gated", "pending generation is absent")
1656                        })?;
1657                    Transition::for_module(&entry.generation.changeset, module)
1658                })
1659                .transpose()?;
1660            let pending_is_deliverable =
1661                versions
1662                    .pending
1663                    .zip(pending_transition)
1664                    .is_some_and(|(generation, transition)| {
1665                        state
1666                            .delivery
1667                            .boundary_open(connection, generation, transition)
1668                    });
1669            if pending_is_deliverable || versions.active != Some(bound.generation) {
1670                return Ok(AppliedDisposition::Ignored);
1671            }
1672            state.delivery.attempts.remove(&connection);
1673            if bound.mark_only {
1674                // Asked only for the mark: a refusal or timeout leaves the
1675                // spawn unmarked and starts no recovery.
1676                return Ok(if ack {
1677                    super::mark::acknowledged(self, &state, bound.spawn.as_ref(), bound.generation);
1678                    AppliedDisposition::CatchupComplete
1679                } else {
1680                    super::mark::acknowledgement_refused(self, &state, bound.spawn.as_ref());
1681                    AppliedDisposition::Ignored
1682                });
1683            }
1684            if ack {
1685                super::mark::acknowledged(self, &state, bound.spawn.as_ref(), bound.generation);
1686            } else {
1687                super::mark::delivery_failed(self, bound.spawn.as_ref());
1688                state.delivery.recoveries.push((
1689                    connection,
1690                    module.into(),
1691                    AppliedDisposition::CatchupFailed {
1692                        active: bound.generation,
1693                    },
1694                ));
1695            }
1696            return Ok(if ack {
1697                AppliedDisposition::CatchupComplete
1698            } else {
1699                AppliedDisposition::CatchupFailed {
1700                    active: bound.generation,
1701                }
1702            });
1703        }
1704        if versions.pending != Some(bound.generation) {
1705            return Ok(AppliedDisposition::Ignored);
1706        }
1707        if ack
1708            && (!bound.ready
1709                || (bound.transition == Transition::DaemonRestart
1710                    && bound.generation > state.delivery.initial_generation))
1711        {
1712            if let Some(bound) = state.delivery.attempts.get_mut(&connection) {
1713                bound.acknowledged = true;
1714            }
1715            return Ok(AppliedDisposition::WaitingForReadiness);
1716        }
1717        if !ack {
1718            state.restore_route_policy(module, bound.generation);
1719        }
1720        let result = state.store.record_delivery(&tuple, bound.generation, ack);
1721        if result.is_ok() && ack {
1722            state.commit_route_policy(module, bound.generation);
1723        }
1724        // A pending-anchor NACK has already restored memory. Discard its attempt
1725        // even though the durable failure must wait for the anchor retry.
1726        if result.is_ok() || !ack {
1727            state.delivery.attempts.remove(&connection);
1728        }
1729        if !ack {
1730            let keep_running =
1731                versions.active.is_none() && state.delivery.stamp_running.contains(&connection);
1732            state.delivery.recoveries.push((
1733                connection,
1734                module.into(),
1735                AppliedDisposition::Failed {
1736                    generation: bound.generation,
1737                    restore: versions.active,
1738                    keep_running,
1739                },
1740            ));
1741        }
1742        if ack && result.is_ok() {
1743            super::mark::acknowledged(self, &state, bound.spawn.as_ref(), bound.generation);
1744        } else if !ack {
1745            super::mark::delivery_failed(self, bound.spawn.as_ref());
1746        }
1747        result?;
1748        Ok(if ack {
1749            AppliedDisposition::Active
1750        } else {
1751            AppliedDisposition::Failed {
1752                generation: bound.generation,
1753                restore: versions.active,
1754                keep_running: versions.active.is_none()
1755                    && state.delivery.stamp_running.contains(&connection),
1756            }
1757        })
1758    }
1759}