Skip to main content

subc_daemon/config_gate/
delivery.rs

1//! Connection-bound delivery attempts; durable vectors remain in ConfigStore.
2
3use std::collections::HashSet;
4use subc_protocol::config::{ConfigAppliedRequest, ConfigGetReply, ConfigGetRequest};
5
6use super::*;
7use crate::config_gate::apply::Transition;
8use crate::config_store::Tuple;
9
10pub(crate) const CONFIG_LIVE_ACK_TIMEOUT: Duration = Duration::from_secs(30);
11pub(crate) const CONFIG_WARMUP_SLACK: Duration = Duration::from_secs(10);
12
13#[allow(clippy::too_many_arguments)]
14fn allocate_spawn(
15    state: &mut OwnerState,
16    module: &str,
17    generation: u64,
18    at: tokio::time::Instant,
19    register: Duration,
20    health: Duration,
21    authority: Option<(
22        std::sync::Arc<crate::spawn_authority::SpawnAuthorityRecord>,
23        bool,
24    )>,
25    program: Option<super::admission::AdmittedProgram>,
26) -> Result<DeliverySpawn> {
27    #[cfg(test)]
28    if state.delivery.failed_spawn_module.as_deref() == Some(module) {
29        return Err(GateError::new(
30            "delivery_spawn_exhausted",
31            "delivery spawn identifiers exhausted",
32        ));
33    }
34    state.delivery.next_spawn = state.delivery.next_spawn.checked_add(1).ok_or_else(|| {
35        GateError::new(
36            "delivery_spawn_exhausted",
37            "delivery spawn identifiers exhausted",
38        )
39    })?;
40    let (authority, stamped) =
41        authority.map_or((None, false), |(record, stamped)| (Some(record), stamped));
42    Ok(DeliverySpawn {
43        id: state.delivery.next_spawn,
44        module: module.into(),
45        generation,
46        deadline: at + register + health + CONFIG_WARMUP_SLACK,
47        authority,
48        stamped,
49        program,
50    })
51}
52
53#[derive(Clone, Debug)]
54struct Attempt {
55    module: String,
56    generation: u64,
57    id: u64,
58    transition: Transition,
59    ready: bool,
60    acknowledged: bool,
61    delivered: bool,
62    deadline: Option<tokio::time::Instant>,
63    // Catch-up belongs to one promoted child, not to the durable generation.
64    // A daemon restart drops it: every fresh launch receives the active vector.
65    catchup: bool,
66    write_deadline: Option<tokio::time::Instant>,
67    policy_probe_requested: bool,
68    /// The spawn this attempt was assigned to, when it carries an authority
69    /// record. Its acknowledgement counts toward that spawn's first-party
70    /// mark only, never another spawn's.
71    spawn: Option<DeliverySpawn>,
72    /// A catch-up assigned only so a fresh spawn acknowledges the active
73    /// generation for its first-party mark. Its refusal or timeout leaves the
74    /// spawn unmarked and starts no recovery.
75    mark_only: bool,
76}
77
78#[derive(Clone, Debug)]
79pub struct DeliverySpawn {
80    id: u64,
81    module: String,
82    generation: u64,
83    deadline: tokio::time::Instant,
84    /// The launch's spawn authority record, when the supervisor launched it.
85    authority: Option<std::sync::Arc<crate::spawn_authority::SpawnAuthorityRecord>>,
86    /// The stamped store gated this module when the spawn was admitted. Only
87    /// such a spawn can ever carry the first-party mark.
88    stamped: bool,
89    /// The program the spawn was admitted under, when the stamped store
90    /// gated its module. A placement or approval that supersedes it rules
91    /// the spawn out of the mark.
92    program: Option<super::admission::AdmittedProgram>,
93}
94
95impl DeliverySpawn {
96    pub(crate) fn module(&self) -> &str {
97        &self.module
98    }
99    pub(crate) fn generation(&self) -> u64 {
100        self.generation
101    }
102    pub(crate) fn authority(
103        &self,
104    ) -> Option<&std::sync::Arc<crate::spawn_authority::SpawnAuthorityRecord>> {
105        self.authority.as_ref()
106    }
107    pub(crate) fn stamped(&self) -> bool {
108        self.stamped
109    }
110    pub(super) fn admitted_program(&self) -> Option<&super::admission::AdmittedProgram> {
111        self.program.as_ref()
112    }
113    /// Attach a launch's authority record, as the supervisor's admission path
114    /// does, to a spawn a test allocated directly.
115    #[cfg(test)]
116    pub(crate) fn with_authority_for_test(
117        mut self,
118        record: std::sync::Arc<crate::spawn_authority::SpawnAuthorityRecord>,
119        stamped: bool,
120    ) -> Self {
121        self.authority = Some(record);
122        self.stamped = stamped;
123        self
124    }
125}
126
127#[derive(Default)]
128pub(super) struct DeliveryState {
129    #[cfg(test)]
130    failed_spawn_module: Option<String>,
131    next_spawn: u64,
132    next_attempt: u64,
133    policy_epochs: HashMap<String, u64>,
134    attempts: HashMap<ConnectionId, Attempt>,
135    served: HashMap<ConnectionId, (Option<u64>, Option<u64>)>,
136    pub(super) unstamped_views: HashMap<ConnectionId, std::sync::Arc<ConfigView>>,
137    stamp_running: HashSet<ConnectionId>,
138    initial_generation: u64,
139    children: HashMap<ConnectionId, DeliverySpawn>,
140    healthy_children: HashSet<(ConnectionId, u64)>,
141    readiness_observed: HashSet<(ConnectionId, u64)>,
142    unregistered_children: HashMap<(String, bool), DeliverySpawn>,
143    recoveries: Vec<(ConnectionId, String, AppliedDisposition)>,
144    exited: HashMap<String, ExitedDelivery>,
145    // Transport cleanup may precede the process-exit observation. Retain the
146    // last active child's assigned generation, not an unattempted deferred one.
147    last_target_attempt: HashMap<String, (u64, u64)>,
148    /// Every spawn admitted under the stamp, which carries or may still earn
149    /// the first-party mark. Unlike the rest of this state it survives
150    /// `reset_for_authority_exit`, so a committed unstamp still finds the
151    /// spawns whose marks must be lowered. Each is kept with the program it
152    /// was admitted under.
153    mark_records: Vec<(
154        std::sync::Weak<crate::spawn_authority::SpawnAuthorityRecord>,
155        Option<super::admission::AdmittedProgram>,
156    )>,
157}
158
159impl DeliveryState {
160    pub(super) fn reset_for_authority_exit(&mut self, generation: u64) {
161        *self = Self {
162            next_spawn: self.next_spawn,
163            next_attempt: self.next_attempt,
164            mark_records: std::mem::take(&mut self.mark_records),
165            ..Self::new(generation)
166        };
167    }
168    /// Remember `spawn` for the mark sweep when it was admitted under the
169    /// stamp with an authority record.
170    pub(super) fn track_mark(&mut self, spawn: &DeliverySpawn) {
171        let Some(record) = spawn.authority.as_ref().filter(|_| spawn.stamped) else {
172            return;
173        };
174        if !self
175            .mark_records
176            .iter()
177            .any(|(known, _)| std::ptr::eq(known.as_ptr(), std::sync::Arc::as_ptr(record)))
178        {
179            self.mark_records
180                .push((std::sync::Arc::downgrade(record), spawn.program.clone()));
181        }
182    }
183    /// The live spawns remembered for the mark sweep, each with the program
184    /// it was admitted under; forgets dead ones.
185    pub(super) fn mark_records(
186        &mut self,
187    ) -> Vec<(
188        std::sync::Arc<crate::spawn_authority::SpawnAuthorityRecord>,
189        Option<super::admission::AdmittedProgram>,
190    )> {
191        self.mark_records
192            .retain(|(record, _)| record.upgrade().is_some_and(|record| record.alive()));
193        self.mark_records
194            .iter()
195            .filter_map(|(record, program)| Some((record.upgrade()?, program.clone())))
196            .collect()
197    }
198    pub(super) fn started_at_generation(&self) -> u64 {
199        self.initial_generation
200    }
201    pub(super) fn policy_authorization(
202        &self,
203        module: &str,
204        generation: u64,
205        connection: ConnectionId,
206    ) -> Option<(u64, tokio::time::Instant)> {
207        let attempt = self.attempts.get(&connection)?;
208        let deadline = attempt.deadline?;
209        (attempt.module == module
210            && attempt.generation == generation
211            && attempt.delivered
212            && attempt.acknowledged
213            && tokio::time::Instant::now() < deadline)
214            .then_some((attempt.id, deadline))
215    }
216    pub(super) fn fenced_modules(&self) -> impl Iterator<Item = String> + '_ {
217        self.exited.keys().cloned()
218    }
219}
220
221/// Captured before a health RPC; only the same child and installed policy may
222/// consume its result. The fields are private so a completion cannot retarget it.
223#[derive(Clone, Debug)]
224pub struct DeliveryProbe {
225    connection: ConnectionId,
226    module: String,
227    generation: u64,
228    attempt: Option<u64>,
229    policy_epoch: u64,
230    route_readiness: bool,
231}
232
233impl DeliveryProbe {
234    pub(crate) fn connection(&self) -> ConnectionId {
235        self.connection
236    }
237    pub(crate) fn owns_failure(&self) -> bool {
238        self.route_readiness
239    }
240}
241
242#[derive(Clone)]
243pub(crate) struct ExitedDelivery {
244    pub module: String,
245    spawn: u64,
246    generation: u64,
247    changed: tokio::sync::watch::Sender<bool>,
248}
249
250#[derive(Clone, Debug, PartialEq, Eq)]
251pub enum AppliedDisposition {
252    Ignored,
253    WaitingForReadiness,
254    Active,
255    CatchupComplete,
256    Failed {
257        generation: u64,
258        restore: Option<u64>,
259        keep_running: bool,
260    },
261    CatchupFailed {
262        active: u64,
263    },
264}
265
266#[derive(Clone, Copy)]
267pub(crate) struct DeliveryWindow {
268    pub generation: u64,
269    pub attempt: u64,
270    pub delivered: bool,
271    pub deadline: Option<tokio::time::Instant>,
272}
273
274impl DeliveryState {
275    pub(super) fn invalidate_policy(&mut self, module: &str, generation: u64) {
276        for attempt in self
277            .attempts
278            .values_mut()
279            .filter(|attempt| attempt.module == module && attempt.generation == generation)
280        {
281            attempt.ready = false;
282            attempt.policy_probe_requested = false;
283        }
284    }
285    pub(super) fn policy_installed(
286        &mut self,
287        module: &str,
288        generation: u64,
289        target: Option<ConnectionId>,
290    ) -> Result<()> {
291        // The generation may be unchanged after a restore and retry. Use a
292        // separate epoch so a probe begun before this installation cannot count.
293        let epoch = self.policy_epochs.entry(module.into()).or_default();
294        *epoch = epoch.checked_add(1).ok_or_else(|| {
295            GateError::new(
296                "delivery_probe_exhausted",
297                "policy readiness epochs exhausted",
298            )
299        })?;
300        self.invalidate_policy(module, generation);
301        let connections: HashSet<_> = self
302            .children
303            .iter()
304            .filter(|(_, spawn)| spawn.module == module)
305            .map(|(connection, _)| *connection)
306            .chain(
307                self.attempts
308                    .iter()
309                    .filter(|(_, attempt)| attempt.module == module)
310                    .map(|(connection, _)| *connection),
311            )
312            .chain(target)
313            .collect();
314        self.readiness_observed.retain(|(connection, observed)| {
315            *observed != generation || !connections.contains(connection)
316        });
317        self.healthy_children.retain(|(connection, observed)| {
318            *observed != generation || !connections.contains(connection)
319        });
320        Ok(())
321    }
322    pub(super) fn declared(&mut self, connection: ConnectionId, module: &str, active: bool) {
323        if let Some(spawn) = self.unregistered_children.remove(&(module.into(), active)) {
324            self.children.insert(connection, spawn);
325        }
326    }
327    pub(super) fn new(initial_generation: u64) -> Self {
328        Self {
329            initial_generation,
330            ..Self::default()
331        }
332    }
333
334    /// Forget `connection`'s delivery state. Returns whether it had any.
335    pub(super) fn disconnected(&mut self, connection: ConnectionId) -> bool {
336        let mut held = self.attempts.remove(&connection).is_some();
337        held |= self.served.remove(&connection).is_some();
338        held |= self.unstamped_views.remove(&connection).is_some();
339        held |= self.stamp_running.remove(&connection);
340        held |= self.children.remove(&connection).is_some();
341        let observed = self.healthy_children.len() + self.readiness_observed.len();
342        self.healthy_children
343            .retain(|(child, _)| *child != connection);
344        self.readiness_observed
345            .retain(|(child, _)| *child != connection);
346        held || self.healthy_children.len() + self.readiness_observed.len() != observed
347    }
348
349    pub(super) fn remember_stamp_connections(&mut self, connections: Vec<ConnectionId>) {
350        self.stamp_running.extend(connections);
351    }
352
353    fn boundary_open(
354        &self,
355        connection: ConnectionId,
356        generation: u64,
357        transition: Transition,
358    ) -> bool {
359        if transition == Transition::DaemonRestart && generation > self.initial_generation {
360            return false;
361        }
362        !transition.requires_spawn()
363            || self
364                .children
365                .get(&connection)
366                .is_some_and(|spawn| spawn.generation == generation)
367    }
368
369    fn allocate(
370        &mut self,
371        module: &str,
372        generation: u64,
373        transition: Transition,
374    ) -> Result<Attempt> {
375        self.next_attempt = self.next_attempt.checked_add(1).ok_or_else(|| {
376            GateError::new(
377                "delivery_attempt_exhausted",
378                "delivery attempt ids exhausted",
379            )
380        })?;
381        Ok(Attempt {
382            module: module.into(),
383            generation,
384            id: self.next_attempt,
385            transition,
386            ready: transition == Transition::Live,
387            acknowledged: false,
388            delivered: false,
389            deadline: None,
390            catchup: false,
391            write_deadline: None,
392            policy_probe_requested: false,
393            spawn: None,
394            mark_only: false,
395        })
396    }
397}
398
399pub(super) fn visible_reply(
400    state: &OwnerState,
401    connection: ConnectionId,
402    mut reply: ConfigGetReply,
403    transition: Transition,
404) -> Result<ConfigGetReply> {
405    if reply.pending.as_ref().is_some_and(|view| {
406        !state
407            .delivery
408            .boundary_open(connection, view.generation, transition)
409    }) {
410        reply.pending = None;
411    }
412    if reply.active.is_none() && reply.pending.is_none() {
413        return Err(GateError::new(
414            "config_gated",
415            "generation waits for its required supervision or daemon boundary",
416        ));
417    }
418    Ok(reply)
419}
420
421pub(super) fn bind_get(
422    state: &mut OwnerState,
423    connection: ConnectionId,
424    module: &str,
425    request: &ConfigGetRequest,
426    reply: ConfigGetReply,
427    transition: Transition,
428) -> Result<ConfigGetReply> {
429    let mut reply = visible_reply(state, connection, reply, transition)?;
430    match (request.generation, request.attempt) {
431        (None, None) => {}
432        (Some(generation), Some(attempt))
433            if state
434                .delivery
435                .attempts
436                .get(&connection)
437                .is_some_and(|bound| {
438                    bound.module == module && bound.generation == generation && bound.id == attempt
439                }) => {}
440        _ => {
441            return Err(GateError::new(
442                "config_gated",
443                "requested generation and attempt are not assigned to this connection",
444            ))
445        }
446    }
447    if reply.pending.is_none()
448        && state
449            .delivery
450            .attempts
451            .get(&connection)
452            .is_some_and(|bound| {
453                bound.catchup
454                    && reply
455                        .active
456                        .as_ref()
457                        .is_some_and(|view| view.generation == bound.generation)
458            })
459    {
460        reply.pending = reply.active.clone();
461        if let Some(view) = &mut reply.pending {
462            view.status = ConfigStatus::Pending;
463        }
464    }
465    if reply.pending.is_none() && request.generation.is_none() {
466        let generation = reply.active.as_ref().map(|view| view.generation);
467        if let Some(attempt) = generation
468            .map(|generation| mark_acknowledgement(state, connection, module, generation))
469            .transpose()?
470            .flatten()
471        {
472            state.delivery.attempts.insert(connection, attempt);
473            reply.pending = reply.active.clone();
474            if let Some(view) = &mut reply.pending {
475                view.status = ConfigStatus::Pending;
476            }
477        }
478    }
479    if let Some(pending) = &mut reply.pending {
480        let matches = state
481            .delivery
482            .attempts
483            .get(&connection)
484            .is_some_and(|bound| bound.module == module && bound.generation == pending.generation);
485        if !matches {
486            let mut attempt = state
487                .delivery
488                .allocate(module, pending.generation, transition)?;
489            if transition.requires_spawn() {
490                attempt.ready = state
491                    .delivery
492                    .healthy_children
493                    .contains(&(connection, pending.generation));
494                attempt.deadline = state
495                    .delivery
496                    .children
497                    .get(&connection)
498                    .filter(|warmup| warmup.generation == pending.generation)
499                    .map(|warmup| warmup.deadline);
500            }
501            attempt.ready &= super::route_policy::ready(state, module, pending.generation)?;
502            attempt.spawn = state.delivery.children.get(&connection).cloned();
503            state.delivery.attempts.insert(connection, attempt);
504        }
505        pending.attempt = state
506            .delivery
507            .attempts
508            .get(&connection)
509            .map(|bound| bound.id);
510        let mark_only = state
511            .delivery
512            .attempts
513            .get(&connection)
514            .is_some_and(|bound| bound.mark_only);
515        if state.active_declarations.get(module) == Some(&connection) && !mark_only {
516            if let Some(spawn) = state.delivery.children.get(&connection) {
517                let id = spawn.id;
518                state
519                    .delivery
520                    .last_target_attempt
521                    .insert(module.into(), (id, pending.generation));
522            }
523        }
524    } else {
525        state.delivery.attempts.remove(&connection);
526    }
527    if request
528        .generation
529        .is_some_and(|id| reply.pending.as_ref().map(|view| view.generation) != Some(id))
530    {
531        return Err(GateError::new(
532            "config_gated",
533            "selected delivery has been superseded",
534        ));
535    }
536    Ok(reply)
537}
538
539/// The catch-up attempt that asks a fresh spawn to acknowledge its module's
540/// active generation, so the spawn can earn the first-party mark: a spawn is
541/// marked only after it acknowledged the active generation and reported
542/// ready, on every start path. Delivering
543/// the active view at launch is not proof the spawn applied it, and without a
544/// pending change a crash, health, manual or boot restart is otherwise never
545/// asked for an acknowledgement.
546///
547/// Assigned only to the active target of a stamped, gated module whose spawn
548/// carries an authority record that could still be marked, at most once per
549/// active generation; never when a pending change already has an attempt.
550/// Unstamped and ungated modules are never asked, so their delivery is
551/// unchanged. A refusal or timeout leaves the spawn unmarked and starts no
552/// recovery.
553fn mark_acknowledgement(
554    state: &mut OwnerState,
555    connection: ConnectionId,
556    module: &str,
557    generation: u64,
558) -> Result<Option<Attempt>> {
559    if state.store.loaded_admission_state().0 != crate::config_store::StoreState::Stamped
560        || state.active_declarations.get(module) != Some(&connection)
561        || state.delivery.attempts.contains_key(&connection)
562    {
563        return Ok(None);
564    }
565    let Some(spawn) = state
566        .delivery
567        .children
568        .get(&connection)
569        .filter(|spawn| spawn.stamped && spawn.module == module)
570        .cloned()
571    else {
572        return Ok(None);
573    };
574    let Some(record) = spawn.authority.as_ref() else {
575        return Ok(None);
576    };
577    let facts = record.mark_facts();
578    if !facts.unmarked || !facts.origin_qualifies || !record.request_acknowledgement(generation) {
579        return Ok(None);
580    }
581    let mut attempt = state
582        .delivery
583        .allocate(module, generation, Transition::Live)?;
584    attempt.catchup = true;
585    attempt.mark_only = true;
586    attempt.spawn = Some(spawn);
587    Ok(Some(attempt))
588}
589
590impl RuntimeOwner {
591    pub(crate) fn policy_probe_due(&self, connection: ConnectionId) -> Result<Option<(u64, u64)>> {
592        let mut state = self.lock()?;
593        let Some(bound) = state.delivery.attempts.get(&connection) else {
594            return Ok(None);
595        };
596        let (module, generation, attempt) = (bound.module.clone(), bound.generation, bound.id);
597        // A mark-only attempt asks for an acknowledgement and nothing else;
598        // readiness and route policy belong to the spawn's ordinary delivery.
599        if bound.ready
600            || bound.mark_only
601            || bound.policy_probe_requested
602            || state.active_declarations.get(&module) != Some(&connection)
603            || !state
604                .store
605                .body()?
606                .generations
607                .get(&generation)
608                .is_some_and(|entry| {
609                    super::route_policy::required(&entry.generation.changeset, &module)
610                })
611            || !super::route_policy::ready(&mut state, &module, generation)?
612        {
613            return Ok(None);
614        }
615        state
616            .delivery
617            .attempts
618            .get_mut(&connection)
619            .unwrap()
620            .policy_probe_requested = true;
621        Ok(Some((generation, attempt)))
622    }
623
624    pub(crate) fn policy_probe_pending(
625        &self,
626        connection: ConnectionId,
627        generation: u64,
628        attempt: u64,
629    ) -> Result<bool> {
630        let mut state = self.lock()?;
631        let Some(bound) = state.delivery.attempts.get(&connection) else {
632            return Ok(false);
633        };
634        let module = bound.module.clone();
635        Ok(bound.generation == generation
636            && bound.id == attempt
637            && !bound.ready
638            && state.active_declarations.get(&module) == Some(&connection)
639            && super::route_policy::ready(&mut state, &module, generation)?)
640    }
641    pub(crate) fn delivery_health_probe(
642        &self,
643        connection: ConnectionId,
644        module: &str,
645    ) -> Result<Option<DeliveryProbe>> {
646        let mut state = self.lock()?;
647        if state.active_declarations.get(module) != Some(&connection) {
648            return Ok(None);
649        }
650        let attempt = state
651            .delivery
652            .attempts
653            .get(&connection)
654            .map(|attempt| (attempt.generation, attempt.id));
655        let generation = attempt.map(|attempt| attempt.0).or_else(|| {
656            state
657                .delivery
658                .children
659                .get(&connection)
660                .map(|child| child.generation)
661        });
662        let route_readiness = match generation {
663            Some(generation) if attempt.is_some() => state
664                .store
665                .body()?
666                .generations
667                .get(&generation)
668                .is_some_and(|entry| {
669                    super::route_policy::required(&entry.generation.changeset, module)
670                }),
671            _ => false,
672        };
673        Ok(generation.map(|generation| DeliveryProbe {
674            connection,
675            module: module.into(),
676            generation,
677            attempt: attempt.map(|attempt| attempt.1),
678            route_readiness,
679            policy_epoch: state
680                .delivery
681                .policy_epochs
682                .get(module)
683                .copied()
684                .unwrap_or_default(),
685        }))
686    }
687
688    pub(crate) fn delivery_health_result(
689        &self,
690        probe: DeliveryProbe,
691        healthy: bool,
692    ) -> Result<Option<AppliedDisposition>> {
693        let attempt = {
694            let mut state = self.lock()?;
695            if state.active_declarations.get(&probe.module) != Some(&probe.connection)
696                || state
697                    .delivery
698                    .policy_epochs
699                    .get(&probe.module)
700                    .copied()
701                    .unwrap_or_default()
702                    != probe.policy_epoch
703            {
704                return Ok(None);
705            }
706            let current = state.delivery.attempts.get(&probe.connection);
707            if probe
708                .attempt
709                .is_some_and(|id| current.is_none_or(|bound| bound.id != id))
710                || current.is_some_and(|bound| bound.generation != probe.generation)
711            {
712                return Ok(None);
713            }
714            // A warm-up probe answering not ready is no answer to a mark-only
715            // attempt: only the spawn's own acknowledgement settles it, so a
716            // spawn still warming up is never refused the mark by a probe.
717            let attempt = current
718                .filter(|bound| !bound.mark_only)
719                .map(|bound| bound.id);
720            if !healthy
721                && (!super::route_policy::ready(&mut state, &probe.module, probe.generation)?
722                    || !state
723                        .store
724                        .body()?
725                        .generations
726                        .get(&probe.generation)
727                        .is_some_and(|entry| {
728                            super::route_policy::required(
729                                &entry.generation.changeset,
730                                &probe.module,
731                            )
732                        }))
733            {
734                return Ok(None);
735            }
736            attempt
737        };
738        if healthy {
739            self.delivery_readiness_complete(probe.connection, &probe.module, probe.generation)
740        } else if let Some(attempt) = attempt {
741            self.module_outcome(
742                probe.connection,
743                &probe.module,
744                ConfigAppliedRequest::new(
745                    probe.generation,
746                    attempt,
747                    subc_protocol::config::ConfigApplyResult::Nack,
748                    vec![],
749                ),
750                true,
751            )
752            .map(Some)
753        } else {
754            Ok(None)
755        }
756    }
757
758    #[cfg(test)]
759    pub(super) fn ready(
760        &self,
761        connection: ConnectionId,
762        module: &str,
763        generation: u64,
764    ) -> Result<Option<AppliedDisposition>> {
765        self.delivery_readiness_complete(connection, module, generation)
766    }
767
768    pub(crate) fn pending_config_restarts(&self) -> Result<Vec<(String, u64)>> {
769        let mut state = self.lock()?;
770        state.store_read(ConfigStore::delivery_ready)?;
771        let body = state.checked_body()?;
772        body.versions
773            .iter()
774            .filter(|(tuple, _)| tuple.project.is_none())
775            .filter_map(|(tuple, versions)| {
776                let generation = versions.pending?;
777                let entry = body.generations.get(&generation)?;
778                Some(
779                    Transition::for_module(&entry.generation.changeset, &tuple.module).map(
780                        |transition| {
781                            (transition == Transition::ModuleRestart)
782                                .then(|| (tuple.module.clone(), generation))
783                        },
784                    ),
785                )
786            })
787            .collect::<Result<Vec<_>>>()
788            .map(|entries| entries.into_iter().flatten().collect())
789    }
790
791    pub(crate) fn requires_fresher_spawn(
792        &self,
793        module: &str,
794        spawn: Option<&DeliverySpawn>,
795    ) -> Result<Option<u64>> {
796        let mut state = self.lock()?;
797        state.store.delivery_ready()?;
798        Self::requires_fresher_spawn_loaded(&state, module, spawn)
799    }
800
801    pub(super) fn requires_fresher_spawn_loaded(
802        state: &OwnerState,
803        module: &str,
804        spawn: Option<&DeliverySpawn>,
805    ) -> Result<Option<u64>> {
806        let body = state.store.loaded_admission_state().1;
807        let pending = body
808            .versions
809            .iter()
810            .find(|(tuple, _)| tuple.module == module && tuple.project.is_none())
811            .and_then(|(_, versions)| versions.pending);
812        let Some(generation) = pending else {
813            return Ok(None);
814        };
815        let entry = body
816            .generations
817            .get(&generation)
818            .ok_or_else(|| GateError::new("config_gated", "pending generation is absent"))?;
819        let transition = Transition::for_module(&entry.generation.changeset, module)?;
820        let deferred_daemon = transition == Transition::DaemonRestart
821            && generation > state.delivery.initial_generation;
822        Ok((transition.requires_spawn()
823            && !deferred_daemon
824            && spawn
825                .is_none_or(|spawn| spawn.module != module || spawn.generation() != generation))
826        .then_some(generation))
827    }
828
829    /// A supervised process exit, not a transport disconnect, fails warm-up.
830    /// The synchronous fence closes the gap before the async failure commit.
831    pub(crate) fn delivery_child_exited(&self, spawn: &DeliverySpawn) -> Result<()> {
832        let mut state = self.lock()?;
833        let pending = state
834            .store
835            .body()?
836            .versions
837            .iter()
838            .find(|(tuple, _)| tuple.module == spawn.module && tuple.project.is_none())
839            .and_then(|(_, versions)| versions.pending);
840        let pending = pending.filter(|generation| {
841            state
842                .store
843                .loaded_admission_state()
844                .1
845                .generations
846                .get(generation)
847                .and_then(|entry| entry.delivery.as_ref())
848                .is_some_and(|delivery| delivery.declarations.contains_key(&spawn.module))
849        });
850        if let Some(generation) = pending.filter(|generation| {
851            *generation == spawn.generation
852                || state.delivery.last_target_attempt.get(&spawn.module)
853                    == Some(&(spawn.id, *generation))
854        }) {
855            state
856                .delivery
857                .exited
858                .entry(spawn.module.clone())
859                .or_insert_with(|| ExitedDelivery {
860                    module: spawn.module.clone(),
861                    spawn: spawn.id,
862                    generation,
863                    changed: tokio::sync::watch::channel(true).0,
864                });
865        }
866        self.notify_delivery_changed();
867        Ok(())
868    }
869
870    pub(crate) fn delivery_exit_fence(
871        &self,
872        module: &str,
873    ) -> Result<Option<tokio::sync::watch::Receiver<bool>>> {
874        Ok(self
875            .lock()?
876            .delivery
877            .exited
878            .get(module)
879            .map(|entry| entry.changed.subscribe()))
880    }
881
882    pub(crate) fn delivery_exits(&self) -> Result<Vec<ExitedDelivery>> {
883        Ok(self.lock()?.delivery.exited.values().cloned().collect())
884    }
885
886    /// The delivery worker holds this module's asynchronous configuration lock
887    /// while recording the failure. Respawn stays blocked until the failure has
888    /// restored the previous active generation or, if none exists, the supervisor
889    /// has canceled respawn and stopped the module.
890    /// `AnchorPending` still restores the previous active generation in memory;
891    /// its durable failure record is retried before delivery can resume.
892    pub(crate) fn delivery_record_exit(&self, exited: &ExitedDelivery) -> Result<bool> {
893        #[cfg(test)]
894        self.exit_record_attempts
895            .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
896        let mut state = self.lock()?;
897        if !state
898            .delivery
899            .exited
900            .get(&exited.module)
901            .is_some_and(|entry| entry.spawn == exited.spawn)
902        {
903            return Ok(false);
904        }
905        let tuple = Tuple {
906            module: exited.module.clone(),
907            project: None,
908        };
909        let versions = state
910            .store
911            .body()?
912            .versions
913            .iter()
914            .find(|(key, _)| key == &tuple)
915            .map(|(_, versions)| versions.clone())
916            .unwrap_or_default();
917        if versions
918            .pending
919            .is_some_and(|generation| generation != exited.generation)
920        {
921            return Ok(false);
922        }
923        if versions.pending == Some(exited.generation) {
924            state.restore_route_policy(&exited.module, exited.generation);
925            let result = state
926                .store
927                .record_delivery(&tuple, exited.generation, false);
928            if let Err(error) = result {
929                if error.code != crate::config_store::StoreErrorCode::AnchorPending {
930                    return Err(error.into());
931                }
932            }
933            state.delivery.attempts.retain(|_, attempt| {
934                attempt.module != exited.module || attempt.generation != exited.generation
935            });
936        }
937        Ok(versions.active.is_none())
938    }
939
940    #[cfg(all(test, unix))]
941    pub(crate) fn exit_record_attempts_for_test(&self) -> usize {
942        self.exit_record_attempts
943            .load(std::sync::atomic::Ordering::SeqCst)
944    }
945
946    pub(crate) fn delivery_release_exit(&self, exited: &ExitedDelivery) -> Result<()> {
947        let mut state = self.lock()?;
948        if state
949            .delivery
950            .exited
951            .get(&exited.module)
952            .is_some_and(|entry| entry.spawn == exited.spawn)
953        {
954            if let Some(entry) = state.delivery.exited.remove(&exited.module) {
955                entry.changed.send_replace(false);
956                // A prior release may have reached the worker while this fence
957                // still blocked admission. Recheck it after the fence is gone;
958                // the worker starts only enabled modules with no remaining hold.
959                state.admission_releases.push(exited.module.clone());
960            }
961        }
962        self.notify_delivery_changed();
963        Ok(())
964    }
965    pub(crate) fn delivery_abandon_spawn(&self, spawn: &DeliverySpawn, active: bool) {
966        if let Ok(mut state) = self.lock() {
967            let key = (spawn.module.clone(), active);
968            if state
969                .delivery
970                .unregistered_children
971                .get(&key)
972                .is_some_and(|current| current.id == spawn.id)
973            {
974                state.delivery.unregistered_children.remove(&key);
975            }
976        }
977    }
978    pub(crate) fn delivery_queue_abandoned_spawn(&self, spawn: DeliverySpawn, active: bool) {
979        if let Ok(mut abandoned) = self.abandoned_spawns.lock() {
980            abandoned.push((spawn, active));
981        }
982        self.notify_delivery_changed();
983    }
984    /// Forget spawns abandoned while the owner was busy. Returns whether any
985    /// was queued.
986    pub(super) fn drain_abandoned_spawns(&self, state: &mut OwnerState) -> bool {
987        let mut drained = false;
988        if let Ok(mut abandoned) = self.abandoned_spawns.lock() {
989            for (spawn, active) in abandoned.drain(..) {
990                drained = true;
991                let key = (spawn.module.clone(), active);
992                if state
993                    .delivery
994                    .unregistered_children
995                    .get(&key)
996                    .is_some_and(|current| current.id == spawn.id)
997                {
998                    state.delivery.unregistered_children.remove(&key);
999                }
1000            }
1001        }
1002        drained
1003    }
1004
1005    /// The full authority check and reservation ran on a blocking worker while
1006    /// the caller owned the module gate. Recheck only that exact witness here;
1007    /// contention asks the async caller to wait off the runtime thread.
1008    pub(crate) fn delivery_launch_current(
1009        &self,
1010        spawn: &DeliverySpawn,
1011        active: bool,
1012    ) -> Result<Option<bool>> {
1013        let state = match self.inner.try_lock() {
1014            Ok(state) => state,
1015            Err(std::sync::TryLockError::WouldBlock) => return Ok(None),
1016            Err(_) => {
1017                return Err(GateError::new(
1018                    "config_unavailable",
1019                    "configuration owner lock is poisoned",
1020                ))
1021            }
1022        };
1023        Ok(Some(
1024            super::admission::AdmissionCache::from_state(&state)
1025                .spawn_generation(&spawn.module)
1026                .is_ok_and(|generation| generation == spawn.generation)
1027                && state
1028                    .delivery
1029                    .unregistered_children
1030                    .get(&(spawn.module.clone(), active))
1031                    .is_some_and(|current| current.id == spawn.id)
1032                && !state.delivery.exited.contains_key(&spawn.module),
1033        ))
1034    }
1035    #[cfg(test)]
1036    pub(crate) fn pause_spawn_preflight_for_test(&self) {
1037        let hook = self.spawn_preflight_hook.lock().unwrap().take();
1038        if let Some(hook) = hook {
1039            hook();
1040        }
1041    }
1042    #[cfg(all(test, unix))]
1043    pub(crate) fn exhaust_delivery_spawns_for_test(&self) {
1044        self.lock().unwrap().delivery.next_spawn = u64::MAX;
1045    }
1046    #[cfg(all(test, unix))]
1047    pub(crate) fn restore_unused_delivery_spawns_for_test(&self) {
1048        self.lock().unwrap().delivery.next_spawn = 1000;
1049    }
1050    #[cfg(all(test, unix))]
1051    pub(crate) fn fail_delivery_spawns_for_module_for_test(&self, module: &str) {
1052        self.lock().unwrap().delivery.failed_spawn_module = Some(module.into());
1053    }
1054    #[cfg(all(test, unix))]
1055    pub(crate) fn clear_delivery_spawn_failure_for_test(&self) {
1056        self.lock().unwrap().delivery.failed_spawn_module = None;
1057    }
1058    #[cfg(all(test, unix))]
1059    pub(crate) fn set_spawn_preflight_hook_for_test(&self, hook: impl FnOnce() + Send + 'static) {
1060        *self.spawn_preflight_hook.lock().unwrap() = Some(Box::new(hook));
1061    }
1062    #[cfg(all(test, unix))]
1063    pub(crate) fn unregistered_spawn_count_for_test(&self) -> usize {
1064        self.lock().unwrap().delivery.unregistered_children.len()
1065    }
1066    #[allow(clippy::too_many_arguments)]
1067    pub(crate) fn delivery_spawned_for_admission_in_slot(
1068        &self,
1069        module: &str,
1070        admission: &super::admission::SpawnAdmission,
1071        at: tokio::time::Instant,
1072        register: Duration,
1073        health: Duration,
1074        active: bool,
1075        authority: Option<std::sync::Arc<crate::spawn_authority::SpawnAuthorityRecord>>,
1076    ) -> Result<Option<DeliverySpawn>> {
1077        let mut state = OwnerGuard::new(
1078            self.inner.try_lock().map_err(|error| {
1079                if matches!(error, std::sync::TryLockError::Poisoned(_)) {
1080                    return GateError::new("config_unavailable", "configuration owner is poisoned");
1081                }
1082                GateError::new(
1083                    "config_delivery_deferred",
1084                    "configuration publication is busy",
1085                )
1086            })?,
1087            self,
1088        );
1089        state.store.check()?;
1090        // Admission under the stamped gate is the only way to the mark: a
1091        // stamp never marks a child admitted before it.
1092        let stamped = admission.gated()
1093            && state.store.loaded_admission_state().0 == crate::config_store::StoreState::Stamped;
1094        let spawn = allocate_spawn(
1095            &mut state,
1096            module,
1097            admission.generation(),
1098            at,
1099            register,
1100            health,
1101            authority.map(|record| (record, stamped)),
1102            admission.admitted_program(),
1103        )?;
1104        state.delivery.track_mark(&spawn);
1105        state
1106            .delivery
1107            .unregistered_children
1108            .insert((module.into(), active), spawn.clone());
1109        Ok(Some(spawn))
1110    }
1111
1112    #[cfg(test)]
1113    pub(crate) fn delivery_spawned_in_slot(
1114        &self,
1115        module: &str,
1116        at: tokio::time::Instant,
1117        register: Duration,
1118        health: Duration,
1119        active: bool,
1120    ) -> Result<Option<DeliverySpawn>> {
1121        let spawn = self.delivery_spawned(module, at, register, health)?;
1122        if let Some(spawn) = &spawn {
1123            self.lock()?
1124                .delivery
1125                .unregistered_children
1126                .insert((module.into(), active), spawn.clone());
1127        }
1128        Ok(spawn)
1129    }
1130    pub(crate) fn delivery_take_recoveries(
1131        &self,
1132    ) -> Result<Vec<(ConnectionId, String, AppliedDisposition)>> {
1133        let mut state = self.lock()?;
1134        if state.delivery.recoveries.is_empty() {
1135            // Nothing to take: leave the owner unchanged.
1136            return Ok(Vec::new());
1137        }
1138        Ok(std::mem::take(&mut state.delivery.recoveries))
1139    }
1140
1141    pub(crate) fn delivery_window(
1142        &self,
1143        connection: ConnectionId,
1144    ) -> Result<Option<DeliveryWindow>> {
1145        Ok(self
1146            .lock()?
1147            .delivery
1148            .attempts
1149            .get(&connection)
1150            .map(|bound| DeliveryWindow {
1151                generation: bound.generation,
1152                attempt: bound.id,
1153                delivered: bound.delivered,
1154                deadline: bound.deadline,
1155            }))
1156    }
1157
1158    pub(crate) fn delivery_write_started(
1159        &self,
1160        connection: ConnectionId,
1161        generation: u64,
1162        attempt: u64,
1163        now: tokio::time::Instant,
1164    ) -> Result<()> {
1165        if let Some(bound) = self
1166            .lock()?
1167            .delivery
1168            .attempts
1169            .get_mut(&connection)
1170            .filter(|bound| {
1171                bound.generation == generation && bound.id == attempt && !bound.delivered
1172            })
1173        {
1174            bound
1175                .write_deadline
1176                .get_or_insert(now + CONFIG_LIVE_ACK_TIMEOUT);
1177        }
1178        Ok(())
1179    }
1180
1181    pub(crate) fn delivery_timeout(
1182        &self,
1183        connection: ConnectionId,
1184        module: &str,
1185        generation: u64,
1186        attempt: u64,
1187    ) -> Result<AppliedDisposition> {
1188        self.module_outcome(
1189            connection,
1190            module,
1191            ConfigAppliedRequest::new(
1192                generation,
1193                attempt,
1194                subc_protocol::config::ConfigApplyResult::Nack,
1195                vec![],
1196            ),
1197            true,
1198        )
1199    }
1200
1201    /// Capture the common spawn instant, not the later HELLO instant, so a slow
1202    /// registration cannot extend the approved generation's warm-up deadline.
1203    pub(crate) fn delivery_spawned(
1204        &self,
1205        module: &str,
1206        spawned_at: tokio::time::Instant,
1207        register_timeout: Duration,
1208        first_health_budget: Duration,
1209    ) -> Result<Option<DeliverySpawn>> {
1210        let mut state = self.lock()?;
1211        state.store.check()?;
1212        let admission = super::admission::AdmissionCache::from_state(&state);
1213        let generation = admission.spawn_generation(module)?;
1214        allocate_spawn(
1215            &mut state,
1216            module,
1217            generation,
1218            spawned_at,
1219            register_timeout,
1220            first_health_budget,
1221            None,
1222            admission.admitted_program(module),
1223        )
1224        .map(Some)
1225    }
1226
1227    pub(crate) fn delivery_child_registered(
1228        &self,
1229        connection: ConnectionId,
1230        spawn: &DeliverySpawn,
1231    ) -> Result<()> {
1232        let mut state = self.lock()?;
1233        state.delivery.track_mark(spawn);
1234        state.delivery.children.insert(connection, spawn.clone());
1235        if let Some(bound) = state
1236            .delivery
1237            .attempts
1238            .get_mut(&connection)
1239            .filter(|bound| bound.module == spawn.module && bound.generation == spawn.generation)
1240        {
1241            if bound.transition.requires_spawn() {
1242                bound.deadline = Some(spawn.deadline);
1243            }
1244        }
1245        Ok(())
1246    }
1247
1248    pub(crate) fn module_view_delivered(
1249        &self,
1250        connection: ConnectionId,
1251        module: &str,
1252        reply: &ConfigGetReply,
1253        written_at: tokio::time::Instant,
1254        active: bool,
1255    ) -> Result<()> {
1256        {
1257            let mut state = self.lock()?;
1258            if !state
1259                .declarations
1260                .get(&connection)
1261                .is_some_and(|(id, _)| id == module)
1262            {
1263                return Err(GateError::new(
1264                    "not_delivery_target",
1265                    "configuration connection has ended",
1266                ));
1267            }
1268            let body = state.store.body()?;
1269            if body.stamped {
1270                let versions = body
1271                    .versions
1272                    .iter()
1273                    .find(|(tuple, _)| tuple.module == module && tuple.project.is_none())
1274                    .map(|(_, versions)| versions.clone())
1275                    .unwrap_or_default();
1276                if reply
1277                    .active
1278                    .as_ref()
1279                    .is_some_and(|view| versions.active != Some(view.generation))
1280                    || reply.pending.as_ref().is_some_and(|view| {
1281                        // A promoted child catches up to the durable active
1282                        // vector through its own pending, child-local attempt.
1283                        let catchup = versions.active == Some(view.generation)
1284                            && state
1285                                .delivery
1286                                .attempts
1287                                .get(&connection)
1288                                .is_some_and(|bound| {
1289                                    bound.catchup
1290                                        && bound.generation == view.generation
1291                                        && Some(bound.id) == view.attempt
1292                                });
1293                        versions.pending != Some(view.generation) && !catchup
1294                    })
1295                {
1296                    return Err(GateError::new(
1297                        "delivery_superseded",
1298                        "configuration changed while its view was being written",
1299                    ));
1300                }
1301            }
1302        }
1303        if active {
1304            if let Some(view) = &reply.pending {
1305                if let Some(attempt) = view.attempt {
1306                    self.delivery_written(connection, view.generation, attempt, written_at)?;
1307                }
1308            }
1309        }
1310        let mut state = self.lock()?;
1311        if !state
1312            .declarations
1313            .get(&connection)
1314            .is_some_and(|(id, _)| id == module)
1315        {
1316            return Err(GateError::new(
1317                "not_delivery_target",
1318                "configuration connection has ended",
1319            ));
1320        }
1321        state.delivery.served.insert(
1322            connection,
1323            (
1324                reply.active.as_ref().map(|view| view.generation),
1325                reply.pending.as_ref().map(|view| view.generation),
1326            ),
1327        );
1328        if let Some(view) = reply
1329            .active
1330            .as_ref()
1331            .filter(|view| view.status == ConfigStatus::Unstamped)
1332        {
1333            state
1334                .delivery
1335                .unstamped_views
1336                .insert(connection, std::sync::Arc::new(view.clone()));
1337        } else {
1338            state.delivery.unstamped_views.remove(&connection);
1339        }
1340        Ok(())
1341    }
1342    /// Completion is supplied by the supervisor after registration, the first
1343    /// healthy probe and any required route teardown, never by a module body.
1344    pub(crate) fn delivery_readiness_complete(
1345        &self,
1346        connection: ConnectionId,
1347        module: &str,
1348        generation: u64,
1349    ) -> Result<Option<AppliedDisposition>> {
1350        let attempt = {
1351            let mut state = self.lock()?;
1352            if state.active_declarations.get(module) != Some(&connection) {
1353                return Err(GateError::new(
1354                    "not_delivery_target",
1355                    "readiness belongs to another connection",
1356                ));
1357            }
1358            let started_for_generation = state
1359                .delivery
1360                .children
1361                .get(&connection)
1362                .is_some_and(|warmup| warmup.generation == generation);
1363            if started_for_generation {
1364                state
1365                    .delivery
1366                    .healthy_children
1367                    .insert((connection, generation));
1368            }
1369            let policy_ready = super::route_policy::ready(&mut state, module, generation)?;
1370            let Some(bound) = state
1371                .delivery
1372                .attempts
1373                .get_mut(&connection)
1374                .filter(|bound| bound.module == module && bound.generation == generation)
1375            else {
1376                return Ok(None);
1377            };
1378            if bound.transition.requires_spawn() && !started_for_generation {
1379                return Ok(None);
1380            }
1381            bound.ready = policy_ready;
1382            let acknowledged = bound.ready && bound.acknowledged;
1383            let id = bound.id;
1384            state
1385                .delivery
1386                .readiness_observed
1387                .insert((connection, generation));
1388            acknowledged.then_some(id)
1389        };
1390        attempt
1391            .map(|attempt| {
1392                self.module_applied(
1393                    connection,
1394                    module,
1395                    ConfigAppliedRequest::new(
1396                        generation,
1397                        attempt,
1398                        subc_protocol::config::ConfigApplyResult::Ack,
1399                        vec![],
1400                    ),
1401                )
1402            })
1403            .transpose()
1404    }
1405
1406    /// Both forwarding and registry promotion happen under the caller's module
1407    /// gate before this connection becomes an acknowledgement target.
1408    pub(crate) fn delivery_promoted(
1409        &self,
1410        connection: ConnectionId,
1411        module: &str,
1412        cutover_at: tokio::time::Instant,
1413    ) -> Result<Option<(u64, u64)>> {
1414        let mut state = self.lock()?;
1415        state.store.delivery_ready()?;
1416        Self::promote_loaded(&mut state, connection, module, cutover_at)
1417    }
1418
1419    pub(super) fn promote_loaded(
1420        state: &mut OwnerState,
1421        connection: ConnectionId,
1422        module: &str,
1423        cutover_at: tokio::time::Instant,
1424    ) -> Result<Option<(u64, u64)>> {
1425        if state.store.loaded_admission_state().0 == crate::config_store::StoreState::Unstamped {
1426            if state
1427                .declarations
1428                .get(&connection)
1429                .is_some_and(|(id, _)| id == module)
1430            {
1431                state.phone_target_changed(module, connection);
1432                state.active_declarations.insert(module.into(), connection);
1433            }
1434            return Ok(None);
1435        }
1436        let Some((id, declaration)) = state
1437            .declarations
1438            .get(&connection)
1439            .cloned()
1440            .filter(|(id, _)| id == module)
1441        else {
1442            return Err(GateError::new(
1443                "config_gated",
1444                "candidate has not declared its approved configuration",
1445            ));
1446        };
1447        let tuple = Tuple {
1448            module: module.into(),
1449            project: None,
1450        };
1451        let body = state.store.loaded_admission_state().1;
1452        let versions = body
1453            .versions
1454            .iter()
1455            .find(|(key, _)| key == &tuple)
1456            .map(|(_, versions)| versions.clone())
1457            .unwrap_or_default();
1458        let transition = versions
1459            .pending
1460            .and_then(|id| body.generations.get(&id))
1461            .map(|entry| Transition::for_module(&entry.generation.changeset, module))
1462            .transpose()?
1463            .unwrap_or(Transition::Live);
1464        let pending_declaration = versions
1465            .pending
1466            .and_then(|id| body.generations.get(&id))
1467            .and_then(|entry| entry.delivery.as_ref())
1468            .and_then(|snapshot| snapshot.declarations.get(module))
1469            .cloned();
1470        let active_declaration = versions
1471            .active
1472            .and_then(|id| body.generations.get(&id))
1473            .and_then(|entry| entry.delivery.as_ref())
1474            .and_then(|snapshot| snapshot.declarations.get(module))
1475            .cloned();
1476        let pending = versions.pending.filter(|generation| {
1477            state
1478                .delivery
1479                .boundary_open(connection, *generation, transition)
1480        });
1481        if pending != versions.pending && transition != Transition::DaemonRestart {
1482            return Err(GateError::new(
1483                "config_gated",
1484                "candidate launch was superseded by a configuration requiring a fresh start",
1485            ));
1486        }
1487        let expected = if pending.is_some() {
1488            pending_declaration
1489        } else {
1490            active_declaration
1491        };
1492        if expected.as_ref() != Some(&declaration) {
1493            return Err(GateError::new(
1494                "config_gated",
1495                "candidate declaration differs from the approved declaration",
1496            ));
1497        }
1498        let transition = if pending.is_some() {
1499            transition
1500        } else {
1501            Transition::Live
1502        };
1503        let vector = (versions.active, pending);
1504        let catchup = pending.is_none()
1505            && versions.active.is_some()
1506            && state.delivery.served.get(&connection) != Some(&vector);
1507        state.phone_target_changed(module, connection);
1508        state.active_declarations.insert(id, connection);
1509        state.delivery.attempts.remove(&connection);
1510        let Some(generation) = pending.or_else(|| catchup.then_some(versions.active).flatten())
1511        else {
1512            return Ok(None);
1513        };
1514        let mut attempt = state.delivery.allocate(module, generation, transition)?;
1515        attempt.catchup = catchup;
1516        attempt.spawn = state.delivery.children.get(&connection).cloned();
1517        // Swap readiness completed before promotion. Its new target now gets
1518        // a full live ACK window, independently of the finished spawn budget.
1519        // A route obligation still needs its own post-install probe: the
1520        // candidate's earlier HEALTHY did not bind the installed policy epoch.
1521        attempt.ready = super::route_policy::ready_loaded(state, module, generation)
1522            && !state
1523                .store
1524                .loaded_admission_state()
1525                .1
1526                .generations
1527                .get(&generation)
1528                .is_some_and(|entry| {
1529                    super::route_policy::required(&entry.generation.changeset, module)
1530                });
1531        attempt.deadline = Some(cutover_at + CONFIG_LIVE_ACK_TIMEOUT);
1532        let assignment = (generation, attempt.id);
1533        state.delivery.attempts.insert(connection, attempt);
1534        state
1535            .delivery
1536            .readiness_observed
1537            .insert((connection, generation));
1538        Ok(Some(assignment))
1539    }
1540
1541    /// The control adapter calls this only for the active target while holding
1542    /// its module gate, after the delivery frame's writer flush succeeds.
1543    pub(crate) fn delivery_written(
1544        &self,
1545        connection: ConnectionId,
1546        generation: u64,
1547        attempt: u64,
1548        written_at: tokio::time::Instant,
1549    ) -> Result<()> {
1550        let mut state = self.lock()?;
1551        let bound = state
1552            .delivery
1553            .attempts
1554            .get_mut(&connection)
1555            .filter(|bound| bound.generation == generation && bound.id == attempt)
1556            .ok_or_else(|| {
1557                GateError::new(
1558                    "delivery_superseded",
1559                    "writer receipt no longer names the current delivery attempt",
1560                )
1561            })?;
1562        if bound
1563            .write_deadline
1564            .is_some_and(|deadline| written_at >= deadline)
1565            || bound
1566                .deadline
1567                .is_some_and(|deadline| written_at >= deadline)
1568        {
1569            return Err(GateError::new(
1570                "delivery_expired",
1571                "configuration delivery timed out",
1572            ));
1573        }
1574        bound.delivered = true;
1575        if !bound.transition.requires_spawn() && bound.deadline.is_none() {
1576            bound.deadline = Some(written_at + CONFIG_LIVE_ACK_TIMEOUT);
1577        }
1578        Ok(())
1579    }
1580
1581    pub(crate) fn module_applied(
1582        &self,
1583        connection: ConnectionId,
1584        module: &str,
1585        request: ConfigAppliedRequest,
1586    ) -> Result<AppliedDisposition> {
1587        self.module_outcome(connection, module, request, false)
1588    }
1589
1590    fn module_outcome(
1591        &self,
1592        connection: ConnectionId,
1593        module: &str,
1594        request: ConfigAppliedRequest,
1595        timed_out: bool,
1596    ) -> Result<AppliedDisposition> {
1597        let mut state = self.lock()?;
1598        if !state
1599            .declarations
1600            .get(&connection)
1601            .is_some_and(|(id, _)| id == module)
1602        {
1603            return Err(GateError::new(
1604                "declaration_order",
1605                "config.declare must precede get or applied on this connection",
1606            ));
1607        }
1608        if request.project_id.is_some() {
1609            return Err(GateError::new(
1610                "config_gated",
1611                "project-tuple delivery is not supported yet",
1612            ));
1613        }
1614        if state.active_declarations.get(module) != Some(&connection) {
1615            return Err(GateError::new(
1616                "not_delivery_target",
1617                "connection is not the active delivery target",
1618            ));
1619        }
1620        state.store.check()?;
1621        let Some(bound) = state
1622            .delivery
1623            .attempts
1624            .get(&connection)
1625            .cloned()
1626            .filter(|bound| {
1627                bound.module == module
1628                    && bound.generation == request.generation
1629                    && bound.id == request.attempt
1630            })
1631        else {
1632            return Ok(AppliedDisposition::Ignored);
1633        };
1634        if !bound.delivered && !timed_out {
1635            return Ok(AppliedDisposition::Ignored);
1636        }
1637        let tuple = Tuple {
1638            module: module.into(),
1639            project: None,
1640        };
1641        let versions = state
1642            .store
1643            .body()?
1644            .versions
1645            .iter()
1646            .find(|(key, _)| key == &tuple)
1647            .map(|(_, versions)| versions.clone())
1648            .unwrap_or_default();
1649        let ack = request.result.is_ack()
1650            && !bound
1651                .deadline
1652                .is_some_and(|deadline| tokio::time::Instant::now() >= deadline);
1653        if bound.catchup {
1654            // A later approval waiting for a new daemon cannot replace the
1655            // active configuration this child is catching up to now.
1656            let pending_transition = versions
1657                .pending
1658                .map(|generation| {
1659                    let entry = state
1660                        .store
1661                        .body()?
1662                        .generations
1663                        .get(&generation)
1664                        .ok_or_else(|| {
1665                            GateError::new("config_gated", "pending generation is absent")
1666                        })?;
1667                    Transition::for_module(&entry.generation.changeset, module)
1668                })
1669                .transpose()?;
1670            let pending_is_deliverable =
1671                versions
1672                    .pending
1673                    .zip(pending_transition)
1674                    .is_some_and(|(generation, transition)| {
1675                        state
1676                            .delivery
1677                            .boundary_open(connection, generation, transition)
1678                    });
1679            if pending_is_deliverable || versions.active != Some(bound.generation) {
1680                return Ok(AppliedDisposition::Ignored);
1681            }
1682            state.delivery.attempts.remove(&connection);
1683            if bound.mark_only {
1684                // Asked only for the mark: a refusal or timeout leaves the
1685                // spawn unmarked and starts no recovery.
1686                return Ok(if ack {
1687                    super::mark::acknowledged(self, &state, bound.spawn.as_ref(), bound.generation);
1688                    AppliedDisposition::CatchupComplete
1689                } else {
1690                    super::mark::acknowledgement_refused(self, &state, bound.spawn.as_ref());
1691                    AppliedDisposition::Ignored
1692                });
1693            }
1694            if ack {
1695                super::mark::acknowledged(self, &state, bound.spawn.as_ref(), bound.generation);
1696            } else {
1697                super::mark::delivery_failed(self, bound.spawn.as_ref());
1698                state.delivery.recoveries.push((
1699                    connection,
1700                    module.into(),
1701                    AppliedDisposition::CatchupFailed {
1702                        active: bound.generation,
1703                    },
1704                ));
1705            }
1706            return Ok(if ack {
1707                AppliedDisposition::CatchupComplete
1708            } else {
1709                AppliedDisposition::CatchupFailed {
1710                    active: bound.generation,
1711                }
1712            });
1713        }
1714        if versions.pending != Some(bound.generation) {
1715            return Ok(AppliedDisposition::Ignored);
1716        }
1717        if ack
1718            && (!bound.ready
1719                || (bound.transition == Transition::DaemonRestart
1720                    && bound.generation > state.delivery.initial_generation))
1721        {
1722            if let Some(bound) = state.delivery.attempts.get_mut(&connection) {
1723                bound.acknowledged = true;
1724            }
1725            return Ok(AppliedDisposition::WaitingForReadiness);
1726        }
1727        if !ack {
1728            state.restore_route_policy(module, bound.generation);
1729        }
1730        let result = state.store.record_delivery(&tuple, bound.generation, ack);
1731        if result.is_ok() && ack {
1732            state.commit_route_policy(module, bound.generation);
1733        }
1734        // A pending-anchor NACK has already restored memory. Discard its attempt
1735        // even though the durable failure must wait for the anchor retry.
1736        if result.is_ok() || !ack {
1737            state.delivery.attempts.remove(&connection);
1738        }
1739        if !ack {
1740            let keep_running =
1741                versions.active.is_none() && state.delivery.stamp_running.contains(&connection);
1742            state.delivery.recoveries.push((
1743                connection,
1744                module.into(),
1745                AppliedDisposition::Failed {
1746                    generation: bound.generation,
1747                    restore: versions.active,
1748                    keep_running,
1749                },
1750            ));
1751        }
1752        if ack && result.is_ok() {
1753            super::mark::acknowledged(self, &state, bound.spawn.as_ref(), bound.generation);
1754        } else if !ack {
1755            super::mark::delivery_failed(self, bound.spawn.as_ref());
1756        }
1757        result?;
1758        Ok(if ack {
1759            AppliedDisposition::Active
1760        } else {
1761            AppliedDisposition::Failed {
1762                generation: bound.generation,
1763                restore: versions.active,
1764                keep_running: versions.active.is_none()
1765                    && state.delivery.stamp_running.contains(&connection),
1766            }
1767        })
1768    }
1769}