1use std::collections::HashSet;
4use subc_protocol::config::{ConfigAppliedRequest, ConfigGetReply, ConfigGetRequest};
5
6use super::*;
7use crate::config_gate::apply::Transition;
8use crate::config_store::Tuple;
9
10pub(crate) const CONFIG_LIVE_ACK_TIMEOUT: Duration = Duration::from_secs(30);
11pub(crate) const CONFIG_WARMUP_SLACK: Duration = Duration::from_secs(10);
12
13#[allow(clippy::too_many_arguments)]
14fn allocate_spawn(
15 state: &mut OwnerState,
16 module: &str,
17 generation: u64,
18 at: tokio::time::Instant,
19 register: Duration,
20 health: Duration,
21 authority: Option<(
22 std::sync::Arc<crate::spawn_authority::SpawnAuthorityRecord>,
23 bool,
24 )>,
25 program: Option<super::admission::AdmittedProgram>,
26) -> Result<DeliverySpawn> {
27 #[cfg(test)]
28 if state.delivery.failed_spawn_module.as_deref() == Some(module) {
29 return Err(GateError::new(
30 "delivery_spawn_exhausted",
31 "delivery spawn identifiers exhausted",
32 ));
33 }
34 state.delivery.next_spawn = state.delivery.next_spawn.checked_add(1).ok_or_else(|| {
35 GateError::new(
36 "delivery_spawn_exhausted",
37 "delivery spawn identifiers exhausted",
38 )
39 })?;
40 let (authority, stamped) =
41 authority.map_or((None, false), |(record, stamped)| (Some(record), stamped));
42 Ok(DeliverySpawn {
43 id: state.delivery.next_spawn,
44 module: module.into(),
45 generation,
46 deadline: at + register + health + CONFIG_WARMUP_SLACK,
47 authority,
48 stamped,
49 program,
50 })
51}
52
53#[derive(Clone, Debug)]
54struct Attempt {
55 module: String,
56 generation: u64,
57 id: u64,
58 transition: Transition,
59 ready: bool,
60 acknowledged: bool,
61 delivered: bool,
62 deadline: Option<tokio::time::Instant>,
63 catchup: bool,
66 write_deadline: Option<tokio::time::Instant>,
67 policy_probe_requested: bool,
68 spawn: Option<DeliverySpawn>,
72 mark_only: bool,
76}
77
78#[derive(Clone, Debug)]
79pub struct DeliverySpawn {
80 id: u64,
81 module: String,
82 generation: u64,
83 deadline: tokio::time::Instant,
84 authority: Option<std::sync::Arc<crate::spawn_authority::SpawnAuthorityRecord>>,
86 stamped: bool,
89 program: Option<super::admission::AdmittedProgram>,
93}
94
95impl DeliverySpawn {
96 pub(crate) fn module(&self) -> &str {
97 &self.module
98 }
99 pub(crate) fn generation(&self) -> u64 {
100 self.generation
101 }
102 pub(crate) fn authority(
103 &self,
104 ) -> Option<&std::sync::Arc<crate::spawn_authority::SpawnAuthorityRecord>> {
105 self.authority.as_ref()
106 }
107 pub(crate) fn stamped(&self) -> bool {
108 self.stamped
109 }
110 pub(super) fn admitted_program(&self) -> Option<&super::admission::AdmittedProgram> {
111 self.program.as_ref()
112 }
113 #[cfg(test)]
116 pub(crate) fn with_authority_for_test(
117 mut self,
118 record: std::sync::Arc<crate::spawn_authority::SpawnAuthorityRecord>,
119 stamped: bool,
120 ) -> Self {
121 self.authority = Some(record);
122 self.stamped = stamped;
123 self
124 }
125}
126
127#[derive(Default)]
128pub(super) struct DeliveryState {
129 #[cfg(test)]
130 failed_spawn_module: Option<String>,
131 next_spawn: u64,
132 next_attempt: u64,
133 policy_epochs: HashMap<String, u64>,
134 attempts: HashMap<ConnectionId, Attempt>,
135 served: HashMap<ConnectionId, (Option<u64>, Option<u64>)>,
136 pub(super) unstamped_views: HashMap<ConnectionId, std::sync::Arc<ConfigView>>,
137 stamp_running: HashSet<ConnectionId>,
138 initial_generation: u64,
139 children: HashMap<ConnectionId, DeliverySpawn>,
140 healthy_children: HashSet<(ConnectionId, u64)>,
141 readiness_observed: HashSet<(ConnectionId, u64)>,
142 unregistered_children: HashMap<(String, bool), DeliverySpawn>,
143 recoveries: Vec<(ConnectionId, String, AppliedDisposition)>,
144 exited: HashMap<String, ExitedDelivery>,
145 last_target_attempt: HashMap<String, (u64, u64)>,
148 mark_records: Vec<(
154 std::sync::Weak<crate::spawn_authority::SpawnAuthorityRecord>,
155 Option<super::admission::AdmittedProgram>,
156 )>,
157}
158
159impl DeliveryState {
160 pub(super) fn reset_for_authority_exit(&mut self, generation: u64) {
161 *self = Self {
162 next_spawn: self.next_spawn,
163 next_attempt: self.next_attempt,
164 mark_records: std::mem::take(&mut self.mark_records),
165 ..Self::new(generation)
166 };
167 }
168 pub(super) fn track_mark(&mut self, spawn: &DeliverySpawn) {
171 let Some(record) = spawn.authority.as_ref().filter(|_| spawn.stamped) else {
172 return;
173 };
174 if !self
175 .mark_records
176 .iter()
177 .any(|(known, _)| std::ptr::eq(known.as_ptr(), std::sync::Arc::as_ptr(record)))
178 {
179 self.mark_records
180 .push((std::sync::Arc::downgrade(record), spawn.program.clone()));
181 }
182 }
183 pub(super) fn mark_records(
186 &mut self,
187 ) -> Vec<(
188 std::sync::Arc<crate::spawn_authority::SpawnAuthorityRecord>,
189 Option<super::admission::AdmittedProgram>,
190 )> {
191 self.mark_records
192 .retain(|(record, _)| record.upgrade().is_some_and(|record| record.alive()));
193 self.mark_records
194 .iter()
195 .filter_map(|(record, program)| Some((record.upgrade()?, program.clone())))
196 .collect()
197 }
198 pub(super) fn started_at_generation(&self) -> u64 {
199 self.initial_generation
200 }
201 pub(super) fn policy_authorization(
202 &self,
203 module: &str,
204 generation: u64,
205 connection: ConnectionId,
206 ) -> Option<(u64, tokio::time::Instant)> {
207 let attempt = self.attempts.get(&connection)?;
208 let deadline = attempt.deadline?;
209 (attempt.module == module
210 && attempt.generation == generation
211 && attempt.delivered
212 && attempt.acknowledged
213 && tokio::time::Instant::now() < deadline)
214 .then_some((attempt.id, deadline))
215 }
216 pub(super) fn fenced_modules(&self) -> impl Iterator<Item = String> + '_ {
217 self.exited.keys().cloned()
218 }
219}
220
221#[derive(Clone, Debug)]
224pub struct DeliveryProbe {
225 connection: ConnectionId,
226 module: String,
227 generation: u64,
228 attempt: Option<u64>,
229 policy_epoch: u64,
230 route_readiness: bool,
231}
232
233impl DeliveryProbe {
234 pub(crate) fn connection(&self) -> ConnectionId {
235 self.connection
236 }
237 pub(crate) fn owns_failure(&self) -> bool {
238 self.route_readiness
239 }
240}
241
242#[derive(Clone)]
243pub(crate) struct ExitedDelivery {
244 pub module: String,
245 spawn: u64,
246 generation: u64,
247 changed: tokio::sync::watch::Sender<bool>,
248}
249
250#[derive(Clone, Debug, PartialEq, Eq)]
251pub enum AppliedDisposition {
252 Ignored,
253 WaitingForReadiness,
254 Active,
255 CatchupComplete,
256 Failed {
257 generation: u64,
258 restore: Option<u64>,
259 keep_running: bool,
260 },
261 CatchupFailed {
262 active: u64,
263 },
264}
265
266#[derive(Clone, Copy)]
267pub(crate) struct DeliveryWindow {
268 pub generation: u64,
269 pub attempt: u64,
270 pub delivered: bool,
271 pub deadline: Option<tokio::time::Instant>,
272}
273
274impl DeliveryState {
275 pub(super) fn invalidate_policy(&mut self, module: &str, generation: u64) {
276 for attempt in self
277 .attempts
278 .values_mut()
279 .filter(|attempt| attempt.module == module && attempt.generation == generation)
280 {
281 attempt.ready = false;
282 attempt.policy_probe_requested = false;
283 }
284 }
285 pub(super) fn policy_installed(
286 &mut self,
287 module: &str,
288 generation: u64,
289 target: Option<ConnectionId>,
290 ) -> Result<()> {
291 let epoch = self.policy_epochs.entry(module.into()).or_default();
294 *epoch = epoch.checked_add(1).ok_or_else(|| {
295 GateError::new(
296 "delivery_probe_exhausted",
297 "policy readiness epochs exhausted",
298 )
299 })?;
300 self.invalidate_policy(module, generation);
301 let connections: HashSet<_> = self
302 .children
303 .iter()
304 .filter(|(_, spawn)| spawn.module == module)
305 .map(|(connection, _)| *connection)
306 .chain(
307 self.attempts
308 .iter()
309 .filter(|(_, attempt)| attempt.module == module)
310 .map(|(connection, _)| *connection),
311 )
312 .chain(target)
313 .collect();
314 self.readiness_observed.retain(|(connection, observed)| {
315 *observed != generation || !connections.contains(connection)
316 });
317 self.healthy_children.retain(|(connection, observed)| {
318 *observed != generation || !connections.contains(connection)
319 });
320 Ok(())
321 }
322 pub(super) fn declared(&mut self, connection: ConnectionId, module: &str, active: bool) {
323 if let Some(spawn) = self.unregistered_children.remove(&(module.into(), active)) {
324 self.children.insert(connection, spawn);
325 }
326 }
327 pub(super) fn new(initial_generation: u64) -> Self {
328 Self {
329 initial_generation,
330 ..Self::default()
331 }
332 }
333
334 pub(super) fn disconnected(&mut self, connection: ConnectionId) {
335 self.attempts.remove(&connection);
336 self.served.remove(&connection);
337 self.unstamped_views.remove(&connection);
338 self.stamp_running.remove(&connection);
339 self.children.remove(&connection);
340 self.healthy_children
341 .retain(|(child, _)| *child != connection);
342 self.readiness_observed
343 .retain(|(child, _)| *child != connection);
344 }
345
346 pub(super) fn remember_stamp_connections(&mut self, connections: Vec<ConnectionId>) {
347 self.stamp_running.extend(connections);
348 }
349
350 fn boundary_open(
351 &self,
352 connection: ConnectionId,
353 generation: u64,
354 transition: Transition,
355 ) -> bool {
356 if transition == Transition::DaemonRestart && generation > self.initial_generation {
357 return false;
358 }
359 !transition.requires_spawn()
360 || self
361 .children
362 .get(&connection)
363 .is_some_and(|spawn| spawn.generation == generation)
364 }
365
366 fn allocate(
367 &mut self,
368 module: &str,
369 generation: u64,
370 transition: Transition,
371 ) -> Result<Attempt> {
372 self.next_attempt = self.next_attempt.checked_add(1).ok_or_else(|| {
373 GateError::new(
374 "delivery_attempt_exhausted",
375 "delivery attempt ids exhausted",
376 )
377 })?;
378 Ok(Attempt {
379 module: module.into(),
380 generation,
381 id: self.next_attempt,
382 transition,
383 ready: transition == Transition::Live,
384 acknowledged: false,
385 delivered: false,
386 deadline: None,
387 catchup: false,
388 write_deadline: None,
389 policy_probe_requested: false,
390 spawn: None,
391 mark_only: false,
392 })
393 }
394}
395
396pub(super) fn visible_reply(
397 state: &OwnerState,
398 connection: ConnectionId,
399 mut reply: ConfigGetReply,
400 transition: Transition,
401) -> Result<ConfigGetReply> {
402 if reply.pending.as_ref().is_some_and(|view| {
403 !state
404 .delivery
405 .boundary_open(connection, view.generation, transition)
406 }) {
407 reply.pending = None;
408 }
409 if reply.active.is_none() && reply.pending.is_none() {
410 return Err(GateError::new(
411 "config_gated",
412 "generation waits for its required supervision or daemon boundary",
413 ));
414 }
415 Ok(reply)
416}
417
418pub(super) fn bind_get(
419 state: &mut OwnerState,
420 connection: ConnectionId,
421 module: &str,
422 request: &ConfigGetRequest,
423 reply: ConfigGetReply,
424 transition: Transition,
425) -> Result<ConfigGetReply> {
426 let mut reply = visible_reply(state, connection, reply, transition)?;
427 match (request.generation, request.attempt) {
428 (None, None) => {}
429 (Some(generation), Some(attempt))
430 if state
431 .delivery
432 .attempts
433 .get(&connection)
434 .is_some_and(|bound| {
435 bound.module == module && bound.generation == generation && bound.id == attempt
436 }) => {}
437 _ => {
438 return Err(GateError::new(
439 "config_gated",
440 "requested generation and attempt are not assigned to this connection",
441 ))
442 }
443 }
444 if reply.pending.is_none()
445 && state
446 .delivery
447 .attempts
448 .get(&connection)
449 .is_some_and(|bound| {
450 bound.catchup
451 && reply
452 .active
453 .as_ref()
454 .is_some_and(|view| view.generation == bound.generation)
455 })
456 {
457 reply.pending = reply.active.clone();
458 if let Some(view) = &mut reply.pending {
459 view.status = ConfigStatus::Pending;
460 }
461 }
462 if reply.pending.is_none() && request.generation.is_none() {
463 let generation = reply.active.as_ref().map(|view| view.generation);
464 if let Some(attempt) = generation
465 .map(|generation| mark_acknowledgement(state, connection, module, generation))
466 .transpose()?
467 .flatten()
468 {
469 state.delivery.attempts.insert(connection, attempt);
470 reply.pending = reply.active.clone();
471 if let Some(view) = &mut reply.pending {
472 view.status = ConfigStatus::Pending;
473 }
474 }
475 }
476 if let Some(pending) = &mut reply.pending {
477 let matches = state
478 .delivery
479 .attempts
480 .get(&connection)
481 .is_some_and(|bound| bound.module == module && bound.generation == pending.generation);
482 if !matches {
483 let mut attempt = state
484 .delivery
485 .allocate(module, pending.generation, transition)?;
486 if transition.requires_spawn() {
487 attempt.ready = state
488 .delivery
489 .healthy_children
490 .contains(&(connection, pending.generation));
491 attempt.deadline = state
492 .delivery
493 .children
494 .get(&connection)
495 .filter(|warmup| warmup.generation == pending.generation)
496 .map(|warmup| warmup.deadline);
497 }
498 attempt.ready &= super::route_policy::ready(state, module, pending.generation)?;
499 attempt.spawn = state.delivery.children.get(&connection).cloned();
500 state.delivery.attempts.insert(connection, attempt);
501 }
502 pending.attempt = state
503 .delivery
504 .attempts
505 .get(&connection)
506 .map(|bound| bound.id);
507 let mark_only = state
508 .delivery
509 .attempts
510 .get(&connection)
511 .is_some_and(|bound| bound.mark_only);
512 if state.active_declarations.get(module) == Some(&connection) && !mark_only {
513 if let Some(spawn) = state.delivery.children.get(&connection) {
514 let id = spawn.id;
515 state
516 .delivery
517 .last_target_attempt
518 .insert(module.into(), (id, pending.generation));
519 }
520 }
521 } else {
522 state.delivery.attempts.remove(&connection);
523 }
524 if request
525 .generation
526 .is_some_and(|id| reply.pending.as_ref().map(|view| view.generation) != Some(id))
527 {
528 return Err(GateError::new(
529 "config_gated",
530 "selected delivery has been superseded",
531 ));
532 }
533 Ok(reply)
534}
535
536fn mark_acknowledgement(
551 state: &mut OwnerState,
552 connection: ConnectionId,
553 module: &str,
554 generation: u64,
555) -> Result<Option<Attempt>> {
556 if state.store.loaded_admission_state().0 != crate::config_store::StoreState::Stamped
557 || state.active_declarations.get(module) != Some(&connection)
558 || state.delivery.attempts.contains_key(&connection)
559 {
560 return Ok(None);
561 }
562 let Some(spawn) = state
563 .delivery
564 .children
565 .get(&connection)
566 .filter(|spawn| spawn.stamped && spawn.module == module)
567 .cloned()
568 else {
569 return Ok(None);
570 };
571 let Some(record) = spawn.authority.as_ref() else {
572 return Ok(None);
573 };
574 let facts = record.mark_facts();
575 if !facts.unmarked || !facts.origin_qualifies || !record.request_acknowledgement(generation) {
576 return Ok(None);
577 }
578 let mut attempt = state
579 .delivery
580 .allocate(module, generation, Transition::Live)?;
581 attempt.catchup = true;
582 attempt.mark_only = true;
583 attempt.spawn = Some(spawn);
584 Ok(Some(attempt))
585}
586
587impl RuntimeOwner {
588 pub(crate) fn policy_probe_due(&self, connection: ConnectionId) -> Result<Option<(u64, u64)>> {
589 let mut state = self.lock()?;
590 let Some(bound) = state.delivery.attempts.get(&connection) else {
591 return Ok(None);
592 };
593 let (module, generation, attempt) = (bound.module.clone(), bound.generation, bound.id);
594 if bound.ready
597 || bound.mark_only
598 || bound.policy_probe_requested
599 || state.active_declarations.get(&module) != Some(&connection)
600 || !state
601 .store
602 .body()?
603 .generations
604 .get(&generation)
605 .is_some_and(|entry| {
606 super::route_policy::required(&entry.generation.changeset, &module)
607 })
608 || !super::route_policy::ready(&mut state, &module, generation)?
609 {
610 return Ok(None);
611 }
612 state
613 .delivery
614 .attempts
615 .get_mut(&connection)
616 .unwrap()
617 .policy_probe_requested = true;
618 Ok(Some((generation, attempt)))
619 }
620
621 pub(crate) fn policy_probe_pending(
622 &self,
623 connection: ConnectionId,
624 generation: u64,
625 attempt: u64,
626 ) -> Result<bool> {
627 let mut state = self.lock()?;
628 let Some(bound) = state.delivery.attempts.get(&connection) else {
629 return Ok(false);
630 };
631 let module = bound.module.clone();
632 Ok(bound.generation == generation
633 && bound.id == attempt
634 && !bound.ready
635 && state.active_declarations.get(&module) == Some(&connection)
636 && super::route_policy::ready(&mut state, &module, generation)?)
637 }
638 pub(crate) fn delivery_health_probe(
639 &self,
640 connection: ConnectionId,
641 module: &str,
642 ) -> Result<Option<DeliveryProbe>> {
643 let mut state = self.lock()?;
644 if state.active_declarations.get(module) != Some(&connection) {
645 return Ok(None);
646 }
647 let attempt = state
648 .delivery
649 .attempts
650 .get(&connection)
651 .map(|attempt| (attempt.generation, attempt.id));
652 let generation = attempt.map(|attempt| attempt.0).or_else(|| {
653 state
654 .delivery
655 .children
656 .get(&connection)
657 .map(|child| child.generation)
658 });
659 let route_readiness = match generation {
660 Some(generation) if attempt.is_some() => state
661 .store
662 .body()?
663 .generations
664 .get(&generation)
665 .is_some_and(|entry| {
666 super::route_policy::required(&entry.generation.changeset, module)
667 }),
668 _ => false,
669 };
670 Ok(generation.map(|generation| DeliveryProbe {
671 connection,
672 module: module.into(),
673 generation,
674 attempt: attempt.map(|attempt| attempt.1),
675 route_readiness,
676 policy_epoch: state
677 .delivery
678 .policy_epochs
679 .get(module)
680 .copied()
681 .unwrap_or_default(),
682 }))
683 }
684
685 pub(crate) fn delivery_health_result(
686 &self,
687 probe: DeliveryProbe,
688 healthy: bool,
689 ) -> Result<Option<AppliedDisposition>> {
690 let attempt = {
691 let mut state = self.lock()?;
692 if state.active_declarations.get(&probe.module) != Some(&probe.connection)
693 || state
694 .delivery
695 .policy_epochs
696 .get(&probe.module)
697 .copied()
698 .unwrap_or_default()
699 != probe.policy_epoch
700 {
701 return Ok(None);
702 }
703 let current = state.delivery.attempts.get(&probe.connection);
704 if probe
705 .attempt
706 .is_some_and(|id| current.is_none_or(|bound| bound.id != id))
707 || current.is_some_and(|bound| bound.generation != probe.generation)
708 {
709 return Ok(None);
710 }
711 let attempt = current
715 .filter(|bound| !bound.mark_only)
716 .map(|bound| bound.id);
717 if !healthy
718 && (!super::route_policy::ready(&mut state, &probe.module, probe.generation)?
719 || !state
720 .store
721 .body()?
722 .generations
723 .get(&probe.generation)
724 .is_some_and(|entry| {
725 super::route_policy::required(
726 &entry.generation.changeset,
727 &probe.module,
728 )
729 }))
730 {
731 return Ok(None);
732 }
733 attempt
734 };
735 if healthy {
736 self.delivery_readiness_complete(probe.connection, &probe.module, probe.generation)
737 } else if let Some(attempt) = attempt {
738 self.module_outcome(
739 probe.connection,
740 &probe.module,
741 ConfigAppliedRequest::new(
742 probe.generation,
743 attempt,
744 subc_protocol::config::ConfigApplyResult::Nack,
745 vec![],
746 ),
747 true,
748 )
749 .map(Some)
750 } else {
751 Ok(None)
752 }
753 }
754
755 #[cfg(test)]
756 pub(super) fn ready(
757 &self,
758 connection: ConnectionId,
759 module: &str,
760 generation: u64,
761 ) -> Result<Option<AppliedDisposition>> {
762 self.delivery_readiness_complete(connection, module, generation)
763 }
764
765 pub(crate) fn pending_config_restarts(&self) -> Result<Vec<(String, u64)>> {
766 let mut state = self.lock()?;
767 state.store.delivery_ready()?;
768 let body = state.store.body()?;
769 body.versions
770 .iter()
771 .filter(|(tuple, _)| tuple.project.is_none())
772 .filter_map(|(tuple, versions)| {
773 let generation = versions.pending?;
774 let entry = body.generations.get(&generation)?;
775 Some(
776 Transition::for_module(&entry.generation.changeset, &tuple.module).map(
777 |transition| {
778 (transition == Transition::ModuleRestart)
779 .then(|| (tuple.module.clone(), generation))
780 },
781 ),
782 )
783 })
784 .collect::<Result<Vec<_>>>()
785 .map(|entries| entries.into_iter().flatten().collect())
786 }
787
788 pub(crate) fn requires_fresher_spawn(
789 &self,
790 module: &str,
791 spawn: Option<&DeliverySpawn>,
792 ) -> Result<Option<u64>> {
793 let mut state = self.lock()?;
794 state.store.delivery_ready()?;
795 Self::requires_fresher_spawn_loaded(&state, module, spawn)
796 }
797
798 pub(super) fn requires_fresher_spawn_loaded(
799 state: &OwnerState,
800 module: &str,
801 spawn: Option<&DeliverySpawn>,
802 ) -> Result<Option<u64>> {
803 let body = state.store.loaded_admission_state().1;
804 let pending = body
805 .versions
806 .iter()
807 .find(|(tuple, _)| tuple.module == module && tuple.project.is_none())
808 .and_then(|(_, versions)| versions.pending);
809 let Some(generation) = pending else {
810 return Ok(None);
811 };
812 let entry = body
813 .generations
814 .get(&generation)
815 .ok_or_else(|| GateError::new("config_gated", "pending generation is absent"))?;
816 let transition = Transition::for_module(&entry.generation.changeset, module)?;
817 let deferred_daemon = transition == Transition::DaemonRestart
818 && generation > state.delivery.initial_generation;
819 Ok((transition.requires_spawn()
820 && !deferred_daemon
821 && spawn
822 .is_none_or(|spawn| spawn.module != module || spawn.generation() != generation))
823 .then_some(generation))
824 }
825
826 pub(crate) fn delivery_child_exited(&self, spawn: &DeliverySpawn) -> Result<()> {
829 let mut state = self.lock()?;
830 let pending = state
831 .store
832 .body()?
833 .versions
834 .iter()
835 .find(|(tuple, _)| tuple.module == spawn.module && tuple.project.is_none())
836 .and_then(|(_, versions)| versions.pending);
837 let pending = pending.filter(|generation| {
838 state
839 .store
840 .loaded_admission_state()
841 .1
842 .generations
843 .get(generation)
844 .and_then(|entry| entry.delivery.as_ref())
845 .is_some_and(|delivery| delivery.declarations.contains_key(&spawn.module))
846 });
847 if let Some(generation) = pending.filter(|generation| {
848 *generation == spawn.generation
849 || state.delivery.last_target_attempt.get(&spawn.module)
850 == Some(&(spawn.id, *generation))
851 }) {
852 state
853 .delivery
854 .exited
855 .entry(spawn.module.clone())
856 .or_insert_with(|| ExitedDelivery {
857 module: spawn.module.clone(),
858 spawn: spawn.id,
859 generation,
860 changed: tokio::sync::watch::channel(true).0,
861 });
862 }
863 self.notify_delivery_changed();
864 Ok(())
865 }
866
867 pub(crate) fn delivery_exit_fence(
868 &self,
869 module: &str,
870 ) -> Result<Option<tokio::sync::watch::Receiver<bool>>> {
871 Ok(self
872 .lock()?
873 .delivery
874 .exited
875 .get(module)
876 .map(|entry| entry.changed.subscribe()))
877 }
878
879 pub(crate) fn delivery_exits(&self) -> Result<Vec<ExitedDelivery>> {
880 Ok(self.lock()?.delivery.exited.values().cloned().collect())
881 }
882
883 pub(crate) fn delivery_record_exit(&self, exited: &ExitedDelivery) -> Result<bool> {
890 #[cfg(test)]
891 self.exit_record_attempts
892 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
893 let mut state = self.lock()?;
894 if !state
895 .delivery
896 .exited
897 .get(&exited.module)
898 .is_some_and(|entry| entry.spawn == exited.spawn)
899 {
900 return Ok(false);
901 }
902 let tuple = Tuple {
903 module: exited.module.clone(),
904 project: None,
905 };
906 let versions = state
907 .store
908 .body()?
909 .versions
910 .iter()
911 .find(|(key, _)| key == &tuple)
912 .map(|(_, versions)| versions.clone())
913 .unwrap_or_default();
914 if versions
915 .pending
916 .is_some_and(|generation| generation != exited.generation)
917 {
918 return Ok(false);
919 }
920 if versions.pending == Some(exited.generation) {
921 state.restore_route_policy(&exited.module, exited.generation);
922 let result = state
923 .store
924 .record_delivery(&tuple, exited.generation, false);
925 if let Err(error) = result {
926 if error.code != crate::config_store::StoreErrorCode::AnchorPending {
927 return Err(error.into());
928 }
929 }
930 state.delivery.attempts.retain(|_, attempt| {
931 attempt.module != exited.module || attempt.generation != exited.generation
932 });
933 }
934 Ok(versions.active.is_none())
935 }
936
937 #[cfg(all(test, unix))]
938 pub(crate) fn exit_record_attempts_for_test(&self) -> usize {
939 self.exit_record_attempts
940 .load(std::sync::atomic::Ordering::SeqCst)
941 }
942
943 pub(crate) fn delivery_release_exit(&self, exited: &ExitedDelivery) -> Result<()> {
944 let mut state = self.lock()?;
945 if state
946 .delivery
947 .exited
948 .get(&exited.module)
949 .is_some_and(|entry| entry.spawn == exited.spawn)
950 {
951 if let Some(entry) = state.delivery.exited.remove(&exited.module) {
952 entry.changed.send_replace(false);
953 state.admission_releases.push(exited.module.clone());
957 }
958 }
959 self.notify_delivery_changed();
960 Ok(())
961 }
962 pub(crate) fn delivery_abandon_spawn(&self, spawn: &DeliverySpawn, active: bool) {
963 if let Ok(mut state) = self.lock() {
964 let key = (spawn.module.clone(), active);
965 if state
966 .delivery
967 .unregistered_children
968 .get(&key)
969 .is_some_and(|current| current.id == spawn.id)
970 {
971 state.delivery.unregistered_children.remove(&key);
972 }
973 }
974 }
975 pub(crate) fn delivery_queue_abandoned_spawn(&self, spawn: DeliverySpawn, active: bool) {
976 if let Ok(mut abandoned) = self.abandoned_spawns.lock() {
977 abandoned.push((spawn, active));
978 }
979 self.notify_delivery_changed();
980 }
981 pub(super) fn drain_abandoned_spawns(&self, state: &mut OwnerState) {
982 if let Ok(mut abandoned) = self.abandoned_spawns.lock() {
983 for (spawn, active) in abandoned.drain(..) {
984 let key = (spawn.module.clone(), active);
985 if state
986 .delivery
987 .unregistered_children
988 .get(&key)
989 .is_some_and(|current| current.id == spawn.id)
990 {
991 state.delivery.unregistered_children.remove(&key);
992 }
993 }
994 }
995 }
996
997 pub(crate) fn delivery_launch_current(
1001 &self,
1002 spawn: &DeliverySpawn,
1003 active: bool,
1004 ) -> Result<Option<bool>> {
1005 let state = match self.inner.try_lock() {
1006 Ok(state) => state,
1007 Err(std::sync::TryLockError::WouldBlock) => return Ok(None),
1008 Err(_) => {
1009 return Err(GateError::new(
1010 "config_unavailable",
1011 "configuration owner lock is poisoned",
1012 ))
1013 }
1014 };
1015 Ok(Some(
1016 super::admission::AdmissionCache::from_state(&state)
1017 .spawn_generation(&spawn.module)
1018 .is_ok_and(|generation| generation == spawn.generation)
1019 && state
1020 .delivery
1021 .unregistered_children
1022 .get(&(spawn.module.clone(), active))
1023 .is_some_and(|current| current.id == spawn.id)
1024 && !state.delivery.exited.contains_key(&spawn.module),
1025 ))
1026 }
1027 #[cfg(test)]
1028 pub(crate) fn pause_spawn_preflight_for_test(&self) {
1029 let hook = self.spawn_preflight_hook.lock().unwrap().take();
1030 if let Some(hook) = hook {
1031 hook();
1032 }
1033 }
1034 #[cfg(all(test, unix))]
1035 pub(crate) fn exhaust_delivery_spawns_for_test(&self) {
1036 self.lock().unwrap().delivery.next_spawn = u64::MAX;
1037 }
1038 #[cfg(all(test, unix))]
1039 pub(crate) fn restore_unused_delivery_spawns_for_test(&self) {
1040 self.lock().unwrap().delivery.next_spawn = 1000;
1041 }
1042 #[cfg(all(test, unix))]
1043 pub(crate) fn fail_delivery_spawns_for_module_for_test(&self, module: &str) {
1044 self.lock().unwrap().delivery.failed_spawn_module = Some(module.into());
1045 }
1046 #[cfg(all(test, unix))]
1047 pub(crate) fn clear_delivery_spawn_failure_for_test(&self) {
1048 self.lock().unwrap().delivery.failed_spawn_module = None;
1049 }
1050 #[cfg(all(test, unix))]
1051 pub(crate) fn set_spawn_preflight_hook_for_test(&self, hook: impl FnOnce() + Send + 'static) {
1052 *self.spawn_preflight_hook.lock().unwrap() = Some(Box::new(hook));
1053 }
1054 #[cfg(all(test, unix))]
1055 pub(crate) fn unregistered_spawn_count_for_test(&self) -> usize {
1056 self.lock().unwrap().delivery.unregistered_children.len()
1057 }
1058 #[allow(clippy::too_many_arguments)]
1059 pub(crate) fn delivery_spawned_for_admission_in_slot(
1060 &self,
1061 module: &str,
1062 admission: &super::admission::SpawnAdmission,
1063 at: tokio::time::Instant,
1064 register: Duration,
1065 health: Duration,
1066 active: bool,
1067 authority: Option<std::sync::Arc<crate::spawn_authority::SpawnAuthorityRecord>>,
1068 ) -> Result<Option<DeliverySpawn>> {
1069 let mut state = OwnerGuard(
1070 self.inner.try_lock().map_err(|error| {
1071 if matches!(error, std::sync::TryLockError::Poisoned(_)) {
1072 return GateError::new("config_unavailable", "configuration owner is poisoned");
1073 }
1074 GateError::new(
1075 "config_delivery_deferred",
1076 "configuration publication is busy",
1077 )
1078 })?,
1079 &self.published,
1080 &self.delivery_changed,
1081 &self.module_gates,
1082 self,
1083 );
1084 state.store.check()?;
1085 let stamped = admission.gated()
1088 && state.store.loaded_admission_state().0 == crate::config_store::StoreState::Stamped;
1089 let spawn = allocate_spawn(
1090 &mut state,
1091 module,
1092 admission.generation(),
1093 at,
1094 register,
1095 health,
1096 authority.map(|record| (record, stamped)),
1097 admission.admitted_program(),
1098 )?;
1099 state.delivery.track_mark(&spawn);
1100 state
1101 .delivery
1102 .unregistered_children
1103 .insert((module.into(), active), spawn.clone());
1104 Ok(Some(spawn))
1105 }
1106
1107 #[cfg(test)]
1108 pub(crate) fn delivery_spawned_in_slot(
1109 &self,
1110 module: &str,
1111 at: tokio::time::Instant,
1112 register: Duration,
1113 health: Duration,
1114 active: bool,
1115 ) -> Result<Option<DeliverySpawn>> {
1116 let spawn = self.delivery_spawned(module, at, register, health)?;
1117 if let Some(spawn) = &spawn {
1118 self.lock()?
1119 .delivery
1120 .unregistered_children
1121 .insert((module.into(), active), spawn.clone());
1122 }
1123 Ok(spawn)
1124 }
1125 pub(crate) fn delivery_take_recoveries(
1126 &self,
1127 ) -> Result<Vec<(ConnectionId, String, AppliedDisposition)>> {
1128 Ok(std::mem::take(&mut self.lock()?.delivery.recoveries))
1129 }
1130
1131 pub(crate) fn delivery_window(
1132 &self,
1133 connection: ConnectionId,
1134 ) -> Result<Option<DeliveryWindow>> {
1135 Ok(self
1136 .lock()?
1137 .delivery
1138 .attempts
1139 .get(&connection)
1140 .map(|bound| DeliveryWindow {
1141 generation: bound.generation,
1142 attempt: bound.id,
1143 delivered: bound.delivered,
1144 deadline: bound.deadline,
1145 }))
1146 }
1147
1148 pub(crate) fn delivery_write_started(
1149 &self,
1150 connection: ConnectionId,
1151 generation: u64,
1152 attempt: u64,
1153 now: tokio::time::Instant,
1154 ) -> Result<()> {
1155 if let Some(bound) = self
1156 .lock()?
1157 .delivery
1158 .attempts
1159 .get_mut(&connection)
1160 .filter(|bound| {
1161 bound.generation == generation && bound.id == attempt && !bound.delivered
1162 })
1163 {
1164 bound
1165 .write_deadline
1166 .get_or_insert(now + CONFIG_LIVE_ACK_TIMEOUT);
1167 }
1168 Ok(())
1169 }
1170
1171 pub(crate) fn delivery_timeout(
1172 &self,
1173 connection: ConnectionId,
1174 module: &str,
1175 generation: u64,
1176 attempt: u64,
1177 ) -> Result<AppliedDisposition> {
1178 self.module_outcome(
1179 connection,
1180 module,
1181 ConfigAppliedRequest::new(
1182 generation,
1183 attempt,
1184 subc_protocol::config::ConfigApplyResult::Nack,
1185 vec![],
1186 ),
1187 true,
1188 )
1189 }
1190
1191 pub(crate) fn delivery_spawned(
1194 &self,
1195 module: &str,
1196 spawned_at: tokio::time::Instant,
1197 register_timeout: Duration,
1198 first_health_budget: Duration,
1199 ) -> Result<Option<DeliverySpawn>> {
1200 let mut state = self.lock()?;
1201 state.store.check()?;
1202 let admission = super::admission::AdmissionCache::from_state(&state);
1203 let generation = admission.spawn_generation(module)?;
1204 allocate_spawn(
1205 &mut state,
1206 module,
1207 generation,
1208 spawned_at,
1209 register_timeout,
1210 first_health_budget,
1211 None,
1212 admission.admitted_program(module),
1213 )
1214 .map(Some)
1215 }
1216
1217 pub(crate) fn delivery_child_registered(
1218 &self,
1219 connection: ConnectionId,
1220 spawn: &DeliverySpawn,
1221 ) -> Result<()> {
1222 let mut state = self.lock()?;
1223 state.delivery.track_mark(spawn);
1224 state.delivery.children.insert(connection, spawn.clone());
1225 if let Some(bound) = state
1226 .delivery
1227 .attempts
1228 .get_mut(&connection)
1229 .filter(|bound| bound.module == spawn.module && bound.generation == spawn.generation)
1230 {
1231 if bound.transition.requires_spawn() {
1232 bound.deadline = Some(spawn.deadline);
1233 }
1234 }
1235 Ok(())
1236 }
1237
1238 pub(crate) fn module_view_delivered(
1239 &self,
1240 connection: ConnectionId,
1241 module: &str,
1242 reply: &ConfigGetReply,
1243 written_at: tokio::time::Instant,
1244 active: bool,
1245 ) -> Result<()> {
1246 {
1247 let mut state = self.lock()?;
1248 if !state
1249 .declarations
1250 .get(&connection)
1251 .is_some_and(|(id, _)| id == module)
1252 {
1253 return Err(GateError::new(
1254 "not_delivery_target",
1255 "configuration connection has ended",
1256 ));
1257 }
1258 let body = state.store.body()?;
1259 if body.stamped {
1260 let versions = body
1261 .versions
1262 .iter()
1263 .find(|(tuple, _)| tuple.module == module && tuple.project.is_none())
1264 .map(|(_, versions)| versions.clone())
1265 .unwrap_or_default();
1266 if reply
1267 .active
1268 .as_ref()
1269 .is_some_and(|view| versions.active != Some(view.generation))
1270 || reply.pending.as_ref().is_some_and(|view| {
1271 let catchup = versions.active == Some(view.generation)
1274 && state
1275 .delivery
1276 .attempts
1277 .get(&connection)
1278 .is_some_and(|bound| {
1279 bound.catchup
1280 && bound.generation == view.generation
1281 && Some(bound.id) == view.attempt
1282 });
1283 versions.pending != Some(view.generation) && !catchup
1284 })
1285 {
1286 return Err(GateError::new(
1287 "delivery_superseded",
1288 "configuration changed while its view was being written",
1289 ));
1290 }
1291 }
1292 }
1293 if active {
1294 if let Some(view) = &reply.pending {
1295 if let Some(attempt) = view.attempt {
1296 self.delivery_written(connection, view.generation, attempt, written_at)?;
1297 }
1298 }
1299 }
1300 let mut state = self.lock()?;
1301 if !state
1302 .declarations
1303 .get(&connection)
1304 .is_some_and(|(id, _)| id == module)
1305 {
1306 return Err(GateError::new(
1307 "not_delivery_target",
1308 "configuration connection has ended",
1309 ));
1310 }
1311 state.delivery.served.insert(
1312 connection,
1313 (
1314 reply.active.as_ref().map(|view| view.generation),
1315 reply.pending.as_ref().map(|view| view.generation),
1316 ),
1317 );
1318 if let Some(view) = reply
1319 .active
1320 .as_ref()
1321 .filter(|view| view.status == ConfigStatus::Unstamped)
1322 {
1323 state
1324 .delivery
1325 .unstamped_views
1326 .insert(connection, std::sync::Arc::new(view.clone()));
1327 } else {
1328 state.delivery.unstamped_views.remove(&connection);
1329 }
1330 Ok(())
1331 }
1332 pub(crate) fn delivery_readiness_complete(
1335 &self,
1336 connection: ConnectionId,
1337 module: &str,
1338 generation: u64,
1339 ) -> Result<Option<AppliedDisposition>> {
1340 let attempt = {
1341 let mut state = self.lock()?;
1342 if state.active_declarations.get(module) != Some(&connection) {
1343 return Err(GateError::new(
1344 "not_delivery_target",
1345 "readiness belongs to another connection",
1346 ));
1347 }
1348 let started_for_generation = state
1349 .delivery
1350 .children
1351 .get(&connection)
1352 .is_some_and(|warmup| warmup.generation == generation);
1353 if started_for_generation {
1354 state
1355 .delivery
1356 .healthy_children
1357 .insert((connection, generation));
1358 }
1359 let policy_ready = super::route_policy::ready(&mut state, module, generation)?;
1360 let Some(bound) = state
1361 .delivery
1362 .attempts
1363 .get_mut(&connection)
1364 .filter(|bound| bound.module == module && bound.generation == generation)
1365 else {
1366 return Ok(None);
1367 };
1368 if bound.transition.requires_spawn() && !started_for_generation {
1369 return Ok(None);
1370 }
1371 bound.ready = policy_ready;
1372 let acknowledged = bound.ready && bound.acknowledged;
1373 let id = bound.id;
1374 state
1375 .delivery
1376 .readiness_observed
1377 .insert((connection, generation));
1378 acknowledged.then_some(id)
1379 };
1380 attempt
1381 .map(|attempt| {
1382 self.module_applied(
1383 connection,
1384 module,
1385 ConfigAppliedRequest::new(
1386 generation,
1387 attempt,
1388 subc_protocol::config::ConfigApplyResult::Ack,
1389 vec![],
1390 ),
1391 )
1392 })
1393 .transpose()
1394 }
1395
1396 pub(crate) fn delivery_promoted(
1399 &self,
1400 connection: ConnectionId,
1401 module: &str,
1402 cutover_at: tokio::time::Instant,
1403 ) -> Result<Option<(u64, u64)>> {
1404 let mut state = self.lock()?;
1405 state.store.delivery_ready()?;
1406 Self::promote_loaded(&mut state, connection, module, cutover_at)
1407 }
1408
1409 pub(super) fn promote_loaded(
1410 state: &mut OwnerState,
1411 connection: ConnectionId,
1412 module: &str,
1413 cutover_at: tokio::time::Instant,
1414 ) -> Result<Option<(u64, u64)>> {
1415 if state.store.loaded_admission_state().0 == crate::config_store::StoreState::Unstamped {
1416 if state
1417 .declarations
1418 .get(&connection)
1419 .is_some_and(|(id, _)| id == module)
1420 {
1421 state.phone_target_changed(module, connection);
1422 state.active_declarations.insert(module.into(), connection);
1423 }
1424 return Ok(None);
1425 }
1426 let Some((id, declaration)) = state
1427 .declarations
1428 .get(&connection)
1429 .cloned()
1430 .filter(|(id, _)| id == module)
1431 else {
1432 return Err(GateError::new(
1433 "config_gated",
1434 "candidate has not declared its approved configuration",
1435 ));
1436 };
1437 let tuple = Tuple {
1438 module: module.into(),
1439 project: None,
1440 };
1441 let body = state.store.loaded_admission_state().1;
1442 let versions = body
1443 .versions
1444 .iter()
1445 .find(|(key, _)| key == &tuple)
1446 .map(|(_, versions)| versions.clone())
1447 .unwrap_or_default();
1448 let transition = versions
1449 .pending
1450 .and_then(|id| body.generations.get(&id))
1451 .map(|entry| Transition::for_module(&entry.generation.changeset, module))
1452 .transpose()?
1453 .unwrap_or(Transition::Live);
1454 let pending_declaration = versions
1455 .pending
1456 .and_then(|id| body.generations.get(&id))
1457 .and_then(|entry| entry.delivery.as_ref())
1458 .and_then(|snapshot| snapshot.declarations.get(module))
1459 .cloned();
1460 let active_declaration = versions
1461 .active
1462 .and_then(|id| body.generations.get(&id))
1463 .and_then(|entry| entry.delivery.as_ref())
1464 .and_then(|snapshot| snapshot.declarations.get(module))
1465 .cloned();
1466 let pending = versions.pending.filter(|generation| {
1467 state
1468 .delivery
1469 .boundary_open(connection, *generation, transition)
1470 });
1471 if pending != versions.pending && transition != Transition::DaemonRestart {
1472 return Err(GateError::new(
1473 "config_gated",
1474 "candidate launch was superseded by a configuration requiring a fresh start",
1475 ));
1476 }
1477 let expected = if pending.is_some() {
1478 pending_declaration
1479 } else {
1480 active_declaration
1481 };
1482 if expected.as_ref() != Some(&declaration) {
1483 return Err(GateError::new(
1484 "config_gated",
1485 "candidate declaration differs from the approved declaration",
1486 ));
1487 }
1488 let transition = if pending.is_some() {
1489 transition
1490 } else {
1491 Transition::Live
1492 };
1493 let vector = (versions.active, pending);
1494 let catchup = pending.is_none()
1495 && versions.active.is_some()
1496 && state.delivery.served.get(&connection) != Some(&vector);
1497 state.phone_target_changed(module, connection);
1498 state.active_declarations.insert(id, connection);
1499 state.delivery.attempts.remove(&connection);
1500 let Some(generation) = pending.or_else(|| catchup.then_some(versions.active).flatten())
1501 else {
1502 return Ok(None);
1503 };
1504 let mut attempt = state.delivery.allocate(module, generation, transition)?;
1505 attempt.catchup = catchup;
1506 attempt.spawn = state.delivery.children.get(&connection).cloned();
1507 attempt.ready = super::route_policy::ready_loaded(state, module, generation)
1512 && !state
1513 .store
1514 .loaded_admission_state()
1515 .1
1516 .generations
1517 .get(&generation)
1518 .is_some_and(|entry| {
1519 super::route_policy::required(&entry.generation.changeset, module)
1520 });
1521 attempt.deadline = Some(cutover_at + CONFIG_LIVE_ACK_TIMEOUT);
1522 let assignment = (generation, attempt.id);
1523 state.delivery.attempts.insert(connection, attempt);
1524 state
1525 .delivery
1526 .readiness_observed
1527 .insert((connection, generation));
1528 Ok(Some(assignment))
1529 }
1530
1531 pub(crate) fn delivery_written(
1534 &self,
1535 connection: ConnectionId,
1536 generation: u64,
1537 attempt: u64,
1538 written_at: tokio::time::Instant,
1539 ) -> Result<()> {
1540 let mut state = self.lock()?;
1541 let bound = state
1542 .delivery
1543 .attempts
1544 .get_mut(&connection)
1545 .filter(|bound| bound.generation == generation && bound.id == attempt)
1546 .ok_or_else(|| {
1547 GateError::new(
1548 "delivery_superseded",
1549 "writer receipt no longer names the current delivery attempt",
1550 )
1551 })?;
1552 if bound
1553 .write_deadline
1554 .is_some_and(|deadline| written_at >= deadline)
1555 || bound
1556 .deadline
1557 .is_some_and(|deadline| written_at >= deadline)
1558 {
1559 return Err(GateError::new(
1560 "delivery_expired",
1561 "configuration delivery timed out",
1562 ));
1563 }
1564 bound.delivered = true;
1565 if !bound.transition.requires_spawn() && bound.deadline.is_none() {
1566 bound.deadline = Some(written_at + CONFIG_LIVE_ACK_TIMEOUT);
1567 }
1568 Ok(())
1569 }
1570
1571 pub(crate) fn module_applied(
1572 &self,
1573 connection: ConnectionId,
1574 module: &str,
1575 request: ConfigAppliedRequest,
1576 ) -> Result<AppliedDisposition> {
1577 self.module_outcome(connection, module, request, false)
1578 }
1579
1580 fn module_outcome(
1581 &self,
1582 connection: ConnectionId,
1583 module: &str,
1584 request: ConfigAppliedRequest,
1585 timed_out: bool,
1586 ) -> Result<AppliedDisposition> {
1587 let mut state = self.lock()?;
1588 if !state
1589 .declarations
1590 .get(&connection)
1591 .is_some_and(|(id, _)| id == module)
1592 {
1593 return Err(GateError::new(
1594 "declaration_order",
1595 "config.declare must precede get or applied on this connection",
1596 ));
1597 }
1598 if request.project_id.is_some() {
1599 return Err(GateError::new(
1600 "config_gated",
1601 "project-tuple delivery is not supported yet",
1602 ));
1603 }
1604 if state.active_declarations.get(module) != Some(&connection) {
1605 return Err(GateError::new(
1606 "not_delivery_target",
1607 "connection is not the active delivery target",
1608 ));
1609 }
1610 state.store.check()?;
1611 let Some(bound) = state
1612 .delivery
1613 .attempts
1614 .get(&connection)
1615 .cloned()
1616 .filter(|bound| {
1617 bound.module == module
1618 && bound.generation == request.generation
1619 && bound.id == request.attempt
1620 })
1621 else {
1622 return Ok(AppliedDisposition::Ignored);
1623 };
1624 if !bound.delivered && !timed_out {
1625 return Ok(AppliedDisposition::Ignored);
1626 }
1627 let tuple = Tuple {
1628 module: module.into(),
1629 project: None,
1630 };
1631 let versions = state
1632 .store
1633 .body()?
1634 .versions
1635 .iter()
1636 .find(|(key, _)| key == &tuple)
1637 .map(|(_, versions)| versions.clone())
1638 .unwrap_or_default();
1639 let ack = request.result.is_ack()
1640 && !bound
1641 .deadline
1642 .is_some_and(|deadline| tokio::time::Instant::now() >= deadline);
1643 if bound.catchup {
1644 let pending_transition = versions
1647 .pending
1648 .map(|generation| {
1649 let entry = state
1650 .store
1651 .body()?
1652 .generations
1653 .get(&generation)
1654 .ok_or_else(|| {
1655 GateError::new("config_gated", "pending generation is absent")
1656 })?;
1657 Transition::for_module(&entry.generation.changeset, module)
1658 })
1659 .transpose()?;
1660 let pending_is_deliverable =
1661 versions
1662 .pending
1663 .zip(pending_transition)
1664 .is_some_and(|(generation, transition)| {
1665 state
1666 .delivery
1667 .boundary_open(connection, generation, transition)
1668 });
1669 if pending_is_deliverable || versions.active != Some(bound.generation) {
1670 return Ok(AppliedDisposition::Ignored);
1671 }
1672 state.delivery.attempts.remove(&connection);
1673 if bound.mark_only {
1674 return Ok(if ack {
1677 super::mark::acknowledged(self, &state, bound.spawn.as_ref(), bound.generation);
1678 AppliedDisposition::CatchupComplete
1679 } else {
1680 super::mark::acknowledgement_refused(self, &state, bound.spawn.as_ref());
1681 AppliedDisposition::Ignored
1682 });
1683 }
1684 if ack {
1685 super::mark::acknowledged(self, &state, bound.spawn.as_ref(), bound.generation);
1686 } else {
1687 super::mark::delivery_failed(self, bound.spawn.as_ref());
1688 state.delivery.recoveries.push((
1689 connection,
1690 module.into(),
1691 AppliedDisposition::CatchupFailed {
1692 active: bound.generation,
1693 },
1694 ));
1695 }
1696 return Ok(if ack {
1697 AppliedDisposition::CatchupComplete
1698 } else {
1699 AppliedDisposition::CatchupFailed {
1700 active: bound.generation,
1701 }
1702 });
1703 }
1704 if versions.pending != Some(bound.generation) {
1705 return Ok(AppliedDisposition::Ignored);
1706 }
1707 if ack
1708 && (!bound.ready
1709 || (bound.transition == Transition::DaemonRestart
1710 && bound.generation > state.delivery.initial_generation))
1711 {
1712 if let Some(bound) = state.delivery.attempts.get_mut(&connection) {
1713 bound.acknowledged = true;
1714 }
1715 return Ok(AppliedDisposition::WaitingForReadiness);
1716 }
1717 if !ack {
1718 state.restore_route_policy(module, bound.generation);
1719 }
1720 let result = state.store.record_delivery(&tuple, bound.generation, ack);
1721 if result.is_ok() && ack {
1722 state.commit_route_policy(module, bound.generation);
1723 }
1724 if result.is_ok() || !ack {
1727 state.delivery.attempts.remove(&connection);
1728 }
1729 if !ack {
1730 let keep_running =
1731 versions.active.is_none() && state.delivery.stamp_running.contains(&connection);
1732 state.delivery.recoveries.push((
1733 connection,
1734 module.into(),
1735 AppliedDisposition::Failed {
1736 generation: bound.generation,
1737 restore: versions.active,
1738 keep_running,
1739 },
1740 ));
1741 }
1742 if ack && result.is_ok() {
1743 super::mark::acknowledged(self, &state, bound.spawn.as_ref(), bound.generation);
1744 } else if !ack {
1745 super::mark::delivery_failed(self, bound.spawn.as_ref());
1746 }
1747 result?;
1748 Ok(if ack {
1749 AppliedDisposition::Active
1750 } else {
1751 AppliedDisposition::Failed {
1752 generation: bound.generation,
1753 restore: versions.active,
1754 keep_running: versions.active.is_none()
1755 && state.delivery.stamp_running.contains(&connection),
1756 }
1757 })
1758 }
1759}