1use std::collections::HashSet;
4use subc_protocol::config::{ConfigAppliedRequest, ConfigGetReply, ConfigGetRequest};
5
6use super::*;
7use crate::config_gate::apply::Transition;
8use crate::config_store::Tuple;
9
10pub(crate) const CONFIG_LIVE_ACK_TIMEOUT: Duration = Duration::from_secs(30);
11pub(crate) const CONFIG_WARMUP_SLACK: Duration = Duration::from_secs(10);
12
13#[allow(clippy::too_many_arguments)]
14fn allocate_spawn(
15 state: &mut OwnerState,
16 module: &str,
17 generation: u64,
18 at: tokio::time::Instant,
19 register: Duration,
20 health: Duration,
21 authority: Option<(
22 std::sync::Arc<crate::spawn_authority::SpawnAuthorityRecord>,
23 bool,
24 )>,
25 program: Option<super::admission::AdmittedProgram>,
26) -> Result<DeliverySpawn> {
27 #[cfg(test)]
28 if state.delivery.failed_spawn_module.as_deref() == Some(module) {
29 return Err(GateError::new(
30 "delivery_spawn_exhausted",
31 "delivery spawn identifiers exhausted",
32 ));
33 }
34 state.delivery.next_spawn = state.delivery.next_spawn.checked_add(1).ok_or_else(|| {
35 GateError::new(
36 "delivery_spawn_exhausted",
37 "delivery spawn identifiers exhausted",
38 )
39 })?;
40 let (authority, stamped) =
41 authority.map_or((None, false), |(record, stamped)| (Some(record), stamped));
42 Ok(DeliverySpawn {
43 id: state.delivery.next_spawn,
44 module: module.into(),
45 generation,
46 deadline: at + register + health + CONFIG_WARMUP_SLACK,
47 authority,
48 stamped,
49 program,
50 })
51}
52
53#[derive(Clone, Debug)]
54struct Attempt {
55 module: String,
56 generation: u64,
57 id: u64,
58 transition: Transition,
59 ready: bool,
60 acknowledged: bool,
61 delivered: bool,
62 deadline: Option<tokio::time::Instant>,
63 catchup: bool,
66 write_deadline: Option<tokio::time::Instant>,
67 policy_probe_requested: bool,
68 spawn: Option<DeliverySpawn>,
72 mark_only: bool,
76}
77
78#[derive(Clone, Debug)]
79pub struct DeliverySpawn {
80 id: u64,
81 module: String,
82 generation: u64,
83 deadline: tokio::time::Instant,
84 authority: Option<std::sync::Arc<crate::spawn_authority::SpawnAuthorityRecord>>,
86 stamped: bool,
89 program: Option<super::admission::AdmittedProgram>,
93}
94
95impl DeliverySpawn {
96 pub(crate) fn module(&self) -> &str {
97 &self.module
98 }
99 pub(crate) fn generation(&self) -> u64 {
100 self.generation
101 }
102 pub(crate) fn authority(
103 &self,
104 ) -> Option<&std::sync::Arc<crate::spawn_authority::SpawnAuthorityRecord>> {
105 self.authority.as_ref()
106 }
107 pub(crate) fn stamped(&self) -> bool {
108 self.stamped
109 }
110 pub(super) fn admitted_program(&self) -> Option<&super::admission::AdmittedProgram> {
111 self.program.as_ref()
112 }
113 #[cfg(test)]
116 pub(crate) fn with_authority_for_test(
117 mut self,
118 record: std::sync::Arc<crate::spawn_authority::SpawnAuthorityRecord>,
119 stamped: bool,
120 ) -> Self {
121 self.authority = Some(record);
122 self.stamped = stamped;
123 self
124 }
125}
126
127#[derive(Default)]
128pub(super) struct DeliveryState {
129 #[cfg(test)]
130 failed_spawn_module: Option<String>,
131 next_spawn: u64,
132 next_attempt: u64,
133 policy_epochs: HashMap<String, u64>,
134 attempts: HashMap<ConnectionId, Attempt>,
135 served: HashMap<ConnectionId, (Option<u64>, Option<u64>)>,
136 pub(super) unstamped_views: HashMap<ConnectionId, std::sync::Arc<ConfigView>>,
137 stamp_running: HashSet<ConnectionId>,
138 initial_generation: u64,
139 children: HashMap<ConnectionId, DeliverySpawn>,
140 healthy_children: HashSet<(ConnectionId, u64)>,
141 readiness_observed: HashSet<(ConnectionId, u64)>,
142 unregistered_children: HashMap<(String, bool), DeliverySpawn>,
143 recoveries: Vec<(ConnectionId, String, AppliedDisposition)>,
144 exited: HashMap<String, ExitedDelivery>,
145 last_target_attempt: HashMap<String, (u64, u64)>,
148 mark_records: Vec<(
154 std::sync::Weak<crate::spawn_authority::SpawnAuthorityRecord>,
155 Option<super::admission::AdmittedProgram>,
156 )>,
157}
158
159impl DeliveryState {
160 pub(super) fn reset_for_authority_exit(&mut self, generation: u64) {
161 *self = Self {
162 next_spawn: self.next_spawn,
163 next_attempt: self.next_attempt,
164 mark_records: std::mem::take(&mut self.mark_records),
165 ..Self::new(generation)
166 };
167 }
168 pub(super) fn track_mark(&mut self, spawn: &DeliverySpawn) {
171 let Some(record) = spawn.authority.as_ref().filter(|_| spawn.stamped) else {
172 return;
173 };
174 if !self
175 .mark_records
176 .iter()
177 .any(|(known, _)| std::ptr::eq(known.as_ptr(), std::sync::Arc::as_ptr(record)))
178 {
179 self.mark_records
180 .push((std::sync::Arc::downgrade(record), spawn.program.clone()));
181 }
182 }
183 pub(super) fn mark_records(
186 &mut self,
187 ) -> Vec<(
188 std::sync::Arc<crate::spawn_authority::SpawnAuthorityRecord>,
189 Option<super::admission::AdmittedProgram>,
190 )> {
191 self.mark_records
192 .retain(|(record, _)| record.upgrade().is_some_and(|record| record.alive()));
193 self.mark_records
194 .iter()
195 .filter_map(|(record, program)| Some((record.upgrade()?, program.clone())))
196 .collect()
197 }
198 pub(super) fn started_at_generation(&self) -> u64 {
199 self.initial_generation
200 }
201 pub(super) fn policy_authorization(
202 &self,
203 module: &str,
204 generation: u64,
205 connection: ConnectionId,
206 ) -> Option<(u64, tokio::time::Instant)> {
207 let attempt = self.attempts.get(&connection)?;
208 let deadline = attempt.deadline?;
209 (attempt.module == module
210 && attempt.generation == generation
211 && attempt.delivered
212 && attempt.acknowledged
213 && tokio::time::Instant::now() < deadline)
214 .then_some((attempt.id, deadline))
215 }
216 pub(super) fn fenced_modules(&self) -> impl Iterator<Item = String> + '_ {
217 self.exited.keys().cloned()
218 }
219}
220
221#[derive(Clone, Debug)]
224pub struct DeliveryProbe {
225 connection: ConnectionId,
226 module: String,
227 generation: u64,
228 attempt: Option<u64>,
229 policy_epoch: u64,
230 route_readiness: bool,
231}
232
233impl DeliveryProbe {
234 pub(crate) fn connection(&self) -> ConnectionId {
235 self.connection
236 }
237 pub(crate) fn owns_failure(&self) -> bool {
238 self.route_readiness
239 }
240}
241
242#[derive(Clone)]
243pub(crate) struct ExitedDelivery {
244 pub module: String,
245 spawn: u64,
246 generation: u64,
247 changed: tokio::sync::watch::Sender<bool>,
248}
249
250#[derive(Clone, Debug, PartialEq, Eq)]
251pub enum AppliedDisposition {
252 Ignored,
253 WaitingForReadiness,
254 Active,
255 CatchupComplete,
256 Failed {
257 generation: u64,
258 restore: Option<u64>,
259 keep_running: bool,
260 },
261 CatchupFailed {
262 active: u64,
263 },
264}
265
266#[derive(Clone, Copy)]
267pub(crate) struct DeliveryWindow {
268 pub generation: u64,
269 pub attempt: u64,
270 pub delivered: bool,
271 pub deadline: Option<tokio::time::Instant>,
272}
273
274impl DeliveryState {
275 pub(super) fn invalidate_policy(&mut self, module: &str, generation: u64) {
276 for attempt in self
277 .attempts
278 .values_mut()
279 .filter(|attempt| attempt.module == module && attempt.generation == generation)
280 {
281 attempt.ready = false;
282 attempt.policy_probe_requested = false;
283 }
284 }
285 pub(super) fn policy_installed(
286 &mut self,
287 module: &str,
288 generation: u64,
289 target: Option<ConnectionId>,
290 ) -> Result<()> {
291 let epoch = self.policy_epochs.entry(module.into()).or_default();
294 *epoch = epoch.checked_add(1).ok_or_else(|| {
295 GateError::new(
296 "delivery_probe_exhausted",
297 "policy readiness epochs exhausted",
298 )
299 })?;
300 self.invalidate_policy(module, generation);
301 let connections: HashSet<_> = self
302 .children
303 .iter()
304 .filter(|(_, spawn)| spawn.module == module)
305 .map(|(connection, _)| *connection)
306 .chain(
307 self.attempts
308 .iter()
309 .filter(|(_, attempt)| attempt.module == module)
310 .map(|(connection, _)| *connection),
311 )
312 .chain(target)
313 .collect();
314 self.readiness_observed.retain(|(connection, observed)| {
315 *observed != generation || !connections.contains(connection)
316 });
317 self.healthy_children.retain(|(connection, observed)| {
318 *observed != generation || !connections.contains(connection)
319 });
320 Ok(())
321 }
322 pub(super) fn declared(&mut self, connection: ConnectionId, module: &str, active: bool) {
323 if let Some(spawn) = self.unregistered_children.remove(&(module.into(), active)) {
324 self.children.insert(connection, spawn);
325 }
326 }
327 pub(super) fn new(initial_generation: u64) -> Self {
328 Self {
329 initial_generation,
330 ..Self::default()
331 }
332 }
333
334 pub(super) fn disconnected(&mut self, connection: ConnectionId) -> bool {
336 let mut held = self.attempts.remove(&connection).is_some();
337 held |= self.served.remove(&connection).is_some();
338 held |= self.unstamped_views.remove(&connection).is_some();
339 held |= self.stamp_running.remove(&connection);
340 held |= self.children.remove(&connection).is_some();
341 let observed = self.healthy_children.len() + self.readiness_observed.len();
342 self.healthy_children
343 .retain(|(child, _)| *child != connection);
344 self.readiness_observed
345 .retain(|(child, _)| *child != connection);
346 held || self.healthy_children.len() + self.readiness_observed.len() != observed
347 }
348
349 pub(super) fn remember_stamp_connections(&mut self, connections: Vec<ConnectionId>) {
350 self.stamp_running.extend(connections);
351 }
352
353 fn boundary_open(
354 &self,
355 connection: ConnectionId,
356 generation: u64,
357 transition: Transition,
358 ) -> bool {
359 if transition == Transition::DaemonRestart && generation > self.initial_generation {
360 return false;
361 }
362 !transition.requires_spawn()
363 || self
364 .children
365 .get(&connection)
366 .is_some_and(|spawn| spawn.generation == generation)
367 }
368
369 fn allocate(
370 &mut self,
371 module: &str,
372 generation: u64,
373 transition: Transition,
374 ) -> Result<Attempt> {
375 self.next_attempt = self.next_attempt.checked_add(1).ok_or_else(|| {
376 GateError::new(
377 "delivery_attempt_exhausted",
378 "delivery attempt ids exhausted",
379 )
380 })?;
381 Ok(Attempt {
382 module: module.into(),
383 generation,
384 id: self.next_attempt,
385 transition,
386 ready: transition == Transition::Live,
387 acknowledged: false,
388 delivered: false,
389 deadline: None,
390 catchup: false,
391 write_deadline: None,
392 policy_probe_requested: false,
393 spawn: None,
394 mark_only: false,
395 })
396 }
397}
398
399pub(super) fn visible_reply(
400 state: &OwnerState,
401 connection: ConnectionId,
402 mut reply: ConfigGetReply,
403 transition: Transition,
404) -> Result<ConfigGetReply> {
405 if reply.pending.as_ref().is_some_and(|view| {
406 !state
407 .delivery
408 .boundary_open(connection, view.generation, transition)
409 }) {
410 reply.pending = None;
411 }
412 if reply.active.is_none() && reply.pending.is_none() {
413 return Err(GateError::new(
414 "config_gated",
415 "generation waits for its required supervision or daemon boundary",
416 ));
417 }
418 Ok(reply)
419}
420
421pub(super) fn bind_get(
422 state: &mut OwnerState,
423 connection: ConnectionId,
424 module: &str,
425 request: &ConfigGetRequest,
426 reply: ConfigGetReply,
427 transition: Transition,
428) -> Result<ConfigGetReply> {
429 let mut reply = visible_reply(state, connection, reply, transition)?;
430 match (request.generation, request.attempt) {
431 (None, None) => {}
432 (Some(generation), Some(attempt))
433 if state
434 .delivery
435 .attempts
436 .get(&connection)
437 .is_some_and(|bound| {
438 bound.module == module && bound.generation == generation && bound.id == attempt
439 }) => {}
440 _ => {
441 return Err(GateError::new(
442 "config_gated",
443 "requested generation and attempt are not assigned to this connection",
444 ))
445 }
446 }
447 if reply.pending.is_none()
448 && state
449 .delivery
450 .attempts
451 .get(&connection)
452 .is_some_and(|bound| {
453 bound.catchup
454 && reply
455 .active
456 .as_ref()
457 .is_some_and(|view| view.generation == bound.generation)
458 })
459 {
460 reply.pending = reply.active.clone();
461 if let Some(view) = &mut reply.pending {
462 view.status = ConfigStatus::Pending;
463 }
464 }
465 if reply.pending.is_none() && request.generation.is_none() {
466 let generation = reply.active.as_ref().map(|view| view.generation);
467 if let Some(attempt) = generation
468 .map(|generation| mark_acknowledgement(state, connection, module, generation))
469 .transpose()?
470 .flatten()
471 {
472 state.delivery.attempts.insert(connection, attempt);
473 reply.pending = reply.active.clone();
474 if let Some(view) = &mut reply.pending {
475 view.status = ConfigStatus::Pending;
476 }
477 }
478 }
479 if let Some(pending) = &mut reply.pending {
480 let matches = state
481 .delivery
482 .attempts
483 .get(&connection)
484 .is_some_and(|bound| bound.module == module && bound.generation == pending.generation);
485 if !matches {
486 let mut attempt = state
487 .delivery
488 .allocate(module, pending.generation, transition)?;
489 if transition.requires_spawn() {
490 attempt.ready = state
491 .delivery
492 .healthy_children
493 .contains(&(connection, pending.generation));
494 attempt.deadline = state
495 .delivery
496 .children
497 .get(&connection)
498 .filter(|warmup| warmup.generation == pending.generation)
499 .map(|warmup| warmup.deadline);
500 }
501 attempt.ready &= super::route_policy::ready(state, module, pending.generation)?;
502 attempt.spawn = state.delivery.children.get(&connection).cloned();
503 state.delivery.attempts.insert(connection, attempt);
504 }
505 pending.attempt = state
506 .delivery
507 .attempts
508 .get(&connection)
509 .map(|bound| bound.id);
510 let mark_only = state
511 .delivery
512 .attempts
513 .get(&connection)
514 .is_some_and(|bound| bound.mark_only);
515 if state.active_declarations.get(module) == Some(&connection) && !mark_only {
516 if let Some(spawn) = state.delivery.children.get(&connection) {
517 let id = spawn.id;
518 state
519 .delivery
520 .last_target_attempt
521 .insert(module.into(), (id, pending.generation));
522 }
523 }
524 } else {
525 state.delivery.attempts.remove(&connection);
526 }
527 if request
528 .generation
529 .is_some_and(|id| reply.pending.as_ref().map(|view| view.generation) != Some(id))
530 {
531 return Err(GateError::new(
532 "config_gated",
533 "selected delivery has been superseded",
534 ));
535 }
536 Ok(reply)
537}
538
539fn mark_acknowledgement(
554 state: &mut OwnerState,
555 connection: ConnectionId,
556 module: &str,
557 generation: u64,
558) -> Result<Option<Attempt>> {
559 if state.store.loaded_admission_state().0 != crate::config_store::StoreState::Stamped
560 || state.active_declarations.get(module) != Some(&connection)
561 || state.delivery.attempts.contains_key(&connection)
562 {
563 return Ok(None);
564 }
565 let Some(spawn) = state
566 .delivery
567 .children
568 .get(&connection)
569 .filter(|spawn| spawn.stamped && spawn.module == module)
570 .cloned()
571 else {
572 return Ok(None);
573 };
574 let Some(record) = spawn.authority.as_ref() else {
575 return Ok(None);
576 };
577 let facts = record.mark_facts();
578 if !facts.unmarked || !facts.origin_qualifies || !record.request_acknowledgement(generation) {
579 return Ok(None);
580 }
581 let mut attempt = state
582 .delivery
583 .allocate(module, generation, Transition::Live)?;
584 attempt.catchup = true;
585 attempt.mark_only = true;
586 attempt.spawn = Some(spawn);
587 Ok(Some(attempt))
588}
589
590impl RuntimeOwner {
591 pub(crate) fn policy_probe_due(&self, connection: ConnectionId) -> Result<Option<(u64, u64)>> {
592 let mut state = self.lock()?;
593 let Some(bound) = state.delivery.attempts.get(&connection) else {
594 return Ok(None);
595 };
596 let (module, generation, attempt) = (bound.module.clone(), bound.generation, bound.id);
597 if bound.ready
600 || bound.mark_only
601 || bound.policy_probe_requested
602 || state.active_declarations.get(&module) != Some(&connection)
603 || !state
604 .store
605 .body()?
606 .generations
607 .get(&generation)
608 .is_some_and(|entry| {
609 super::route_policy::required(&entry.generation.changeset, &module)
610 })
611 || !super::route_policy::ready(&mut state, &module, generation)?
612 {
613 return Ok(None);
614 }
615 state
616 .delivery
617 .attempts
618 .get_mut(&connection)
619 .unwrap()
620 .policy_probe_requested = true;
621 Ok(Some((generation, attempt)))
622 }
623
624 pub(crate) fn policy_probe_pending(
625 &self,
626 connection: ConnectionId,
627 generation: u64,
628 attempt: u64,
629 ) -> Result<bool> {
630 let mut state = self.lock()?;
631 let Some(bound) = state.delivery.attempts.get(&connection) else {
632 return Ok(false);
633 };
634 let module = bound.module.clone();
635 Ok(bound.generation == generation
636 && bound.id == attempt
637 && !bound.ready
638 && state.active_declarations.get(&module) == Some(&connection)
639 && super::route_policy::ready(&mut state, &module, generation)?)
640 }
641 pub(crate) fn delivery_health_probe(
642 &self,
643 connection: ConnectionId,
644 module: &str,
645 ) -> Result<Option<DeliveryProbe>> {
646 let mut state = self.lock()?;
647 if state.active_declarations.get(module) != Some(&connection) {
648 return Ok(None);
649 }
650 let attempt = state
651 .delivery
652 .attempts
653 .get(&connection)
654 .map(|attempt| (attempt.generation, attempt.id));
655 let generation = attempt.map(|attempt| attempt.0).or_else(|| {
656 state
657 .delivery
658 .children
659 .get(&connection)
660 .map(|child| child.generation)
661 });
662 let route_readiness = match generation {
663 Some(generation) if attempt.is_some() => state
664 .store
665 .body()?
666 .generations
667 .get(&generation)
668 .is_some_and(|entry| {
669 super::route_policy::required(&entry.generation.changeset, module)
670 }),
671 _ => false,
672 };
673 Ok(generation.map(|generation| DeliveryProbe {
674 connection,
675 module: module.into(),
676 generation,
677 attempt: attempt.map(|attempt| attempt.1),
678 route_readiness,
679 policy_epoch: state
680 .delivery
681 .policy_epochs
682 .get(module)
683 .copied()
684 .unwrap_or_default(),
685 }))
686 }
687
688 pub(crate) fn delivery_health_result(
689 &self,
690 probe: DeliveryProbe,
691 healthy: bool,
692 ) -> Result<Option<AppliedDisposition>> {
693 let attempt = {
694 let mut state = self.lock()?;
695 if state.active_declarations.get(&probe.module) != Some(&probe.connection)
696 || state
697 .delivery
698 .policy_epochs
699 .get(&probe.module)
700 .copied()
701 .unwrap_or_default()
702 != probe.policy_epoch
703 {
704 return Ok(None);
705 }
706 let current = state.delivery.attempts.get(&probe.connection);
707 if probe
708 .attempt
709 .is_some_and(|id| current.is_none_or(|bound| bound.id != id))
710 || current.is_some_and(|bound| bound.generation != probe.generation)
711 {
712 return Ok(None);
713 }
714 let attempt = current
718 .filter(|bound| !bound.mark_only)
719 .map(|bound| bound.id);
720 if !healthy
721 && (!super::route_policy::ready(&mut state, &probe.module, probe.generation)?
722 || !state
723 .store
724 .body()?
725 .generations
726 .get(&probe.generation)
727 .is_some_and(|entry| {
728 super::route_policy::required(
729 &entry.generation.changeset,
730 &probe.module,
731 )
732 }))
733 {
734 return Ok(None);
735 }
736 attempt
737 };
738 if healthy {
739 self.delivery_readiness_complete(probe.connection, &probe.module, probe.generation)
740 } else if let Some(attempt) = attempt {
741 self.module_outcome(
742 probe.connection,
743 &probe.module,
744 ConfigAppliedRequest::new(
745 probe.generation,
746 attempt,
747 subc_protocol::config::ConfigApplyResult::Nack,
748 vec![],
749 ),
750 true,
751 )
752 .map(Some)
753 } else {
754 Ok(None)
755 }
756 }
757
758 #[cfg(test)]
759 pub(super) fn ready(
760 &self,
761 connection: ConnectionId,
762 module: &str,
763 generation: u64,
764 ) -> Result<Option<AppliedDisposition>> {
765 self.delivery_readiness_complete(connection, module, generation)
766 }
767
768 pub(crate) fn pending_config_restarts(&self) -> Result<Vec<(String, u64)>> {
769 let mut state = self.lock()?;
770 state.store_read(ConfigStore::delivery_ready)?;
771 let body = state.checked_body()?;
772 body.versions
773 .iter()
774 .filter(|(tuple, _)| tuple.project.is_none())
775 .filter_map(|(tuple, versions)| {
776 let generation = versions.pending?;
777 let entry = body.generations.get(&generation)?;
778 Some(
779 Transition::for_module(&entry.generation.changeset, &tuple.module).map(
780 |transition| {
781 (transition == Transition::ModuleRestart)
782 .then(|| (tuple.module.clone(), generation))
783 },
784 ),
785 )
786 })
787 .collect::<Result<Vec<_>>>()
788 .map(|entries| entries.into_iter().flatten().collect())
789 }
790
791 pub(crate) fn requires_fresher_spawn(
792 &self,
793 module: &str,
794 spawn: Option<&DeliverySpawn>,
795 ) -> Result<Option<u64>> {
796 let mut state = self.lock()?;
797 state.store.delivery_ready()?;
798 Self::requires_fresher_spawn_loaded(&state, module, spawn)
799 }
800
801 pub(super) fn requires_fresher_spawn_loaded(
802 state: &OwnerState,
803 module: &str,
804 spawn: Option<&DeliverySpawn>,
805 ) -> Result<Option<u64>> {
806 let body = state.store.loaded_admission_state().1;
807 let pending = body
808 .versions
809 .iter()
810 .find(|(tuple, _)| tuple.module == module && tuple.project.is_none())
811 .and_then(|(_, versions)| versions.pending);
812 let Some(generation) = pending else {
813 return Ok(None);
814 };
815 let entry = body
816 .generations
817 .get(&generation)
818 .ok_or_else(|| GateError::new("config_gated", "pending generation is absent"))?;
819 let transition = Transition::for_module(&entry.generation.changeset, module)?;
820 let deferred_daemon = transition == Transition::DaemonRestart
821 && generation > state.delivery.initial_generation;
822 Ok((transition.requires_spawn()
823 && !deferred_daemon
824 && spawn
825 .is_none_or(|spawn| spawn.module != module || spawn.generation() != generation))
826 .then_some(generation))
827 }
828
829 pub(crate) fn delivery_child_exited(&self, spawn: &DeliverySpawn) -> Result<()> {
832 let mut state = self.lock()?;
833 let pending = state
834 .store
835 .body()?
836 .versions
837 .iter()
838 .find(|(tuple, _)| tuple.module == spawn.module && tuple.project.is_none())
839 .and_then(|(_, versions)| versions.pending);
840 let pending = pending.filter(|generation| {
841 state
842 .store
843 .loaded_admission_state()
844 .1
845 .generations
846 .get(generation)
847 .and_then(|entry| entry.delivery.as_ref())
848 .is_some_and(|delivery| delivery.declarations.contains_key(&spawn.module))
849 });
850 if let Some(generation) = pending.filter(|generation| {
851 *generation == spawn.generation
852 || state.delivery.last_target_attempt.get(&spawn.module)
853 == Some(&(spawn.id, *generation))
854 }) {
855 state
856 .delivery
857 .exited
858 .entry(spawn.module.clone())
859 .or_insert_with(|| ExitedDelivery {
860 module: spawn.module.clone(),
861 spawn: spawn.id,
862 generation,
863 changed: tokio::sync::watch::channel(true).0,
864 });
865 }
866 self.notify_delivery_changed();
867 Ok(())
868 }
869
870 pub(crate) fn delivery_exit_fence(
871 &self,
872 module: &str,
873 ) -> Result<Option<tokio::sync::watch::Receiver<bool>>> {
874 Ok(self
875 .lock()?
876 .delivery
877 .exited
878 .get(module)
879 .map(|entry| entry.changed.subscribe()))
880 }
881
882 pub(crate) fn delivery_exits(&self) -> Result<Vec<ExitedDelivery>> {
883 Ok(self.lock()?.delivery.exited.values().cloned().collect())
884 }
885
886 pub(crate) fn delivery_record_exit(&self, exited: &ExitedDelivery) -> Result<bool> {
893 #[cfg(test)]
894 self.exit_record_attempts
895 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
896 let mut state = self.lock()?;
897 if !state
898 .delivery
899 .exited
900 .get(&exited.module)
901 .is_some_and(|entry| entry.spawn == exited.spawn)
902 {
903 return Ok(false);
904 }
905 let tuple = Tuple {
906 module: exited.module.clone(),
907 project: None,
908 };
909 let versions = state
910 .store
911 .body()?
912 .versions
913 .iter()
914 .find(|(key, _)| key == &tuple)
915 .map(|(_, versions)| versions.clone())
916 .unwrap_or_default();
917 if versions
918 .pending
919 .is_some_and(|generation| generation != exited.generation)
920 {
921 return Ok(false);
922 }
923 if versions.pending == Some(exited.generation) {
924 state.restore_route_policy(&exited.module, exited.generation);
925 let result = state
926 .store
927 .record_delivery(&tuple, exited.generation, false);
928 if let Err(error) = result {
929 if error.code != crate::config_store::StoreErrorCode::AnchorPending {
930 return Err(error.into());
931 }
932 }
933 state.delivery.attempts.retain(|_, attempt| {
934 attempt.module != exited.module || attempt.generation != exited.generation
935 });
936 }
937 Ok(versions.active.is_none())
938 }
939
940 #[cfg(all(test, unix))]
941 pub(crate) fn exit_record_attempts_for_test(&self) -> usize {
942 self.exit_record_attempts
943 .load(std::sync::atomic::Ordering::SeqCst)
944 }
945
946 pub(crate) fn delivery_release_exit(&self, exited: &ExitedDelivery) -> Result<()> {
947 let mut state = self.lock()?;
948 if state
949 .delivery
950 .exited
951 .get(&exited.module)
952 .is_some_and(|entry| entry.spawn == exited.spawn)
953 {
954 if let Some(entry) = state.delivery.exited.remove(&exited.module) {
955 entry.changed.send_replace(false);
956 state.admission_releases.push(exited.module.clone());
960 }
961 }
962 self.notify_delivery_changed();
963 Ok(())
964 }
965 pub(crate) fn delivery_abandon_spawn(&self, spawn: &DeliverySpawn, active: bool) {
966 if let Ok(mut state) = self.lock() {
967 let key = (spawn.module.clone(), active);
968 if state
969 .delivery
970 .unregistered_children
971 .get(&key)
972 .is_some_and(|current| current.id == spawn.id)
973 {
974 state.delivery.unregistered_children.remove(&key);
975 }
976 }
977 }
978 pub(crate) fn delivery_queue_abandoned_spawn(&self, spawn: DeliverySpawn, active: bool) {
979 if let Ok(mut abandoned) = self.abandoned_spawns.lock() {
980 abandoned.push((spawn, active));
981 }
982 self.notify_delivery_changed();
983 }
984 pub(super) fn drain_abandoned_spawns(&self, state: &mut OwnerState) -> bool {
987 let mut drained = false;
988 if let Ok(mut abandoned) = self.abandoned_spawns.lock() {
989 for (spawn, active) in abandoned.drain(..) {
990 drained = true;
991 let key = (spawn.module.clone(), active);
992 if state
993 .delivery
994 .unregistered_children
995 .get(&key)
996 .is_some_and(|current| current.id == spawn.id)
997 {
998 state.delivery.unregistered_children.remove(&key);
999 }
1000 }
1001 }
1002 drained
1003 }
1004
1005 pub(crate) fn delivery_launch_current(
1009 &self,
1010 spawn: &DeliverySpawn,
1011 active: bool,
1012 ) -> Result<Option<bool>> {
1013 let state = match self.inner.try_lock() {
1014 Ok(state) => state,
1015 Err(std::sync::TryLockError::WouldBlock) => return Ok(None),
1016 Err(_) => {
1017 return Err(GateError::new(
1018 "config_unavailable",
1019 "configuration owner lock is poisoned",
1020 ))
1021 }
1022 };
1023 Ok(Some(
1024 super::admission::AdmissionCache::from_state(&state)
1025 .spawn_generation(&spawn.module)
1026 .is_ok_and(|generation| generation == spawn.generation)
1027 && state
1028 .delivery
1029 .unregistered_children
1030 .get(&(spawn.module.clone(), active))
1031 .is_some_and(|current| current.id == spawn.id)
1032 && !state.delivery.exited.contains_key(&spawn.module),
1033 ))
1034 }
1035 #[cfg(test)]
1036 pub(crate) fn pause_spawn_preflight_for_test(&self) {
1037 let hook = self.spawn_preflight_hook.lock().unwrap().take();
1038 if let Some(hook) = hook {
1039 hook();
1040 }
1041 }
1042 #[cfg(all(test, unix))]
1043 pub(crate) fn exhaust_delivery_spawns_for_test(&self) {
1044 self.lock().unwrap().delivery.next_spawn = u64::MAX;
1045 }
1046 #[cfg(all(test, unix))]
1047 pub(crate) fn restore_unused_delivery_spawns_for_test(&self) {
1048 self.lock().unwrap().delivery.next_spawn = 1000;
1049 }
1050 #[cfg(all(test, unix))]
1051 pub(crate) fn fail_delivery_spawns_for_module_for_test(&self, module: &str) {
1052 self.lock().unwrap().delivery.failed_spawn_module = Some(module.into());
1053 }
1054 #[cfg(all(test, unix))]
1055 pub(crate) fn clear_delivery_spawn_failure_for_test(&self) {
1056 self.lock().unwrap().delivery.failed_spawn_module = None;
1057 }
1058 #[cfg(all(test, unix))]
1059 pub(crate) fn set_spawn_preflight_hook_for_test(&self, hook: impl FnOnce() + Send + 'static) {
1060 *self.spawn_preflight_hook.lock().unwrap() = Some(Box::new(hook));
1061 }
1062 #[cfg(all(test, unix))]
1063 pub(crate) fn unregistered_spawn_count_for_test(&self) -> usize {
1064 self.lock().unwrap().delivery.unregistered_children.len()
1065 }
1066 #[allow(clippy::too_many_arguments)]
1067 pub(crate) fn delivery_spawned_for_admission_in_slot(
1068 &self,
1069 module: &str,
1070 admission: &super::admission::SpawnAdmission,
1071 at: tokio::time::Instant,
1072 register: Duration,
1073 health: Duration,
1074 active: bool,
1075 authority: Option<std::sync::Arc<crate::spawn_authority::SpawnAuthorityRecord>>,
1076 ) -> Result<Option<DeliverySpawn>> {
1077 let mut state = OwnerGuard::new(
1078 self.inner.try_lock().map_err(|error| {
1079 if matches!(error, std::sync::TryLockError::Poisoned(_)) {
1080 return GateError::new("config_unavailable", "configuration owner is poisoned");
1081 }
1082 GateError::new(
1083 "config_delivery_deferred",
1084 "configuration publication is busy",
1085 )
1086 })?,
1087 self,
1088 );
1089 state.store.check()?;
1090 let stamped = admission.gated()
1093 && state.store.loaded_admission_state().0 == crate::config_store::StoreState::Stamped;
1094 let spawn = allocate_spawn(
1095 &mut state,
1096 module,
1097 admission.generation(),
1098 at,
1099 register,
1100 health,
1101 authority.map(|record| (record, stamped)),
1102 admission.admitted_program(),
1103 )?;
1104 state.delivery.track_mark(&spawn);
1105 state
1106 .delivery
1107 .unregistered_children
1108 .insert((module.into(), active), spawn.clone());
1109 Ok(Some(spawn))
1110 }
1111
1112 #[cfg(test)]
1113 pub(crate) fn delivery_spawned_in_slot(
1114 &self,
1115 module: &str,
1116 at: tokio::time::Instant,
1117 register: Duration,
1118 health: Duration,
1119 active: bool,
1120 ) -> Result<Option<DeliverySpawn>> {
1121 let spawn = self.delivery_spawned(module, at, register, health)?;
1122 if let Some(spawn) = &spawn {
1123 self.lock()?
1124 .delivery
1125 .unregistered_children
1126 .insert((module.into(), active), spawn.clone());
1127 }
1128 Ok(spawn)
1129 }
1130 pub(crate) fn delivery_take_recoveries(
1131 &self,
1132 ) -> Result<Vec<(ConnectionId, String, AppliedDisposition)>> {
1133 let mut state = self.lock()?;
1134 if state.delivery.recoveries.is_empty() {
1135 return Ok(Vec::new());
1137 }
1138 Ok(std::mem::take(&mut state.delivery.recoveries))
1139 }
1140
1141 pub(crate) fn delivery_window(
1142 &self,
1143 connection: ConnectionId,
1144 ) -> Result<Option<DeliveryWindow>> {
1145 Ok(self
1146 .lock()?
1147 .delivery
1148 .attempts
1149 .get(&connection)
1150 .map(|bound| DeliveryWindow {
1151 generation: bound.generation,
1152 attempt: bound.id,
1153 delivered: bound.delivered,
1154 deadline: bound.deadline,
1155 }))
1156 }
1157
1158 pub(crate) fn delivery_write_started(
1159 &self,
1160 connection: ConnectionId,
1161 generation: u64,
1162 attempt: u64,
1163 now: tokio::time::Instant,
1164 ) -> Result<()> {
1165 if let Some(bound) = self
1166 .lock()?
1167 .delivery
1168 .attempts
1169 .get_mut(&connection)
1170 .filter(|bound| {
1171 bound.generation == generation && bound.id == attempt && !bound.delivered
1172 })
1173 {
1174 bound
1175 .write_deadline
1176 .get_or_insert(now + CONFIG_LIVE_ACK_TIMEOUT);
1177 }
1178 Ok(())
1179 }
1180
1181 pub(crate) fn delivery_timeout(
1182 &self,
1183 connection: ConnectionId,
1184 module: &str,
1185 generation: u64,
1186 attempt: u64,
1187 ) -> Result<AppliedDisposition> {
1188 self.module_outcome(
1189 connection,
1190 module,
1191 ConfigAppliedRequest::new(
1192 generation,
1193 attempt,
1194 subc_protocol::config::ConfigApplyResult::Nack,
1195 vec![],
1196 ),
1197 true,
1198 )
1199 }
1200
1201 pub(crate) fn delivery_spawned(
1204 &self,
1205 module: &str,
1206 spawned_at: tokio::time::Instant,
1207 register_timeout: Duration,
1208 first_health_budget: Duration,
1209 ) -> Result<Option<DeliverySpawn>> {
1210 let mut state = self.lock()?;
1211 state.store.check()?;
1212 let admission = super::admission::AdmissionCache::from_state(&state);
1213 let generation = admission.spawn_generation(module)?;
1214 allocate_spawn(
1215 &mut state,
1216 module,
1217 generation,
1218 spawned_at,
1219 register_timeout,
1220 first_health_budget,
1221 None,
1222 admission.admitted_program(module),
1223 )
1224 .map(Some)
1225 }
1226
1227 pub(crate) fn delivery_child_registered(
1228 &self,
1229 connection: ConnectionId,
1230 spawn: &DeliverySpawn,
1231 ) -> Result<()> {
1232 let mut state = self.lock()?;
1233 state.delivery.track_mark(spawn);
1234 state.delivery.children.insert(connection, spawn.clone());
1235 if let Some(bound) = state
1236 .delivery
1237 .attempts
1238 .get_mut(&connection)
1239 .filter(|bound| bound.module == spawn.module && bound.generation == spawn.generation)
1240 {
1241 if bound.transition.requires_spawn() {
1242 bound.deadline = Some(spawn.deadline);
1243 }
1244 }
1245 Ok(())
1246 }
1247
1248 pub(crate) fn module_view_delivered(
1249 &self,
1250 connection: ConnectionId,
1251 module: &str,
1252 reply: &ConfigGetReply,
1253 written_at: tokio::time::Instant,
1254 active: bool,
1255 ) -> Result<()> {
1256 {
1257 let mut state = self.lock()?;
1258 if !state
1259 .declarations
1260 .get(&connection)
1261 .is_some_and(|(id, _)| id == module)
1262 {
1263 return Err(GateError::new(
1264 "not_delivery_target",
1265 "configuration connection has ended",
1266 ));
1267 }
1268 let body = state.store.body()?;
1269 if body.stamped {
1270 let versions = body
1271 .versions
1272 .iter()
1273 .find(|(tuple, _)| tuple.module == module && tuple.project.is_none())
1274 .map(|(_, versions)| versions.clone())
1275 .unwrap_or_default();
1276 if reply
1277 .active
1278 .as_ref()
1279 .is_some_and(|view| versions.active != Some(view.generation))
1280 || reply.pending.as_ref().is_some_and(|view| {
1281 let catchup = versions.active == Some(view.generation)
1284 && state
1285 .delivery
1286 .attempts
1287 .get(&connection)
1288 .is_some_and(|bound| {
1289 bound.catchup
1290 && bound.generation == view.generation
1291 && Some(bound.id) == view.attempt
1292 });
1293 versions.pending != Some(view.generation) && !catchup
1294 })
1295 {
1296 return Err(GateError::new(
1297 "delivery_superseded",
1298 "configuration changed while its view was being written",
1299 ));
1300 }
1301 }
1302 }
1303 if active {
1304 if let Some(view) = &reply.pending {
1305 if let Some(attempt) = view.attempt {
1306 self.delivery_written(connection, view.generation, attempt, written_at)?;
1307 }
1308 }
1309 }
1310 let mut state = self.lock()?;
1311 if !state
1312 .declarations
1313 .get(&connection)
1314 .is_some_and(|(id, _)| id == module)
1315 {
1316 return Err(GateError::new(
1317 "not_delivery_target",
1318 "configuration connection has ended",
1319 ));
1320 }
1321 state.delivery.served.insert(
1322 connection,
1323 (
1324 reply.active.as_ref().map(|view| view.generation),
1325 reply.pending.as_ref().map(|view| view.generation),
1326 ),
1327 );
1328 if let Some(view) = reply
1329 .active
1330 .as_ref()
1331 .filter(|view| view.status == ConfigStatus::Unstamped)
1332 {
1333 state
1334 .delivery
1335 .unstamped_views
1336 .insert(connection, std::sync::Arc::new(view.clone()));
1337 } else {
1338 state.delivery.unstamped_views.remove(&connection);
1339 }
1340 Ok(())
1341 }
1342 pub(crate) fn delivery_readiness_complete(
1345 &self,
1346 connection: ConnectionId,
1347 module: &str,
1348 generation: u64,
1349 ) -> Result<Option<AppliedDisposition>> {
1350 let attempt = {
1351 let mut state = self.lock()?;
1352 if state.active_declarations.get(module) != Some(&connection) {
1353 return Err(GateError::new(
1354 "not_delivery_target",
1355 "readiness belongs to another connection",
1356 ));
1357 }
1358 let started_for_generation = state
1359 .delivery
1360 .children
1361 .get(&connection)
1362 .is_some_and(|warmup| warmup.generation == generation);
1363 if started_for_generation {
1364 state
1365 .delivery
1366 .healthy_children
1367 .insert((connection, generation));
1368 }
1369 let policy_ready = super::route_policy::ready(&mut state, module, generation)?;
1370 let Some(bound) = state
1371 .delivery
1372 .attempts
1373 .get_mut(&connection)
1374 .filter(|bound| bound.module == module && bound.generation == generation)
1375 else {
1376 return Ok(None);
1377 };
1378 if bound.transition.requires_spawn() && !started_for_generation {
1379 return Ok(None);
1380 }
1381 bound.ready = policy_ready;
1382 let acknowledged = bound.ready && bound.acknowledged;
1383 let id = bound.id;
1384 state
1385 .delivery
1386 .readiness_observed
1387 .insert((connection, generation));
1388 acknowledged.then_some(id)
1389 };
1390 attempt
1391 .map(|attempt| {
1392 self.module_applied(
1393 connection,
1394 module,
1395 ConfigAppliedRequest::new(
1396 generation,
1397 attempt,
1398 subc_protocol::config::ConfigApplyResult::Ack,
1399 vec![],
1400 ),
1401 )
1402 })
1403 .transpose()
1404 }
1405
1406 pub(crate) fn delivery_promoted(
1409 &self,
1410 connection: ConnectionId,
1411 module: &str,
1412 cutover_at: tokio::time::Instant,
1413 ) -> Result<Option<(u64, u64)>> {
1414 let mut state = self.lock()?;
1415 state.store.delivery_ready()?;
1416 Self::promote_loaded(&mut state, connection, module, cutover_at)
1417 }
1418
1419 pub(super) fn promote_loaded(
1420 state: &mut OwnerState,
1421 connection: ConnectionId,
1422 module: &str,
1423 cutover_at: tokio::time::Instant,
1424 ) -> Result<Option<(u64, u64)>> {
1425 if state.store.loaded_admission_state().0 == crate::config_store::StoreState::Unstamped {
1426 if state
1427 .declarations
1428 .get(&connection)
1429 .is_some_and(|(id, _)| id == module)
1430 {
1431 state.phone_target_changed(module, connection);
1432 state.active_declarations.insert(module.into(), connection);
1433 }
1434 return Ok(None);
1435 }
1436 let Some((id, declaration)) = state
1437 .declarations
1438 .get(&connection)
1439 .cloned()
1440 .filter(|(id, _)| id == module)
1441 else {
1442 return Err(GateError::new(
1443 "config_gated",
1444 "candidate has not declared its approved configuration",
1445 ));
1446 };
1447 let tuple = Tuple {
1448 module: module.into(),
1449 project: None,
1450 };
1451 let body = state.store.loaded_admission_state().1;
1452 let versions = body
1453 .versions
1454 .iter()
1455 .find(|(key, _)| key == &tuple)
1456 .map(|(_, versions)| versions.clone())
1457 .unwrap_or_default();
1458 let transition = versions
1459 .pending
1460 .and_then(|id| body.generations.get(&id))
1461 .map(|entry| Transition::for_module(&entry.generation.changeset, module))
1462 .transpose()?
1463 .unwrap_or(Transition::Live);
1464 let pending_declaration = versions
1465 .pending
1466 .and_then(|id| body.generations.get(&id))
1467 .and_then(|entry| entry.delivery.as_ref())
1468 .and_then(|snapshot| snapshot.declarations.get(module))
1469 .cloned();
1470 let active_declaration = versions
1471 .active
1472 .and_then(|id| body.generations.get(&id))
1473 .and_then(|entry| entry.delivery.as_ref())
1474 .and_then(|snapshot| snapshot.declarations.get(module))
1475 .cloned();
1476 let pending = versions.pending.filter(|generation| {
1477 state
1478 .delivery
1479 .boundary_open(connection, *generation, transition)
1480 });
1481 if pending != versions.pending && transition != Transition::DaemonRestart {
1482 return Err(GateError::new(
1483 "config_gated",
1484 "candidate launch was superseded by a configuration requiring a fresh start",
1485 ));
1486 }
1487 let expected = if pending.is_some() {
1488 pending_declaration
1489 } else {
1490 active_declaration
1491 };
1492 if expected.as_ref() != Some(&declaration) {
1493 return Err(GateError::new(
1494 "config_gated",
1495 "candidate declaration differs from the approved declaration",
1496 ));
1497 }
1498 let transition = if pending.is_some() {
1499 transition
1500 } else {
1501 Transition::Live
1502 };
1503 let vector = (versions.active, pending);
1504 let catchup = pending.is_none()
1505 && versions.active.is_some()
1506 && state.delivery.served.get(&connection) != Some(&vector);
1507 state.phone_target_changed(module, connection);
1508 state.active_declarations.insert(id, connection);
1509 state.delivery.attempts.remove(&connection);
1510 let Some(generation) = pending.or_else(|| catchup.then_some(versions.active).flatten())
1511 else {
1512 return Ok(None);
1513 };
1514 let mut attempt = state.delivery.allocate(module, generation, transition)?;
1515 attempt.catchup = catchup;
1516 attempt.spawn = state.delivery.children.get(&connection).cloned();
1517 attempt.ready = super::route_policy::ready_loaded(state, module, generation)
1522 && !state
1523 .store
1524 .loaded_admission_state()
1525 .1
1526 .generations
1527 .get(&generation)
1528 .is_some_and(|entry| {
1529 super::route_policy::required(&entry.generation.changeset, module)
1530 });
1531 attempt.deadline = Some(cutover_at + CONFIG_LIVE_ACK_TIMEOUT);
1532 let assignment = (generation, attempt.id);
1533 state.delivery.attempts.insert(connection, attempt);
1534 state
1535 .delivery
1536 .readiness_observed
1537 .insert((connection, generation));
1538 Ok(Some(assignment))
1539 }
1540
1541 pub(crate) fn delivery_written(
1544 &self,
1545 connection: ConnectionId,
1546 generation: u64,
1547 attempt: u64,
1548 written_at: tokio::time::Instant,
1549 ) -> Result<()> {
1550 let mut state = self.lock()?;
1551 let bound = state
1552 .delivery
1553 .attempts
1554 .get_mut(&connection)
1555 .filter(|bound| bound.generation == generation && bound.id == attempt)
1556 .ok_or_else(|| {
1557 GateError::new(
1558 "delivery_superseded",
1559 "writer receipt no longer names the current delivery attempt",
1560 )
1561 })?;
1562 if bound
1563 .write_deadline
1564 .is_some_and(|deadline| written_at >= deadline)
1565 || bound
1566 .deadline
1567 .is_some_and(|deadline| written_at >= deadline)
1568 {
1569 return Err(GateError::new(
1570 "delivery_expired",
1571 "configuration delivery timed out",
1572 ));
1573 }
1574 bound.delivered = true;
1575 if !bound.transition.requires_spawn() && bound.deadline.is_none() {
1576 bound.deadline = Some(written_at + CONFIG_LIVE_ACK_TIMEOUT);
1577 }
1578 Ok(())
1579 }
1580
1581 pub(crate) fn module_applied(
1582 &self,
1583 connection: ConnectionId,
1584 module: &str,
1585 request: ConfigAppliedRequest,
1586 ) -> Result<AppliedDisposition> {
1587 self.module_outcome(connection, module, request, false)
1588 }
1589
1590 fn module_outcome(
1591 &self,
1592 connection: ConnectionId,
1593 module: &str,
1594 request: ConfigAppliedRequest,
1595 timed_out: bool,
1596 ) -> Result<AppliedDisposition> {
1597 let mut state = self.lock()?;
1598 if !state
1599 .declarations
1600 .get(&connection)
1601 .is_some_and(|(id, _)| id == module)
1602 {
1603 return Err(GateError::new(
1604 "declaration_order",
1605 "config.declare must precede get or applied on this connection",
1606 ));
1607 }
1608 if request.project_id.is_some() {
1609 return Err(GateError::new(
1610 "config_gated",
1611 "project-tuple delivery is not supported yet",
1612 ));
1613 }
1614 if state.active_declarations.get(module) != Some(&connection) {
1615 return Err(GateError::new(
1616 "not_delivery_target",
1617 "connection is not the active delivery target",
1618 ));
1619 }
1620 state.store.check()?;
1621 let Some(bound) = state
1622 .delivery
1623 .attempts
1624 .get(&connection)
1625 .cloned()
1626 .filter(|bound| {
1627 bound.module == module
1628 && bound.generation == request.generation
1629 && bound.id == request.attempt
1630 })
1631 else {
1632 return Ok(AppliedDisposition::Ignored);
1633 };
1634 if !bound.delivered && !timed_out {
1635 return Ok(AppliedDisposition::Ignored);
1636 }
1637 let tuple = Tuple {
1638 module: module.into(),
1639 project: None,
1640 };
1641 let versions = state
1642 .store
1643 .body()?
1644 .versions
1645 .iter()
1646 .find(|(key, _)| key == &tuple)
1647 .map(|(_, versions)| versions.clone())
1648 .unwrap_or_default();
1649 let ack = request.result.is_ack()
1650 && !bound
1651 .deadline
1652 .is_some_and(|deadline| tokio::time::Instant::now() >= deadline);
1653 if bound.catchup {
1654 let pending_transition = versions
1657 .pending
1658 .map(|generation| {
1659 let entry = state
1660 .store
1661 .body()?
1662 .generations
1663 .get(&generation)
1664 .ok_or_else(|| {
1665 GateError::new("config_gated", "pending generation is absent")
1666 })?;
1667 Transition::for_module(&entry.generation.changeset, module)
1668 })
1669 .transpose()?;
1670 let pending_is_deliverable =
1671 versions
1672 .pending
1673 .zip(pending_transition)
1674 .is_some_and(|(generation, transition)| {
1675 state
1676 .delivery
1677 .boundary_open(connection, generation, transition)
1678 });
1679 if pending_is_deliverable || versions.active != Some(bound.generation) {
1680 return Ok(AppliedDisposition::Ignored);
1681 }
1682 state.delivery.attempts.remove(&connection);
1683 if bound.mark_only {
1684 return Ok(if ack {
1687 super::mark::acknowledged(self, &state, bound.spawn.as_ref(), bound.generation);
1688 AppliedDisposition::CatchupComplete
1689 } else {
1690 super::mark::acknowledgement_refused(self, &state, bound.spawn.as_ref());
1691 AppliedDisposition::Ignored
1692 });
1693 }
1694 if ack {
1695 super::mark::acknowledged(self, &state, bound.spawn.as_ref(), bound.generation);
1696 } else {
1697 super::mark::delivery_failed(self, bound.spawn.as_ref());
1698 state.delivery.recoveries.push((
1699 connection,
1700 module.into(),
1701 AppliedDisposition::CatchupFailed {
1702 active: bound.generation,
1703 },
1704 ));
1705 }
1706 return Ok(if ack {
1707 AppliedDisposition::CatchupComplete
1708 } else {
1709 AppliedDisposition::CatchupFailed {
1710 active: bound.generation,
1711 }
1712 });
1713 }
1714 if versions.pending != Some(bound.generation) {
1715 return Ok(AppliedDisposition::Ignored);
1716 }
1717 if ack
1718 && (!bound.ready
1719 || (bound.transition == Transition::DaemonRestart
1720 && bound.generation > state.delivery.initial_generation))
1721 {
1722 if let Some(bound) = state.delivery.attempts.get_mut(&connection) {
1723 bound.acknowledged = true;
1724 }
1725 return Ok(AppliedDisposition::WaitingForReadiness);
1726 }
1727 if !ack {
1728 state.restore_route_policy(module, bound.generation);
1729 }
1730 let result = state.store.record_delivery(&tuple, bound.generation, ack);
1731 if result.is_ok() && ack {
1732 state.commit_route_policy(module, bound.generation);
1733 }
1734 if result.is_ok() || !ack {
1737 state.delivery.attempts.remove(&connection);
1738 }
1739 if !ack {
1740 let keep_running =
1741 versions.active.is_none() && state.delivery.stamp_running.contains(&connection);
1742 state.delivery.recoveries.push((
1743 connection,
1744 module.into(),
1745 AppliedDisposition::Failed {
1746 generation: bound.generation,
1747 restore: versions.active,
1748 keep_running,
1749 },
1750 ));
1751 }
1752 if ack && result.is_ok() {
1753 super::mark::acknowledged(self, &state, bound.spawn.as_ref(), bound.generation);
1754 } else if !ack {
1755 super::mark::delivery_failed(self, bound.spawn.as_ref());
1756 }
1757 result?;
1758 Ok(if ack {
1759 AppliedDisposition::Active
1760 } else {
1761 AppliedDisposition::Failed {
1762 generation: bound.generation,
1763 restore: versions.active,
1764 keep_running: versions.active.is_none()
1765 && state.delivery.stamp_running.contains(&connection),
1766 }
1767 })
1768 }
1769}