1use std::collections::HashSet;
4use subc_protocol::config::{ConfigAppliedRequest, ConfigGetReply, ConfigGetRequest};
5
6use super::*;
7use crate::config_gate::apply::Transition;
8use crate::config_store::Tuple;
9
10pub(crate) const CONFIG_LIVE_ACK_TIMEOUT: Duration = Duration::from_secs(30);
11pub(crate) const CONFIG_WARMUP_SLACK: Duration = Duration::from_secs(10);
12
13fn allocate_spawn(
14 state: &mut OwnerState,
15 module: &str,
16 generation: u64,
17 at: tokio::time::Instant,
18 register: Duration,
19 health: Duration,
20) -> Result<DeliverySpawn> {
21 #[cfg(test)]
22 if state.delivery.failed_spawn_module.as_deref() == Some(module) {
23 return Err(GateError::new(
24 "delivery_spawn_exhausted",
25 "delivery spawn identifiers exhausted",
26 ));
27 }
28 state.delivery.next_spawn = state.delivery.next_spawn.checked_add(1).ok_or_else(|| {
29 GateError::new(
30 "delivery_spawn_exhausted",
31 "delivery spawn identifiers exhausted",
32 )
33 })?;
34 Ok(DeliverySpawn {
35 id: state.delivery.next_spawn,
36 module: module.into(),
37 generation,
38 deadline: at + register + health + CONFIG_WARMUP_SLACK,
39 })
40}
41
42#[derive(Clone, Debug)]
43struct Attempt {
44 module: String,
45 generation: u64,
46 id: u64,
47 transition: Transition,
48 ready: bool,
49 acknowledged: bool,
50 delivered: bool,
51 deadline: Option<tokio::time::Instant>,
52 catchup: bool,
55 write_deadline: Option<tokio::time::Instant>,
56 policy_probe_requested: bool,
57}
58
59#[derive(Clone, Debug)]
60pub struct DeliverySpawn {
61 id: u64,
62 module: String,
63 generation: u64,
64 deadline: tokio::time::Instant,
65}
66
67impl DeliverySpawn {
68 pub(crate) fn module(&self) -> &str {
69 &self.module
70 }
71 pub(crate) fn generation(&self) -> u64 {
72 self.generation
73 }
74}
75
76#[derive(Default)]
77pub(super) struct DeliveryState {
78 #[cfg(test)]
79 failed_spawn_module: Option<String>,
80 next_spawn: u64,
81 next_attempt: u64,
82 policy_epochs: HashMap<String, u64>,
83 attempts: HashMap<ConnectionId, Attempt>,
84 served: HashMap<ConnectionId, (Option<u64>, Option<u64>)>,
85 stamp_running: HashSet<ConnectionId>,
86 initial_generation: u64,
87 children: HashMap<ConnectionId, DeliverySpawn>,
88 healthy_children: HashSet<(ConnectionId, u64)>,
89 readiness_observed: HashSet<(ConnectionId, u64)>,
90 unregistered_children: HashMap<(String, bool), DeliverySpawn>,
91 recoveries: Vec<(ConnectionId, String, AppliedDisposition)>,
92 exited: HashMap<String, ExitedDelivery>,
93 last_target_attempt: HashMap<String, (u64, u64)>,
96}
97
98impl DeliveryState {
99 pub(super) fn reset_for_authority_exit(&mut self, generation: u64) {
100 *self = Self {
101 next_spawn: self.next_spawn,
102 next_attempt: self.next_attempt,
103 ..Self::new(generation)
104 };
105 }
106 pub(super) fn started_at_generation(&self) -> u64 {
107 self.initial_generation
108 }
109 pub(super) fn policy_authorization(
110 &self,
111 module: &str,
112 generation: u64,
113 connection: ConnectionId,
114 ) -> Option<(u64, tokio::time::Instant)> {
115 let attempt = self.attempts.get(&connection)?;
116 let deadline = attempt.deadline?;
117 (attempt.module == module
118 && attempt.generation == generation
119 && attempt.delivered
120 && attempt.acknowledged
121 && tokio::time::Instant::now() < deadline)
122 .then_some((attempt.id, deadline))
123 }
124 pub(super) fn fenced_modules(&self) -> impl Iterator<Item = String> + '_ {
125 self.exited.keys().cloned()
126 }
127}
128
129#[derive(Clone, Debug)]
132pub struct DeliveryProbe {
133 connection: ConnectionId,
134 module: String,
135 generation: u64,
136 attempt: Option<u64>,
137 policy_epoch: u64,
138 route_readiness: bool,
139}
140
141impl DeliveryProbe {
142 pub(crate) fn connection(&self) -> ConnectionId {
143 self.connection
144 }
145 pub(crate) fn owns_failure(&self) -> bool {
146 self.route_readiness
147 }
148}
149
150#[derive(Clone)]
151pub(crate) struct ExitedDelivery {
152 pub module: String,
153 spawn: u64,
154 generation: u64,
155 changed: tokio::sync::watch::Sender<bool>,
156}
157
158#[derive(Clone, Debug, PartialEq, Eq)]
159pub enum AppliedDisposition {
160 Ignored,
161 WaitingForReadiness,
162 Active,
163 CatchupComplete,
164 Failed {
165 generation: u64,
166 restore: Option<u64>,
167 keep_running: bool,
168 },
169 CatchupFailed {
170 active: u64,
171 },
172}
173
174#[derive(Clone, Copy)]
175pub(crate) struct DeliveryWindow {
176 pub generation: u64,
177 pub attempt: u64,
178 pub delivered: bool,
179 pub deadline: Option<tokio::time::Instant>,
180}
181
182impl DeliveryState {
183 pub(super) fn invalidate_policy(&mut self, module: &str, generation: u64) {
184 for attempt in self
185 .attempts
186 .values_mut()
187 .filter(|attempt| attempt.module == module && attempt.generation == generation)
188 {
189 attempt.ready = false;
190 attempt.policy_probe_requested = false;
191 }
192 }
193 pub(super) fn policy_installed(
194 &mut self,
195 module: &str,
196 generation: u64,
197 target: Option<ConnectionId>,
198 ) -> Result<()> {
199 let epoch = self.policy_epochs.entry(module.into()).or_default();
202 *epoch = epoch.checked_add(1).ok_or_else(|| {
203 GateError::new(
204 "delivery_probe_exhausted",
205 "policy readiness epochs exhausted",
206 )
207 })?;
208 self.invalidate_policy(module, generation);
209 let connections: HashSet<_> = self
210 .children
211 .iter()
212 .filter(|(_, spawn)| spawn.module == module)
213 .map(|(connection, _)| *connection)
214 .chain(
215 self.attempts
216 .iter()
217 .filter(|(_, attempt)| attempt.module == module)
218 .map(|(connection, _)| *connection),
219 )
220 .chain(target)
221 .collect();
222 self.readiness_observed.retain(|(connection, observed)| {
223 *observed != generation || !connections.contains(connection)
224 });
225 self.healthy_children.retain(|(connection, observed)| {
226 *observed != generation || !connections.contains(connection)
227 });
228 Ok(())
229 }
230 pub(super) fn declared(&mut self, connection: ConnectionId, module: &str, active: bool) {
231 if let Some(spawn) = self.unregistered_children.remove(&(module.into(), active)) {
232 self.children.insert(connection, spawn);
233 }
234 }
235 pub(super) fn new(initial_generation: u64) -> Self {
236 Self {
237 initial_generation,
238 ..Self::default()
239 }
240 }
241
242 pub(super) fn disconnected(&mut self, connection: ConnectionId) {
243 self.attempts.remove(&connection);
244 self.served.remove(&connection);
245 self.stamp_running.remove(&connection);
246 self.children.remove(&connection);
247 self.healthy_children
248 .retain(|(child, _)| *child != connection);
249 self.readiness_observed
250 .retain(|(child, _)| *child != connection);
251 }
252
253 pub(super) fn remember_stamp_connections(&mut self, connections: Vec<ConnectionId>) {
254 self.stamp_running.extend(connections);
255 }
256
257 fn boundary_open(
258 &self,
259 connection: ConnectionId,
260 generation: u64,
261 transition: Transition,
262 ) -> bool {
263 if transition == Transition::DaemonRestart && generation > self.initial_generation {
264 return false;
265 }
266 !transition.requires_spawn()
267 || self
268 .children
269 .get(&connection)
270 .is_some_and(|spawn| spawn.generation == generation)
271 }
272
273 fn allocate(
274 &mut self,
275 module: &str,
276 generation: u64,
277 transition: Transition,
278 ) -> Result<Attempt> {
279 self.next_attempt = self.next_attempt.checked_add(1).ok_or_else(|| {
280 GateError::new(
281 "delivery_attempt_exhausted",
282 "delivery attempt ids exhausted",
283 )
284 })?;
285 Ok(Attempt {
286 module: module.into(),
287 generation,
288 id: self.next_attempt,
289 transition,
290 ready: transition == Transition::Live,
291 acknowledged: false,
292 delivered: false,
293 deadline: None,
294 catchup: false,
295 write_deadline: None,
296 policy_probe_requested: false,
297 })
298 }
299}
300
301pub(super) fn visible_reply(
302 state: &OwnerState,
303 connection: ConnectionId,
304 mut reply: ConfigGetReply,
305 transition: Transition,
306) -> Result<ConfigGetReply> {
307 if reply.pending.as_ref().is_some_and(|view| {
308 !state
309 .delivery
310 .boundary_open(connection, view.generation, transition)
311 }) {
312 reply.pending = None;
313 }
314 if reply.active.is_none() && reply.pending.is_none() {
315 return Err(GateError::new(
316 "config_gated",
317 "generation waits for its required supervision or daemon boundary",
318 ));
319 }
320 Ok(reply)
321}
322
323pub(super) fn bind_get(
324 state: &mut OwnerState,
325 connection: ConnectionId,
326 module: &str,
327 request: &ConfigGetRequest,
328 reply: ConfigGetReply,
329 transition: Transition,
330) -> Result<ConfigGetReply> {
331 let mut reply = visible_reply(state, connection, reply, transition)?;
332 match (request.generation, request.attempt) {
333 (None, None) => {}
334 (Some(generation), Some(attempt))
335 if state
336 .delivery
337 .attempts
338 .get(&connection)
339 .is_some_and(|bound| {
340 bound.module == module && bound.generation == generation && bound.id == attempt
341 }) => {}
342 _ => {
343 return Err(GateError::new(
344 "config_gated",
345 "requested generation and attempt are not assigned to this connection",
346 ))
347 }
348 }
349 if reply.pending.is_none()
350 && state
351 .delivery
352 .attempts
353 .get(&connection)
354 .is_some_and(|bound| {
355 bound.catchup
356 && reply
357 .active
358 .as_ref()
359 .is_some_and(|view| view.generation == bound.generation)
360 })
361 {
362 reply.pending = reply.active.clone();
363 if let Some(view) = &mut reply.pending {
364 view.status = ConfigStatus::Pending;
365 }
366 }
367 if let Some(pending) = &mut reply.pending {
368 let matches = state
369 .delivery
370 .attempts
371 .get(&connection)
372 .is_some_and(|bound| bound.module == module && bound.generation == pending.generation);
373 if !matches {
374 let mut attempt = state
375 .delivery
376 .allocate(module, pending.generation, transition)?;
377 if transition.requires_spawn() {
378 attempt.ready = state
379 .delivery
380 .healthy_children
381 .contains(&(connection, pending.generation));
382 attempt.deadline = state
383 .delivery
384 .children
385 .get(&connection)
386 .filter(|warmup| warmup.generation == pending.generation)
387 .map(|warmup| warmup.deadline);
388 }
389 attempt.ready &= super::route_policy::ready(state, module, pending.generation)?;
390 state.delivery.attempts.insert(connection, attempt);
391 }
392 pending.attempt = state
393 .delivery
394 .attempts
395 .get(&connection)
396 .map(|bound| bound.id);
397 if state.active_declarations.get(module) == Some(&connection) {
398 if let Some(spawn) = state.delivery.children.get(&connection) {
399 let id = spawn.id;
400 state
401 .delivery
402 .last_target_attempt
403 .insert(module.into(), (id, pending.generation));
404 }
405 }
406 } else {
407 state.delivery.attempts.remove(&connection);
408 }
409 if request
410 .generation
411 .is_some_and(|id| reply.pending.as_ref().map(|view| view.generation) != Some(id))
412 {
413 return Err(GateError::new(
414 "config_gated",
415 "selected delivery has been superseded",
416 ));
417 }
418 Ok(reply)
419}
420
421impl RuntimeOwner {
422 pub(crate) fn policy_probe_due(&self, connection: ConnectionId) -> Result<Option<(u64, u64)>> {
423 let mut state = self.lock()?;
424 let Some(bound) = state.delivery.attempts.get(&connection) else {
425 return Ok(None);
426 };
427 let (module, generation, attempt) = (bound.module.clone(), bound.generation, bound.id);
428 if bound.ready
429 || bound.policy_probe_requested
430 || state.active_declarations.get(&module) != Some(&connection)
431 || !state
432 .store
433 .body()?
434 .generations
435 .get(&generation)
436 .is_some_and(|entry| {
437 super::route_policy::required(&entry.generation.changeset, &module)
438 })
439 || !super::route_policy::ready(&mut state, &module, generation)?
440 {
441 return Ok(None);
442 }
443 state
444 .delivery
445 .attempts
446 .get_mut(&connection)
447 .unwrap()
448 .policy_probe_requested = true;
449 Ok(Some((generation, attempt)))
450 }
451
452 pub(crate) fn policy_probe_pending(
453 &self,
454 connection: ConnectionId,
455 generation: u64,
456 attempt: u64,
457 ) -> Result<bool> {
458 let mut state = self.lock()?;
459 let Some(bound) = state.delivery.attempts.get(&connection) else {
460 return Ok(false);
461 };
462 let module = bound.module.clone();
463 Ok(bound.generation == generation
464 && bound.id == attempt
465 && !bound.ready
466 && state.active_declarations.get(&module) == Some(&connection)
467 && super::route_policy::ready(&mut state, &module, generation)?)
468 }
469 pub(crate) fn delivery_health_probe(
470 &self,
471 connection: ConnectionId,
472 module: &str,
473 ) -> Result<Option<DeliveryProbe>> {
474 let mut state = self.lock()?;
475 if state.active_declarations.get(module) != Some(&connection) {
476 return Ok(None);
477 }
478 let attempt = state
479 .delivery
480 .attempts
481 .get(&connection)
482 .map(|attempt| (attempt.generation, attempt.id));
483 let generation = attempt.map(|attempt| attempt.0).or_else(|| {
484 state
485 .delivery
486 .children
487 .get(&connection)
488 .map(|child| child.generation)
489 });
490 let route_readiness = match generation {
491 Some(generation) if attempt.is_some() => state
492 .store
493 .body()?
494 .generations
495 .get(&generation)
496 .is_some_and(|entry| {
497 super::route_policy::required(&entry.generation.changeset, module)
498 }),
499 _ => false,
500 };
501 Ok(generation.map(|generation| DeliveryProbe {
502 connection,
503 module: module.into(),
504 generation,
505 attempt: attempt.map(|attempt| attempt.1),
506 route_readiness,
507 policy_epoch: state
508 .delivery
509 .policy_epochs
510 .get(module)
511 .copied()
512 .unwrap_or_default(),
513 }))
514 }
515
516 pub(crate) fn delivery_health_result(
517 &self,
518 probe: DeliveryProbe,
519 healthy: bool,
520 ) -> Result<Option<AppliedDisposition>> {
521 let attempt = {
522 let mut state = self.lock()?;
523 if state.active_declarations.get(&probe.module) != Some(&probe.connection)
524 || state
525 .delivery
526 .policy_epochs
527 .get(&probe.module)
528 .copied()
529 .unwrap_or_default()
530 != probe.policy_epoch
531 {
532 return Ok(None);
533 }
534 let current = state.delivery.attempts.get(&probe.connection);
535 if probe
536 .attempt
537 .is_some_and(|id| current.is_none_or(|bound| bound.id != id))
538 || current.is_some_and(|bound| bound.generation != probe.generation)
539 {
540 return Ok(None);
541 }
542 let attempt = current.map(|bound| bound.id);
543 if !healthy
544 && (!super::route_policy::ready(&mut state, &probe.module, probe.generation)?
545 || !state
546 .store
547 .body()?
548 .generations
549 .get(&probe.generation)
550 .is_some_and(|entry| {
551 super::route_policy::required(
552 &entry.generation.changeset,
553 &probe.module,
554 )
555 }))
556 {
557 return Ok(None);
558 }
559 attempt
560 };
561 if healthy {
562 self.delivery_readiness_complete(probe.connection, &probe.module, probe.generation)
563 } else if let Some(attempt) = attempt {
564 self.module_outcome(
565 probe.connection,
566 &probe.module,
567 ConfigAppliedRequest::new(
568 probe.generation,
569 attempt,
570 subc_protocol::config::ConfigApplyResult::Nack,
571 vec![],
572 ),
573 true,
574 )
575 .map(Some)
576 } else {
577 Ok(None)
578 }
579 }
580
581 #[cfg(test)]
582 pub(super) fn ready(
583 &self,
584 connection: ConnectionId,
585 module: &str,
586 generation: u64,
587 ) -> Result<Option<AppliedDisposition>> {
588 self.delivery_readiness_complete(connection, module, generation)
589 }
590
591 pub(crate) fn pending_config_restarts(&self) -> Result<Vec<(String, u64)>> {
592 let mut state = self.lock()?;
593 state.store.delivery_ready()?;
594 let body = state.store.body()?;
595 body.versions
596 .iter()
597 .filter(|(tuple, _)| tuple.project.is_none())
598 .filter_map(|(tuple, versions)| {
599 let generation = versions.pending?;
600 let entry = body.generations.get(&generation)?;
601 Some(
602 Transition::for_module(&entry.generation.changeset, &tuple.module).map(
603 |transition| {
604 (transition == Transition::ModuleRestart)
605 .then(|| (tuple.module.clone(), generation))
606 },
607 ),
608 )
609 })
610 .collect::<Result<Vec<_>>>()
611 .map(|entries| entries.into_iter().flatten().collect())
612 }
613
614 pub(crate) fn requires_fresher_spawn(
615 &self,
616 module: &str,
617 spawn: Option<&DeliverySpawn>,
618 ) -> Result<Option<u64>> {
619 let mut state = self.lock()?;
620 state.store.delivery_ready()?;
621 Self::requires_fresher_spawn_loaded(&state, module, spawn)
622 }
623
624 pub(super) fn requires_fresher_spawn_loaded(
625 state: &OwnerState,
626 module: &str,
627 spawn: Option<&DeliverySpawn>,
628 ) -> Result<Option<u64>> {
629 let body = state.store.loaded_admission_state().1;
630 let pending = body
631 .versions
632 .iter()
633 .find(|(tuple, _)| tuple.module == module && tuple.project.is_none())
634 .and_then(|(_, versions)| versions.pending);
635 let Some(generation) = pending else {
636 return Ok(None);
637 };
638 let entry = body
639 .generations
640 .get(&generation)
641 .ok_or_else(|| GateError::new("config_gated", "pending generation is absent"))?;
642 let transition = Transition::for_module(&entry.generation.changeset, module)?;
643 let deferred_daemon = transition == Transition::DaemonRestart
644 && generation > state.delivery.initial_generation;
645 Ok((transition.requires_spawn()
646 && !deferred_daemon
647 && spawn
648 .is_none_or(|spawn| spawn.module != module || spawn.generation() != generation))
649 .then_some(generation))
650 }
651
652 pub(crate) fn delivery_child_exited(&self, spawn: &DeliverySpawn) -> Result<()> {
655 let mut state = self.lock()?;
656 let pending = state
657 .store
658 .body()?
659 .versions
660 .iter()
661 .find(|(tuple, _)| tuple.module == spawn.module && tuple.project.is_none())
662 .and_then(|(_, versions)| versions.pending);
663 let pending = pending.filter(|generation| {
664 state
665 .store
666 .loaded_admission_state()
667 .1
668 .generations
669 .get(generation)
670 .and_then(|entry| entry.delivery.as_ref())
671 .is_some_and(|delivery| delivery.declarations.contains_key(&spawn.module))
672 });
673 if let Some(generation) = pending.filter(|generation| {
674 *generation == spawn.generation
675 || state.delivery.last_target_attempt.get(&spawn.module)
676 == Some(&(spawn.id, *generation))
677 }) {
678 state
679 .delivery
680 .exited
681 .entry(spawn.module.clone())
682 .or_insert_with(|| ExitedDelivery {
683 module: spawn.module.clone(),
684 spawn: spawn.id,
685 generation,
686 changed: tokio::sync::watch::channel(true).0,
687 });
688 }
689 self.notify_delivery_changed();
690 Ok(())
691 }
692
693 pub(crate) fn delivery_exit_fence(
694 &self,
695 module: &str,
696 ) -> Result<Option<tokio::sync::watch::Receiver<bool>>> {
697 Ok(self
698 .lock()?
699 .delivery
700 .exited
701 .get(module)
702 .map(|entry| entry.changed.subscribe()))
703 }
704
705 pub(crate) fn delivery_exits(&self) -> Result<Vec<ExitedDelivery>> {
706 Ok(self.lock()?.delivery.exited.values().cloned().collect())
707 }
708
709 pub(crate) fn delivery_record_exit(&self, exited: &ExitedDelivery) -> Result<bool> {
716 #[cfg(test)]
717 self.exit_record_attempts
718 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
719 let mut state = self.lock()?;
720 if !state
721 .delivery
722 .exited
723 .get(&exited.module)
724 .is_some_and(|entry| entry.spawn == exited.spawn)
725 {
726 return Ok(false);
727 }
728 let tuple = Tuple {
729 module: exited.module.clone(),
730 project: None,
731 };
732 let versions = state
733 .store
734 .body()?
735 .versions
736 .iter()
737 .find(|(key, _)| key == &tuple)
738 .map(|(_, versions)| versions.clone())
739 .unwrap_or_default();
740 if versions
741 .pending
742 .is_some_and(|generation| generation != exited.generation)
743 {
744 return Ok(false);
745 }
746 if versions.pending == Some(exited.generation) {
747 state.restore_route_policy(&exited.module, exited.generation);
748 let result = state
749 .store
750 .record_delivery(&tuple, exited.generation, false);
751 if let Err(error) = result {
752 if error.code != crate::config_store::StoreErrorCode::AnchorPending {
753 return Err(error.into());
754 }
755 }
756 state.delivery.attempts.retain(|_, attempt| {
757 attempt.module != exited.module || attempt.generation != exited.generation
758 });
759 }
760 Ok(versions.active.is_none())
761 }
762
763 #[cfg(all(test, unix))]
764 pub(crate) fn exit_record_attempts_for_test(&self) -> usize {
765 self.exit_record_attempts
766 .load(std::sync::atomic::Ordering::SeqCst)
767 }
768
769 pub(crate) fn delivery_release_exit(&self, exited: &ExitedDelivery) -> Result<()> {
770 let mut state = self.lock()?;
771 if state
772 .delivery
773 .exited
774 .get(&exited.module)
775 .is_some_and(|entry| entry.spawn == exited.spawn)
776 {
777 if let Some(entry) = state.delivery.exited.remove(&exited.module) {
778 entry.changed.send_replace(false);
779 state.admission_releases.push(exited.module.clone());
783 }
784 }
785 self.notify_delivery_changed();
786 Ok(())
787 }
788 pub(crate) fn delivery_abandon_spawn(&self, spawn: &DeliverySpawn, active: bool) {
789 if let Ok(mut state) = self.lock() {
790 let key = (spawn.module.clone(), active);
791 if state
792 .delivery
793 .unregistered_children
794 .get(&key)
795 .is_some_and(|current| current.id == spawn.id)
796 {
797 state.delivery.unregistered_children.remove(&key);
798 }
799 }
800 }
801 pub(crate) fn delivery_queue_abandoned_spawn(&self, spawn: DeliverySpawn, active: bool) {
802 if let Ok(mut abandoned) = self.abandoned_spawns.lock() {
803 abandoned.push((spawn, active));
804 }
805 self.notify_delivery_changed();
806 }
807 pub(super) fn drain_abandoned_spawns(&self, state: &mut OwnerState) {
808 if let Ok(mut abandoned) = self.abandoned_spawns.lock() {
809 for (spawn, active) in abandoned.drain(..) {
810 let key = (spawn.module.clone(), active);
811 if state
812 .delivery
813 .unregistered_children
814 .get(&key)
815 .is_some_and(|current| current.id == spawn.id)
816 {
817 state.delivery.unregistered_children.remove(&key);
818 }
819 }
820 }
821 }
822
823 pub(crate) fn delivery_launch_current(
827 &self,
828 spawn: &DeliverySpawn,
829 active: bool,
830 ) -> Result<Option<bool>> {
831 let state = match self.inner.try_lock() {
832 Ok(state) => state,
833 Err(std::sync::TryLockError::WouldBlock) => return Ok(None),
834 Err(_) => {
835 return Err(GateError::new(
836 "config_unavailable",
837 "configuration owner lock is poisoned",
838 ))
839 }
840 };
841 Ok(Some(
842 super::admission::AdmissionCache::from_state(&state)
843 .spawn_generation(&spawn.module)
844 .is_ok_and(|generation| generation == spawn.generation)
845 && state
846 .delivery
847 .unregistered_children
848 .get(&(spawn.module.clone(), active))
849 .is_some_and(|current| current.id == spawn.id)
850 && !state.delivery.exited.contains_key(&spawn.module),
851 ))
852 }
853 #[cfg(test)]
854 pub(crate) fn pause_spawn_preflight_for_test(&self) {
855 let hook = self.spawn_preflight_hook.lock().unwrap().take();
856 if let Some(hook) = hook {
857 hook();
858 }
859 }
860 #[cfg(all(test, unix))]
861 pub(crate) fn exhaust_delivery_spawns_for_test(&self) {
862 self.lock().unwrap().delivery.next_spawn = u64::MAX;
863 }
864 #[cfg(all(test, unix))]
865 pub(crate) fn restore_unused_delivery_spawns_for_test(&self) {
866 self.lock().unwrap().delivery.next_spawn = 1000;
867 }
868 #[cfg(all(test, unix))]
869 pub(crate) fn fail_delivery_spawns_for_module_for_test(&self, module: &str) {
870 self.lock().unwrap().delivery.failed_spawn_module = Some(module.into());
871 }
872 #[cfg(all(test, unix))]
873 pub(crate) fn clear_delivery_spawn_failure_for_test(&self) {
874 self.lock().unwrap().delivery.failed_spawn_module = None;
875 }
876 #[cfg(all(test, unix))]
877 pub(crate) fn set_spawn_preflight_hook_for_test(&self, hook: impl FnOnce() + Send + 'static) {
878 *self.spawn_preflight_hook.lock().unwrap() = Some(Box::new(hook));
879 }
880 #[cfg(all(test, unix))]
881 pub(crate) fn unregistered_spawn_count_for_test(&self) -> usize {
882 self.lock().unwrap().delivery.unregistered_children.len()
883 }
884 pub(crate) fn delivery_spawned_for_admission_in_slot(
885 &self,
886 module: &str,
887 admission: &super::admission::SpawnAdmission,
888 at: tokio::time::Instant,
889 register: Duration,
890 health: Duration,
891 active: bool,
892 ) -> Result<Option<DeliverySpawn>> {
893 let mut state = OwnerGuard(
894 self.inner.try_lock().map_err(|error| {
895 if matches!(error, std::sync::TryLockError::Poisoned(_)) {
896 return GateError::new("config_unavailable", "configuration owner is poisoned");
897 }
898 GateError::new(
899 "config_delivery_deferred",
900 "configuration publication is busy",
901 )
902 })?,
903 &self.published,
904 &self.delivery_changed,
905 &self.module_gates,
906 self,
907 );
908 state.store.check()?;
909 let spawn = allocate_spawn(
910 &mut state,
911 module,
912 admission.generation(),
913 at,
914 register,
915 health,
916 )?;
917 state
918 .delivery
919 .unregistered_children
920 .insert((module.into(), active), spawn.clone());
921 Ok(Some(spawn))
922 }
923
924 #[cfg(test)]
925 pub(crate) fn delivery_spawned_in_slot(
926 &self,
927 module: &str,
928 at: tokio::time::Instant,
929 register: Duration,
930 health: Duration,
931 active: bool,
932 ) -> Result<Option<DeliverySpawn>> {
933 let spawn = self.delivery_spawned(module, at, register, health)?;
934 if let Some(spawn) = &spawn {
935 self.lock()?
936 .delivery
937 .unregistered_children
938 .insert((module.into(), active), spawn.clone());
939 }
940 Ok(spawn)
941 }
942 pub(crate) fn delivery_take_recoveries(
943 &self,
944 ) -> Result<Vec<(ConnectionId, String, AppliedDisposition)>> {
945 Ok(std::mem::take(&mut self.lock()?.delivery.recoveries))
946 }
947
948 pub(crate) fn delivery_window(
949 &self,
950 connection: ConnectionId,
951 ) -> Result<Option<DeliveryWindow>> {
952 Ok(self
953 .lock()?
954 .delivery
955 .attempts
956 .get(&connection)
957 .map(|bound| DeliveryWindow {
958 generation: bound.generation,
959 attempt: bound.id,
960 delivered: bound.delivered,
961 deadline: bound.deadline,
962 }))
963 }
964
965 pub(crate) fn delivery_write_started(
966 &self,
967 connection: ConnectionId,
968 generation: u64,
969 attempt: u64,
970 now: tokio::time::Instant,
971 ) -> Result<()> {
972 if let Some(bound) = self
973 .lock()?
974 .delivery
975 .attempts
976 .get_mut(&connection)
977 .filter(|bound| {
978 bound.generation == generation && bound.id == attempt && !bound.delivered
979 })
980 {
981 bound
982 .write_deadline
983 .get_or_insert(now + CONFIG_LIVE_ACK_TIMEOUT);
984 }
985 Ok(())
986 }
987
988 pub(crate) fn delivery_timeout(
989 &self,
990 connection: ConnectionId,
991 module: &str,
992 generation: u64,
993 attempt: u64,
994 ) -> Result<AppliedDisposition> {
995 self.module_outcome(
996 connection,
997 module,
998 ConfigAppliedRequest::new(
999 generation,
1000 attempt,
1001 subc_protocol::config::ConfigApplyResult::Nack,
1002 vec![],
1003 ),
1004 true,
1005 )
1006 }
1007
1008 pub(crate) fn delivery_spawned(
1011 &self,
1012 module: &str,
1013 spawned_at: tokio::time::Instant,
1014 register_timeout: Duration,
1015 first_health_budget: Duration,
1016 ) -> Result<Option<DeliverySpawn>> {
1017 let mut state = self.lock()?;
1018 state.store.check()?;
1019 let generation =
1020 super::admission::AdmissionCache::from_state(&state).spawn_generation(module)?;
1021 allocate_spawn(
1022 &mut state,
1023 module,
1024 generation,
1025 spawned_at,
1026 register_timeout,
1027 first_health_budget,
1028 )
1029 .map(Some)
1030 }
1031
1032 pub(crate) fn delivery_child_registered(
1033 &self,
1034 connection: ConnectionId,
1035 spawn: &DeliverySpawn,
1036 ) -> Result<()> {
1037 let mut state = self.lock()?;
1038 state.delivery.children.insert(connection, spawn.clone());
1039 if let Some(bound) = state
1040 .delivery
1041 .attempts
1042 .get_mut(&connection)
1043 .filter(|bound| bound.module == spawn.module && bound.generation == spawn.generation)
1044 {
1045 if bound.transition.requires_spawn() {
1046 bound.deadline = Some(spawn.deadline);
1047 }
1048 }
1049 Ok(())
1050 }
1051
1052 pub(crate) fn module_view_delivered(
1053 &self,
1054 connection: ConnectionId,
1055 module: &str,
1056 reply: &ConfigGetReply,
1057 written_at: tokio::time::Instant,
1058 active: bool,
1059 ) -> Result<()> {
1060 {
1061 let mut state = self.lock()?;
1062 if !state
1063 .declarations
1064 .get(&connection)
1065 .is_some_and(|(id, _)| id == module)
1066 {
1067 return Err(GateError::new(
1068 "not_delivery_target",
1069 "configuration connection has ended",
1070 ));
1071 }
1072 let body = state.store.body()?;
1073 if body.stamped {
1074 let versions = body
1075 .versions
1076 .iter()
1077 .find(|(tuple, _)| tuple.module == module && tuple.project.is_none())
1078 .map(|(_, versions)| versions.clone())
1079 .unwrap_or_default();
1080 if reply
1081 .active
1082 .as_ref()
1083 .is_some_and(|view| versions.active != Some(view.generation))
1084 || reply.pending.as_ref().is_some_and(|view| {
1085 let catchup = versions.active == Some(view.generation)
1088 && state
1089 .delivery
1090 .attempts
1091 .get(&connection)
1092 .is_some_and(|bound| {
1093 bound.catchup
1094 && bound.generation == view.generation
1095 && Some(bound.id) == view.attempt
1096 });
1097 versions.pending != Some(view.generation) && !catchup
1098 })
1099 {
1100 return Err(GateError::new(
1101 "delivery_superseded",
1102 "configuration changed while its view was being written",
1103 ));
1104 }
1105 }
1106 }
1107 if active {
1108 if let Some(view) = &reply.pending {
1109 if let Some(attempt) = view.attempt {
1110 self.delivery_written(connection, view.generation, attempt, written_at)?;
1111 }
1112 }
1113 }
1114 let mut state = self.lock()?;
1115 if !state
1116 .declarations
1117 .get(&connection)
1118 .is_some_and(|(id, _)| id == module)
1119 {
1120 return Err(GateError::new(
1121 "not_delivery_target",
1122 "configuration connection has ended",
1123 ));
1124 }
1125 state.delivery.served.insert(
1126 connection,
1127 (
1128 reply.active.as_ref().map(|view| view.generation),
1129 reply.pending.as_ref().map(|view| view.generation),
1130 ),
1131 );
1132 Ok(())
1133 }
1134 pub(crate) fn delivery_readiness_complete(
1137 &self,
1138 connection: ConnectionId,
1139 module: &str,
1140 generation: u64,
1141 ) -> Result<Option<AppliedDisposition>> {
1142 let attempt = {
1143 let mut state = self.lock()?;
1144 if state.active_declarations.get(module) != Some(&connection) {
1145 return Err(GateError::new(
1146 "not_delivery_target",
1147 "readiness belongs to another connection",
1148 ));
1149 }
1150 let started_for_generation = state
1151 .delivery
1152 .children
1153 .get(&connection)
1154 .is_some_and(|warmup| warmup.generation == generation);
1155 if started_for_generation {
1156 state
1157 .delivery
1158 .healthy_children
1159 .insert((connection, generation));
1160 }
1161 let policy_ready = super::route_policy::ready(&mut state, module, generation)?;
1162 let Some(bound) = state
1163 .delivery
1164 .attempts
1165 .get_mut(&connection)
1166 .filter(|bound| bound.module == module && bound.generation == generation)
1167 else {
1168 return Ok(None);
1169 };
1170 if bound.transition.requires_spawn() && !started_for_generation {
1171 return Ok(None);
1172 }
1173 bound.ready = policy_ready;
1174 let acknowledged = bound.ready && bound.acknowledged;
1175 let id = bound.id;
1176 state
1177 .delivery
1178 .readiness_observed
1179 .insert((connection, generation));
1180 acknowledged.then_some(id)
1181 };
1182 attempt
1183 .map(|attempt| {
1184 self.module_applied(
1185 connection,
1186 module,
1187 ConfigAppliedRequest::new(
1188 generation,
1189 attempt,
1190 subc_protocol::config::ConfigApplyResult::Ack,
1191 vec![],
1192 ),
1193 )
1194 })
1195 .transpose()
1196 }
1197
1198 pub(crate) fn delivery_promoted(
1201 &self,
1202 connection: ConnectionId,
1203 module: &str,
1204 cutover_at: tokio::time::Instant,
1205 ) -> Result<Option<(u64, u64)>> {
1206 let mut state = self.lock()?;
1207 state.store.delivery_ready()?;
1208 Self::promote_loaded(&mut state, connection, module, cutover_at)
1209 }
1210
1211 pub(super) fn promote_loaded(
1212 state: &mut OwnerState,
1213 connection: ConnectionId,
1214 module: &str,
1215 cutover_at: tokio::time::Instant,
1216 ) -> Result<Option<(u64, u64)>> {
1217 if state.store.loaded_admission_state().0 == crate::config_store::StoreState::Unstamped {
1218 if state
1219 .declarations
1220 .get(&connection)
1221 .is_some_and(|(id, _)| id == module)
1222 {
1223 state.phone_target_changed(module, connection);
1224 state.active_declarations.insert(module.into(), connection);
1225 }
1226 return Ok(None);
1227 }
1228 let Some((id, declaration)) = state
1229 .declarations
1230 .get(&connection)
1231 .cloned()
1232 .filter(|(id, _)| id == module)
1233 else {
1234 return Err(GateError::new(
1235 "config_gated",
1236 "candidate has not declared its approved configuration",
1237 ));
1238 };
1239 let tuple = Tuple {
1240 module: module.into(),
1241 project: None,
1242 };
1243 let body = state.store.loaded_admission_state().1;
1244 let versions = body
1245 .versions
1246 .iter()
1247 .find(|(key, _)| key == &tuple)
1248 .map(|(_, versions)| versions.clone())
1249 .unwrap_or_default();
1250 let transition = versions
1251 .pending
1252 .and_then(|id| body.generations.get(&id))
1253 .map(|entry| Transition::for_module(&entry.generation.changeset, module))
1254 .transpose()?
1255 .unwrap_or(Transition::Live);
1256 let pending_declaration = versions
1257 .pending
1258 .and_then(|id| body.generations.get(&id))
1259 .and_then(|entry| entry.delivery.as_ref())
1260 .and_then(|snapshot| snapshot.declarations.get(module))
1261 .cloned();
1262 let active_declaration = versions
1263 .active
1264 .and_then(|id| body.generations.get(&id))
1265 .and_then(|entry| entry.delivery.as_ref())
1266 .and_then(|snapshot| snapshot.declarations.get(module))
1267 .cloned();
1268 let pending = versions.pending.filter(|generation| {
1269 state
1270 .delivery
1271 .boundary_open(connection, *generation, transition)
1272 });
1273 if pending != versions.pending && transition != Transition::DaemonRestart {
1274 return Err(GateError::new(
1275 "config_gated",
1276 "candidate launch was superseded by a configuration requiring a fresh start",
1277 ));
1278 }
1279 let expected = if pending.is_some() {
1280 pending_declaration
1281 } else {
1282 active_declaration
1283 };
1284 if expected.as_ref() != Some(&declaration) {
1285 return Err(GateError::new(
1286 "config_gated",
1287 "candidate declaration differs from the approved declaration",
1288 ));
1289 }
1290 let transition = if pending.is_some() {
1291 transition
1292 } else {
1293 Transition::Live
1294 };
1295 let vector = (versions.active, pending);
1296 let catchup = pending.is_none()
1297 && versions.active.is_some()
1298 && state.delivery.served.get(&connection) != Some(&vector);
1299 state.phone_target_changed(module, connection);
1300 state.active_declarations.insert(id, connection);
1301 state.delivery.attempts.remove(&connection);
1302 let Some(generation) = pending.or_else(|| catchup.then_some(versions.active).flatten())
1303 else {
1304 return Ok(None);
1305 };
1306 let mut attempt = state.delivery.allocate(module, generation, transition)?;
1307 attempt.catchup = catchup;
1308 attempt.ready = super::route_policy::ready_loaded(state, module, generation)
1313 && !state
1314 .store
1315 .loaded_admission_state()
1316 .1
1317 .generations
1318 .get(&generation)
1319 .is_some_and(|entry| {
1320 super::route_policy::required(&entry.generation.changeset, module)
1321 });
1322 attempt.deadline = Some(cutover_at + CONFIG_LIVE_ACK_TIMEOUT);
1323 let assignment = (generation, attempt.id);
1324 state.delivery.attempts.insert(connection, attempt);
1325 state
1326 .delivery
1327 .readiness_observed
1328 .insert((connection, generation));
1329 Ok(Some(assignment))
1330 }
1331
1332 pub(crate) fn delivery_written(
1335 &self,
1336 connection: ConnectionId,
1337 generation: u64,
1338 attempt: u64,
1339 written_at: tokio::time::Instant,
1340 ) -> Result<()> {
1341 let mut state = self.lock()?;
1342 let bound = state
1343 .delivery
1344 .attempts
1345 .get_mut(&connection)
1346 .filter(|bound| bound.generation == generation && bound.id == attempt)
1347 .ok_or_else(|| {
1348 GateError::new(
1349 "delivery_superseded",
1350 "writer receipt no longer names the current delivery attempt",
1351 )
1352 })?;
1353 if bound
1354 .write_deadline
1355 .is_some_and(|deadline| written_at >= deadline)
1356 || bound
1357 .deadline
1358 .is_some_and(|deadline| written_at >= deadline)
1359 {
1360 return Err(GateError::new(
1361 "delivery_expired",
1362 "configuration delivery timed out",
1363 ));
1364 }
1365 bound.delivered = true;
1366 if !bound.transition.requires_spawn() && bound.deadline.is_none() {
1367 bound.deadline = Some(written_at + CONFIG_LIVE_ACK_TIMEOUT);
1368 }
1369 Ok(())
1370 }
1371
1372 pub(crate) fn module_applied(
1373 &self,
1374 connection: ConnectionId,
1375 module: &str,
1376 request: ConfigAppliedRequest,
1377 ) -> Result<AppliedDisposition> {
1378 self.module_outcome(connection, module, request, false)
1379 }
1380
1381 fn module_outcome(
1382 &self,
1383 connection: ConnectionId,
1384 module: &str,
1385 request: ConfigAppliedRequest,
1386 timed_out: bool,
1387 ) -> Result<AppliedDisposition> {
1388 let mut state = self.lock()?;
1389 if !state
1390 .declarations
1391 .get(&connection)
1392 .is_some_and(|(id, _)| id == module)
1393 {
1394 return Err(GateError::new(
1395 "declaration_order",
1396 "config.declare must precede get or applied on this connection",
1397 ));
1398 }
1399 if request.project_id.is_some() {
1400 return Err(GateError::new(
1401 "project_registry_unavailable",
1402 "project delivery requires the authoritative project registry",
1403 ));
1404 }
1405 if state.active_declarations.get(module) != Some(&connection) {
1406 return Err(GateError::new(
1407 "not_delivery_target",
1408 "connection is not the active delivery target",
1409 ));
1410 }
1411 state.store.check()?;
1412 let Some(bound) = state
1413 .delivery
1414 .attempts
1415 .get(&connection)
1416 .cloned()
1417 .filter(|bound| {
1418 bound.module == module
1419 && bound.generation == request.generation
1420 && bound.id == request.attempt
1421 })
1422 else {
1423 return Ok(AppliedDisposition::Ignored);
1424 };
1425 if !bound.delivered && !timed_out {
1426 return Ok(AppliedDisposition::Ignored);
1427 }
1428 let tuple = Tuple {
1429 module: module.into(),
1430 project: None,
1431 };
1432 let versions = state
1433 .store
1434 .body()?
1435 .versions
1436 .iter()
1437 .find(|(key, _)| key == &tuple)
1438 .map(|(_, versions)| versions.clone())
1439 .unwrap_or_default();
1440 let ack = request.result.is_ack()
1441 && !bound
1442 .deadline
1443 .is_some_and(|deadline| tokio::time::Instant::now() >= deadline);
1444 if bound.catchup {
1445 let pending_transition = versions
1448 .pending
1449 .map(|generation| {
1450 let entry = state
1451 .store
1452 .body()?
1453 .generations
1454 .get(&generation)
1455 .ok_or_else(|| {
1456 GateError::new("config_gated", "pending generation is absent")
1457 })?;
1458 Transition::for_module(&entry.generation.changeset, module)
1459 })
1460 .transpose()?;
1461 let pending_is_deliverable =
1462 versions
1463 .pending
1464 .zip(pending_transition)
1465 .is_some_and(|(generation, transition)| {
1466 state
1467 .delivery
1468 .boundary_open(connection, generation, transition)
1469 });
1470 if pending_is_deliverable || versions.active != Some(bound.generation) {
1471 return Ok(AppliedDisposition::Ignored);
1472 }
1473 state.delivery.attempts.remove(&connection);
1474 if !ack {
1475 state.delivery.recoveries.push((
1476 connection,
1477 module.into(),
1478 AppliedDisposition::CatchupFailed {
1479 active: bound.generation,
1480 },
1481 ));
1482 }
1483 return Ok(if ack {
1484 AppliedDisposition::CatchupComplete
1485 } else {
1486 AppliedDisposition::CatchupFailed {
1487 active: bound.generation,
1488 }
1489 });
1490 }
1491 if versions.pending != Some(bound.generation) {
1492 return Ok(AppliedDisposition::Ignored);
1493 }
1494 if ack
1495 && (!bound.ready
1496 || (bound.transition == Transition::DaemonRestart
1497 && bound.generation > state.delivery.initial_generation))
1498 {
1499 if let Some(bound) = state.delivery.attempts.get_mut(&connection) {
1500 bound.acknowledged = true;
1501 }
1502 return Ok(AppliedDisposition::WaitingForReadiness);
1503 }
1504 if !ack {
1505 state.restore_route_policy(module, bound.generation);
1506 }
1507 let result = state.store.record_delivery(&tuple, bound.generation, ack);
1508 if result.is_ok() && ack {
1509 state.commit_route_policy(module, bound.generation);
1510 }
1511 if result.is_ok() || !ack {
1514 state.delivery.attempts.remove(&connection);
1515 }
1516 if !ack {
1517 let keep_running =
1518 versions.active.is_none() && state.delivery.stamp_running.contains(&connection);
1519 state.delivery.recoveries.push((
1520 connection,
1521 module.into(),
1522 AppliedDisposition::Failed {
1523 generation: bound.generation,
1524 restore: versions.active,
1525 keep_running,
1526 },
1527 ));
1528 }
1529 result?;
1530 Ok(if ack {
1531 AppliedDisposition::Active
1532 } else {
1533 AppliedDisposition::Failed {
1534 generation: bound.generation,
1535 restore: versions.active,
1536 keep_running: versions.active.is_none()
1537 && state.delivery.stamp_running.contains(&connection),
1538 }
1539 })
1540 }
1541}