1use std::{
4 collections::{BTreeMap, BTreeSet, HashMap, VecDeque},
5 sync::Mutex,
6 time::{Duration, Instant},
7};
8
9use subc_config::{Changeset, Digest};
10use subc_control::config::ConfigPlanView;
11use subc_protocol::config::{
12 ConfigDeclareRequest, ConfigLayer, ConfigLayerSnapshot, ConfigStatus, ConfigView,
13};
14
15use super::{
16 capture::{CapturedConfiguration, ConfigCapturer},
17 ApprovalChallenge, ApprovalFacts, ApprovalReply, ConfigGate, GateError, PlanKind, Result,
18};
19use crate::{
20 config_store::{ConfigStore, DeliverySnapshot, StoreActivity, StoreState},
21 ConnectionId,
22};
23
24#[cfg(test)]
25type UserFileTestIdle =
26 tokio::sync::mpsc::UnboundedSender<(ConnectionId, tokio::sync::oneshot::Sender<()>)>;
27
28pub struct RuntimeOwner {
29 #[cfg(test)]
30 pub(crate) user_file_test_idle: Mutex<Option<UserFileTestIdle>>,
31 #[cfg(test)]
32 config_park_hook: Mutex<Option<Box<dyn FnOnce() + Send>>>,
33 #[cfg(test)]
34 pub(crate) content_kill_hook: Mutex<Option<Box<dyn FnOnce() + Send>>>,
35 #[cfg(test)]
36 polled_capture_hook: Mutex<Option<Box<dyn FnOnce() + Send>>>,
37 #[cfg(test)]
38 polled_capture_count: std::sync::atomic::AtomicUsize,
39 #[cfg(test)]
40 census_probe: Mutex<Option<std::sync::Arc<dyn Fn() + Send + Sync>>>,
41 #[cfg(test)]
42 agent_validation_window: Mutex<Option<Duration>>,
43 #[cfg(test)]
44 pub(crate) warm_restart_pause: Mutex<Option<WarmRestartPause>>,
45 #[cfg(test)]
46 pub(crate) warm_restart_busy: Mutex<Option<tokio::sync::oneshot::Sender<()>>>,
47 #[cfg(test)]
48 pub(crate) swap_preflight_attempts: std::sync::atomic::AtomicUsize,
49 #[cfg(feature = "test-support")]
50 test_hooks: Option<crate::config_store::testing::TestSupport>,
51 #[cfg(test)]
52 spawn_exec_hook: Mutex<Option<Box<dyn FnOnce() + Send>>>,
53 published: std::sync::RwLock<std::sync::Arc<Published>>,
54 configured_modules: std::sync::RwLock<Option<admission::ConfiguredModuleSnapshot>>,
55 inner: Mutex<OwnerState>,
56 capturer: ConfigCapturer,
57 startup_overrides: Vec<String>,
58 module_gates: Mutex<HashMap<String, std::sync::Arc<tokio::sync::Mutex<()>>>>,
59 delivery_changed: std::sync::Arc<tokio::sync::Notify>,
60 route_policy_authority: std::sync::OnceLock<std::sync::Arc<dyn RoutePolicyAuthority>>,
61 mark_publisher: std::sync::OnceLock<std::sync::Arc<dyn mark::MarkPublisher>>,
62 binding_retirements: Mutex<BTreeMap<usize, Box<dyn Fn() + Send + Sync>>>,
63 canceled_approvals: Mutex<Vec<u64>>,
64 approval_request_index: Mutex<HashMap<(ConnectionId, u64), std::sync::Arc<ApprovalRequest>>>,
67 disconnected_pending: Mutex<Vec<ConnectionId>>,
68 abandoned_spawns: Mutex<Vec<(DeliverySpawn, bool)>>,
69 phone_withdrawals: Mutex<phone::PhoneWithdrawals>,
70 #[cfg(any(test, feature = "test-support"))]
71 os_prompt_policy: std::sync::atomic::AtomicBool,
72 #[cfg(test)]
73 spawn_preflight_hook: Mutex<Option<Box<dyn FnOnce() + Send>>>,
74 #[cfg(test)]
75 exit_record_attempts: std::sync::atomic::AtomicUsize,
76 #[cfg(test)]
77 phone_bind_hook: Mutex<Option<Box<dyn FnOnce() + Send>>>,
78 #[cfg(test)]
79 begin_approval_hook: Mutex<Option<Box<dyn FnOnce() + Send>>>,
80 #[cfg(test)]
81 begin_approval_completions: std::sync::atomic::AtomicUsize,
82 #[cfg(test)]
83 desktop_receipt_hook: Mutex<Option<Box<dyn FnOnce() + Send>>>,
84 #[cfg(test)]
85 prepare_capture_hook: Mutex<Option<Box<dyn FnOnce() + Send>>>,
86 #[cfg(all(test, unix))]
87 approval_claim_pause: Mutex<Option<ApprovalClaimPause>>,
88}
89
90pub(crate) struct SwapConfiguration<'a> {
93 owner: &'a RuntimeOwner,
94 state: OwnerGuard<'a>,
95}
96
97#[cfg(test)]
98pub(crate) struct WarmRestartPause {
99 pub(crate) reached: tokio::sync::oneshot::Sender<()>,
100 pub(crate) resume: tokio::sync::oneshot::Receiver<()>,
101}
102
103impl SwapConfiguration<'_> {
104 pub(crate) fn before_cutover(
105 &mut self,
106 module: &str,
107 connection: ConnectionId,
108 spawn: Option<&DeliverySpawn>,
109 ) -> Result<Option<u64>> {
110 match self.state.store.loaded_admission_state().0 {
111 StoreState::Unstamped => return Ok(None),
112 StoreState::Stamped => {}
113 _ => {
114 return Err(GateError::new(
115 "config_gated",
116 "configuration store is in recovery",
117 ))
118 }
119 }
120 let newer = RuntimeOwner::requires_fresher_spawn_loaded(&self.state, module, spawn)?;
121 if newer.is_none() {
122 self.owner.module_get_locked(
123 &mut self.state,
124 connection,
125 module,
126 subc_protocol::config::ConfigGetRequest::new(),
127 false,
128 )?;
129 }
130 Ok(newer)
131 }
132 pub(crate) fn promoted(
133 &mut self,
134 module: &str,
135 connection: ConnectionId,
136 at: tokio::time::Instant,
137 ) -> Result<()> {
138 RuntimeOwner::promote_loaded(&mut self.state, connection, module, at).map(|_| ())
139 }
140}
141
142#[cfg(all(test, unix))]
143struct ApprovalClaimPause {
144 reached: tokio::sync::oneshot::Sender<()>,
145 release: tokio::sync::oneshot::Receiver<()>,
146}
147
148pub(crate) struct ApprovalCommitWindow {
151 pub deadline: tokio::time::Instant,
152 phase: std::sync::atomic::AtomicU8,
153}
154
155impl ApprovalCommitWindow {
156 pub(crate) fn new(deadline: tokio::time::Instant) -> Self {
157 Self {
158 deadline,
159 phase: std::sync::atomic::AtomicU8::new(0),
160 }
161 }
162
163 pub(crate) fn expire(&self) -> bool {
164 self.phase
165 .compare_exchange(
166 0,
167 2,
168 std::sync::atomic::Ordering::AcqRel,
169 std::sync::atomic::Ordering::Acquire,
170 )
171 .is_ok()
172 }
173
174 fn begin_publication(&self) -> Result<()> {
175 if tokio::time::Instant::now() >= self.deadline {
176 self.expire();
177 return Err(GateError::commit_timeout());
178 }
179 self.phase
180 .compare_exchange(
181 0,
182 1,
183 std::sync::atomic::Ordering::AcqRel,
184 std::sync::atomic::Ordering::Acquire,
185 )
186 .map(|_| ())
187 .map_err(|_| GateError::commit_timeout())
188 }
189}
190
191pub use route_policy::{
192 ApprovedRoutePolicy, RoutePolicyAuthority, RoutePolicyInstall, RoutePolicyInstallation,
193 RoutePolicyReceipt,
194};
195
196pub use delivery::{AppliedDisposition, DeliveryProbe, DeliverySpawn};
197
198pub trait DeliveryObserver: Send + Sync {
204 fn spawned(
205 &self,
206 module: &str,
207 spawned_at: tokio::time::Instant,
208 register_timeout: std::time::Duration,
209 first_health_budget: std::time::Duration,
210 ) -> Result<Option<DeliverySpawn>>;
211 fn registered(&self, connection: ConnectionId, spawn: &DeliverySpawn) -> Result<()>;
212 fn health_probe(&self, connection: ConnectionId, module: &str)
213 -> Result<Option<DeliveryProbe>>;
214 fn health_result(
215 &self,
216 probe: DeliveryProbe,
217 healthy: bool,
218 ) -> Result<Option<AppliedDisposition>>;
219 fn promoted(
220 &self,
221 connection: ConnectionId,
222 module: &str,
223 cutover_at: tokio::time::Instant,
224 ) -> Result<Option<(u64, u64)>>;
225 fn retry_anchor(&self) -> Result<Option<tokio::time::Instant>>;
228}
229
230impl DeliveryObserver for RuntimeOwner {
231 fn spawned(
232 &self,
233 module: &str,
234 spawned_at: tokio::time::Instant,
235 register_timeout: std::time::Duration,
236 first_health_budget: std::time::Duration,
237 ) -> Result<Option<DeliverySpawn>> {
238 self.delivery_spawned(module, spawned_at, register_timeout, first_health_budget)
239 }
240
241 fn registered(&self, connection: ConnectionId, spawn: &DeliverySpawn) -> Result<()> {
242 self.delivery_child_registered(connection, spawn)
243 }
244
245 fn health_probe(
248 &self,
249 connection: ConnectionId,
250 module: &str,
251 ) -> Result<Option<DeliveryProbe>> {
252 self.delivery_health_probe(connection, module)
253 }
254
255 fn health_result(
258 &self,
259 probe: DeliveryProbe,
260 healthy: bool,
261 ) -> Result<Option<AppliedDisposition>> {
262 self.delivery_health_result(probe, healthy)
263 }
264
265 fn promoted(
266 &self,
267 connection: ConnectionId,
268 module: &str,
269 cutover_at: tokio::time::Instant,
270 ) -> Result<Option<(u64, u64)>> {
271 self.delivery_promoted(connection, module, cutover_at)
272 }
273
274 fn retry_anchor(&self) -> Result<Option<tokio::time::Instant>> {
275 self.poll_anchor_retry()
276 }
277}
278
279const PLAN_IDLE_TIMEOUT: Duration = Duration::from_secs(60 * 60);
281pub(crate) const PLAN_SWEEP_INTERVAL: Duration = Duration::from_secs(60);
283const EXPIRED_PLAN_IDS: usize = 64;
285
286pub(crate) fn daemon_review(
287 old: &super::GateSnapshot,
288 new: &super::GateSnapshot,
289) -> Result<Changeset> {
290 let mut changeset = Changeset::new();
291 for key in old.changed_keys(new) {
292 match key {
293 super::InputKey::Entry(module) | super::InputKey::Program { module, .. } => {
294 let field = match key {
295 super::InputKey::Entry(_) => "entry".into(),
296 super::InputKey::Program { path, .. } => format!("program:{path}"),
297 _ => unreachable!(),
298 };
299 let manifest = changeset
300 .launch_manifests
301 .entry(module.clone())
302 .or_insert_with(|| serde_json::json!({"origin": "other"}));
303 manifest.as_object_mut().expect("launch manifest is an object").insert(field,serde_json::json!({"old":old.inputs.get(key).map(|v| &v.value),"new":new.inputs.get(key).map(|v| &v.value)}));
304 }
305 super::InputKey::Declaration(module) => {
306 changeset.policy_changes.push(serde_json::json!({"module_id":module,"path":"declaration","old":old.inputs.get(key).map(|v| &v.value),"new":new.inputs.get(key).map(|v| &v.value)}));
307 }
308 super::InputKey::Daemon => {
309 changeset.policy_changes.push(serde_json::json!({
310 "path":"subc.jsonc",
311 "old":old.inputs.get(key).map(|v| &v.value),
312 "new":new.inputs.get(key).map(|v| &v.value),
313 }));
314 changeset.apply_plan = serde_json::json!({"supervisor_reconcile":true});
315 }
316 _ => {}
317 }
318 }
319 Ok(changeset)
320}
321
322struct OwnerState {
323 authority_incarnation: std::sync::Arc<()>,
324 observed_snapshot: Option<super::GateSnapshot>,
325 stamp_channel_cutover: bool,
326 gate: ConfigGate,
327 store: ConfigStore,
328 _writer_lock: Option<crate::config_store::StoreWriterLock>,
329 plan_deadlines: BTreeMap<Digest, Instant>,
330 desktop_views: BTreeMap<u64, (Digest, u64)>,
333 phone_waiters: BTreeMap<u64, tokio::sync::oneshot::Sender<()>>,
334 approval_requests: HashMap<(ConnectionId, u64), u64>,
335 delivery: delivery::DeliveryState,
336 anchor_retry: super::anchor_retry::AnchorRetry,
337 retained_plan_files: BTreeMap<Digest, Vec<std::fs::File>>,
340 expired_plans: VecDeque<Digest>,
341 declarations: HashMap<ConnectionId, (String, ConfigDeclareRequest)>,
342 active_declarations: BTreeMap<String, ConnectionId>,
343 activity: Option<crate::activity::ActivityLedger>,
344 refused_generations: BTreeMap<String, Digest>,
345 admission_holds: Vec<String>,
346 admission_releases: Vec<String>,
347 authority_exit: Option<u64>,
348 reconciled_entries: BTreeMap<String, crate::daemon_config::ConfiguredModule>,
350 recovery_exit_hold: bool,
351 unstamp_hints: std::collections::HashSet<ConnectionId>,
352 anchor_waiters: std::collections::HashSet<ConnectionId>,
353 anchor_hints: std::collections::HashSet<ConnectionId>,
354 anchor_notify: std::sync::Arc<tokio::sync::Notify>,
355 installed_route_policies: BTreeMap<String, u64>,
356 route_policy_receipts: BTreeMap<(String, u64), route_policy::PendingRoutePolicy>,
357 #[cfg(test)]
358 publication_hook: Option<Box<dyn FnOnce() + Send>>,
359 #[cfg(test)]
360 prepublication_hook: Option<Box<dyn FnOnce() + Send>>,
361 #[cfg(test)]
362 after_approval_drain_hook: Option<Box<dyn FnOnce() + Send>>,
363}
364
365impl OwnerState {
366 fn retry_anchor(&mut self, operator_command: bool) -> Option<tokio::time::Instant> {
367 let pending = self.store.status().anchor_pending;
368 let events = self.anchor_retry.poll(
369 pending,
370 tokio::time::Instant::now(),
371 operator_command,
372 || {
373 let error = self
374 .store
375 .retry_anchor()
376 .err()
377 .map(|error| error.to_string());
378 (self.store.status().anchor_pending, error)
379 },
380 );
381 for event in events {
382 match event {
383 super::anchor_retry::Event::Pending => {
384 tracing::info!("configuration anchor became pending")
385 }
386 super::anchor_retry::Event::Failed(reason) => {
387 tracing::warn!(%reason, "configuration anchor retry failed")
388 }
389 super::anchor_retry::Event::Recovered => {
390 tracing::info!("configuration anchor recovered");
391 self.queue_anchor_hints();
392 }
393 }
394 }
395 self.commit_anchored_route_policies();
396 self.anchor_retry.next()
397 }
398 fn with_desktop_view(&self, digest: &Digest, facts: &ApprovalFacts) -> ApprovalFacts {
399 let mut checked = facts.clone();
400 let bound = self
401 .desktop_views
402 .get(&facts.connection)
403 .is_some_and(|(shown, revision)| {
404 shown == digest
405 && self
406 .gate
407 .plans
408 .get(digest)
409 .is_some_and(|plan| plan.view.base_revision == *revision)
410 });
411 if !bound {
412 checked.desktop_app = false;
413 checked.attested_operator_app = false;
414 }
415 checked
416 }
417 fn expire_idle(&mut self, now: Instant) {
418 let expired: Vec<_> = self
419 .plan_deadlines
420 .iter()
421 .filter(|(digest, deadline)| **deadline <= now && !self.gate.has_open_approval(digest))
422 .map(|(digest, _)| digest.clone())
423 .collect();
424 for digest in expired {
425 self.plan_deadlines.remove(&digest);
426 self.retained_plan_files.remove(&digest);
427 self.desktop_views.retain(|_, (shown, _)| shown != &digest);
428 if self.gate.expire_plan(&digest) {
429 self.expired_plans.push_back(digest);
430 if self.expired_plans.len() > EXPIRED_PLAN_IDS {
431 self.expired_plans.pop_front();
432 }
433 }
434 }
435 }
436 fn reviewed(&mut self, digest: &Digest, now: Instant) {
437 self.plan_deadlines
438 .insert(digest.clone(), now + PLAN_IDLE_TIMEOUT);
439 self.expired_plans.retain(|expired| expired != digest);
440 }
441}
442
443struct OwnerGuard<'a>(
444 std::sync::MutexGuard<'a, OwnerState>,
445 &'a std::sync::RwLock<std::sync::Arc<Published>>,
446 &'a tokio::sync::Notify,
447 &'a Mutex<HashMap<String, std::sync::Arc<tokio::sync::Mutex<()>>>>,
448 &'a RuntimeOwner,
449);
450impl std::ops::Deref for OwnerGuard<'_> {
451 type Target = OwnerState;
452 fn deref(&self) -> &Self::Target {
453 &self.0
454 }
455}
456impl std::ops::DerefMut for OwnerGuard<'_> {
457 fn deref_mut(&mut self) -> &mut Self::Target {
458 &mut self.0
459 }
460}
461impl Drop for OwnerGuard<'_> {
462 fn drop(&mut self) {
463 let previous = self
467 .1
468 .read()
469 .map(|view| view.approved_daemon.clone())
470 .unwrap_or(ApprovedDaemonView::Unavailable);
471 let approved_daemon =
472 ApprovedDaemonView::from_store(&self.0.store, self.0.recovery_exit_hold, &previous);
473 let (_, body) = self.0.store.loaded_admission_state();
477 let valid: std::collections::BTreeSet<_> = body
478 .program_refusals
479 .iter()
480 .map(|refusal| (refusal.module.clone(), refusal.approved_base.clone()))
481 .collect();
482 self.0
483 .refused_generations
484 .retain(|module, base| valid.contains(&(module.clone(), base.clone())));
485 let next = admission::AdmissionCache::from_state(&self.0);
486 mark::sweep(self.4, &mut self.0, &next);
490 let observed = self.0.observed_snapshot.clone();
491 let projection = self.4.status_projection(&mut self.0, observed, &next);
492 if let Ok(mut cache) = self.1.write() {
493 let (holds, releases) = next.transitions_from(&cache);
494 let changed = next.is_recovering() != cache.is_recovering()
495 || !holds.is_empty()
496 || !releases.is_empty();
497 self.0.admission_holds.extend(holds);
498 self.0.admission_releases.extend(releases);
499 *cache = std::sync::Arc::new(Published {
500 admission: next,
501 approved_daemon,
502 projection,
503 });
504 if changed {
505 self.2.notify_one();
506 }
507 }
508 let Some(ledger) = self.0.activity.clone() else {
509 return;
510 };
511 let Ok(gates) = self.3.try_lock() else {
512 return;
513 };
514 let held: Vec<_> = gates
515 .values()
516 .filter(|gate| gate.try_lock().is_err())
517 .cloned()
518 .collect();
519 drop(gates);
520 if held.is_empty() {
521 for activity in self.0.store.take_activities() {
522 ledger.config_store_activity(activity);
523 }
524 } else if let Ok(runtime) = tokio::runtime::Handle::try_current() {
525 let activities = self.0.store.take_activities();
526 if !activities.is_empty() {
527 runtime.spawn(async move {
528 for gate in held {
531 drop(gate.lock_owned().await);
532 }
533 for activity in activities {
534 ledger.config_store_activity(activity);
535 }
536 });
537 }
538 }
539 }
540}
541
542pub struct PreparedPlan {
543 pub candidate: CapturedConfiguration,
544 pub bases: BTreeMap<String, ConfigView>,
545 pub base_schemas: BTreeMap<String, subc_config::Schema>,
546 pub kind: PlanKind,
547 pub review_modules: std::collections::BTreeSet<String>,
548 pub rereview_modules: std::collections::BTreeSet<String>,
552 base_revision: u64,
553 active_generations: BTreeMap<crate::config_store::Tuple, Option<u64>>,
554 touched_modules: BTreeSet<String>,
555 read_set: subc_config::InputVector,
556 program_keys: Vec<super::InputKey>,
557 observed_handles: Vec<std::fs::File>,
558}
559
560pub struct ApprovedRollbackCommands {
562 pub generation: u64,
563 pub daemon: ConfigLayerSnapshot,
564 pub programs: BTreeMap<String, ApprovedRollbackProgram>,
565}
566
567#[derive(Clone, Debug)]
571pub(crate) struct ApprovedDaemonConfiguration {
572 generation: u64,
573 approval_digest: Digest,
574 authority_incarnation: std::sync::Arc<()>,
575 daemon: std::sync::Arc<ConfigLayerSnapshot>,
576 changed_entries: BTreeSet<String>,
577}
578
579#[derive(Clone, Debug)]
580pub(crate) struct ApprovalIdentity {
581 generation: u64,
582 approval_digest: Digest,
583 authority_incarnation: std::sync::Arc<()>,
587}
588
589impl PartialEq for ApprovalIdentity {
590 fn eq(&self, other: &Self) -> bool {
591 self.generation == other.generation
592 && self.approval_digest == other.approval_digest
593 && std::sync::Arc::ptr_eq(&self.authority_incarnation, &other.authority_incarnation)
594 }
595}
596
597impl Eq for ApprovalIdentity {}
598
599pub struct ApprovedRollbackProgram {
600 pub path: String,
601 pub identity: super::CapturedInput,
602}
603
604#[cfg(test)]
605#[path = "runtime_tests.rs"]
606mod tests;
607
608#[cfg(test)]
609#[path = "commit_pause.rs"]
610pub(crate) mod commit_pause;
611
612#[path = "delivery.rs"]
613pub(crate) mod delivery;
614
615#[path = "admission.rs"]
616mod admission;
617#[path = "mark.rs"]
618pub(crate) mod mark;
619#[path = "recovery.rs"]
620mod recovery;
621#[path = "route_policy.rs"]
622mod route_policy;
623pub(crate) use admission::SpawnAdmission;
624
625#[path = "approval_request.rs"]
626mod approval_request;
627#[path = "daemon_readers.rs"]
628mod daemon_readers;
629pub(crate) use daemon_readers::ApprovedDaemonView;
630#[path = "phone.rs"]
631mod phone;
632#[path = "placement.rs"]
633mod placement;
634#[path = "polled.rs"]
635mod polled;
636#[path = "published.rs"]
637mod published;
638#[path = "unstamped.rs"]
639pub(crate) mod unstamped;
640#[path = "widening.rs"]
641mod widening;
642pub(crate) use published::Published;
643#[path = "status_inputs.rs"]
644mod status_inputs;
645pub(crate) use approval_request::{ApprovalRequest, ApprovalRequestGuard, CancelApprovalRequest};
646
647impl RuntimeOwner {
648 pub(crate) async fn on_blocking<T: Send + 'static>(
650 self: &std::sync::Arc<Self>,
651 work: impl FnOnce(std::sync::Arc<Self>) -> Result<T> + Send + 'static,
652 ) -> Result<T> {
653 let owner = self.clone();
654 tokio::task::spawn_blocking(move || work(owner))
655 .await
656 .map_err(|_| GateError::new("config_unavailable", "configuration worker stopped"))?
657 }
658
659 #[cfg(feature = "test-support")]
660 pub(crate) fn pause_spawn_before_exec(&self) -> Result<()> {
661 if let Some(hooks) = &self.test_hooks {
662 hooks
663 .pause_spawn_before_exec()
664 .map_err(|error| GateError::new("config_test_pause", error.to_string()))?;
665 }
666 Ok(())
667 }
668
669 #[cfg(feature = "test-support")]
670 pub(crate) fn pause_swap_before_promotion(&self) -> Result<()> {
671 if let Some(hooks) = &self.test_hooks {
672 hooks
673 .pause_swap_before_promotion()
674 .map_err(|error| GateError::new("config_test_pause", error.to_string()))?;
675 }
676 Ok(())
677 }
678 #[cfg(test)]
681 pub(crate) fn desktop_view_delivered(
682 &self,
683 connection: ConnectionId,
684 view: &subc_control::config::ConfigPlanView,
685 ) -> Result<()> {
686 self.desktop_view_delivered_when(connection, view, || true)
687 }
688
689 pub(crate) fn desktop_view_delivered_when(
690 &self,
691 connection: ConnectionId,
692 view: &subc_control::config::ConfigPlanView,
693 is_live: impl FnOnce() -> bool,
694 ) -> Result<()> {
695 #[cfg(test)]
696 if let Some(hook) = self.desktop_receipt_hook.lock().unwrap().take() {
697 hook();
698 }
699 let mut state = self.lock()?;
700 if !is_live() {
703 return Ok(());
704 }
705 let plan = state
706 .gate
707 .plans
708 .get(&view.digest)
709 .ok_or_else(|| GateError::new("plan_unknown", "plan is absent or expired"))?;
710 if plan.view != *view {
711 return Err(GateError::new(
712 "plan_stale",
713 "delivered view differs from the immutable plan",
714 ));
715 }
716 state
717 .desktop_views
718 .insert(connection.get(), (view.digest.clone(), view.base_revision));
719 Ok(())
720 }
721 pub fn new(incarnation: String, mut store: ConfigStore, capturer: ConfigCapturer) -> Self {
722 let initial_generation = match store.booted_generation() {
724 Some(booted) => booted,
725 None => store.body().map(|body| body.last_generation).unwrap_or(0),
726 };
727 let admission = admission::AdmissionCache::from_store(&store);
728 let approved_daemon =
729 ApprovedDaemonView::from_store(&store, false, &ApprovedDaemonView::Unavailable);
730 let reconciled_entries = match &approved_daemon {
731 ApprovedDaemonView::Approved { config, .. } => config
732 .modules
733 .iter()
734 .map(|module| (module.module_id.clone(), module.clone()))
735 .collect(),
736 _ => BTreeMap::new(),
737 };
738 let published = std::sync::RwLock::new(std::sync::Arc::new(Published {
739 admission,
740 approved_daemon,
741 projection: polled::Projection::default(),
742 }));
743 let delivery_changed = std::sync::Arc::new(tokio::sync::Notify::new());
744 let owner = Self {
745 #[cfg(test)]
746 user_file_test_idle: Mutex::new(None),
747 #[cfg(test)]
748 polled_capture_hook: Mutex::new(None),
749 #[cfg(test)]
750 config_park_hook: Mutex::new(None),
751 #[cfg(test)]
752 content_kill_hook: Mutex::new(None),
753 #[cfg(test)]
754 polled_capture_count: std::sync::atomic::AtomicUsize::new(0),
755 #[cfg(test)]
756 census_probe: Mutex::new(None),
757 #[cfg(test)]
758 agent_validation_window: Mutex::new(None),
759 #[cfg(test)]
760 warm_restart_pause: Mutex::new(None),
761 #[cfg(test)]
762 warm_restart_busy: Mutex::new(None),
763 #[cfg(test)]
764 swap_preflight_attempts: std::sync::atomic::AtomicUsize::new(0),
765 #[cfg(feature = "test-support")]
766 test_hooks: store.test_support().cloned(),
767 #[cfg(test)]
768 spawn_exec_hook: Mutex::new(None),
769 published,
770 configured_modules: std::sync::RwLock::new(None),
771 inner: Mutex::new(OwnerState {
772 authority_incarnation: std::sync::Arc::new(()),
773 observed_snapshot: None,
774 stamp_channel_cutover: false,
775 gate: ConfigGate::new(incarnation),
776 store,
777 _writer_lock: None,
778 plan_deadlines: BTreeMap::new(),
779 desktop_views: BTreeMap::new(),
780 phone_waiters: BTreeMap::new(),
781 approval_requests: HashMap::new(),
782 delivery: delivery::DeliveryState::new(initial_generation),
783 anchor_retry: super::anchor_retry::AnchorRetry::default(),
784 retained_plan_files: BTreeMap::new(),
785 expired_plans: VecDeque::new(),
786 declarations: HashMap::new(),
787 active_declarations: BTreeMap::new(),
788 activity: None,
789 refused_generations: BTreeMap::new(),
790 admission_holds: Vec::new(),
791 admission_releases: Vec::new(),
792 authority_exit: None,
793 reconciled_entries,
794 recovery_exit_hold: false,
795 unstamp_hints: std::collections::HashSet::new(),
796 anchor_waiters: std::collections::HashSet::new(),
797 anchor_hints: std::collections::HashSet::new(),
798 anchor_notify: delivery_changed.clone(),
799 installed_route_policies: BTreeMap::new(),
800 route_policy_receipts: BTreeMap::new(),
801 #[cfg(test)]
802 publication_hook: None,
803 #[cfg(test)]
804 prepublication_hook: None,
805 #[cfg(test)]
806 after_approval_drain_hook: None,
807 }),
808 capturer,
809 startup_overrides: status_inputs::present_overrides(|name| {
810 std::env::var_os(name).is_some()
811 }),
812 module_gates: Mutex::new(HashMap::new()),
813 delivery_changed,
814 route_policy_authority: std::sync::OnceLock::new(),
815 mark_publisher: std::sync::OnceLock::new(),
816 binding_retirements: Mutex::new(BTreeMap::new()),
817 canceled_approvals: Mutex::new(Vec::new()),
818 approval_request_index: Mutex::new(HashMap::new()),
819 disconnected_pending: Mutex::new(Vec::new()),
820 abandoned_spawns: Mutex::new(Vec::new()),
821 phone_withdrawals: Mutex::new(phone::PhoneWithdrawals::default()),
822 #[cfg(any(test, feature = "test-support"))]
823 os_prompt_policy: std::sync::atomic::AtomicBool::new(false),
824 #[cfg(test)]
825 spawn_preflight_hook: Mutex::new(None),
826 #[cfg(test)]
827 exit_record_attempts: std::sync::atomic::AtomicUsize::new(0),
828 #[cfg(test)]
829 phone_bind_hook: Mutex::new(None),
830 #[cfg(test)]
831 begin_approval_hook: Mutex::new(None),
832 #[cfg(test)]
833 begin_approval_completions: std::sync::atomic::AtomicUsize::new(0),
834 #[cfg(test)]
835 desktop_receipt_hook: Mutex::new(None),
836 #[cfg(test)]
837 prepare_capture_hook: Mutex::new(None),
838 #[cfg(all(test, unix))]
839 approval_claim_pause: Mutex::new(None),
840 };
841 if let Ok(state) = owner.lock() {
842 drop(state);
843 }
844 owner
845 }
846
847 pub fn module_gate(&self, module: &str) -> Result<std::sync::Arc<tokio::sync::Mutex<()>>> {
848 let mut gates = self
849 .module_gates
850 .lock()
851 .map_err(|_| GateError::new("config_unavailable", "module gate lock is poisoned"))?;
852 Ok(gates.entry(module.into()).or_default().clone())
853 }
854
855 #[cfg(all(test, unix))]
856 pub(crate) fn set_agent_validation_window_for_test(&self, window: Duration) {
857 *self.agent_validation_window.lock().unwrap() = Some(window);
858 }
859
860 #[cfg(test)]
861 pub(crate) fn agent_validation_window_for_test(&self) -> Option<Duration> {
862 *self.agent_validation_window.lock().unwrap()
863 }
864
865 pub(crate) fn install_binding_retirement(
866 &self,
867 key: usize,
868 retire: impl Fn() + Send + Sync + 'static,
869 ) {
870 self.binding_retirements
871 .lock()
872 .unwrap_or_else(|poisoned| poisoned.into_inner())
873 .insert(key, Box::new(retire));
874 }
875
876 pub(crate) fn delivery_notifications(&self) -> std::sync::Arc<tokio::sync::Notify> {
877 self.delivery_changed.clone()
878 }
879
880 pub(crate) fn notify_delivery_changed(&self) {
881 self.delivery_changed.notify_one();
882 }
883
884 pub(crate) fn retain_writer_lock(
885 self,
886 lock: crate::config_store::StoreWriterLock,
887 ) -> Result<Self> {
888 self.lock()?._writer_lock = Some(lock);
889 Ok(self)
890 }
891
892 fn lock(&self) -> Result<OwnerGuard<'_>> {
893 crate::forwarding::assert_forwarding_unlocked("the configuration owner's state");
896 let mut guard = self
897 .inner
898 .lock()
899 .map(|guard| {
900 OwnerGuard(
901 guard,
902 &self.published,
903 &self.delivery_changed,
904 &self.module_gates,
905 self,
906 )
907 })
908 .map_err(|_| {
909 GateError::new("config_unavailable", "configuration owner lock is poisoned")
910 })?;
911 #[cfg(feature = "test-support")]
912 guard.store.refresh_test_anchor_failure();
913 self.drain_canceled_approvals(&mut guard);
914 self.drain_disconnections(&mut guard);
915 self.drain_abandoned_spawns(&mut guard);
916 guard.expire_idle(Instant::now());
917 Ok(guard)
918 }
919
920 pub(crate) fn try_swap_configuration(&self) -> Result<SwapConfiguration<'_>> {
921 let state = self.inner.try_lock().map_err(|_| {
922 GateError::new(
923 "config_delivery_deferred",
924 "configuration publication is busy or unavailable",
925 )
926 })?;
927 Ok(SwapConfiguration {
928 owner: self,
929 state: OwnerGuard(
930 state,
931 &self.published,
932 &self.delivery_changed,
933 &self.module_gates,
934 self,
935 ),
936 })
937 }
938
939 pub(crate) fn try_requires_fresher_spawn(
940 &self,
941 module: &str,
942 spawn: Option<&DeliverySpawn>,
943 ) -> Result<Option<u64>> {
944 let state = self.inner.try_lock().map_err(|_| {
945 GateError::new(
946 "config_delivery_deferred",
947 "configuration publication is busy or unavailable",
948 )
949 })?;
950 Self::requires_fresher_spawn_loaded(&state, module, spawn)
951 }
952
953 pub(crate) fn sweep_expired(&self) -> Result<()> {
954 drop(self.lock()?);
955 Ok(())
956 }
957
958 pub(crate) fn poll_anchor_retry(&self) -> Result<Option<tokio::time::Instant>> {
959 Ok(self.lock()?.retry_anchor(false))
960 }
961
962 pub(crate) fn set_activity(
963 &self,
964 ledger: Option<crate::activity::ActivityLedger>,
965 ) -> Result<()> {
966 self.lock()?.activity = ledger;
967 Ok(())
968 }
969
970 pub fn take_activities(&self) -> Vec<StoreActivity> {
971 self.inner
972 .lock()
973 .map(|mut state| state.store.take_activities())
974 .unwrap_or_default()
975 }
976
977 pub fn declare(
980 &self,
981 connection: ConnectionId,
982 module: &str,
983 declaration: ConfigDeclareRequest,
984 active: bool,
985 ) -> Result<()> {
986 subc_config::Schema::new(declaration.schema.clone())
987 .map_err(|r| GateError::new("schema_invalid", format!("{module}: {}", r.path)))?;
988 let mut state = self.lock()?;
989 if state.store.status().state == StoreState::StoreMissing
990 || state.store.status().state == StoreState::StoreUnverified
991 {
992 return Err(GateError::new(
993 "config_gated",
994 "configuration store is in recovery",
995 ));
996 }
997 if state.declarations.contains_key(&connection) {
998 return Err(GateError::new(
999 "declaration_order",
1000 "configuration is declared once per connection",
1001 ));
1002 }
1003 state
1004 .declarations
1005 .insert(connection, (module.into(), declaration));
1006 state.delivery.declared(connection, module, active);
1007 if active {
1008 state.active_declarations.insert(module.into(), connection);
1009 }
1010 self.notify_delivery_changed();
1011 Ok(())
1012 }
1013
1014 pub fn disconnected(&self, connection: ConnectionId) {
1015 if let Ok(mut index) = self.approval_request_index.lock() {
1016 index.retain(|(caller, _), request| {
1017 if *caller == connection {
1018 request.cancel();
1019 false
1020 } else {
1021 true
1022 }
1023 });
1024 }
1025 self.notify_delivery_changed();
1026 if let Ok(mut state) = self.inner.try_lock() {
1027 state.disconnected(connection);
1028 } else if let Ok(mut pending) = self.disconnected_pending.lock() {
1029 pending.push(connection);
1030 }
1031 }
1032
1033 pub fn module_get(
1034 &self,
1035 connection: ConnectionId,
1036 module: &str,
1037 request: subc_protocol::config::ConfigGetRequest,
1038 ) -> Result<subc_protocol::config::ConfigGetReply> {
1039 let mut state = self.lock()?;
1040 self.module_get_locked(&mut state, connection, module, request, true)
1041 }
1042
1043 fn module_get_locked(
1044 &self,
1045 state: &mut OwnerState,
1046 connection: ConnectionId,
1047 module: &str,
1048 request: subc_protocol::config::ConfigGetRequest,
1049 check_store: bool,
1050 ) -> Result<subc_protocol::config::ConfigGetReply> {
1051 let declaration = state
1052 .declarations
1053 .get(&connection)
1054 .filter(|(id, _)| id == module)
1055 .map(|(_, declaration)| declaration.clone())
1056 .ok_or_else(|| {
1057 GateError::new(
1058 "declaration_order",
1059 "config.declare must precede get or applied on this connection",
1060 )
1061 })?;
1062 if request.project_id.is_some() {
1063 return Err(GateError::new(
1064 "project_registry_unavailable",
1065 format!("{module}: project layer registry is not connected"),
1066 ));
1067 }
1068 if check_store {
1069 let checked = state.store.check();
1070 if matches!(
1071 state.store.loaded_admission_state().0,
1072 StoreState::StoreMissing | StoreState::StoreUnverified
1073 ) {
1074 return recovery::active_view(state, module, &declaration, &request);
1075 }
1076 checked?;
1077 }
1078 if state.store.loaded_admission_state().0 == StoreState::Unstamped {
1079 if request.generation.is_some() || request.attempt.is_some() {
1080 return Err(GateError::new(
1081 "config_gated",
1082 "unstamped reads have no assigned generation or delivery attempt",
1083 ));
1084 }
1085 let mut declarations = Self::declarations(state)?;
1086 declarations.insert(module.into(), declaration);
1087 let mut captured = self
1088 .capturer
1089 .capture_modules(&declarations, &BTreeSet::from([module.to_owned()]))?;
1090 let view = captured
1091 .modules
1092 .remove(module)
1093 .ok_or_else(|| GateError::new("config_gated", "module declaration is unavailable"))?
1094 .view;
1095 return Ok(subc_protocol::config::ConfigGetReply::new(Some(view), None));
1096 }
1097 if check_store {
1098 state.store.delivery_ready()?;
1099 }
1100 let body = if check_store {
1101 state.store.body()?
1102 } else {
1103 state.store.loaded_admission_state().1
1104 };
1105 let tuple = crate::config_store::Tuple {
1106 module: module.into(),
1107 project: None,
1108 };
1109 let versions = body
1110 .versions
1111 .iter()
1112 .find(|(entry, _)| entry == &tuple)
1113 .map(|(_, versions)| versions)
1114 .ok_or_else(|| {
1115 GateError::new("config_gated", "module is not in an approved generation")
1116 })?;
1117 let approved_declaration = [versions.pending, versions.active]
1118 .into_iter()
1119 .flatten()
1120 .any(|generation| {
1121 body.generations
1122 .get(&generation)
1123 .and_then(|entry| entry.delivery.as_ref())
1124 .and_then(|snapshot| snapshot.declarations.get(module))
1125 == Some(&declaration)
1126 });
1127 if !approved_declaration {
1128 return Err(GateError::new(
1129 "config_gated",
1130 "connection declaration differs from approved declaration",
1131 ));
1132 }
1133 let view = |generation: Option<u64>, status: ConfigStatus| -> Result<Option<ConfigView>> {
1134 let Some(generation) = generation else {
1135 return Ok(None);
1136 };
1137 let delivery = body
1138 .generations
1139 .get(&generation)
1140 .and_then(|entry| entry.delivery.as_ref())
1141 .ok_or_else(|| {
1142 GateError::new("config_gated", "approved delivery bytes are absent")
1143 })?;
1144 let mut view =
1145 delivery.modules.get(module).cloned().ok_or_else(|| {
1146 GateError::new("config_gated", "approved module view is absent")
1147 })?;
1148 view.generation = generation;
1149 view.status = status;
1150 Ok(Some(view))
1151 };
1152 let active = view(versions.active, ConfigStatus::Active)?;
1153 let pending = view(versions.pending, ConfigStatus::Pending)?;
1154 let transition = versions
1155 .pending
1156 .and_then(|id| body.generations.get(&id))
1157 .map(|entry| super::apply::Transition::for_module(&entry.generation.changeset, module))
1158 .transpose()?
1159 .unwrap_or(super::apply::Transition::Live);
1160 if active.is_none() && pending.is_none() {
1161 return Err(GateError::new(
1162 "config_gated",
1163 "module has no active or deliverable generation",
1164 ));
1165 }
1166 if !check_store {
1167 return delivery::visible_reply(
1168 state,
1169 connection,
1170 subc_protocol::config::ConfigGetReply::new(active, pending),
1171 transition,
1172 );
1173 }
1174 delivery::bind_get(
1175 state,
1176 connection,
1177 module,
1178 &request,
1179 subc_protocol::config::ConfigGetReply::new(active, pending),
1180 transition,
1181 )
1182 }
1183
1184 fn declarations(state: &mut OwnerState) -> Result<BTreeMap<String, ConfigDeclareRequest>> {
1185 let mut declarations = if state.store.status().state == StoreState::Stamped {
1186 state
1187 .store
1188 .body()?
1189 .generations
1190 .values()
1191 .next_back()
1192 .and_then(|entry| entry.delivery.as_ref())
1193 .map(|snapshot| snapshot.declarations.clone())
1194 .unwrap_or_default()
1195 } else {
1196 BTreeMap::new()
1197 };
1198 for (module, connection) in &state.active_declarations {
1199 if let Some((_, declaration)) = state.declarations.get(connection) {
1200 declarations.insert(module.clone(), declaration.clone());
1201 }
1202 }
1203 Ok(declarations)
1204 }
1205
1206 fn capture(&self, state: &mut OwnerState) -> Result<CapturedConfiguration> {
1207 let declarations = Self::declarations(state)?;
1208 let captured = self.capturer.capture(&declarations)?;
1209 state.observed_snapshot = Some(captured.diagnostic.clone());
1210 Ok(captured)
1211 }
1212
1213 fn capture_modules(
1214 &self,
1215 state: &mut OwnerState,
1216 modules: &BTreeSet<String>,
1217 ) -> Result<CapturedConfiguration> {
1218 self.capturer
1219 .capture_modules(&Self::declarations(state)?, modules)
1220 }
1221
1222 fn check_authority(state: &mut OwnerState, recovery: bool) -> Result<()> {
1223 if let Err(error) = state.store.check() {
1224 return if recovery
1225 && matches!(
1226 state.store.loaded_admission_state().0,
1227 StoreState::StoreUnverified | StoreState::StoreMissing
1228 ) {
1229 Ok(())
1230 } else {
1231 Err(error.into())
1232 };
1233 }
1234 state.retry_anchor(true);
1235 match state.store.check() {
1236 Ok(()) => Ok(()),
1237 Err(_)
1238 if recovery
1239 && matches!(
1240 state.store.status().state,
1241 StoreState::StoreUnverified | StoreState::StoreMissing
1242 ) =>
1243 {
1244 Ok(())
1245 }
1246 Err(error) => Err(error.into()),
1247 }
1248 }
1249
1250 pub fn prepare(&self, kind: PlanKind) -> Result<PreparedPlan> {
1257 let mut state = self.lock()?;
1258 Self::check_authority(
1259 &mut state,
1260 matches!(kind, PlanKind::Stamp | PlanKind::Unstamp),
1261 )?;
1262 #[cfg(test)]
1263 if let Some(hook) = self.prepare_capture_hook.lock().unwrap().take() {
1264 hook();
1265 }
1266 let observed = self.capture(&mut state)?;
1267 if observed.daemon.bytes.is_none() {
1268 return Err(GateError::new(
1269 "invalid_daemon_config",
1270 "subc.jsonc is absent; refusing to rescan",
1271 ));
1272 }
1273 if !state.recovery_exit_hold && state.store.status().state == StoreState::Stamped {
1274 state.store.observe(observed.diagnostic.to_store()?)?;
1275 }
1276 let rollback = if let PlanKind::Rollback(id) = kind {
1277 let target = super::GateSnapshot::from_store(&state.store.rollback_configuration(id)?)?;
1278 let delivery = state
1279 .store
1280 .body()?
1281 .generations
1282 .get(&id)
1283 .and_then(|entry| entry.delivery.as_ref())
1284 .ok_or_else(|| {
1285 GateError::new(
1286 "rollback_snapshot_unavailable",
1287 "retained raw configuration is absent",
1288 )
1289 })?;
1290 Some((self.capturer.capture_retained(&target, delivery)?, target))
1291 } else {
1292 None
1293 };
1294 let (candidate, mut observed, target) = if let Some((candidate, target)) = rollback {
1295 (candidate, Some(observed), Some(target))
1296 } else {
1297 (observed, None, None)
1298 };
1299 if kind == PlanKind::Stamp {
1300 super::registration::check_stamp(&candidate.diagnostic)?;
1301 }
1302 let base_revision = state.store.status().base_revision;
1303 let old = if state.store.status().state == StoreState::Stamped {
1304 super::GateSnapshot::from_store(&state.store.body()?.approved)?
1305 } else {
1306 super::GateSnapshot::default()
1307 };
1308 let mut touched_modules: BTreeSet<_> = candidate
1309 .diagnostic
1310 .changed_keys(&old)
1311 .iter()
1312 .filter_map(|key| key.module().map(str::to_owned))
1313 .collect();
1314 if matches!(kind, PlanKind::Stamp | PlanKind::Unstamp) {
1315 touched_modules.extend(candidate.diagnostic.modules());
1316 touched_modules.extend(old.modules());
1317 }
1318 if let Some(target) = target {
1319 touched_modules.extend(
1320 candidate
1321 .diagnostic
1322 .inputs
1323 .iter()
1324 .filter(|(key, value)| {
1325 key.is_program() && target.inputs.get(*key) != Some(*value)
1326 })
1327 .filter_map(|(key, _)| key.module().map(str::to_owned)),
1328 );
1329 }
1330 let mut review_modules: BTreeSet<String> = touched_modules
1331 .iter()
1332 .filter(|module| candidate.modules.contains_key(*module))
1333 .filter(|module|matches!(kind,PlanKind::Stamp|PlanKind::Unstamp) || candidate.diagnostic.changed_keys(&old).iter().any(|key|matches!(key,super::InputKey::Layer {module:owner,..}|super::InputKey::Declaration(owner) if owner==*module)))
1334 .cloned()
1335 .collect();
1336 let mut rereview_modules = BTreeSet::new();
1342 if matches!(kind, PlanKind::Change | PlanKind::Rollback(_))
1343 && state.store.status().state == StoreState::Stamped
1344 {
1345 for (module, rereview) in
1346 super::reviewed_build::pending(state.store.body()?, candidate.modules.keys())?
1347 {
1348 if rereview == super::reviewed_build::Rereview::Ready
1349 && !review_modules.contains(&module)
1350 {
1351 touched_modules.insert(module.clone());
1352 review_modules.insert(module.clone());
1353 rereview_modules.insert(module);
1354 }
1355 }
1356 }
1357 let program_keys: Vec<_> = candidate
1358 .diagnostic
1359 .inputs
1360 .keys()
1361 .filter(|key| {
1362 key.is_program()
1363 && key
1364 .module()
1365 .is_some_and(|module| touched_modules.contains(module))
1366 })
1367 .cloned()
1368 .collect();
1369 let read_set = if let Some(observed) = &mut observed {
1370 observed.capture_programs(&program_keys)?;
1371 if program_keys.iter().any(|key| {
1372 observed.diagnostic.inputs.get(key) != candidate.diagnostic.inputs.get(key)
1373 }) {
1374 return Err(GateError::stale());
1375 }
1376 observed.diagnostic.read_set(&touched_modules)?
1377 } else {
1378 candidate.diagnostic.read_set(&touched_modules)?
1379 };
1380 let observed_handles = observed
1381 .as_mut()
1382 .map(CapturedConfiguration::take_handles)
1383 .unwrap_or_default();
1384 let mut bases = BTreeMap::new();
1385 let mut base_schemas = BTreeMap::new();
1386 let mut active_generations = BTreeMap::new();
1387 if state.store.status().state == StoreState::Stamped {
1388 let body = state.store.body()?;
1389 for (tuple, versions) in &body.versions {
1390 if touched_modules.contains(&tuple.module) {
1391 active_generations.insert(tuple.clone(), versions.active);
1392 }
1393 if tuple.project.is_some() || !review_modules.contains(&tuple.module) {
1394 continue;
1395 }
1396 if let Some(id) = versions.active {
1397 let snapshot = body
1398 .generations
1399 .get(&id)
1400 .and_then(|entry| entry.delivery.as_ref())
1401 .ok_or_else(|| {
1402 GateError::new(
1403 "approved_schema_unavailable",
1404 format!(
1405 "{}: approved old schema is missing or damaged",
1406 tuple.module
1407 ),
1408 )
1409 })?;
1410 let schema = snapshot
1411 .declarations
1412 .get(&tuple.module)
1413 .and_then(|declaration| {
1414 subc_config::Schema::new(declaration.schema.clone()).ok()
1415 })
1416 .ok_or_else(|| {
1417 GateError::new(
1418 "approved_schema_unavailable",
1419 format!(
1420 "{}: approved old schema is missing or damaged",
1421 tuple.module
1422 ),
1423 )
1424 })?;
1425 base_schemas.insert(tuple.module.clone(), schema);
1426 let view = snapshot.modules.get(&tuple.module).ok_or_else(|| {
1427 GateError::new(
1428 "config_gated",
1429 format!("{}: approved active view is absent", tuple.module),
1430 )
1431 })?;
1432 let mut view = view.clone();
1433 view.generation = id;
1434 view.status = ConfigStatus::Active;
1435 bases.insert(tuple.module.clone(), view);
1436 }
1437 }
1438 }
1439 Ok(PreparedPlan {
1440 candidate,
1441 bases,
1442 base_schemas,
1443 kind,
1444 review_modules,
1445 rereview_modules,
1446 base_revision,
1447 active_generations,
1448 touched_modules,
1449 read_set,
1450 program_keys,
1451 observed_handles,
1452 })
1453 }
1454
1455 pub fn finish_plan(
1456 &self,
1457 mut prepared: PreparedPlan,
1458 review: impl FnOnce(&super::GateSnapshot, &super::GateSnapshot) -> Result<Changeset>,
1459 ) -> Result<ConfigPlanView> {
1460 let mut state = self.lock()?;
1461 Self::check_authority(
1462 &mut state,
1463 matches!(prepared.kind, PlanKind::Stamp | PlanKind::Unstamp),
1464 )?;
1465 let active_generations = if state.store.status().state == StoreState::Stamped {
1466 state
1467 .store
1468 .body()?
1469 .versions
1470 .iter()
1471 .filter(|(tuple, _)| prepared.touched_modules.contains(&tuple.module))
1472 .map(|(tuple, versions)| (tuple.clone(), versions.active))
1473 .collect()
1474 } else {
1475 BTreeMap::new()
1476 };
1477 let mut current = self
1478 .capture_modules(&mut state, &prepared.touched_modules)
1479 .map_err(|_| GateError::stale())?;
1480 current
1481 .capture_programs(&prepared.program_keys)
1482 .map_err(|_| GateError::stale())?;
1483 if state.store.status().base_revision != prepared.base_revision
1484 || active_generations != prepared.active_generations
1485 || !prepared
1486 .read_set
1487 .matches(¤t.diagnostic.read_set(&prepared.touched_modules)?)
1488 {
1489 return Err(GateError::stale());
1490 }
1491 let delivery = DeliverySnapshot {
1492 daemon: Some(ConfigLayerSnapshot::file(
1493 "subc",
1494 ConfigLayer::User,
1495 prepared.candidate.daemon.path.to_string_lossy(),
1496 prepared.candidate.daemon.bytes.clone(),
1497 )),
1498 modules: prepared
1499 .candidate
1500 .modules
1501 .iter()
1502 .map(|(module, captured)| (module.clone(), captured.view.clone()))
1503 .collect(),
1504 declarations: prepared
1505 .candidate
1506 .modules
1507 .iter()
1508 .map(|(module, captured)| (module.clone(), captured.declaration.clone()))
1509 .collect(),
1510 };
1511 let mut handles = prepared.candidate.take_handles();
1512 handles.append(&mut prepared.observed_handles);
1513 let OwnerState { gate, store, .. } = &mut *state;
1514 let view = gate.plan_reviewed(
1515 store,
1516 prepared.kind,
1517 prepared.candidate.diagnostic,
1518 ¤t.diagnostic,
1519 super::ReviewRecord::Reviewed(&prepared.review_modules),
1520 review,
1521 )?;
1522 if let Err(error) = self.check_route_policy_authority(&view.plan.changeset) {
1523 gate.expire_plan(&view.digest);
1524 return Err(error);
1525 }
1526 gate.bind_delivery(&view.digest, delivery)?;
1527 state.reviewed(&view.digest, Instant::now());
1528 state
1529 .retained_plan_files
1530 .insert(view.digest.clone(), handles);
1531 Ok(view)
1532 }
1533
1534 pub fn begin_approval(
1535 &self,
1536 digest: &Digest,
1537 facts: &ApprovalFacts,
1538 ) -> Result<ApprovalChallenge> {
1539 self.begin_approval_for_request(digest, facts, None)
1540 }
1541
1542 pub(crate) fn begin_approval_for_request(
1543 &self,
1544 digest: &Digest,
1545 facts: &ApprovalFacts,
1546 request: Option<&ApprovalRequest>,
1547 ) -> Result<ApprovalChallenge> {
1548 #[cfg(test)]
1549 struct Completion<'a>(&'a std::sync::atomic::AtomicUsize);
1550 #[cfg(test)]
1551 impl Drop for Completion<'_> {
1552 fn drop(&mut self) {
1553 self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
1554 }
1555 }
1556 #[cfg(test)]
1557 let _completed = Completion(&self.begin_approval_completions);
1558 #[cfg(test)]
1559 if let Some(hook) = self.begin_approval_hook.lock().unwrap().take() {
1560 hook();
1561 }
1562 let mut state = self.lock()?;
1563 if let Some(request) = request {
1564 request.ensure_live()?;
1565 }
1566 if state.expired_plans.contains(digest) {
1567 return Err(GateError::new(
1568 "plan_expired",
1569 "configuration plan exceeded its one-hour idle lifetime",
1570 ));
1571 }
1572 Self::check_authority(&mut state, true)?;
1573 let modules = state.gate.plan_modules(digest)?;
1574 let program_keys = state.gate.plan_program_keys(digest)?;
1575 let mut current = self
1576 .capture_modules(&mut state, &modules)
1577 .map_err(|_| GateError::stale())?;
1578 current
1579 .capture_programs(&program_keys)
1580 .map_err(|_| GateError::stale())?;
1581 let facts = state.with_desktop_view(digest, facts);
1582 let OwnerState { gate, store, .. } = &mut *state;
1583 if let Some(request) = request {
1584 request.ensure_live()?;
1585 }
1586 let challenge = gate.begin_approval(digest, store, ¤t.diagnostic, &facts)?;
1587 state.reviewed(digest, Instant::now());
1588 Ok(challenge)
1589 }
1590
1591 pub(crate) async fn approval_commit_gates(
1594 self: &std::sync::Arc<Self>,
1595 attempt: u64,
1596 ) -> Result<Vec<tokio::sync::OwnedMutexGuard<()>>> {
1597 let modules = self
1598 .on_blocking(move |owner| owner.lock()?.gate.approval_gate_modules(attempt))
1599 .await?;
1600 let mut guards = Vec::with_capacity(modules.len());
1601 for module in modules {
1602 guards.push(self.module_gate(&module)?.lock_owned().await);
1603 }
1604 Ok(guards)
1605 }
1606
1607 pub fn finish_approval(
1608 &self,
1609 attempt: u64,
1610 facts: &ApprovalFacts,
1611 reply: ApprovalReply,
1612 now_ms: u64,
1613 ) -> Result<u64> {
1614 self.finish_approval_with_commit_window(
1617 attempt,
1618 facts,
1619 reply,
1620 now_ms,
1621 None,
1622 ApprovalRequest::for_attempt(attempt),
1623 )
1624 }
1625
1626 pub(crate) fn finish_approval_with_commit_window(
1627 &self,
1628 attempt: u64,
1629 facts: &ApprovalFacts,
1630 reply: ApprovalReply,
1631 now_ms: u64,
1632 window: Option<std::sync::Arc<ApprovalCommitWindow>>,
1633 cancellation: std::sync::Arc<ApprovalRequest>,
1634 ) -> Result<u64> {
1635 if cancellation.attempt() != attempt {
1636 return Err(GateError::new(
1637 "config_unavailable",
1638 "approval request witness does not match its attempt",
1639 ));
1640 }
1641 let mut state = self.lock()?;
1642 let digest = state.gate.approval_digest(attempt)?;
1643 let declarations_only = state
1644 .gate
1645 .plans
1646 .get(&digest)
1647 .is_some_and(|plan| plan.development_approval.is_some());
1648 state.reviewed(&digest, Instant::now());
1649 let modules = state.gate.approval_modules(attempt)?;
1650 let program_keys = state.gate.plan_program_keys(&digest)?;
1651 let current = match Self::check_authority(&mut state, true).and_then(|_| {
1654 let mut current = self
1655 .capture_modules(&mut state, &modules)
1656 .map_err(|_| GateError::stale())?;
1657 current
1658 .capture_programs(&program_keys)
1659 .map_err(|_| GateError::stale())?;
1660 Ok(current)
1661 }) {
1662 Ok(current) => current,
1663 Err(error) => {
1664 state.gate.cancel_attempt(attempt);
1665 return Err(error);
1666 }
1667 };
1668 let facts = state.with_desktop_view(&digest, facts);
1669 self.drain_canceled_approvals(&mut state);
1670 self.drain_disconnections(&mut state);
1671 #[cfg(test)]
1672 if let Some(hook) = state.after_approval_drain_hook.take() {
1673 hook();
1674 }
1675 let was_unstamped = state.store.status().state == StoreState::Unstamped;
1676 let was_recovery = state.recovery_exit_hold
1677 || matches!(
1678 state.store.status().state,
1679 StoreState::StoreMissing | StoreState::StoreUnverified
1680 );
1681 let unstamp_hints = recovery::changed_file_views(self, &mut state, ¤t);
1682 #[cfg(test)]
1683 let publication_hook = state.publication_hook.take();
1684 #[cfg(test)]
1685 let prepublication_hook = state.prepublication_hook.take();
1686 let publication_before = state.store.publication_sequence();
1687 let OwnerState {
1688 gate,
1689 store,
1690 route_policy_receipts,
1691 installed_route_policies,
1692 delivery,
1693 ..
1694 } = &mut *state;
1695 let result = gate.finish_approval_with_precommit(
1696 attempt,
1697 reply,
1698 store,
1699 ¤t.diagnostic,
1700 &facts,
1701 now_ms,
1702 || {
1703 #[cfg(test)]
1704 if let Some(hook) = prepublication_hook {
1705 hook();
1706 }
1707 if let Some(window) = window {
1708 window.begin_publication()?;
1709 }
1710 {
1711 let _index = self.approval_request_index.lock().map_err(|_| {
1714 GateError::new(
1715 "config_unavailable",
1716 "approval cancellation index is poisoned",
1717 )
1718 })?;
1719 cancellation.claim_publication()?;
1720 }
1721 if !declarations_only {
1722 route_policy::restore_superseded(
1723 route_policy_receipts,
1724 installed_route_policies,
1725 &modules,
1726 delivery,
1727 );
1728 }
1729 #[cfg(test)]
1730 if let Some(hook) = publication_hook {
1731 hook();
1732 }
1733 Ok(())
1734 },
1735 );
1736 let committed_this_call = state.store.publication_sequence() != publication_before;
1737 let stamped_files = committed_this_call
1738 && (was_unstamped || was_recovery)
1739 && state.store.body().is_ok_and(|body| {
1740 body.generations
1741 .get(&body.last_generation)
1742 .is_some_and(|entry| {
1743 entry.generation.plan_digest == digest
1744 && entry.kind == crate::config_store::CommitKind::Stamp
1745 })
1746 });
1747 if result.is_ok() {
1748 state.plan_deadlines.remove(&digest);
1749 state.retained_plan_files.remove(&digest);
1750 state.desktop_views.retain(|_, (shown, _)| shown != &digest);
1751 }
1752 let unstamped = result.is_ok() && state.store.status().state == StoreState::Unstamped;
1753 let committed_exit = state
1754 .store
1755 .loaded_admission_state()
1756 .1
1757 .generations
1758 .iter()
1759 .next_back()
1760 .filter(|(_, entry)| {
1761 committed_this_call
1762 && entry.generation.plan_digest == digest
1763 && entry.kind != crate::config_store::CommitKind::DeclarationApproval
1764 && (was_recovery
1765 || state.store.loaded_admission_state().1.stamped
1766 || entry.kind == crate::config_store::CommitKind::Unstamp)
1767 })
1768 .map(|(generation, entry)| {
1769 (
1770 *generation,
1771 entry.kind == crate::config_store::CommitKind::Unstamp,
1772 )
1773 });
1774 if committed_this_call && was_recovery {
1775 state.authority_incarnation = std::sync::Arc::new(());
1776 }
1777 if let Some((generation, unstamp)) = committed_exit {
1778 state.authority_exit = Some(generation);
1779 state.recovery_exit_hold = was_recovery;
1780 if was_recovery || unstamp {
1781 state
1782 .delivery
1783 .reset_for_authority_exit(if unstamp { generation } else { 0 });
1784 }
1785 if unstamp {
1786 state.stamp_channel_cutover = false;
1787 state.reconciled_entries.clear();
1788 state.unstamp_hints.extend(unstamp_hints);
1789 }
1790 } else if committed_this_call {
1791 state.authority_exit = None;
1792 }
1793 if stamped_files {
1794 state.stamp_channel_cutover = true;
1795 let connections: Vec<_> = state.active_declarations.values().copied().collect();
1796 state.delivery.remember_stamp_connections(connections);
1797 }
1798 drop(state);
1799 if unstamped {
1800 self.retire_unstamped_bindings();
1801 }
1802 result
1803 }
1804
1805 #[cfg(all(test, unix))]
1806 pub(crate) fn set_publication_hook(&self, hook: impl FnOnce() + Send + 'static) {
1807 self.inner.lock().unwrap().publication_hook = Some(Box::new(hook));
1808 }
1809
1810 #[cfg(all(test, unix))]
1811 pub(crate) fn set_prepublication_hook(&self, hook: impl FnOnce() + Send + 'static) {
1812 self.inner.lock().unwrap().prepublication_hook = Some(Box::new(hook));
1813 }
1814
1815 #[cfg(all(test, unix))]
1816 pub(crate) fn set_after_approval_drain_hook(&self, hook: impl FnOnce() + Send + 'static) {
1817 self.inner.lock().unwrap().after_approval_drain_hook = Some(Box::new(hook));
1818 }
1819
1820 #[cfg(all(test, unix))]
1821 pub(crate) fn approval_request_count_for_test(&self) -> usize {
1822 self.approval_request_index.lock().unwrap().len()
1823 }
1824
1825 #[cfg(all(test, unix))]
1826 pub(crate) fn poison_approval_request_index_for_test(&self) {
1827 let poisoned = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
1828 let _guard = self.approval_request_index.lock().unwrap();
1829 panic!("injected approval index poison");
1830 }));
1831 assert!(poisoned.is_err());
1832 }
1833
1834 #[cfg(all(test, unix))]
1835 pub(crate) fn pause_next_approval_claim_for_test(
1836 &self,
1837 ) -> (
1838 tokio::sync::oneshot::Receiver<()>,
1839 tokio::sync::oneshot::Sender<()>,
1840 ) {
1841 let (reached, observed) = tokio::sync::oneshot::channel();
1842 let (release, wait) = tokio::sync::oneshot::channel();
1843 *self.approval_claim_pause.lock().unwrap() = Some(ApprovalClaimPause {
1844 reached,
1845 release: wait,
1846 });
1847 (observed, release)
1848 }
1849
1850 #[cfg(all(test, unix))]
1851 pub(crate) async fn pause_approval_claim_for_test(&self) {
1852 let pause = self.approval_claim_pause.lock().unwrap().take();
1853 if let Some(pause) = pause {
1854 let _ = pause.reached.send(());
1855 pause.release.await.unwrap();
1856 }
1857 }
1858
1859 #[cfg(any(all(test, unix), feature = "test-support"))]
1862 pub(crate) fn set_os_prompt_policy_for_tests(&self) {
1863 self.os_prompt_policy
1864 .store(true, std::sync::atomic::Ordering::Relaxed);
1865 }
1866
1867 #[cfg(any(test, feature = "test-support"))]
1868 pub(crate) fn os_prompt_policy_for_tests(&self) -> bool {
1869 self.os_prompt_policy
1870 .load(std::sync::atomic::Ordering::Relaxed)
1871 }
1872
1873 pub fn status(&self) -> Result<subc_control::config::ConfigStatus> {
1874 let mut state = self.lock()?;
1875 let mut current_snapshot = None;
1876 state.retry_anchor(true);
1877 let verified = state.store.check().is_ok();
1878 if verified
1879 && !state.recovery_exit_hold
1880 && !state.store.status().anchor_pending
1881 && matches!(
1882 state.store.status().state,
1883 StoreState::Stamped | StoreState::Unstamped
1884 )
1885 {
1886 let captured = self.capture(&mut state)?;
1887 current_snapshot = Some(captured.diagnostic.clone());
1888 state.store.observe(captured.diagnostic.to_store()?)?;
1889 }
1890 let store_status = state.store.status();
1891 let mut status = subc_control::config::ConfigStatus::default();
1892 status.state = match store_status.state {
1893 StoreState::Stamped => "stamped",
1894 StoreState::Unstamped => "unstamped",
1895 StoreState::StoreMissing => "store_missing",
1896 StoreState::StoreUnverified => "store_unverified",
1897 }
1898 .into();
1899 if state.recovery_exit_hold {
1900 if store_status.state != StoreState::Stamped {
1901 status.state = "store_unverified".into();
1902 }
1903 status
1904 .warnings
1905 .push("approved recovery exit is waiting for reconciliation".into());
1906 }
1907 status.integrity = match store_status.integrity {
1908 crate::config_store::IntegrityMode::DigestOnly => "digest_only",
1909 crate::config_store::IntegrityMode::Sealed => "sealed",
1910 }
1911 .into();
1912 status.revision = store_status.revision;
1913 status.base_revision = store_status.base_revision;
1914 status.anchor_pending = store_status.anchor_pending;
1915 status.ungated_inputs = self.startup_overrides.clone();
1916 #[cfg(feature = "test-support")]
1917 if state.store.uses_config_gate_test_seam() {
1918 status
1919 .development_switches
1920 .push("SUBC_TEST_CONFIG_GATE".into());
1921 }
1922 status.warnings.extend(store_status.warnings);
1923 if cfg!(feature = "dev-desktop-approval") {
1924 status
1925 .development_switches
1926 .push("dev-desktop-approval".into());
1927 }
1928 if cfg!(feature = "dev-placement") {
1929 status.development_switches.push("dev-placement".into());
1930 }
1931 let declared: Vec<String> = Self::declarations(&mut state)
1932 .map(|declarations| declarations.into_keys().collect())
1933 .unwrap_or_default();
1934 if let Ok(body) = state.store.body() {
1935 if body.stamped {
1936 for (module, rereview) in
1937 super::reviewed_build::pending(body, &declared).unwrap_or_default()
1938 {
1939 status.warnings.push(match rereview {
1940 super::reviewed_build::Rereview::WaitingForProgram => format!(
1941 "module {module}: needs configuration re-review: its approved program has not reviewed its approved layers; the re-review waits until that program is running and active"
1942 ),
1943 super::reviewed_build::Rereview::Ready => format!(
1944 "module {module}: needs configuration re-review: its approved program has not reviewed its approved layers; the next configuration plan re-reviews them with the running program"
1945 ),
1946 });
1947 }
1948 }
1949 if body.stamped {
1950 current_snapshot = super::GateSnapshot::from_store(&body.approved).ok();
1951 }
1952 for entry in body.generations.values().rev().take_while(|entry| {
1953 matches!(
1954 entry.kind,
1955 crate::config_store::CommitKind::Development
1956 | crate::config_store::CommitKind::DeclarationApproval
1957 )
1958 }) {
1959 if entry.kind != crate::config_store::CommitKind::Development {
1960 continue;
1961 }
1962 status.warnings.push(format!(
1963 "development generation {}: development placement (no approval)",
1964 entry.generation.id
1965 ));
1966 }
1967 status.approval_channels = body
1968 .generations
1969 .iter()
1970 .map(|(id, entry)| {
1971 subc_control::config::ConfigApprovalRecord::new(
1972 *id,
1973 entry.generation.approval_channel,
1974 )
1975 })
1976 .chain(body.pruned.iter().map(|(id, entry)| {
1977 subc_control::config::ConfigApprovalRecord::new(*id, entry.approval_channel)
1978 }))
1979 .collect();
1980 status
1981 .approval_channels
1982 .sort_by_key(|record| record.generation);
1983 status.pending_proposals = u64::from(body.proposal.is_some());
1984 for (tuple, versions) in &body.versions {
1985 if let Some(generation) = versions.pending {
1986 let entry = body.generations.get(&generation);
1987 let transition = entry.and_then(|entry| {
1988 super::apply::Transition::for_module(
1989 &entry.generation.changeset,
1990 &tuple.module,
1991 )
1992 .ok()
1993 });
1994 status.approved_but_inactive.push(serde_json::json!({
1995 "module_id":tuple.module,"project_id":tuple.project,"generation":generation,
1996 "digest":entry.map(|entry| &entry.generation.plan_digest),
1997 "waiting_for":match transition {
1998 Some(super::apply::Transition::DaemonRestart) => "daemon_restart",
1999 Some(super::apply::Transition::NextSupervision) => "next_supervision",
2000 _ => "delivery",
2001 },
2002 }));
2003 }
2004 }
2005 }
2006 if let Some(snapshot) = current_snapshot {
2007 status_inputs::report_snapshot(&snapshot, &mut status);
2008 }
2009 status.gated_modules = admission::AdmissionCache::from_state(&state).held_modules();
2010 if state.store.loaded_admission_state().0 == StoreState::Stamped {
2011 let declarations: Vec<_> = state
2012 .declarations
2013 .iter()
2014 .map(|(connection, (module, _))| (*connection, module.clone()))
2015 .collect();
2016 for (connection, module) in declarations {
2017 if self
2020 .module_get_locked(
2021 &mut state,
2022 connection,
2023 &module,
2024 subc_protocol::config::ConfigGetRequest::new(),
2025 false,
2026 )
2027 .is_err_and(|error| error.code == "config_gated")
2028 {
2029 status.gated_modules.push(module);
2030 }
2031 }
2032 status.gated_modules.sort();
2033 status.gated_modules.dedup();
2034 }
2035 Ok(status)
2036 }
2037
2038 pub fn history(&self) -> Result<(Vec<subc_config::Generation>, Vec<serde_json::Value>)> {
2039 let mut state = self.lock()?;
2040 state.retry_anchor(true);
2041 state.store.check()?;
2042 let body = state.store.body()?;
2043 Ok((
2044 body.generations
2045 .values()
2046 .map(|entry| entry.generation.clone())
2047 .collect(),
2048 body.pruned
2049 .values()
2050 .map(serde_json::to_value)
2051 .collect::<std::result::Result<_, _>>()
2052 .map_err(GateError::invalid)?,
2053 ))
2054 }
2055
2056 pub fn approved_snapshot(&self) -> Result<DeliverySnapshot> {
2059 let mut state = self.lock()?;
2060 state.store.delivery_ready()?;
2061 state
2062 .store
2063 .body()?
2064 .generations
2065 .values()
2066 .next_back()
2067 .and_then(|entry| entry.delivery.clone())
2068 .ok_or_else(|| GateError::new("config_gated", "approved delivery snapshot is absent"))
2069 }
2070
2071 pub fn approved_rollback_commands(&self, generation: u64) -> Result<ApprovedRollbackCommands> {
2079 let mut state = self.lock()?;
2080 state.store.delivery_ready()?;
2081 let configuration = state.store.rollback_configuration(generation)?;
2082 let entry = &state.store.body()?.generations[&generation];
2083 if entry.kind != crate::config_store::CommitKind::Rollback {
2084 return Err(GateError::new(
2085 "rollback_generation_required",
2086 "generation is not an approved rollback",
2087 ));
2088 }
2089 let daemon = entry
2090 .delivery
2091 .as_ref()
2092 .and_then(|snapshot| snapshot.daemon.clone())
2093 .ok_or_else(|| {
2094 GateError::new(
2095 "rollback_snapshot_unavailable",
2096 "approved rollback command input is absent",
2097 )
2098 })?;
2099 let snapshot = super::GateSnapshot::from_store(&configuration)?;
2100 let mut programs = BTreeMap::new();
2101 for (key, identity) in snapshot.inputs {
2102 if let super::InputKey::Program { module, path } = key {
2103 if programs
2104 .insert(module, ApprovedRollbackProgram { path, identity })
2105 .is_some()
2106 {
2107 return Err(GateError::new(
2108 "rollback_snapshot_unavailable",
2109 "rollback contains conflicting program identities",
2110 ));
2111 }
2112 }
2113 }
2114 Ok(ApprovedRollbackCommands {
2115 generation,
2116 daemon,
2117 programs,
2118 })
2119 }
2120}