1use std::{
6 collections::{HashMap, VecDeque},
7 path::PathBuf,
8 sync::{Arc, Mutex},
9 time::Duration,
10};
11
12use subc_protocol::{
13 error_codes,
14 session::{OperatorConfirmReply, OperatorConfirmRequest},
15 ErrorBody, Flags, FrameType, Principal, Priority,
16};
17use tokio::{
18 sync::{oneshot, Notify},
19 time::{timeout_at, Instant},
20};
21
22use crate::{
23 caller_identity::CallerIdentity,
24 forwarding::{ModuleEndpointId, ModuleRouteKey, RouteBinding},
25 router::RouteCtx,
26 ConnectionId, Frame,
27};
28
29mod shortcut;
30pub(crate) use shortcut::ShortcutContext;
31
32const DECLINE_LIMIT: usize = 3;
35const DECLINE_WINDOW: Duration = Duration::from_secs(10 * 60);
36const DECLINE_COOLDOWN: Duration = Duration::from_secs(10 * 60);
39
40pub(crate) type CooldownClock = Arc<dyn Fn() -> Instant + Send + Sync>;
43
44#[derive(Clone, Debug, PartialEq, Eq, Hash)]
49pub(crate) enum DeclineKey {
50 Executable(PathBuf),
51 Connection(ConnectionId),
52}
53
54impl DeclineKey {
55 pub(crate) fn for_caller(caller: &CallerIdentity, connection: ConnectionId) -> Self {
56 match caller.executable() {
57 Some(executable) => Self::Executable(executable.to_path_buf()),
58 None => Self::Connection(connection),
59 }
60 }
61}
62
63impl std::fmt::Display for DeclineKey {
64 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
65 match self {
66 Self::Executable(path) => write!(f, "{}", path.display()),
67 Self::Connection(connection) => write!(
68 f,
69 "an unidentified local process on connection {}",
70 connection.get()
71 ),
72 }
73 }
74}
75
76pub trait OperatorProvider: Send + Sync + 'static {
80 fn config_gate_ready(&self) -> bool {
83 false
84 }
85 fn prompt(
86 &self,
87 text: &str,
88 publish: Box<dyn FnOnce(Arc<dyn OperatorWithdraw>) + Send>,
89 ) -> ProviderResult;
90
91 fn prompt_gate(
94 &self,
95 text: &str,
96 publish: Box<dyn FnOnce(Arc<dyn OperatorWithdraw>) + Send>,
97 ) -> ProviderResult {
98 self.prompt(text, publish)
99 }
100}
101
102pub trait OperatorWithdraw: Send + Sync + 'static {
104 fn withdraw(&self, reason: &str);
105}
106
107#[derive(Debug, Clone, PartialEq, Eq)]
108pub enum ProviderResult {
109 Approved,
110 Declined,
111 Unavailable,
112 ProviderError {
113 reason: &'static str,
114 },
115 UnsupportedPlatform,
116 NoPresence,
117 SetupRequired,
118 LocalAuthenticationError {
119 failure: OperatorOsErrorKind,
120 domain: String,
121 code: isize,
122 },
123}
124
125#[derive(Debug, Clone, Copy, PartialEq, Eq)]
126pub enum OperatorOsErrorKind {
127 Declined,
128 NoPresence,
129 ProviderError,
130}
131
132#[derive(Debug, Clone, Copy)]
133pub struct OperatorLimits {
134 pub queue_wait: Duration,
135 pub prompt_timeout: Duration,
136 pub stuck_grace: Duration,
137}
138
139impl Default for OperatorLimits {
140 fn default() -> Self {
141 Self {
142 queue_wait: Duration::from_secs(150),
143 prompt_timeout: Duration::from_secs(120),
144 stuck_grace: Duration::from_secs(10),
145 }
146 }
147}
148
149impl OperatorLimits {
150 fn delivery_budget(self) -> Duration {
151 self.queue_wait + self.prompt_timeout + self.stuck_grace + Duration::from_secs(10)
152 }
153}
154
155struct OsProvider;
156
157#[cfg(any(target_os = "macos", windows, target_os = "linux"))]
158struct OsWithdraw(Arc<dyn subc_presence::Withdraw>);
159#[cfg(any(target_os = "macos", windows, target_os = "linux"))]
160impl OperatorWithdraw for OsWithdraw {
161 fn withdraw(&self, _: &str) {
162 self.0.withdraw();
163 }
164}
165
166impl OperatorProvider for OsProvider {
167 fn config_gate_ready(&self) -> bool {
168 cfg!(any(target_os = "macos", windows, target_os = "linux"))
169 }
170 fn prompt_gate(
171 &self,
172 text: &str,
173 publish: Box<dyn FnOnce(Arc<dyn OperatorWithdraw>) + Send>,
174 ) -> ProviderResult {
175 #[cfg(any(target_os = "macos", windows, target_os = "linux"))]
176 {
177 gate_os_outcome(subc_presence::prompt(
178 text,
179 Box::new(move |handle| publish(Arc::new(OsWithdraw(handle)))),
180 ))
181 }
182 #[cfg(not(any(target_os = "macos", windows, target_os = "linux")))]
183 {
184 let _ = (text, publish);
185 ProviderResult::Unavailable
186 }
187 }
188
189 fn prompt(
190 &self,
191 text: &str,
192 publish: Box<dyn FnOnce(Arc<dyn OperatorWithdraw>) + Send>,
193 ) -> ProviderResult {
194 #[cfg(any(target_os = "macos", windows, target_os = "linux"))]
195 {
196 os_outcome(subc_presence::prompt(
197 text,
198 Box::new(move |handle| publish(Arc::new(OsWithdraw(handle)))),
199 ))
200 }
201 #[cfg(not(any(target_os = "macos", windows, target_os = "linux")))]
202 {
203 let _ = (text, publish);
204 ProviderResult::Unavailable
205 }
206 }
207}
208
209#[cfg(any(target_os = "macos", windows, target_os = "linux"))]
215fn os_outcome(outcome: subc_presence::Outcome) -> ProviderResult {
216 match outcome {
217 subc_presence::Outcome::Approved => ProviderResult::Approved,
218 subc_presence::Outcome::AuthorizedWithoutPrompt => ProviderResult::Approved,
219 subc_presence::Outcome::Declined => ProviderResult::Declined,
220 subc_presence::Outcome::ProviderError => ProviderResult::Unavailable,
221 subc_presence::Outcome::ProviderErrorWithReason { reason } => {
222 ProviderResult::ProviderError { reason }
223 }
224 subc_presence::Outcome::NoPresence => ProviderResult::NoPresence,
225 subc_presence::Outcome::SetupRequired => ProviderResult::SetupRequired,
226 subc_presence::Outcome::UnsupportedPlatform => ProviderResult::UnsupportedPlatform,
227 subc_presence::Outcome::LocalAuthenticationError {
228 failure,
229 domain,
230 code,
231 } => ProviderResult::LocalAuthenticationError {
232 failure: match failure {
233 subc_presence::LocalAuthenticationFailure::Declined => {
234 OperatorOsErrorKind::Declined
235 }
236 subc_presence::LocalAuthenticationFailure::NoPresence => {
237 OperatorOsErrorKind::NoPresence
238 }
239 subc_presence::LocalAuthenticationFailure::ProviderError => {
240 OperatorOsErrorKind::ProviderError
241 }
242 },
243 domain,
244 code,
245 },
246 _ => ProviderResult::Unavailable,
248 }
249}
250
251#[cfg(any(target_os = "macos", windows, target_os = "linux"))]
252fn gate_os_outcome(outcome: subc_presence::Outcome) -> ProviderResult {
253 match outcome {
254 subc_presence::Outcome::AuthorizedWithoutPrompt => ProviderResult::Unavailable,
255 other => os_outcome(other),
256 }
257}
258
259#[cfg(all(test, any(target_os = "macos", windows, target_os = "linux")))]
260mod os_outcome_tests {
261 use super::*;
262
263 #[test]
264 fn config_gate_os_outcomes_require_interactive_confirmation() {
265 use crate::config_gate::PresenceDecision as Gate;
266 use subc_presence::{LocalAuthenticationFailure as Failure, Outcome as Os};
267 for (outcome, expected) in [
268 (Os::Approved, Gate::InteractiveApproval),
269 (Os::AuthorizedWithoutPrompt, Gate::Unavailable),
270 (Os::Declined, Gate::Declined),
271 (Os::NoPresence, Gate::Unavailable),
272 (Os::SetupRequired, Gate::Unavailable),
273 (Os::UnsupportedPlatform, Gate::Unavailable),
274 (Os::ProviderError, Gate::Unavailable),
275 (
276 Os::ProviderErrorWithReason { reason: "no_agent" },
277 Gate::Unavailable,
278 ),
279 (
280 Os::LocalAuthenticationError {
281 failure: Failure::Declined,
282 domain: "LA".into(),
283 code: -2,
284 },
285 Gate::Declined,
286 ),
287 (
288 Os::LocalAuthenticationError {
289 failure: Failure::NoPresence,
290 domain: "LA".into(),
291 code: -4,
292 },
293 Gate::Unavailable,
294 ),
295 (
296 Os::LocalAuthenticationError {
297 failure: Failure::ProviderError,
298 domain: "LA".into(),
299 code: -1,
300 },
301 Gate::Unavailable,
302 ),
303 ] {
304 assert_eq!(
305 Gate::from_operator_outcome(&Outcome::from_provider(gate_os_outcome(outcome))),
306 expected
307 );
308 }
309 assert_eq!(
310 Gate::from_operator_outcome(&Outcome::declined("backoff")),
311 Gate::Declined
312 );
313 assert_eq!(
314 Gate::from_operator_outcome(&Outcome::unavailable("queue_full")),
315 Gate::Unavailable
316 );
317 }
318
319 #[test]
320 fn policy_authorization_still_approves_legacy_prompts() {
321 assert_eq!(
322 os_outcome(subc_presence::Outcome::AuthorizedWithoutPrompt),
323 ProviderResult::Approved
324 );
325 }
326
327 #[test]
328 fn only_an_os_approval_becomes_approved() {
329 use subc_presence::Outcome as Os;
330 assert_eq!(os_outcome(Os::Approved), ProviderResult::Approved);
331 assert_eq!(os_outcome(Os::Declined), ProviderResult::Declined);
332 assert_eq!(os_outcome(Os::ProviderError), ProviderResult::Unavailable);
333 let root_error = os_outcome(Os::ProviderErrorWithReason {
334 reason: "root_user",
335 });
336 assert_eq!(
337 root_error,
338 ProviderResult::ProviderError {
339 reason: "root_user"
340 }
341 );
342 let refusal = Outcome::from_provider(root_error);
343 assert_eq!(refusal.code, error_codes::OPERATOR_PRESENCE_UNAVAILABLE);
344 assert_eq!(refusal.reason, "root_user");
345 assert_eq!(os_outcome(Os::NoPresence), ProviderResult::NoPresence);
346 assert_eq!(os_outcome(Os::SetupRequired), ProviderResult::SetupRequired);
347 let refusal = Outcome::from_provider(ProviderResult::SetupRequired);
348 assert_eq!(refusal.code, error_codes::OPERATOR_PRESENCE_UNAVAILABLE);
349 assert_eq!(refusal.reason, "operator_prompt_setup_required");
350 assert_eq!(
351 os_outcome(Os::UnsupportedPlatform),
352 ProviderResult::UnsupportedPlatform
353 );
354 assert_eq!(
355 os_outcome(Os::LocalAuthenticationError {
356 failure: subc_presence::LocalAuthenticationFailure::NoPresence,
357 domain: "com.apple.LocalAuthentication".into(),
358 code: -4,
359 }),
360 ProviderResult::LocalAuthenticationError {
361 failure: OperatorOsErrorKind::NoPresence,
362 domain: "com.apple.LocalAuthentication".into(),
363 code: -4,
364 }
365 );
366 }
367}
368
369#[derive(Clone, Debug)]
370pub(crate) struct Outcome {
371 pub code: &'static str,
372 pub reason: &'static str,
373 retry_after_ms: Option<u64>,
374 os_error_domain: Option<String>,
375 os_error_code: Option<isize>,
376}
377impl Outcome {
378 pub(crate) fn unavailable(reason: &'static str) -> Self {
379 Self {
380 code: error_codes::OPERATOR_PRESENCE_UNAVAILABLE,
381 reason,
382 retry_after_ms: None,
383 os_error_domain: None,
384 os_error_code: None,
385 }
386 }
387 fn declined(reason: &'static str) -> Self {
388 Self {
389 code: error_codes::OPERATOR_DECLINED,
390 reason,
391 retry_after_ms: None,
392 os_error_domain: None,
393 os_error_code: None,
394 }
395 }
396 fn with_retry_after_ms(mut self, retry_after_ms: u64) -> Self {
397 self.retry_after_ms = Some(retry_after_ms);
398 self
399 }
400 pub(crate) fn refusal(code: &'static str) -> Self {
401 Self {
402 code,
403 reason: "",
404 retry_after_ms: None,
405 os_error_domain: None,
406 os_error_code: None,
407 }
408 }
409 fn from_provider(result: ProviderResult) -> Self {
410 match result {
411 ProviderResult::Approved => Self {
412 code: "confirmed",
413 reason: "",
414 retry_after_ms: None,
415 os_error_domain: None,
416 os_error_code: None,
417 },
418 ProviderResult::Declined => Self::declined("person"),
419 ProviderResult::Unavailable => Self::unavailable("provider_error"),
420 ProviderResult::ProviderError { reason } => Self::unavailable(reason),
421 ProviderResult::UnsupportedPlatform => Self::unavailable("unsupported_platform"),
422 ProviderResult::NoPresence => Self::unavailable("no_presence"),
423 ProviderResult::SetupRequired => Self::unavailable("operator_prompt_setup_required"),
424 ProviderResult::LocalAuthenticationError {
425 failure,
426 domain,
427 code,
428 } => {
429 let mut outcome = match failure {
430 OperatorOsErrorKind::Declined => Self::declined("person"),
431 OperatorOsErrorKind::NoPresence => Self::unavailable("no_presence"),
432 OperatorOsErrorKind::ProviderError => Self::unavailable("provider_error"),
433 };
434 outcome.os_error_domain = Some(domain);
435 outcome.os_error_code = Some(code);
436 outcome
437 }
438 }
439 }
440 pub(crate) fn frame(self, request: &Frame) -> Frame {
441 let (ty, body, daemon_origin) = if self.code == "confirmed" {
445 (
446 FrameType::Response,
447 serde_json::to_vec(&OperatorConfirmReply::confirmed()).expect("serializable reply"),
448 false,
449 )
450 } else {
451 let mut error = ErrorBody::new(self.code, "operator confirmation refused");
452 if !self.reason.is_empty() {
453 let detail = match self.retry_after_ms {
454 Some(retry_after_ms) => {
455 serde_json::json!({"reason": self.reason, "retry_after_ms": retry_after_ms})
456 }
457 None => serde_json::json!({"reason": self.reason}),
458 };
459 error = error.with_detail(detail);
460 }
461 (
462 FrameType::Error,
463 serde_json::to_vec(&error).expect("serializable error"),
464 true,
465 )
466 };
467 let flags = if daemon_origin {
468 crate::router::daemon_error_flags()
469 } else {
470 Flags::new(false, Priority::Passive, false)
471 };
472 Frame::build_with_version(
473 request.header.ver,
474 ty,
475 flags,
476 0,
477 0,
478 request.header.corr,
479 body,
480 )
481 .expect("bounded control reply")
482 }
483}
484
485impl crate::config_gate::PresenceDecision {
486 pub(crate) fn from_operator_outcome(outcome: &Outcome) -> Self {
487 match outcome.code {
488 "confirmed" => Self::InteractiveApproval,
489 error_codes::OPERATOR_DECLINED => Self::Declined,
490 _ => Self::Unavailable,
491 }
492 }
493}
494
495fn retry_after_ms(remaining: Duration) -> u64 {
496 let whole_milliseconds = remaining.as_millis();
497 let rounded_milliseconds =
498 whole_milliseconds + u128::from(!remaining.subsec_nanos().is_multiple_of(1_000_000));
499 u64::try_from(rounded_milliseconds).expect("operator confirmation backoff fits in u64")
500}
501
502pub(crate) fn audit(
503 module_id: &str,
504 summary: &str,
505 principal: &str,
506 outcome: &Outcome,
507 wait: Duration,
508 prompt: Duration,
509 prompt_shown: bool,
510) {
511 tracing::info!(target: "subc_daemon::operator_confirm", module_id, summary = ?summary,
512 principal, outcome = outcome.code, reason = outcome.reason,
513 os_error_domain = ?outcome.os_error_domain, os_error_code = ?outcome.os_error_code,
514 wait_ms = wait.as_millis() as u64, prompt_ms = prompt.as_millis() as u64,
515 prompt_shown, "operator_confirm_audit");
516}
517
518fn principal_label(principal: &Principal) -> String {
519 match principal {
520 Principal::Direct => "direct".into(),
521 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
522 Principal::Unverified => String::new(),
523 }
524}
525
526pub(crate) fn valid_summary(summary: &str) -> bool {
527 let count = summary.chars().count();
528 (1..=200).contains(&count)
529 && !summary.starts_with(char::is_whitespace)
530 && !summary.ends_with(char::is_whitespace)
531 && !summary.chars().any(|c| {
532 c.is_control()
533 || matches!(c,
534 '\u{200b}'..='\u{200f}' | '\u{202a}'..='\u{202e}' | '\u{2028}' | '\u{2029}' |
535 '\u{2060}'..='\u{2069}' | '\u{feff}')
536 })
537}
538
539enum RequestKind {
540 Module(ModuleRouteKey),
541 Privileged {
542 answer: Mutex<Option<oneshot::Sender<Outcome>>>,
543 key: DeclineKey,
544 },
545}
546
547struct Request {
548 gate_presence: bool,
549 id: u64,
550 module_id: String,
551 summary: String,
552 principal: String,
553 text: String,
554 kind: RequestKind,
555 ctx: RouteCtx,
556 frame: Frame,
557 arrived: Instant,
558 delivery_deadline: Instant,
559 prompt_ready: std::sync::atomic::AtomicBool,
561 shortcut_audit: bool,
562}
563struct Active {
564 request: Arc<Request>,
565 started: Option<Instant>,
566 handle: Option<Arc<dyn OperatorWithdraw>>,
567 committed: bool,
568 withdrawn: Option<(Instant, &'static str)>,
569 withdraw_started: bool,
570}
571struct State {
572 provider: Arc<dyn OperatorProvider>,
573 limits: OperatorLimits,
574 queue: VecDeque<Arc<Request>>,
575 modules: HashMap<String, u64>,
576 backoff: HashMap<(String, String), Instant>,
577 cooldown_clock: CooldownClock,
578 declines: HashMap<DeclineKey, VecDeque<Instant>>,
580 cooldowns: HashMap<DeclineKey, Instant>,
582 prompt: Option<Active>,
583 stuck: bool,
584 running: bool,
585 next_id: u64,
586 shortcuts: HashMap<u64, shortcut::Pending>,
587 #[cfg(test)]
588 deliveries: usize,
589}
590
591pub(crate) struct OperatorConfirms {
592 state: Mutex<State>,
593 wake: Notify,
594}
595
596struct WithdrawOnDrop {
597 confirms: Arc<OperatorConfirms>,
598 id: u64,
599}
600
601impl Drop for WithdrawOnDrop {
602 fn drop(&mut self) {
603 self.confirms
604 .withdraw_matching(|request| request.id == self.id, "caller_cancelled");
605 }
606}
607impl std::fmt::Debug for OperatorConfirms {
608 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
609 f.debug_struct("OperatorConfirms").finish_non_exhaustive()
610 }
611}
612impl Default for OperatorConfirms {
613 fn default() -> Self {
614 Self {
615 state: Mutex::new(State {
616 provider: Arc::new(OsProvider),
617 limits: OperatorLimits::default(),
618 queue: VecDeque::new(),
619 modules: HashMap::new(),
620 backoff: HashMap::new(),
621 cooldown_clock: Arc::new(Instant::now),
622 declines: HashMap::new(),
623 cooldowns: HashMap::new(),
624 prompt: None,
625 stuck: false,
626 running: false,
627 next_id: 0,
628 shortcuts: HashMap::new(),
629 #[cfg(test)]
630 deliveries: 0,
631 }),
632 wake: Notify::new(),
633 }
634 }
635}
636impl OperatorConfirms {
637 pub(crate) fn config_gate_ready(&self) -> bool {
638 let state = self.lock();
639 !state.stuck && state.provider.config_gate_ready()
640 }
641 fn lock(&self) -> std::sync::MutexGuard<'_, State> {
642 self.state.lock().unwrap_or_else(|p| p.into_inner())
643 }
644 pub(crate) fn configure(&self, provider: Arc<dyn OperatorProvider>, limits: OperatorLimits) {
645 let mut state = self.lock();
646 assert!(
647 !state.running && state.modules.is_empty(),
648 "configure before serving"
649 );
650 state.provider = provider;
651 state.limits = limits;
652 }
653 pub(crate) fn configure_bootstrap(&self, config: &crate::bootstrap::BootstrapConfig) {
654 let provider: Arc<dyn OperatorProvider> = Arc::new(OsProvider);
655 #[cfg(feature = "test-support")]
656 let provider = test_provider::select(
657 provider,
658 config.operator_script.as_deref(),
659 config.operator_events.as_deref(),
660 std::env::current_exe().ok().as_deref(),
661 );
662 self.configure(
663 provider,
664 OperatorLimits {
665 queue_wait: config.operator_queue_wait,
666 prompt_timeout: config.operator_timeout,
667 stuck_grace: config.operator_stuck_grace,
668 },
669 );
670 }
671
672 #[cfg(feature = "test-support")]
673 pub(crate) fn configure_script(
674 &self,
675 script: &std::path::Path,
676 events: Option<&std::path::Path>,
677 limits: Option<OperatorLimits>,
678 ) {
679 let provider = test_provider::select(
680 Arc::new(OsProvider),
681 Some(script),
682 events,
683 test_provider::current_executable().as_deref(),
684 );
685 self.configure(provider, limits.unwrap_or_default());
686 }
687
688 #[cfg(test)]
691 pub(crate) fn admit(
692 self: &Arc<Self>,
693 ctx: &RouteCtx,
694 frame: Frame,
695 module_id: String,
696 nonce_proven: bool,
697 request: OperatorConfirmRequest,
698 binding: Option<&RouteBinding>,
699 ) -> Vec<Frame> {
700 self.admit_with_shortcut(ctx, frame, module_id, nonce_proven, request, binding, None)
701 }
702
703 #[allow(clippy::too_many_arguments)]
704 pub(crate) fn admit_with_shortcut(
705 self: &Arc<Self>,
706 ctx: &RouteCtx,
707 frame: Frame,
708 module_id: String,
709 nonce_proven: bool,
710 request: OperatorConfirmRequest,
711 binding: Option<&RouteBinding>,
712 shortcut: Option<ShortcutContext>,
713 ) -> Vec<Frame> {
714 let principal = binding
715 .map(|b| principal_label(&b.principal))
716 .unwrap_or_default();
717 let mut state = self.lock();
718 let now = Instant::now();
719 state.backoff.retain(|_, until| *until > now);
720 let refusal = if !nonce_proven
721 || binding.is_none_or(|b| matches!(b.principal, Principal::Unverified))
722 {
723 Some(Outcome::refusal(
724 error_codes::OPERATOR_REQUEST_NOT_PERMITTED,
725 ))
726 } else if !valid_summary(&request.summary) {
727 Some(Outcome::refusal(error_codes::OPERATOR_SUMMARY_INVALID))
728 } else if let Some(until) = state
729 .backoff
730 .get(&(module_id.clone(), principal.clone()))
731 .copied()
732 .filter(|_| shortcut.is_none())
733 {
734 Some(Outcome::declined("backoff").with_retry_after_ms(retry_after_ms(until - now)))
735 } else if shortcut.is_none() && state.stuck {
736 Some(Outcome::unavailable("provider_stuck"))
737 } else if state.modules.contains_key(&module_id) {
738 Some(Outcome::unavailable("module_limit"))
739 } else if shortcut.is_none() && state.queue.len() >= 4 {
740 Some(Outcome::unavailable("queue_full"))
741 } else {
742 None
743 };
744 if let Some(outcome) = refusal {
745 audit(
746 &module_id,
747 &request.summary,
748 &principal,
749 &outcome,
750 Duration::ZERO,
751 Duration::ZERO,
752 false,
753 );
754 return vec![outcome.frame(&frame)];
755 }
756 let binding = binding.expect("permission checked");
757 let opener = match &binding.principal {
758 Principal::Direct => "a local program",
759 Principal::Reserved { module_id } => module_id,
760 Principal::Unverified => unreachable!(),
761 };
762 state.next_id += 1;
763 let pending = Arc::new(Request {
764 id: state.next_id,
765 text: format!(
766 "approve a request from {module_id}: {} (requested by {opener})",
767 request.summary
768 ),
769 module_id,
770 summary: request.summary,
771 principal,
772 gate_presence: false,
773 kind: RequestKind::Module(ModuleRouteKey {
774 endpoint: binding.module_endpoint,
775 channel: binding.module_channel,
776 }),
777 ctx: ctx.clone(),
778 frame,
779 arrived: now,
780 delivery_deadline: now + state.limits.delivery_budget(),
781 prompt_ready: true.into(),
782 shortcut_audit: shortcut.is_some(),
783 });
784 state.modules.insert(pending.module_id.clone(), pending.id);
785 if let Some(shortcut) = shortcut {
786 self.start_shortcut(&mut state, pending, shortcut);
787 } else {
788 self.enqueue(&mut state, pending);
789 }
790 Vec::new()
791 }
792
793 #[allow(clippy::too_many_arguments)]
798 pub(crate) async fn approve_privileged_with_shortcut(
799 self: &Arc<Self>,
800 ctx: &RouteCtx,
801 frame: &Frame,
802 operation: &str,
803 text: String,
804 caller: String,
805 key: DeclineKey,
806 shortcut: Option<ShortcutContext>,
807 ) -> Outcome {
808 self.approve_client(ctx, frame, operation, text, caller, key, shortcut, false)
809 .await
810 }
811
812 pub(crate) async fn approve_config_gate(
813 self: &Arc<Self>,
814 ctx: &RouteCtx,
815 frame: &Frame,
816 text: String,
817 caller: String,
818 key: DeclineKey,
819 ) -> Outcome {
820 self.approve_client(ctx, frame, "config.apply", text, caller, key, None, true)
822 .await
823 }
824
825 #[allow(clippy::too_many_arguments)]
826 async fn approve_client(
827 self: &Arc<Self>,
828 ctx: &RouteCtx,
829 frame: &Frame,
830 operation: &str,
831 text: String,
832 caller: String,
833 key: DeclineKey,
834 shortcut: Option<ShortcutContext>,
835 gate_presence: bool,
836 ) -> Outcome {
837 let (answer, receive) = oneshot::channel();
838 let id = {
839 let mut state = self.lock();
840 let now = (state.cooldown_clock)();
841 state.cooldowns.retain(|_, until| *until > now);
842 state.declines.retain(|_, declines| {
843 declines.retain(|at| now.saturating_duration_since(*at) < DECLINE_WINDOW);
844 !declines.is_empty()
845 });
846 let refusal = if state.cooldowns.contains_key(&key) {
847 Some(Outcome::declined("backoff"))
848 } else if shortcut.is_none() && state.stuck {
849 Some(Outcome::unavailable("provider_stuck"))
850 } else if shortcut.is_none() && state.queue.len() >= 4 {
851 Some(Outcome::unavailable("queue_full"))
852 } else {
853 None
854 };
855 if let Some(outcome) = refusal {
856 audit(
857 operation,
858 &text,
859 &caller,
860 &outcome,
861 Duration::ZERO,
862 Duration::ZERO,
863 false,
864 );
865 return outcome;
866 }
867 state.next_id += 1;
868 let id = state.next_id;
869 let now = Instant::now();
870 let pending = Arc::new(Request {
871 id,
872 module_id: operation.into(),
873 summary: text.clone(),
874 principal: caller,
875 text,
876 gate_presence,
877 kind: RequestKind::Privileged {
878 answer: Mutex::new(Some(answer)),
879 key,
880 },
881 ctx: ctx.clone(),
882 frame: frame.clone(),
883 arrived: now,
884 delivery_deadline: now + state.limits.delivery_budget(),
885 prompt_ready: true.into(),
886 shortcut_audit: shortcut.is_some(),
887 });
888 if let Some(shortcut) = shortcut {
889 self.start_shortcut(&mut state, pending, shortcut);
890 } else {
891 self.enqueue(&mut state, pending);
892 }
893 id
894 };
895 let _withdraw = WithdrawOnDrop {
896 confirms: Arc::clone(self),
897 id,
898 };
899 receive
900 .await
901 .unwrap_or_else(|_| Outcome::unavailable("provider_error"))
902 }
903
904 fn enqueue(self: &Arc<Self>, state: &mut State, pending: Arc<Request>) {
905 if state.prompt.is_none() && state.queue.is_empty() {
906 state.prompt = Some(Active {
907 request: pending,
908 started: None,
909 handle: None,
910 committed: false,
911 withdrawn: None,
912 withdraw_started: false,
913 });
914 } else {
915 state.queue.push_back(pending);
916 }
917 if !state.running {
918 state.running = true;
919 tokio::spawn(Arc::clone(self).run());
920 }
921 self.wake.notify_one();
922 }
923
924 fn commit(
925 self: &Arc<Self>,
926 state: &mut State,
927 request: Arc<Request>,
928 outcome: Outcome,
929 started: Option<Instant>,
930 shown: bool,
931 ) {
932 let now = Instant::now();
933 let wait = started.unwrap_or(now).duration_since(request.arrived);
934 let prompt = started.map(|at| now.duration_since(at)).unwrap_or_default();
935 if outcome.reason == "person" && matches!(request.kind, RequestKind::Module(_)) {
936 state.backoff.insert(
937 (request.module_id.clone(), request.principal.clone()),
938 now + Duration::from_secs(30),
939 );
940 }
941 audit(
942 &request.module_id,
943 if request.shortcut_audit {
944 "Operator shortcut request"
945 } else {
946 &request.summary
947 },
948 &request.principal,
949 &outcome,
950 wait,
951 prompt,
952 shown,
953 );
954 if let RequestKind::Privileged { answer, key } = &request.kind {
955 let person_declined = outcome.reason == "person";
956 if let Some(answer) = answer.lock().unwrap_or_else(|p| p.into_inner()).take() {
957 let _ = answer.send(outcome);
958 }
959 if person_declined {
960 self.count_person_decline(state, key);
961 }
962 return;
963 }
964 let this = Arc::clone(self);
965 #[cfg(test)]
966 {
967 state.deliveries += 1;
968 }
969 tokio::spawn(async move {
970 let answer = outcome.frame(&request.frame);
971 match timeout_at(request.delivery_deadline, request.ctx.egress.send(answer)).await {
972 Ok(Ok(())) => {}
973 result => {
974 tracing::warn!(target: "subc_daemon::operator_confirm", module_id = request.module_id, ?result, "operator confirm answer dropped")
975 }
976 }
977 let mut state = this.lock();
978 #[cfg(test)]
979 {
980 state.deliveries -= 1;
981 }
982 if state.modules.get(&request.module_id) == Some(&request.id) {
983 state.modules.remove(&request.module_id);
984 }
985 });
986 }
987
988 fn count_person_decline(self: &Arc<Self>, state: &mut State, key: &DeclineKey) {
993 let now = (state.cooldown_clock)();
994 let declines = state.declines.entry(key.clone()).or_default();
995 declines.retain(|at| now.saturating_duration_since(*at) < DECLINE_WINDOW);
996 declines.push_back(now);
997 if declines.len() < DECLINE_LIMIT {
998 return;
999 }
1000 state.declines.remove(key);
1002 state.cooldowns.insert(key.clone(), now + DECLINE_COOLDOWN);
1003 tracing::warn!(target: "subc_daemon::operator_confirm", caller = %key,
1004 cooldown_secs = DECLINE_COOLDOWN.as_secs(),
1005 "privileged requests refused without a prompt after repeated operator declines");
1006 let mut index = 0;
1007 while index < state.queue.len() {
1008 if matches!(&state.queue[index].kind,
1009 RequestKind::Privileged { key: queued, .. } if queued == key)
1010 {
1011 let request = state.queue.remove(index).expect("queue index");
1012 self.commit(state, request, Outcome::declined("backoff"), None, false);
1013 } else {
1014 index += 1;
1015 }
1016 }
1017 }
1018
1019 fn withdraw_matching(
1020 self: &Arc<Self>,
1021 predicate: impl Fn(&Request) -> bool,
1022 reason: &'static str,
1023 ) -> bool {
1024 let mut state = self.lock();
1025 let mut matched = false;
1026 let ids: Vec<_> = state
1027 .shortcuts
1028 .iter()
1029 .filter(|(_, pending)| predicate(&pending.request))
1030 .map(|(id, _)| *id)
1031 .collect();
1032 for id in ids {
1033 self.end_shortcut(&mut state, id, Outcome::declined(reason), "withdrawn");
1034 matched = true;
1035 }
1036 let mut index = 0;
1037 while index < state.queue.len() {
1038 if predicate(&state.queue[index]) {
1039 let request = state.queue.remove(index).expect("queue index");
1040 self.commit(&mut state, request, Outcome::declined(reason), None, false);
1041 matched = true;
1042 } else {
1043 index += 1;
1044 }
1045 }
1046 if let Some(active) = state
1047 .prompt
1048 .as_mut()
1049 .filter(|a| !a.committed && predicate(&a.request))
1050 {
1051 active.committed = true;
1052 active.withdrawn = Some((Instant::now(), reason));
1053 let (request, started, shown) = (
1054 Arc::clone(&active.request),
1055 active.started,
1056 active.handle.is_some(),
1057 );
1058 self.commit(
1059 &mut state,
1060 request,
1061 Outcome::declined(reason),
1062 started,
1063 shown,
1064 );
1065 matched = true;
1066 }
1067 self.wake.notify_one();
1068 matched
1069 }
1070 pub(crate) fn route_closed(self: &Arc<Self>, key: ModuleRouteKey) {
1071 self.withdraw_matching(
1072 |r| matches!(r.kind, RequestKind::Module(k) if k == key),
1073 "route_closed",
1074 );
1075 }
1076 pub(crate) fn module_closed(self: &Arc<Self>, endpoint: ModuleEndpointId) {
1077 self.withdraw_matching(
1078 |r| matches!(r.kind, RequestKind::Module(k) if k.endpoint == endpoint),
1079 "module_closed",
1080 );
1081 }
1082 pub(crate) fn connection_closed(self: &Arc<Self>, connection: crate::ConnectionId) {
1083 self.withdraw_matching(
1084 |r| {
1085 matches!(r.kind, RequestKind::Privileged { .. })
1086 && r.ctx.connection_id == connection
1087 },
1088 "caller_cancelled",
1089 );
1090 }
1091 pub(crate) fn cancel(self: &Arc<Self>, connection: crate::ConnectionId, corr: u64) -> bool {
1092 self.withdraw_matching(
1093 |r| r.ctx.connection_id == connection && r.frame.header.corr == corr,
1094 "caller_cancelled",
1095 )
1096 }
1097
1098 fn publish(self: &Arc<Self>, id: u64, handle: Arc<dyn OperatorWithdraw>) {
1099 let mut state = self.lock();
1100 if let Some(active) = state.prompt.as_mut().filter(|a| a.request.id == id) {
1101 active.handle = Some(Arc::clone(&handle));
1102 if let Some((_, reason)) = active.withdrawn {
1103 active.withdraw_started = true;
1104 tokio::task::spawn_blocking(move || handle.withdraw(reason));
1105 }
1106 }
1107 self.wake.notify_one();
1108 }
1109 fn returned(self: &Arc<Self>, id: u64, result: ProviderResult) {
1110 let mut state = self.lock();
1111 let Some(active) = state.prompt.take() else {
1112 return;
1113 };
1114 assert_eq!(active.request.id, id);
1115 if !active.withdraw_started {
1118 if let (Some(handle), Some((_, reason))) = (active.handle.as_ref(), active.withdrawn) {
1119 let handle = Arc::clone(handle);
1120 tokio::task::spawn_blocking(move || handle.withdraw(reason));
1121 }
1122 }
1123 if !active.committed {
1124 let outcome = if Instant::now() >= active.request.delivery_deadline {
1125 Outcome::unavailable("timeout")
1126 } else {
1127 Outcome::from_provider(result)
1128 };
1129 self.commit(
1130 &mut state,
1131 active.request,
1132 outcome,
1133 active.started,
1134 active.handle.is_some(),
1135 );
1136 } else {
1137 tracing::debug!(target: "subc_daemon::operator_confirm", ?result, "late operator provider result discarded");
1138 }
1139 state.stuck = false;
1140 self.wake.notify_one();
1141 }
1142
1143 async fn run(self: Arc<Self>) {
1144 loop {
1145 let deadline = {
1146 let mut state = self.lock();
1147 let now = Instant::now();
1148 let mut index = 0;
1149 while index < state.queue.len() {
1150 if state.queue[index].arrived + state.limits.queue_wait <= now {
1151 let request = state.queue.remove(index).expect("queue index");
1152 self.commit(
1153 &mut state,
1154 request,
1155 Outcome::unavailable("queue_wait"),
1156 None,
1157 false,
1158 );
1159 } else {
1160 index += 1;
1161 }
1162 }
1163 let timeout = state.limits.prompt_timeout;
1164 if let Some(active) = state.prompt.as_mut().filter(|a| {
1165 !a.committed
1166 && (a.request.delivery_deadline <= now
1167 || a.started.is_some_and(|at| at + timeout <= now))
1168 }) {
1169 active.committed = true;
1170 active.withdrawn = Some((now, "timeout"));
1171 let (request, started, shown) = (
1172 Arc::clone(&active.request),
1173 active.started,
1174 active.handle.is_some(),
1175 );
1176 self.commit(
1177 &mut state,
1178 request,
1179 Outcome::unavailable("timeout"),
1180 started,
1181 shown,
1182 );
1183 }
1184 let grace = state.limits.stuck_grace;
1185 if !state.stuck
1186 && state.prompt.as_ref().is_some_and(|a| {
1187 a.started.is_some() && a.withdrawn.is_some_and(|(at, _)| at + grace <= now)
1188 })
1189 {
1190 state.stuck = true;
1191 tracing::error!(target: "subc_daemon::operator_confirm", "operator provider stuck after withdraw");
1192 while let Some(request) = state.queue.pop_front() {
1193 self.commit(
1194 &mut state,
1195 request,
1196 Outcome::unavailable("provider_stuck"),
1197 None,
1198 false,
1199 );
1200 }
1201 }
1202 if let Some(active) = state
1203 .prompt
1204 .as_mut()
1205 .filter(|a| !a.withdraw_started && a.withdrawn.is_some() && a.handle.is_some())
1206 {
1207 active.withdraw_started = true;
1208 let handle = Arc::clone(active.handle.as_ref().expect("handle checked"));
1209 let reason = active.withdrawn.expect("withdraw checked").1;
1210 tokio::task::spawn_blocking(move || handle.withdraw(reason));
1211 }
1212 if state
1214 .prompt
1215 .as_ref()
1216 .is_some_and(|a| a.started.is_none() && a.committed)
1217 {
1218 state.prompt = None;
1219 }
1220 if state.prompt.is_none() {
1221 if let Some(request) = state.queue.pop_front() {
1222 state.prompt = Some(Active {
1223 request,
1224 started: None,
1225 handle: None,
1226 committed: false,
1227 withdrawn: None,
1228 withdraw_started: false,
1229 });
1230 }
1231 }
1232 if let Some(active) = state.prompt.as_mut().filter(|a| {
1233 a.started.is_none()
1234 && !a.committed
1235 && a.request
1236 .prompt_ready
1237 .load(std::sync::atomic::Ordering::Acquire)
1238 }) {
1239 active.started = Some(now);
1240 let request = Arc::clone(&active.request);
1241 let provider = Arc::clone(&state.provider);
1242 let this = Arc::clone(&self);
1243 tokio::spawn(async move {
1244 let publish_to = Arc::clone(&this);
1245 let id = request.id;
1246 let result = tokio::task::spawn_blocking(move || {
1247 let publish = Box::new(move |handle| publish_to.publish(id, handle));
1248 if request.gate_presence {
1249 provider.prompt_gate(&request.text, publish)
1250 } else {
1251 provider.prompt(&request.text, publish)
1252 }
1253 })
1254 .await
1255 .unwrap_or(ProviderResult::Unavailable);
1256 this.returned(id, result);
1257 });
1258 }
1259 if state.prompt.is_none() && state.queue.is_empty() {
1260 state.running = false;
1261 return;
1262 }
1263 let mut deadline = state
1264 .queue
1265 .iter()
1266 .map(|r| r.arrived + state.limits.queue_wait)
1267 .min();
1268 if let Some(active) = &state.prompt {
1269 let next = if !active.committed {
1270 Some(
1271 active
1272 .started
1273 .map_or(active.request.delivery_deadline, |at| {
1274 (at + timeout).min(active.request.delivery_deadline)
1275 }),
1276 )
1277 } else if !state.stuck {
1278 active.withdrawn.map(|(at, _)| at + grace)
1279 } else {
1280 None
1281 };
1282 if let Some(next) = next {
1283 deadline = Some(deadline.map_or(next, |d| d.min(next)));
1284 }
1285 }
1286 deadline
1287 };
1288 if let Some(at) = deadline {
1289 tokio::select! { _ = self.wake.notified() => {}, _ = tokio::time::sleep_until(at) => {} }
1290 } else {
1291 self.wake.notified().await;
1292 }
1293 }
1294 }
1295}
1296
1297#[cfg(feature = "test-support")]
1298mod test_provider;
1299#[cfg(test)]
1300mod tests;