1use std::{
6 collections::{HashMap, VecDeque},
7 path::PathBuf,
8 sync::{Arc, Mutex},
9 time::Duration,
10};
11
12use subc_protocol::{
13 error_codes,
14 session::{OperatorConfirmReply, OperatorConfirmRequest},
15 ErrorBody, Flags, FrameType, Principal, Priority,
16};
17use tokio::{
18 sync::{oneshot, Notify},
19 time::{timeout_at, Instant},
20};
21
22use crate::{
23 caller_identity::CallerIdentity,
24 forwarding::{ModuleEndpointId, ModuleRouteKey, RouteBinding},
25 router::RouteCtx,
26 ConnectionId, Frame,
27};
28
29mod shortcut;
30pub(crate) use shortcut::ShortcutContext;
31
32const DECLINE_LIMIT: usize = 3;
35const DECLINE_WINDOW: Duration = Duration::from_secs(10 * 60);
36const DECLINE_COOLDOWN: Duration = Duration::from_secs(10 * 60);
39
40pub(crate) type CooldownClock = Arc<dyn Fn() -> Instant + Send + Sync>;
43
44#[derive(Clone, Debug, PartialEq, Eq, Hash)]
49pub(crate) enum DeclineKey {
50 Executable(PathBuf),
51 Connection(ConnectionId),
52}
53
54impl DeclineKey {
55 pub(crate) fn for_caller(caller: &CallerIdentity, connection: ConnectionId) -> Self {
56 match caller.executable() {
57 Some(executable) => Self::Executable(executable.to_path_buf()),
58 None => Self::Connection(connection),
59 }
60 }
61}
62
63impl std::fmt::Display for DeclineKey {
64 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
65 match self {
66 Self::Executable(path) => write!(f, "{}", path.display()),
67 Self::Connection(connection) => write!(
68 f,
69 "an unidentified local process on connection {}",
70 connection.get()
71 ),
72 }
73 }
74}
75
76pub trait OperatorProvider: Send + Sync + 'static {
80 fn prompt(
81 &self,
82 text: &str,
83 publish: Box<dyn FnOnce(Arc<dyn OperatorWithdraw>) + Send>,
84 ) -> ProviderResult;
85}
86
87pub trait OperatorWithdraw: Send + Sync + 'static {
89 fn withdraw(&self, reason: &str);
90}
91
92#[derive(Debug, Clone, PartialEq, Eq)]
93pub enum ProviderResult {
94 Approved,
95 Declined,
96 Unavailable,
97 ProviderError {
98 reason: &'static str,
99 },
100 UnsupportedPlatform,
101 NoPresence,
102 SetupRequired,
103 LocalAuthenticationError {
104 failure: OperatorOsErrorKind,
105 domain: String,
106 code: isize,
107 },
108}
109
110#[derive(Debug, Clone, Copy, PartialEq, Eq)]
111pub enum OperatorOsErrorKind {
112 Declined,
113 NoPresence,
114 ProviderError,
115}
116
117#[derive(Debug, Clone, Copy)]
118pub struct OperatorLimits {
119 pub queue_wait: Duration,
120 pub prompt_timeout: Duration,
121 pub stuck_grace: Duration,
122}
123
124impl Default for OperatorLimits {
125 fn default() -> Self {
126 Self {
127 queue_wait: Duration::from_secs(150),
128 prompt_timeout: Duration::from_secs(120),
129 stuck_grace: Duration::from_secs(10),
130 }
131 }
132}
133
134impl OperatorLimits {
135 fn delivery_budget(self) -> Duration {
136 self.queue_wait + self.prompt_timeout + self.stuck_grace + Duration::from_secs(10)
137 }
138}
139
140struct OsProvider;
141
142#[cfg(any(target_os = "macos", windows, target_os = "linux"))]
143struct OsWithdraw(Arc<dyn subc_presence::Withdraw>);
144#[cfg(any(target_os = "macos", windows, target_os = "linux"))]
145impl OperatorWithdraw for OsWithdraw {
146 fn withdraw(&self, _: &str) {
147 self.0.withdraw();
148 }
149}
150
151impl OperatorProvider for OsProvider {
152 fn prompt(
153 &self,
154 text: &str,
155 publish: Box<dyn FnOnce(Arc<dyn OperatorWithdraw>) + Send>,
156 ) -> ProviderResult {
157 #[cfg(any(target_os = "macos", windows, target_os = "linux"))]
158 {
159 os_outcome(subc_presence::prompt(
160 text,
161 Box::new(move |handle| publish(Arc::new(OsWithdraw(handle)))),
162 ))
163 }
164 #[cfg(not(any(target_os = "macos", windows, target_os = "linux")))]
165 {
166 let _ = (text, publish);
167 ProviderResult::Unavailable
168 }
169 }
170}
171
172#[cfg(any(target_os = "macos", windows, target_os = "linux"))]
177fn os_outcome(outcome: subc_presence::Outcome) -> ProviderResult {
178 match outcome {
179 subc_presence::Outcome::Approved => ProviderResult::Approved,
180 subc_presence::Outcome::Declined => ProviderResult::Declined,
181 subc_presence::Outcome::ProviderError => ProviderResult::Unavailable,
182 subc_presence::Outcome::ProviderErrorWithReason { reason } => {
183 ProviderResult::ProviderError { reason }
184 }
185 subc_presence::Outcome::NoPresence => ProviderResult::NoPresence,
186 subc_presence::Outcome::SetupRequired => ProviderResult::SetupRequired,
187 subc_presence::Outcome::UnsupportedPlatform => ProviderResult::UnsupportedPlatform,
188 subc_presence::Outcome::LocalAuthenticationError {
189 failure,
190 domain,
191 code,
192 } => ProviderResult::LocalAuthenticationError {
193 failure: match failure {
194 subc_presence::LocalAuthenticationFailure::Declined => {
195 OperatorOsErrorKind::Declined
196 }
197 subc_presence::LocalAuthenticationFailure::NoPresence => {
198 OperatorOsErrorKind::NoPresence
199 }
200 subc_presence::LocalAuthenticationFailure::ProviderError => {
201 OperatorOsErrorKind::ProviderError
202 }
203 },
204 domain,
205 code,
206 },
207 _ => ProviderResult::Unavailable,
209 }
210}
211
212#[cfg(all(test, any(target_os = "macos", windows, target_os = "linux")))]
213mod os_outcome_tests {
214 use super::*;
215
216 #[test]
217 fn only_an_os_approval_becomes_approved() {
218 use subc_presence::Outcome as Os;
219 assert_eq!(os_outcome(Os::Approved), ProviderResult::Approved);
220 assert_eq!(os_outcome(Os::Declined), ProviderResult::Declined);
221 assert_eq!(os_outcome(Os::ProviderError), ProviderResult::Unavailable);
222 let root_error = os_outcome(Os::ProviderErrorWithReason {
223 reason: "root_user",
224 });
225 assert_eq!(
226 root_error,
227 ProviderResult::ProviderError {
228 reason: "root_user"
229 }
230 );
231 let refusal = Outcome::from_provider(root_error);
232 assert_eq!(refusal.code, error_codes::OPERATOR_PRESENCE_UNAVAILABLE);
233 assert_eq!(refusal.reason, "root_user");
234 assert_eq!(os_outcome(Os::NoPresence), ProviderResult::NoPresence);
235 assert_eq!(os_outcome(Os::SetupRequired), ProviderResult::SetupRequired);
236 let refusal = Outcome::from_provider(ProviderResult::SetupRequired);
237 assert_eq!(refusal.code, error_codes::OPERATOR_PRESENCE_UNAVAILABLE);
238 assert_eq!(refusal.reason, "operator_prompt_setup_required");
239 assert_eq!(
240 os_outcome(Os::UnsupportedPlatform),
241 ProviderResult::UnsupportedPlatform
242 );
243 assert_eq!(
244 os_outcome(Os::LocalAuthenticationError {
245 failure: subc_presence::LocalAuthenticationFailure::NoPresence,
246 domain: "com.apple.LocalAuthentication".into(),
247 code: -4,
248 }),
249 ProviderResult::LocalAuthenticationError {
250 failure: OperatorOsErrorKind::NoPresence,
251 domain: "com.apple.LocalAuthentication".into(),
252 code: -4,
253 }
254 );
255 }
256}
257
258#[derive(Clone, Debug)]
259pub(crate) struct Outcome {
260 pub code: &'static str,
261 pub reason: &'static str,
262 retry_after_ms: Option<u64>,
263 os_error_domain: Option<String>,
264 os_error_code: Option<isize>,
265}
266impl Outcome {
267 pub(crate) fn unavailable(reason: &'static str) -> Self {
268 Self {
269 code: error_codes::OPERATOR_PRESENCE_UNAVAILABLE,
270 reason,
271 retry_after_ms: None,
272 os_error_domain: None,
273 os_error_code: None,
274 }
275 }
276 fn declined(reason: &'static str) -> Self {
277 Self {
278 code: error_codes::OPERATOR_DECLINED,
279 reason,
280 retry_after_ms: None,
281 os_error_domain: None,
282 os_error_code: None,
283 }
284 }
285 fn with_retry_after_ms(mut self, retry_after_ms: u64) -> Self {
286 self.retry_after_ms = Some(retry_after_ms);
287 self
288 }
289 pub(crate) fn refusal(code: &'static str) -> Self {
290 Self {
291 code,
292 reason: "",
293 retry_after_ms: None,
294 os_error_domain: None,
295 os_error_code: None,
296 }
297 }
298 fn from_provider(result: ProviderResult) -> Self {
299 match result {
300 ProviderResult::Approved => Self {
301 code: "confirmed",
302 reason: "",
303 retry_after_ms: None,
304 os_error_domain: None,
305 os_error_code: None,
306 },
307 ProviderResult::Declined => Self::declined("person"),
308 ProviderResult::Unavailable => Self::unavailable("provider_error"),
309 ProviderResult::ProviderError { reason } => Self::unavailable(reason),
310 ProviderResult::UnsupportedPlatform => Self::unavailable("unsupported_platform"),
311 ProviderResult::NoPresence => Self::unavailable("no_presence"),
312 ProviderResult::SetupRequired => Self::unavailable("operator_prompt_setup_required"),
313 ProviderResult::LocalAuthenticationError {
314 failure,
315 domain,
316 code,
317 } => {
318 let mut outcome = match failure {
319 OperatorOsErrorKind::Declined => Self::declined("person"),
320 OperatorOsErrorKind::NoPresence => Self::unavailable("no_presence"),
321 OperatorOsErrorKind::ProviderError => Self::unavailable("provider_error"),
322 };
323 outcome.os_error_domain = Some(domain);
324 outcome.os_error_code = Some(code);
325 outcome
326 }
327 }
328 }
329 pub(crate) fn frame(self, request: &Frame) -> Frame {
330 let (ty, body, daemon_origin) = if self.code == "confirmed" {
334 (
335 FrameType::Response,
336 serde_json::to_vec(&OperatorConfirmReply::confirmed()).expect("serializable reply"),
337 false,
338 )
339 } else {
340 let mut error = ErrorBody::new(self.code, "operator confirmation refused");
341 if !self.reason.is_empty() {
342 let detail = match self.retry_after_ms {
343 Some(retry_after_ms) => {
344 serde_json::json!({"reason": self.reason, "retry_after_ms": retry_after_ms})
345 }
346 None => serde_json::json!({"reason": self.reason}),
347 };
348 error = error.with_detail(detail);
349 }
350 (
351 FrameType::Error,
352 serde_json::to_vec(&error).expect("serializable error"),
353 true,
354 )
355 };
356 let flags = if daemon_origin {
357 crate::router::daemon_error_flags()
358 } else {
359 Flags::new(false, Priority::Passive, false)
360 };
361 Frame::build_with_version(
362 request.header.ver,
363 ty,
364 flags,
365 0,
366 0,
367 request.header.corr,
368 body,
369 )
370 .expect("bounded control reply")
371 }
372}
373
374fn retry_after_ms(remaining: Duration) -> u64 {
375 let whole_milliseconds = remaining.as_millis();
376 let rounded_milliseconds =
377 whole_milliseconds + u128::from(!remaining.subsec_nanos().is_multiple_of(1_000_000));
378 u64::try_from(rounded_milliseconds).expect("operator confirmation backoff fits in u64")
379}
380
381pub(crate) fn audit(
382 module_id: &str,
383 summary: &str,
384 principal: &str,
385 outcome: &Outcome,
386 wait: Duration,
387 prompt: Duration,
388 prompt_shown: bool,
389) {
390 tracing::info!(target: "subc_daemon::operator_confirm", module_id, summary = ?summary,
391 principal, outcome = outcome.code, reason = outcome.reason,
392 os_error_domain = ?outcome.os_error_domain, os_error_code = ?outcome.os_error_code,
393 wait_ms = wait.as_millis() as u64, prompt_ms = prompt.as_millis() as u64,
394 prompt_shown, "operator_confirm_audit");
395}
396
397fn principal_label(principal: &Principal) -> String {
398 match principal {
399 Principal::Direct => "direct".into(),
400 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
401 Principal::Unverified => String::new(),
402 }
403}
404
405pub(crate) fn valid_summary(summary: &str) -> bool {
406 let count = summary.chars().count();
407 (1..=200).contains(&count)
408 && !summary.starts_with(char::is_whitespace)
409 && !summary.ends_with(char::is_whitespace)
410 && !summary.chars().any(|c| {
411 c.is_control()
412 || matches!(c,
413 '\u{200b}'..='\u{200f}' | '\u{202a}'..='\u{202e}' | '\u{2028}' | '\u{2029}' |
414 '\u{2060}'..='\u{2069}' | '\u{feff}')
415 })
416}
417
418enum RequestKind {
419 Module(ModuleRouteKey),
420 Privileged {
421 answer: Mutex<Option<oneshot::Sender<Outcome>>>,
422 key: DeclineKey,
423 },
424}
425
426struct Request {
427 id: u64,
428 module_id: String,
429 summary: String,
430 principal: String,
431 text: String,
432 kind: RequestKind,
433 ctx: RouteCtx,
434 frame: Frame,
435 arrived: Instant,
436 delivery_deadline: Instant,
437 prompt_ready: std::sync::atomic::AtomicBool,
439 shortcut_audit: bool,
440}
441struct Active {
442 request: Arc<Request>,
443 started: Option<Instant>,
444 handle: Option<Arc<dyn OperatorWithdraw>>,
445 committed: bool,
446 withdrawn: Option<(Instant, &'static str)>,
447 withdraw_started: bool,
448}
449struct State {
450 provider: Arc<dyn OperatorProvider>,
451 limits: OperatorLimits,
452 queue: VecDeque<Arc<Request>>,
453 modules: HashMap<String, u64>,
454 backoff: HashMap<(String, String), Instant>,
455 cooldown_clock: CooldownClock,
456 declines: HashMap<DeclineKey, VecDeque<Instant>>,
458 cooldowns: HashMap<DeclineKey, Instant>,
460 prompt: Option<Active>,
461 stuck: bool,
462 running: bool,
463 next_id: u64,
464 shortcuts: HashMap<u64, shortcut::Pending>,
465 #[cfg(test)]
466 deliveries: usize,
467}
468
469pub(crate) struct OperatorConfirms {
470 state: Mutex<State>,
471 wake: Notify,
472}
473
474struct WithdrawOnDrop {
475 confirms: Arc<OperatorConfirms>,
476 id: u64,
477}
478
479impl Drop for WithdrawOnDrop {
480 fn drop(&mut self) {
481 self.confirms
482 .withdraw_matching(|request| request.id == self.id, "caller_cancelled");
483 }
484}
485impl std::fmt::Debug for OperatorConfirms {
486 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
487 f.debug_struct("OperatorConfirms").finish_non_exhaustive()
488 }
489}
490impl Default for OperatorConfirms {
491 fn default() -> Self {
492 Self {
493 state: Mutex::new(State {
494 provider: Arc::new(OsProvider),
495 limits: OperatorLimits::default(),
496 queue: VecDeque::new(),
497 modules: HashMap::new(),
498 backoff: HashMap::new(),
499 cooldown_clock: Arc::new(Instant::now),
500 declines: HashMap::new(),
501 cooldowns: HashMap::new(),
502 prompt: None,
503 stuck: false,
504 running: false,
505 next_id: 0,
506 shortcuts: HashMap::new(),
507 #[cfg(test)]
508 deliveries: 0,
509 }),
510 wake: Notify::new(),
511 }
512 }
513}
514impl OperatorConfirms {
515 fn lock(&self) -> std::sync::MutexGuard<'_, State> {
516 self.state.lock().unwrap_or_else(|p| p.into_inner())
517 }
518 pub(crate) fn configure(&self, provider: Arc<dyn OperatorProvider>, limits: OperatorLimits) {
519 let mut state = self.lock();
520 assert!(
521 !state.running && state.modules.is_empty(),
522 "configure before serving"
523 );
524 state.provider = provider;
525 state.limits = limits;
526 }
527 pub(crate) fn configure_bootstrap(&self, config: &crate::bootstrap::BootstrapConfig) {
528 let provider: Arc<dyn OperatorProvider> = Arc::new(OsProvider);
529 #[cfg(feature = "test-support")]
530 let provider = test_provider::select(
531 provider,
532 config.operator_script.as_deref(),
533 config.operator_events.as_deref(),
534 std::env::current_exe().ok().as_deref(),
535 );
536 self.configure(
537 provider,
538 OperatorLimits {
539 queue_wait: config.operator_queue_wait,
540 prompt_timeout: config.operator_timeout,
541 stuck_grace: config.operator_stuck_grace,
542 },
543 );
544 }
545
546 #[cfg(feature = "test-support")]
547 pub(crate) fn configure_script(
548 &self,
549 script: &std::path::Path,
550 events: Option<&std::path::Path>,
551 limits: Option<OperatorLimits>,
552 ) {
553 let provider = test_provider::select(
554 Arc::new(OsProvider),
555 Some(script),
556 events,
557 test_provider::current_executable().as_deref(),
558 );
559 self.configure(provider, limits.unwrap_or_default());
560 }
561
562 #[cfg(test)]
565 pub(crate) fn admit(
566 self: &Arc<Self>,
567 ctx: &RouteCtx,
568 frame: Frame,
569 module_id: String,
570 nonce_proven: bool,
571 request: OperatorConfirmRequest,
572 binding: Option<&RouteBinding>,
573 ) -> Vec<Frame> {
574 self.admit_with_shortcut(ctx, frame, module_id, nonce_proven, request, binding, None)
575 }
576
577 #[allow(clippy::too_many_arguments)]
578 pub(crate) fn admit_with_shortcut(
579 self: &Arc<Self>,
580 ctx: &RouteCtx,
581 frame: Frame,
582 module_id: String,
583 nonce_proven: bool,
584 request: OperatorConfirmRequest,
585 binding: Option<&RouteBinding>,
586 shortcut: Option<ShortcutContext>,
587 ) -> Vec<Frame> {
588 let principal = binding
589 .map(|b| principal_label(&b.principal))
590 .unwrap_or_default();
591 let mut state = self.lock();
592 let now = Instant::now();
593 state.backoff.retain(|_, until| *until > now);
594 let refusal = if !nonce_proven
595 || binding.is_none_or(|b| matches!(b.principal, Principal::Unverified))
596 {
597 Some(Outcome::refusal(
598 error_codes::OPERATOR_REQUEST_NOT_PERMITTED,
599 ))
600 } else if !valid_summary(&request.summary) {
601 Some(Outcome::refusal(error_codes::OPERATOR_SUMMARY_INVALID))
602 } else if let Some(until) = state
603 .backoff
604 .get(&(module_id.clone(), principal.clone()))
605 .copied()
606 .filter(|_| shortcut.is_none())
607 {
608 Some(Outcome::declined("backoff").with_retry_after_ms(retry_after_ms(until - now)))
609 } else if shortcut.is_none() && state.stuck {
610 Some(Outcome::unavailable("provider_stuck"))
611 } else if state.modules.contains_key(&module_id) {
612 Some(Outcome::unavailable("module_limit"))
613 } else if shortcut.is_none() && state.queue.len() >= 4 {
614 Some(Outcome::unavailable("queue_full"))
615 } else {
616 None
617 };
618 if let Some(outcome) = refusal {
619 audit(
620 &module_id,
621 &request.summary,
622 &principal,
623 &outcome,
624 Duration::ZERO,
625 Duration::ZERO,
626 false,
627 );
628 return vec![outcome.frame(&frame)];
629 }
630 let binding = binding.expect("permission checked");
631 let opener = match &binding.principal {
632 Principal::Direct => "a local program",
633 Principal::Reserved { module_id } => module_id,
634 Principal::Unverified => unreachable!(),
635 };
636 state.next_id += 1;
637 let pending = Arc::new(Request {
638 id: state.next_id,
639 text: format!(
640 "approve a request from {module_id}: {} (requested by {opener})",
641 request.summary
642 ),
643 module_id,
644 summary: request.summary,
645 principal,
646 kind: RequestKind::Module(ModuleRouteKey {
647 endpoint: binding.module_endpoint,
648 channel: binding.module_channel,
649 }),
650 ctx: ctx.clone(),
651 frame,
652 arrived: now,
653 delivery_deadline: now + state.limits.delivery_budget(),
654 prompt_ready: true.into(),
655 shortcut_audit: shortcut.is_some(),
656 });
657 state.modules.insert(pending.module_id.clone(), pending.id);
658 if let Some(shortcut) = shortcut {
659 self.start_shortcut(&mut state, pending, shortcut);
660 } else {
661 self.enqueue(&mut state, pending);
662 }
663 Vec::new()
664 }
665
666 #[allow(clippy::too_many_arguments)]
671 pub(crate) async fn approve_privileged_with_shortcut(
672 self: &Arc<Self>,
673 ctx: &RouteCtx,
674 frame: &Frame,
675 operation: &str,
676 text: String,
677 caller: String,
678 key: DeclineKey,
679 shortcut: Option<ShortcutContext>,
680 ) -> Outcome {
681 let (answer, receive) = oneshot::channel();
682 let id = {
683 let mut state = self.lock();
684 let now = (state.cooldown_clock)();
685 state.cooldowns.retain(|_, until| *until > now);
686 state.declines.retain(|_, declines| {
687 declines.retain(|at| now.saturating_duration_since(*at) < DECLINE_WINDOW);
688 !declines.is_empty()
689 });
690 let refusal = if state.cooldowns.contains_key(&key) {
691 Some(Outcome::declined("backoff"))
692 } else if shortcut.is_none() && state.stuck {
693 Some(Outcome::unavailable("provider_stuck"))
694 } else if shortcut.is_none() && state.queue.len() >= 4 {
695 Some(Outcome::unavailable("queue_full"))
696 } else {
697 None
698 };
699 if let Some(outcome) = refusal {
700 audit(
701 operation,
702 &text,
703 &caller,
704 &outcome,
705 Duration::ZERO,
706 Duration::ZERO,
707 false,
708 );
709 return outcome;
710 }
711 state.next_id += 1;
712 let id = state.next_id;
713 let now = Instant::now();
714 let pending = Arc::new(Request {
715 id,
716 module_id: operation.into(),
717 summary: text.clone(),
718 principal: caller,
719 text,
720 kind: RequestKind::Privileged {
721 answer: Mutex::new(Some(answer)),
722 key,
723 },
724 ctx: ctx.clone(),
725 frame: frame.clone(),
726 arrived: now,
727 delivery_deadline: now + state.limits.delivery_budget(),
728 prompt_ready: true.into(),
729 shortcut_audit: shortcut.is_some(),
730 });
731 if let Some(shortcut) = shortcut {
732 self.start_shortcut(&mut state, pending, shortcut);
733 } else {
734 self.enqueue(&mut state, pending);
735 }
736 id
737 };
738 let _withdraw = WithdrawOnDrop {
739 confirms: Arc::clone(self),
740 id,
741 };
742 receive
743 .await
744 .unwrap_or_else(|_| Outcome::unavailable("provider_error"))
745 }
746
747 fn enqueue(self: &Arc<Self>, state: &mut State, pending: Arc<Request>) {
748 if state.prompt.is_none() && state.queue.is_empty() {
749 state.prompt = Some(Active {
750 request: pending,
751 started: None,
752 handle: None,
753 committed: false,
754 withdrawn: None,
755 withdraw_started: false,
756 });
757 } else {
758 state.queue.push_back(pending);
759 }
760 if !state.running {
761 state.running = true;
762 tokio::spawn(Arc::clone(self).run());
763 }
764 self.wake.notify_one();
765 }
766
767 fn commit(
768 self: &Arc<Self>,
769 state: &mut State,
770 request: Arc<Request>,
771 outcome: Outcome,
772 started: Option<Instant>,
773 shown: bool,
774 ) {
775 let now = Instant::now();
776 let wait = started.unwrap_or(now).duration_since(request.arrived);
777 let prompt = started.map(|at| now.duration_since(at)).unwrap_or_default();
778 if outcome.reason == "person" && matches!(request.kind, RequestKind::Module(_)) {
779 state.backoff.insert(
780 (request.module_id.clone(), request.principal.clone()),
781 now + Duration::from_secs(30),
782 );
783 }
784 audit(
785 &request.module_id,
786 if request.shortcut_audit {
787 "Operator shortcut request"
788 } else {
789 &request.summary
790 },
791 &request.principal,
792 &outcome,
793 wait,
794 prompt,
795 shown,
796 );
797 if let RequestKind::Privileged { answer, key } = &request.kind {
798 let person_declined = outcome.reason == "person";
799 if let Some(answer) = answer.lock().unwrap_or_else(|p| p.into_inner()).take() {
800 let _ = answer.send(outcome);
801 }
802 if person_declined {
803 self.count_person_decline(state, key);
804 }
805 return;
806 }
807 let this = Arc::clone(self);
808 #[cfg(test)]
809 {
810 state.deliveries += 1;
811 }
812 tokio::spawn(async move {
813 let answer = outcome.frame(&request.frame);
814 match timeout_at(request.delivery_deadline, request.ctx.egress.send(answer)).await {
815 Ok(Ok(())) => {}
816 result => {
817 tracing::warn!(target: "subc_daemon::operator_confirm", module_id = request.module_id, ?result, "operator confirm answer dropped")
818 }
819 }
820 let mut state = this.lock();
821 #[cfg(test)]
822 {
823 state.deliveries -= 1;
824 }
825 if state.modules.get(&request.module_id) == Some(&request.id) {
826 state.modules.remove(&request.module_id);
827 }
828 });
829 }
830
831 fn count_person_decline(self: &Arc<Self>, state: &mut State, key: &DeclineKey) {
836 let now = (state.cooldown_clock)();
837 let declines = state.declines.entry(key.clone()).or_default();
838 declines.retain(|at| now.saturating_duration_since(*at) < DECLINE_WINDOW);
839 declines.push_back(now);
840 if declines.len() < DECLINE_LIMIT {
841 return;
842 }
843 state.declines.remove(key);
845 state.cooldowns.insert(key.clone(), now + DECLINE_COOLDOWN);
846 tracing::warn!(target: "subc_daemon::operator_confirm", caller = %key,
847 cooldown_secs = DECLINE_COOLDOWN.as_secs(),
848 "privileged requests refused without a prompt after repeated operator declines");
849 let mut index = 0;
850 while index < state.queue.len() {
851 if matches!(&state.queue[index].kind,
852 RequestKind::Privileged { key: queued, .. } if queued == key)
853 {
854 let request = state.queue.remove(index).expect("queue index");
855 self.commit(state, request, Outcome::declined("backoff"), None, false);
856 } else {
857 index += 1;
858 }
859 }
860 }
861
862 fn withdraw_matching(
863 self: &Arc<Self>,
864 predicate: impl Fn(&Request) -> bool,
865 reason: &'static str,
866 ) -> bool {
867 let mut state = self.lock();
868 let mut matched = false;
869 let ids: Vec<_> = state
870 .shortcuts
871 .iter()
872 .filter(|(_, pending)| predicate(&pending.request))
873 .map(|(id, _)| *id)
874 .collect();
875 for id in ids {
876 self.end_shortcut(&mut state, id, Outcome::declined(reason), "withdrawn");
877 matched = true;
878 }
879 let mut index = 0;
880 while index < state.queue.len() {
881 if predicate(&state.queue[index]) {
882 let request = state.queue.remove(index).expect("queue index");
883 self.commit(&mut state, request, Outcome::declined(reason), None, false);
884 matched = true;
885 } else {
886 index += 1;
887 }
888 }
889 if let Some(active) = state
890 .prompt
891 .as_mut()
892 .filter(|a| !a.committed && predicate(&a.request))
893 {
894 active.committed = true;
895 active.withdrawn = Some((Instant::now(), reason));
896 let (request, started, shown) = (
897 Arc::clone(&active.request),
898 active.started,
899 active.handle.is_some(),
900 );
901 self.commit(
902 &mut state,
903 request,
904 Outcome::declined(reason),
905 started,
906 shown,
907 );
908 matched = true;
909 }
910 self.wake.notify_one();
911 matched
912 }
913 pub(crate) fn route_closed(self: &Arc<Self>, key: ModuleRouteKey) {
914 self.withdraw_matching(
915 |r| matches!(r.kind, RequestKind::Module(k) if k == key),
916 "route_closed",
917 );
918 }
919 pub(crate) fn module_closed(self: &Arc<Self>, endpoint: ModuleEndpointId) {
920 self.withdraw_matching(
921 |r| matches!(r.kind, RequestKind::Module(k) if k.endpoint == endpoint),
922 "module_closed",
923 );
924 }
925 pub(crate) fn connection_closed(self: &Arc<Self>, connection: crate::ConnectionId) {
926 self.withdraw_matching(
927 |r| {
928 matches!(r.kind, RequestKind::Privileged { .. })
929 && r.ctx.connection_id == connection
930 },
931 "caller_cancelled",
932 );
933 }
934 pub(crate) fn cancel(self: &Arc<Self>, connection: crate::ConnectionId, corr: u64) -> bool {
935 self.withdraw_matching(
936 |r| r.ctx.connection_id == connection && r.frame.header.corr == corr,
937 "caller_cancelled",
938 )
939 }
940
941 fn publish(self: &Arc<Self>, id: u64, handle: Arc<dyn OperatorWithdraw>) {
942 let mut state = self.lock();
943 if let Some(active) = state.prompt.as_mut().filter(|a| a.request.id == id) {
944 active.handle = Some(Arc::clone(&handle));
945 if let Some((_, reason)) = active.withdrawn {
946 active.withdraw_started = true;
947 tokio::task::spawn_blocking(move || handle.withdraw(reason));
948 }
949 }
950 self.wake.notify_one();
951 }
952 fn returned(self: &Arc<Self>, id: u64, result: ProviderResult) {
953 let mut state = self.lock();
954 let Some(active) = state.prompt.take() else {
955 return;
956 };
957 assert_eq!(active.request.id, id);
958 if !active.withdraw_started {
961 if let (Some(handle), Some((_, reason))) = (active.handle.as_ref(), active.withdrawn) {
962 let handle = Arc::clone(handle);
963 tokio::task::spawn_blocking(move || handle.withdraw(reason));
964 }
965 }
966 if !active.committed {
967 let outcome = if Instant::now() >= active.request.delivery_deadline {
968 Outcome::unavailable("timeout")
969 } else {
970 Outcome::from_provider(result)
971 };
972 self.commit(
973 &mut state,
974 active.request,
975 outcome,
976 active.started,
977 active.handle.is_some(),
978 );
979 } else {
980 tracing::debug!(target: "subc_daemon::operator_confirm", ?result, "late operator provider result discarded");
981 }
982 state.stuck = false;
983 self.wake.notify_one();
984 }
985
986 async fn run(self: Arc<Self>) {
987 loop {
988 let deadline = {
989 let mut state = self.lock();
990 let now = Instant::now();
991 let mut index = 0;
992 while index < state.queue.len() {
993 if state.queue[index].arrived + state.limits.queue_wait <= now {
994 let request = state.queue.remove(index).expect("queue index");
995 self.commit(
996 &mut state,
997 request,
998 Outcome::unavailable("queue_wait"),
999 None,
1000 false,
1001 );
1002 } else {
1003 index += 1;
1004 }
1005 }
1006 let timeout = state.limits.prompt_timeout;
1007 if let Some(active) = state.prompt.as_mut().filter(|a| {
1008 !a.committed
1009 && (a.request.delivery_deadline <= now
1010 || a.started.is_some_and(|at| at + timeout <= now))
1011 }) {
1012 active.committed = true;
1013 active.withdrawn = Some((now, "timeout"));
1014 let (request, started, shown) = (
1015 Arc::clone(&active.request),
1016 active.started,
1017 active.handle.is_some(),
1018 );
1019 self.commit(
1020 &mut state,
1021 request,
1022 Outcome::unavailable("timeout"),
1023 started,
1024 shown,
1025 );
1026 }
1027 let grace = state.limits.stuck_grace;
1028 if !state.stuck
1029 && state.prompt.as_ref().is_some_and(|a| {
1030 a.started.is_some() && a.withdrawn.is_some_and(|(at, _)| at + grace <= now)
1031 })
1032 {
1033 state.stuck = true;
1034 tracing::error!(target: "subc_daemon::operator_confirm", "operator provider stuck after withdraw");
1035 while let Some(request) = state.queue.pop_front() {
1036 self.commit(
1037 &mut state,
1038 request,
1039 Outcome::unavailable("provider_stuck"),
1040 None,
1041 false,
1042 );
1043 }
1044 }
1045 if let Some(active) = state
1046 .prompt
1047 .as_mut()
1048 .filter(|a| !a.withdraw_started && a.withdrawn.is_some() && a.handle.is_some())
1049 {
1050 active.withdraw_started = true;
1051 let handle = Arc::clone(active.handle.as_ref().expect("handle checked"));
1052 let reason = active.withdrawn.expect("withdraw checked").1;
1053 tokio::task::spawn_blocking(move || handle.withdraw(reason));
1054 }
1055 if state
1057 .prompt
1058 .as_ref()
1059 .is_some_and(|a| a.started.is_none() && a.committed)
1060 {
1061 state.prompt = None;
1062 }
1063 if state.prompt.is_none() {
1064 if let Some(request) = state.queue.pop_front() {
1065 state.prompt = Some(Active {
1066 request,
1067 started: None,
1068 handle: None,
1069 committed: false,
1070 withdrawn: None,
1071 withdraw_started: false,
1072 });
1073 }
1074 }
1075 if let Some(active) = state.prompt.as_mut().filter(|a| {
1076 a.started.is_none()
1077 && !a.committed
1078 && a.request
1079 .prompt_ready
1080 .load(std::sync::atomic::Ordering::Acquire)
1081 }) {
1082 active.started = Some(now);
1083 let request = Arc::clone(&active.request);
1084 let provider = Arc::clone(&state.provider);
1085 let this = Arc::clone(&self);
1086 tokio::spawn(async move {
1087 let publish_to = Arc::clone(&this);
1088 let id = request.id;
1089 let result = tokio::task::spawn_blocking(move || {
1090 provider.prompt(
1091 &request.text,
1092 Box::new(move |handle| publish_to.publish(id, handle)),
1093 )
1094 })
1095 .await
1096 .unwrap_or(ProviderResult::Unavailable);
1097 this.returned(id, result);
1098 });
1099 }
1100 if state.prompt.is_none() && state.queue.is_empty() {
1101 state.running = false;
1102 return;
1103 }
1104 let mut deadline = state
1105 .queue
1106 .iter()
1107 .map(|r| r.arrived + state.limits.queue_wait)
1108 .min();
1109 if let Some(active) = &state.prompt {
1110 let next = if !active.committed {
1111 Some(
1112 active
1113 .started
1114 .map_or(active.request.delivery_deadline, |at| {
1115 (at + timeout).min(active.request.delivery_deadline)
1116 }),
1117 )
1118 } else if !state.stuck {
1119 active.withdrawn.map(|(at, _)| at + grace)
1120 } else {
1121 None
1122 };
1123 if let Some(next) = next {
1124 deadline = Some(deadline.map_or(next, |d| d.min(next)));
1125 }
1126 }
1127 deadline
1128 };
1129 if let Some(at) = deadline {
1130 tokio::select! { _ = self.wake.notified() => {}, _ = tokio::time::sleep_until(at) => {} }
1131 } else {
1132 self.wake.notified().await;
1133 }
1134 }
1135 }
1136}
1137
1138#[cfg(feature = "test-support")]
1139mod test_provider;
1140#[cfg(test)]
1141mod tests;