1use std::{
6 collections::{HashMap, VecDeque},
7 path::PathBuf,
8 sync::{Arc, Mutex},
9 time::Duration,
10};
11
12use subc_protocol::{
13 error_codes,
14 session::{OperatorConfirmReply, OperatorConfirmRequest},
15 ErrorBody, Flags, FrameType, Principal, Priority,
16};
17use tokio::{
18 sync::{oneshot, Notify},
19 time::{timeout_at, Instant},
20};
21
22use crate::{
23 caller_identity::CallerIdentity,
24 forwarding::{ModuleEndpointId, ModuleRouteKey, RouteBinding},
25 router::RouteCtx,
26 ConnectionId, Frame,
27};
28
29const DECLINE_LIMIT: usize = 3;
32const DECLINE_WINDOW: Duration = Duration::from_secs(10 * 60);
33const DECLINE_COOLDOWN: Duration = Duration::from_secs(10 * 60);
36
37pub(crate) type CooldownClock = Arc<dyn Fn() -> Instant + Send + Sync>;
40
41#[derive(Clone, Debug, PartialEq, Eq, Hash)]
46pub(crate) enum DeclineKey {
47 Executable(PathBuf),
48 Connection(ConnectionId),
49}
50
51impl DeclineKey {
52 pub(crate) fn for_caller(caller: &CallerIdentity, connection: ConnectionId) -> Self {
53 match caller.executable() {
54 Some(executable) => Self::Executable(executable.to_path_buf()),
55 None => Self::Connection(connection),
56 }
57 }
58}
59
60impl std::fmt::Display for DeclineKey {
61 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
62 match self {
63 Self::Executable(path) => write!(f, "{}", path.display()),
64 Self::Connection(connection) => write!(
65 f,
66 "an unidentified local process on connection {}",
67 connection.get()
68 ),
69 }
70 }
71}
72
73pub trait OperatorProvider: Send + Sync + 'static {
77 fn prompt(
78 &self,
79 text: &str,
80 publish: Box<dyn FnOnce(Arc<dyn OperatorWithdraw>) + Send>,
81 ) -> ProviderResult;
82}
83
84pub trait OperatorWithdraw: Send + Sync + 'static {
86 fn withdraw(&self, reason: &str);
87}
88
89#[derive(Debug, Clone, PartialEq, Eq)]
90pub enum ProviderResult {
91 Approved,
92 Declined,
93 Unavailable,
94 UnsupportedPlatform,
95 NoPresence,
96 LocalAuthenticationError {
97 failure: OperatorOsErrorKind,
98 domain: String,
99 code: isize,
100 },
101}
102
103#[derive(Debug, Clone, Copy, PartialEq, Eq)]
104pub enum OperatorOsErrorKind {
105 Declined,
106 NoPresence,
107 ProviderError,
108}
109
110#[derive(Debug, Clone, Copy)]
111pub struct OperatorLimits {
112 pub queue_wait: Duration,
113 pub prompt_timeout: Duration,
114 pub stuck_grace: Duration,
115}
116
117impl Default for OperatorLimits {
118 fn default() -> Self {
119 Self {
120 queue_wait: Duration::from_secs(150),
121 prompt_timeout: Duration::from_secs(120),
122 stuck_grace: Duration::from_secs(10),
123 }
124 }
125}
126
127impl OperatorLimits {
128 fn delivery_budget(self) -> Duration {
129 self.queue_wait + self.prompt_timeout + self.stuck_grace + Duration::from_secs(10)
130 }
131}
132
133struct OsProvider;
134
135#[cfg(any(target_os = "macos", windows))]
136struct OsWithdraw(Arc<dyn subc_presence::Withdraw>);
137#[cfg(any(target_os = "macos", windows))]
138impl OperatorWithdraw for OsWithdraw {
139 fn withdraw(&self, _: &str) {
140 self.0.withdraw();
141 }
142}
143
144impl OperatorProvider for OsProvider {
145 fn prompt(
146 &self,
147 text: &str,
148 publish: Box<dyn FnOnce(Arc<dyn OperatorWithdraw>) + Send>,
149 ) -> ProviderResult {
150 #[cfg(target_os = "linux")]
154 {
155 let _ = (text, publish);
156 ProviderResult::UnsupportedPlatform
157 }
158 #[cfg(any(target_os = "macos", windows))]
159 {
160 os_outcome(subc_presence::prompt(
161 text,
162 Box::new(move |handle| publish(Arc::new(OsWithdraw(handle)))),
163 ))
164 }
165 #[cfg(not(any(target_os = "macos", windows, target_os = "linux")))]
166 {
167 let _ = (text, publish);
168 ProviderResult::Unavailable
169 }
170 }
171}
172
173#[cfg(any(target_os = "macos", windows))]
178fn os_outcome(outcome: subc_presence::Outcome) -> ProviderResult {
179 match outcome {
180 subc_presence::Outcome::Approved => ProviderResult::Approved,
181 subc_presence::Outcome::Declined => ProviderResult::Declined,
182 subc_presence::Outcome::ProviderError => ProviderResult::Unavailable,
183 subc_presence::Outcome::NoPresence => ProviderResult::NoPresence,
184 subc_presence::Outcome::UnsupportedPlatform => ProviderResult::UnsupportedPlatform,
185 subc_presence::Outcome::LocalAuthenticationError {
186 failure,
187 domain,
188 code,
189 } => ProviderResult::LocalAuthenticationError {
190 failure: match failure {
191 subc_presence::LocalAuthenticationFailure::Declined => {
192 OperatorOsErrorKind::Declined
193 }
194 subc_presence::LocalAuthenticationFailure::NoPresence => {
195 OperatorOsErrorKind::NoPresence
196 }
197 subc_presence::LocalAuthenticationFailure::ProviderError => {
198 OperatorOsErrorKind::ProviderError
199 }
200 },
201 domain,
202 code,
203 },
204 }
205}
206
207#[cfg(all(test, any(target_os = "macos", windows)))]
208mod os_outcome_tests {
209 use super::*;
210
211 #[test]
212 fn only_an_os_approval_becomes_approved() {
213 use subc_presence::Outcome as Os;
214 assert_eq!(os_outcome(Os::Approved), ProviderResult::Approved);
215 assert_eq!(os_outcome(Os::Declined), ProviderResult::Declined);
216 assert_eq!(os_outcome(Os::ProviderError), ProviderResult::Unavailable);
217 assert_eq!(os_outcome(Os::NoPresence), ProviderResult::NoPresence);
218 assert_eq!(
219 os_outcome(Os::UnsupportedPlatform),
220 ProviderResult::UnsupportedPlatform
221 );
222 assert_eq!(
223 os_outcome(Os::LocalAuthenticationError {
224 failure: subc_presence::LocalAuthenticationFailure::NoPresence,
225 domain: "com.apple.LocalAuthentication".into(),
226 code: -4,
227 }),
228 ProviderResult::LocalAuthenticationError {
229 failure: OperatorOsErrorKind::NoPresence,
230 domain: "com.apple.LocalAuthentication".into(),
231 code: -4,
232 }
233 );
234 }
235}
236
237#[derive(Clone, Debug)]
238pub(crate) struct Outcome {
239 pub code: &'static str,
240 pub reason: &'static str,
241 retry_after_ms: Option<u64>,
242 os_error_domain: Option<String>,
243 os_error_code: Option<isize>,
244}
245impl Outcome {
246 pub(crate) fn unavailable(reason: &'static str) -> Self {
247 Self {
248 code: error_codes::OPERATOR_PRESENCE_UNAVAILABLE,
249 reason,
250 retry_after_ms: None,
251 os_error_domain: None,
252 os_error_code: None,
253 }
254 }
255 fn declined(reason: &'static str) -> Self {
256 Self {
257 code: error_codes::OPERATOR_DECLINED,
258 reason,
259 retry_after_ms: None,
260 os_error_domain: None,
261 os_error_code: None,
262 }
263 }
264 fn with_retry_after_ms(mut self, retry_after_ms: u64) -> Self {
265 self.retry_after_ms = Some(retry_after_ms);
266 self
267 }
268 pub(crate) fn refusal(code: &'static str) -> Self {
269 Self {
270 code,
271 reason: "",
272 retry_after_ms: None,
273 os_error_domain: None,
274 os_error_code: None,
275 }
276 }
277 fn from_provider(result: ProviderResult) -> Self {
278 match result {
279 ProviderResult::Approved => Self {
280 code: "confirmed",
281 reason: "",
282 retry_after_ms: None,
283 os_error_domain: None,
284 os_error_code: None,
285 },
286 ProviderResult::Declined => Self::declined("person"),
287 ProviderResult::Unavailable => Self::unavailable("provider_error"),
288 ProviderResult::UnsupportedPlatform => Self::unavailable("unsupported_platform"),
289 ProviderResult::NoPresence => Self::unavailable("no_presence"),
290 ProviderResult::LocalAuthenticationError {
291 failure,
292 domain,
293 code,
294 } => {
295 let mut outcome = match failure {
296 OperatorOsErrorKind::Declined => Self::declined("person"),
297 OperatorOsErrorKind::NoPresence => Self::unavailable("no_presence"),
298 OperatorOsErrorKind::ProviderError => Self::unavailable("provider_error"),
299 };
300 outcome.os_error_domain = Some(domain);
301 outcome.os_error_code = Some(code);
302 outcome
303 }
304 }
305 }
306 pub(crate) fn frame(self, request: &Frame) -> Frame {
307 let (ty, body) = if self.code == "confirmed" {
308 (
309 FrameType::Response,
310 serde_json::to_vec(&OperatorConfirmReply::confirmed()).expect("serializable reply"),
311 )
312 } else {
313 let mut error = ErrorBody::new(self.code, "operator confirmation refused");
314 if !self.reason.is_empty() {
315 let detail = match self.retry_after_ms {
316 Some(retry_after_ms) => {
317 serde_json::json!({"reason": self.reason, "retry_after_ms": retry_after_ms})
318 }
319 None => serde_json::json!({"reason": self.reason}),
320 };
321 error = error.with_detail(detail);
322 }
323 (
324 FrameType::Error,
325 serde_json::to_vec(&error).expect("serializable error"),
326 )
327 };
328 Frame::build_with_version(
329 request.header.ver,
330 ty,
331 Flags::new(false, Priority::Passive, false),
332 0,
333 0,
334 request.header.corr,
335 body,
336 )
337 .expect("bounded control reply")
338 }
339}
340
341fn retry_after_ms(remaining: Duration) -> u64 {
342 let whole_milliseconds = remaining.as_millis();
343 let rounded_milliseconds =
344 whole_milliseconds + u128::from(!remaining.subsec_nanos().is_multiple_of(1_000_000));
345 u64::try_from(rounded_milliseconds).expect("operator confirmation backoff fits in u64")
346}
347
348pub(crate) fn audit(
349 module_id: &str,
350 summary: &str,
351 principal: &str,
352 outcome: &Outcome,
353 wait: Duration,
354 prompt: Duration,
355 prompt_shown: bool,
356) {
357 tracing::info!(target: "subc_daemon::operator_confirm", module_id, summary = ?summary,
358 principal, outcome = outcome.code, reason = outcome.reason,
359 os_error_domain = ?outcome.os_error_domain, os_error_code = ?outcome.os_error_code,
360 wait_ms = wait.as_millis() as u64, prompt_ms = prompt.as_millis() as u64,
361 prompt_shown, "operator_confirm_audit");
362}
363
364fn principal_label(principal: &Principal) -> String {
365 match principal {
366 Principal::Direct => "direct".into(),
367 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
368 Principal::Unverified => String::new(),
369 }
370}
371
372fn valid_summary(summary: &str) -> bool {
373 let count = summary.chars().count();
374 (1..=200).contains(&count)
375 && !summary.starts_with(char::is_whitespace)
376 && !summary.ends_with(char::is_whitespace)
377 && !summary.chars().any(|c| {
378 c.is_control()
379 || matches!(c,
380 '\u{200b}'..='\u{200f}' | '\u{202a}'..='\u{202e}' | '\u{2028}' | '\u{2029}' |
381 '\u{2060}'..='\u{2069}' | '\u{feff}')
382 })
383}
384
385enum RequestKind {
386 Module(ModuleRouteKey),
387 Privileged {
388 answer: Mutex<Option<oneshot::Sender<Outcome>>>,
389 key: DeclineKey,
390 },
391}
392
393struct Request {
394 id: u64,
395 module_id: String,
396 summary: String,
397 principal: String,
398 text: String,
399 kind: RequestKind,
400 ctx: RouteCtx,
401 frame: Frame,
402 arrived: Instant,
403 delivery_deadline: Instant,
404}
405struct Active {
406 request: Arc<Request>,
407 started: Option<Instant>,
408 handle: Option<Arc<dyn OperatorWithdraw>>,
409 committed: bool,
410 withdrawn: Option<(Instant, &'static str)>,
411 withdraw_started: bool,
412}
413struct State {
414 provider: Arc<dyn OperatorProvider>,
415 limits: OperatorLimits,
416 queue: VecDeque<Arc<Request>>,
417 modules: HashMap<String, u64>,
418 backoff: HashMap<(String, String), Instant>,
419 cooldown_clock: CooldownClock,
420 declines: HashMap<DeclineKey, VecDeque<Instant>>,
422 cooldowns: HashMap<DeclineKey, Instant>,
424 prompt: Option<Active>,
425 stuck: bool,
426 running: bool,
427 next_id: u64,
428 #[cfg(test)]
429 deliveries: usize,
430}
431
432pub(crate) struct OperatorConfirms {
433 state: Mutex<State>,
434 wake: Notify,
435}
436
437struct WithdrawOnDrop {
438 confirms: Arc<OperatorConfirms>,
439 id: u64,
440}
441
442impl Drop for WithdrawOnDrop {
443 fn drop(&mut self) {
444 self.confirms
445 .withdraw_matching(|request| request.id == self.id, "caller_cancelled");
446 }
447}
448impl std::fmt::Debug for OperatorConfirms {
449 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
450 f.debug_struct("OperatorConfirms").finish_non_exhaustive()
451 }
452}
453impl Default for OperatorConfirms {
454 fn default() -> Self {
455 Self {
456 state: Mutex::new(State {
457 provider: Arc::new(OsProvider),
458 limits: OperatorLimits::default(),
459 queue: VecDeque::new(),
460 modules: HashMap::new(),
461 backoff: HashMap::new(),
462 cooldown_clock: Arc::new(Instant::now),
463 declines: HashMap::new(),
464 cooldowns: HashMap::new(),
465 prompt: None,
466 stuck: false,
467 running: false,
468 next_id: 0,
469 #[cfg(test)]
470 deliveries: 0,
471 }),
472 wake: Notify::new(),
473 }
474 }
475}
476impl OperatorConfirms {
477 fn lock(&self) -> std::sync::MutexGuard<'_, State> {
478 self.state.lock().unwrap_or_else(|p| p.into_inner())
479 }
480 pub(crate) fn configure(&self, provider: Arc<dyn OperatorProvider>, limits: OperatorLimits) {
481 let mut state = self.lock();
482 assert!(
483 !state.running && state.modules.is_empty(),
484 "configure before serving"
485 );
486 state.provider = provider;
487 state.limits = limits;
488 }
489 pub(crate) fn configure_bootstrap(&self, config: &crate::bootstrap::BootstrapConfig) {
490 let provider: Arc<dyn OperatorProvider> = Arc::new(OsProvider);
491 #[cfg(feature = "test-support")]
492 let provider = test_provider::select(
493 provider,
494 config.operator_script.as_deref(),
495 config.operator_events.as_deref(),
496 std::env::current_exe().ok().as_deref(),
497 );
498 self.configure(
499 provider,
500 OperatorLimits {
501 queue_wait: config.operator_queue_wait,
502 prompt_timeout: config.operator_timeout,
503 stuck_grace: config.operator_stuck_grace,
504 },
505 );
506 }
507
508 #[cfg(feature = "test-support")]
509 pub(crate) fn configure_script(
510 &self,
511 script: &std::path::Path,
512 events: Option<&std::path::Path>,
513 limits: Option<OperatorLimits>,
514 ) {
515 let provider = test_provider::select(
516 Arc::new(OsProvider),
517 Some(script),
518 events,
519 test_provider::current_executable().as_deref(),
520 );
521 self.configure(provider, limits.unwrap_or_default());
522 }
523
524 pub(crate) fn admit(
527 self: &Arc<Self>,
528 ctx: &RouteCtx,
529 frame: Frame,
530 module_id: String,
531 nonce_proven: bool,
532 request: OperatorConfirmRequest,
533 binding: Option<&RouteBinding>,
534 ) -> Vec<Frame> {
535 let principal = binding
536 .map(|b| principal_label(&b.principal))
537 .unwrap_or_default();
538 let mut state = self.lock();
539 let now = Instant::now();
540 state.backoff.retain(|_, until| *until > now);
541 let refusal = if !nonce_proven
542 || binding.is_none_or(|b| matches!(b.principal, Principal::Unverified))
543 {
544 Some(Outcome::refusal(
545 error_codes::OPERATOR_REQUEST_NOT_PERMITTED,
546 ))
547 } else if !valid_summary(&request.summary) {
548 Some(Outcome::refusal(error_codes::OPERATOR_SUMMARY_INVALID))
549 } else if let Some(until) = state
550 .backoff
551 .get(&(module_id.clone(), principal.clone()))
552 .copied()
553 {
554 Some(Outcome::declined("backoff").with_retry_after_ms(retry_after_ms(until - now)))
555 } else if state.stuck {
556 Some(Outcome::unavailable("provider_stuck"))
557 } else if state.modules.contains_key(&module_id) {
558 Some(Outcome::unavailable("module_limit"))
559 } else if state.queue.len() >= 4 {
560 Some(Outcome::unavailable("queue_full"))
561 } else {
562 None
563 };
564 if let Some(outcome) = refusal {
565 audit(
566 &module_id,
567 &request.summary,
568 &principal,
569 &outcome,
570 Duration::ZERO,
571 Duration::ZERO,
572 false,
573 );
574 return vec![outcome.frame(&frame)];
575 }
576 let binding = binding.expect("permission checked");
577 let opener = match &binding.principal {
578 Principal::Direct => "a local program",
579 Principal::Reserved { module_id } => module_id,
580 Principal::Unverified => unreachable!(),
581 };
582 state.next_id += 1;
583 let pending = Arc::new(Request {
584 id: state.next_id,
585 text: format!(
586 "approve a request from {module_id}: {} (requested by {opener})",
587 request.summary
588 ),
589 module_id,
590 summary: request.summary,
591 principal,
592 kind: RequestKind::Module(ModuleRouteKey {
593 endpoint: binding.module_endpoint,
594 channel: binding.module_channel,
595 }),
596 ctx: ctx.clone(),
597 frame,
598 arrived: now,
599 delivery_deadline: now + state.limits.delivery_budget(),
600 });
601 state.modules.insert(pending.module_id.clone(), pending.id);
602 self.enqueue(&mut state, pending);
603 Vec::new()
604 }
605
606 pub(crate) async fn approve_privileged(
611 self: &Arc<Self>,
612 ctx: &RouteCtx,
613 frame: &Frame,
614 operation: &str,
615 text: String,
616 caller: String,
617 key: DeclineKey,
618 ) -> Outcome {
619 let (answer, receive) = oneshot::channel();
620 let id = {
621 let mut state = self.lock();
622 let now = (state.cooldown_clock)();
623 state.cooldowns.retain(|_, until| *until > now);
624 state.declines.retain(|_, declines| {
625 declines.retain(|at| now.saturating_duration_since(*at) < DECLINE_WINDOW);
626 !declines.is_empty()
627 });
628 let refusal = if state.cooldowns.contains_key(&key) {
629 Some(Outcome::declined("backoff"))
630 } else if state.stuck {
631 Some(Outcome::unavailable("provider_stuck"))
632 } else if state.queue.len() >= 4 {
633 Some(Outcome::unavailable("queue_full"))
634 } else {
635 None
636 };
637 if let Some(outcome) = refusal {
638 audit(
639 operation,
640 &text,
641 &caller,
642 &outcome,
643 Duration::ZERO,
644 Duration::ZERO,
645 false,
646 );
647 return outcome;
648 }
649 state.next_id += 1;
650 let id = state.next_id;
651 let now = Instant::now();
652 let pending = Arc::new(Request {
653 id,
654 module_id: operation.into(),
655 summary: text.clone(),
656 principal: caller,
657 text,
658 kind: RequestKind::Privileged {
659 answer: Mutex::new(Some(answer)),
660 key,
661 },
662 ctx: ctx.clone(),
663 frame: frame.clone(),
664 arrived: now,
665 delivery_deadline: now + state.limits.delivery_budget(),
666 });
667 self.enqueue(&mut state, pending);
668 id
669 };
670 let _withdraw = WithdrawOnDrop {
671 confirms: Arc::clone(self),
672 id,
673 };
674 receive
675 .await
676 .unwrap_or_else(|_| Outcome::unavailable("provider_error"))
677 }
678
679 fn enqueue(self: &Arc<Self>, state: &mut State, pending: Arc<Request>) {
680 if state.prompt.is_none() && state.queue.is_empty() {
681 state.prompt = Some(Active {
682 request: pending,
683 started: None,
684 handle: None,
685 committed: false,
686 withdrawn: None,
687 withdraw_started: false,
688 });
689 } else {
690 state.queue.push_back(pending);
691 }
692 if !state.running {
693 state.running = true;
694 tokio::spawn(Arc::clone(self).run());
695 }
696 self.wake.notify_one();
697 }
698
699 fn commit(
700 self: &Arc<Self>,
701 state: &mut State,
702 request: Arc<Request>,
703 outcome: Outcome,
704 started: Option<Instant>,
705 shown: bool,
706 ) {
707 let now = Instant::now();
708 let wait = started.unwrap_or(now).duration_since(request.arrived);
709 let prompt = started.map(|at| now.duration_since(at)).unwrap_or_default();
710 if outcome.reason == "person" && matches!(request.kind, RequestKind::Module(_)) {
711 state.backoff.insert(
712 (request.module_id.clone(), request.principal.clone()),
713 now + Duration::from_secs(30),
714 );
715 }
716 audit(
717 &request.module_id,
718 &request.summary,
719 &request.principal,
720 &outcome,
721 wait,
722 prompt,
723 shown,
724 );
725 if let RequestKind::Privileged { answer, key } = &request.kind {
726 let person_declined = outcome.reason == "person";
727 if let Some(answer) = answer.lock().unwrap_or_else(|p| p.into_inner()).take() {
728 let _ = answer.send(outcome);
729 }
730 if person_declined {
731 self.count_person_decline(state, key);
732 }
733 return;
734 }
735 let this = Arc::clone(self);
736 #[cfg(test)]
737 {
738 state.deliveries += 1;
739 }
740 tokio::spawn(async move {
741 let answer = outcome.frame(&request.frame);
742 match timeout_at(request.delivery_deadline, request.ctx.egress.send(answer)).await {
743 Ok(Ok(())) => {}
744 result => {
745 tracing::warn!(target: "subc_daemon::operator_confirm", module_id = request.module_id, ?result, "operator confirm answer dropped")
746 }
747 }
748 let mut state = this.lock();
749 #[cfg(test)]
750 {
751 state.deliveries -= 1;
752 }
753 if state.modules.get(&request.module_id) == Some(&request.id) {
754 state.modules.remove(&request.module_id);
755 }
756 });
757 }
758
759 fn count_person_decline(self: &Arc<Self>, state: &mut State, key: &DeclineKey) {
764 let now = (state.cooldown_clock)();
765 let declines = state.declines.entry(key.clone()).or_default();
766 declines.retain(|at| now.saturating_duration_since(*at) < DECLINE_WINDOW);
767 declines.push_back(now);
768 if declines.len() < DECLINE_LIMIT {
769 return;
770 }
771 state.declines.remove(key);
773 state.cooldowns.insert(key.clone(), now + DECLINE_COOLDOWN);
774 tracing::warn!(target: "subc_daemon::operator_confirm", caller = %key,
775 cooldown_secs = DECLINE_COOLDOWN.as_secs(),
776 "privileged requests refused without a prompt after repeated operator declines");
777 let mut index = 0;
778 while index < state.queue.len() {
779 if matches!(&state.queue[index].kind,
780 RequestKind::Privileged { key: queued, .. } if queued == key)
781 {
782 let request = state.queue.remove(index).expect("queue index");
783 self.commit(state, request, Outcome::declined("backoff"), None, false);
784 } else {
785 index += 1;
786 }
787 }
788 }
789
790 fn withdraw_matching(
791 self: &Arc<Self>,
792 predicate: impl Fn(&Request) -> bool,
793 reason: &'static str,
794 ) -> bool {
795 let mut state = self.lock();
796 let mut matched = false;
797 let mut index = 0;
798 while index < state.queue.len() {
799 if predicate(&state.queue[index]) {
800 let request = state.queue.remove(index).expect("queue index");
801 self.commit(&mut state, request, Outcome::declined(reason), None, false);
802 matched = true;
803 } else {
804 index += 1;
805 }
806 }
807 if let Some(active) = state
808 .prompt
809 .as_mut()
810 .filter(|a| !a.committed && predicate(&a.request))
811 {
812 active.committed = true;
813 active.withdrawn = Some((Instant::now(), reason));
814 let (request, started, shown) = (
815 Arc::clone(&active.request),
816 active.started,
817 active.handle.is_some(),
818 );
819 self.commit(
820 &mut state,
821 request,
822 Outcome::declined(reason),
823 started,
824 shown,
825 );
826 matched = true;
827 }
828 self.wake.notify_one();
829 matched
830 }
831 pub(crate) fn route_closed(self: &Arc<Self>, key: ModuleRouteKey) {
832 self.withdraw_matching(
833 |r| matches!(r.kind, RequestKind::Module(k) if k == key),
834 "route_closed",
835 );
836 }
837 pub(crate) fn module_closed(self: &Arc<Self>, endpoint: ModuleEndpointId) {
838 self.withdraw_matching(
839 |r| matches!(r.kind, RequestKind::Module(k) if k.endpoint == endpoint),
840 "module_closed",
841 );
842 }
843 pub(crate) fn connection_closed(self: &Arc<Self>, connection: crate::ConnectionId) {
844 self.withdraw_matching(
845 |r| {
846 matches!(r.kind, RequestKind::Privileged { .. })
847 && r.ctx.connection_id == connection
848 },
849 "caller_cancelled",
850 );
851 }
852 pub(crate) fn cancel(self: &Arc<Self>, connection: crate::ConnectionId, corr: u64) -> bool {
853 self.withdraw_matching(
854 |r| r.ctx.connection_id == connection && r.frame.header.corr == corr,
855 "caller_cancelled",
856 )
857 }
858
859 fn publish(self: &Arc<Self>, id: u64, handle: Arc<dyn OperatorWithdraw>) {
860 let mut state = self.lock();
861 if let Some(active) = state.prompt.as_mut().filter(|a| a.request.id == id) {
862 active.handle = Some(Arc::clone(&handle));
863 if let Some((_, reason)) = active.withdrawn {
864 active.withdraw_started = true;
865 tokio::task::spawn_blocking(move || handle.withdraw(reason));
866 }
867 }
868 self.wake.notify_one();
869 }
870 fn returned(self: &Arc<Self>, id: u64, result: ProviderResult) {
871 let mut state = self.lock();
872 let Some(active) = state.prompt.take() else {
873 return;
874 };
875 assert_eq!(active.request.id, id);
876 if !active.withdraw_started {
879 if let (Some(handle), Some((_, reason))) = (active.handle.as_ref(), active.withdrawn) {
880 let handle = Arc::clone(handle);
881 tokio::task::spawn_blocking(move || handle.withdraw(reason));
882 }
883 }
884 if !active.committed {
885 self.commit(
886 &mut state,
887 active.request,
888 Outcome::from_provider(result),
889 active.started,
890 active.handle.is_some(),
891 );
892 } else {
893 tracing::debug!(target: "subc_daemon::operator_confirm", ?result, "late operator provider result discarded");
894 }
895 state.stuck = false;
896 self.wake.notify_one();
897 }
898
899 async fn run(self: Arc<Self>) {
900 loop {
901 let deadline = {
902 let mut state = self.lock();
903 let now = Instant::now();
904 let mut index = 0;
905 while index < state.queue.len() {
906 if state.queue[index].arrived + state.limits.queue_wait <= now {
907 let request = state.queue.remove(index).expect("queue index");
908 self.commit(
909 &mut state,
910 request,
911 Outcome::unavailable("queue_wait"),
912 None,
913 false,
914 );
915 } else {
916 index += 1;
917 }
918 }
919 let timeout = state.limits.prompt_timeout;
920 if let Some(active) = state
921 .prompt
922 .as_mut()
923 .filter(|a| !a.committed && a.started.is_some_and(|at| at + timeout <= now))
924 {
925 active.committed = true;
926 active.withdrawn = Some((now, "timeout"));
927 let (request, started, shown) = (
928 Arc::clone(&active.request),
929 active.started,
930 active.handle.is_some(),
931 );
932 self.commit(
933 &mut state,
934 request,
935 Outcome::unavailable("timeout"),
936 started,
937 shown,
938 );
939 }
940 let grace = state.limits.stuck_grace;
941 if !state.stuck
942 && state.prompt.as_ref().is_some_and(|a| {
943 a.started.is_some() && a.withdrawn.is_some_and(|(at, _)| at + grace <= now)
944 })
945 {
946 state.stuck = true;
947 tracing::error!(target: "subc_daemon::operator_confirm", "operator provider stuck after withdraw");
948 while let Some(request) = state.queue.pop_front() {
949 self.commit(
950 &mut state,
951 request,
952 Outcome::unavailable("provider_stuck"),
953 None,
954 false,
955 );
956 }
957 }
958 if let Some(active) = state
959 .prompt
960 .as_mut()
961 .filter(|a| !a.withdraw_started && a.withdrawn.is_some() && a.handle.is_some())
962 {
963 active.withdraw_started = true;
964 let handle = Arc::clone(active.handle.as_ref().expect("handle checked"));
965 let reason = active.withdrawn.expect("withdraw checked").1;
966 tokio::task::spawn_blocking(move || handle.withdraw(reason));
967 }
968 if state
970 .prompt
971 .as_ref()
972 .is_some_and(|a| a.started.is_none() && a.committed)
973 {
974 state.prompt = None;
975 }
976 if state.prompt.is_none() {
977 if let Some(request) = state.queue.pop_front() {
978 state.prompt = Some(Active {
979 request,
980 started: None,
981 handle: None,
982 committed: false,
983 withdrawn: None,
984 withdraw_started: false,
985 });
986 }
987 }
988 if let Some(active) = state.prompt.as_mut().filter(|a| a.started.is_none()) {
989 active.started = Some(now);
990 let request = Arc::clone(&active.request);
991 let provider = Arc::clone(&state.provider);
992 let this = Arc::clone(&self);
993 tokio::spawn(async move {
994 let publish_to = Arc::clone(&this);
995 let id = request.id;
996 let result = tokio::task::spawn_blocking(move || {
997 provider.prompt(
998 &request.text,
999 Box::new(move |handle| publish_to.publish(id, handle)),
1000 )
1001 })
1002 .await
1003 .unwrap_or(ProviderResult::Unavailable);
1004 this.returned(id, result);
1005 });
1006 }
1007 if state.prompt.is_none() && state.queue.is_empty() {
1008 state.running = false;
1009 return;
1010 }
1011 let mut deadline = state
1012 .queue
1013 .iter()
1014 .map(|r| r.arrived + state.limits.queue_wait)
1015 .min();
1016 if let Some(active) = &state.prompt {
1017 let next = if !active.committed {
1018 active.started.map(|at| at + timeout)
1019 } else if !state.stuck {
1020 active.withdrawn.map(|(at, _)| at + grace)
1021 } else {
1022 None
1023 };
1024 if let Some(next) = next {
1025 deadline = Some(deadline.map_or(next, |d| d.min(next)));
1026 }
1027 }
1028 deadline
1029 };
1030 if let Some(at) = deadline {
1031 tokio::select! { _ = self.wake.notified() => {}, _ = tokio::time::sleep_until(at) => {} }
1032 } else {
1033 self.wake.notified().await;
1034 }
1035 }
1036 }
1037}
1038
1039#[cfg(feature = "test-support")]
1040mod test_provider;
1041#[cfg(test)]
1042mod tests;