Skip to main content

subc_daemon/
operator_confirm.rs

1//! Operator confirmation policy and its blocking provider boundary.
2//!
3//! The forwarding lock precedes the state lock. Providers and deliveries never
4//! acquire the forwarding lock, and neither can hold up the policy timers.
5use std::{
6    collections::{HashMap, VecDeque},
7    path::PathBuf,
8    sync::{Arc, Mutex},
9    time::Duration,
10};
11
12use subc_protocol::{
13    error_codes,
14    session::{OperatorConfirmReply, OperatorConfirmRequest},
15    ErrorBody, Flags, FrameType, Principal, Priority,
16};
17use tokio::{
18    sync::{oneshot, Notify},
19    time::{timeout_at, Instant},
20};
21
22use crate::{
23    caller_identity::CallerIdentity,
24    forwarding::{ModuleEndpointId, ModuleRouteKey, RouteBinding},
25    router::RouteCtx,
26    ConnectionId, Frame,
27};
28
29/// Person declines of privileged client requests that start a cooldown when
30/// they all fall within `DECLINE_WINDOW` of the newest one.
31const DECLINE_LIMIT: usize = 3;
32const DECLINE_WINDOW: Duration = Duration::from_secs(10 * 60);
33/// Measured from the decline that started the cooldown. Refusals during the
34/// cooldown are not declines by a person, so they never extend it.
35const DECLINE_COOLDOWN: Duration = Duration::from_secs(10 * 60);
36
37/// Monotonic time for the repeated-decline cooldown. Tests replace it so the
38/// ten-minute window and cooldown can be crossed without waiting.
39pub(crate) type CooldownClock = Arc<dyn Fn() -> Instant + Send + Sync>;
40
41/// Who repeated person declines of privileged requests are counted against.
42/// The executable path keeps a reconnecting program under its cooldown; a
43/// caller whose executable could not be resolved is counted per connection,
44/// because nothing else the daemon owns ties its connections together.
45#[derive(Clone, Debug, PartialEq, Eq, Hash)]
46pub(crate) enum DeclineKey {
47    Executable(PathBuf),
48    Connection(ConnectionId),
49}
50
51impl DeclineKey {
52    pub(crate) fn for_caller(caller: &CallerIdentity, connection: ConnectionId) -> Self {
53        match caller.executable() {
54            Some(executable) => Self::Executable(executable.to_path_buf()),
55            None => Self::Connection(connection),
56        }
57    }
58}
59
60impl std::fmt::Display for DeclineKey {
61    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
62        match self {
63            Self::Executable(path) => write!(f, "{}", path.display()),
64            Self::Connection(connection) => write!(
65                f,
66                "an unidentified local process on connection {}",
67                connection.get()
68            ),
69        }
70    }
71}
72
73/// A blocking provider publishes a withdraw handle as soon as one exists, then
74/// returns only when its prompt has ended. Publishing is allowed to be delayed:
75/// the daemon's timeout also covers setup and a delayed handle is still withdrawn.
76pub trait OperatorProvider: Send + Sync + 'static {
77    fn prompt(
78        &self,
79        text: &str,
80        publish: Box<dyn FnOnce(Arc<dyn OperatorWithdraw>) + Send>,
81    ) -> ProviderResult;
82}
83
84/// Called on a blocking thread, never while a daemon lock is held.
85pub trait OperatorWithdraw: Send + Sync + 'static {
86    fn withdraw(&self, reason: &str);
87}
88
89#[derive(Debug, Clone, PartialEq, Eq)]
90pub enum ProviderResult {
91    Approved,
92    Declined,
93    Unavailable,
94    UnsupportedPlatform,
95    NoPresence,
96    LocalAuthenticationError {
97        failure: OperatorOsErrorKind,
98        domain: String,
99        code: isize,
100    },
101}
102
103#[derive(Debug, Clone, Copy, PartialEq, Eq)]
104pub enum OperatorOsErrorKind {
105    Declined,
106    NoPresence,
107    ProviderError,
108}
109
110#[derive(Debug, Clone, Copy)]
111pub struct OperatorLimits {
112    pub queue_wait: Duration,
113    pub prompt_timeout: Duration,
114    pub stuck_grace: Duration,
115}
116
117impl Default for OperatorLimits {
118    fn default() -> Self {
119        Self {
120            queue_wait: Duration::from_secs(150),
121            prompt_timeout: Duration::from_secs(120),
122            stuck_grace: Duration::from_secs(10),
123        }
124    }
125}
126
127impl OperatorLimits {
128    fn delivery_budget(self) -> Duration {
129        self.queue_wait + self.prompt_timeout + self.stuck_grace + Duration::from_secs(10)
130    }
131}
132
133struct OsProvider;
134
135#[cfg(any(target_os = "macos", windows))]
136struct OsWithdraw(Arc<dyn subc_presence::Withdraw>);
137#[cfg(any(target_os = "macos", windows))]
138impl OperatorWithdraw for OsWithdraw {
139    fn withdraw(&self, _: &str) {
140        self.0.withdraw();
141    }
142}
143
144impl OperatorProvider for OsProvider {
145    fn prompt(
146        &self,
147        text: &str,
148        publish: Box<dyn FnOnce(Arc<dyn OperatorWithdraw>) + Send>,
149    ) -> ProviderResult {
150        // On Linux the prompt comes from polkit, whose agents may show only the
151        // fixed message of an installed action file, not this request's summary.
152        // A person must see what they approve, so Linux refuses rather than prompts.
153        #[cfg(target_os = "linux")]
154        {
155            let _ = (text, publish);
156            ProviderResult::UnsupportedPlatform
157        }
158        #[cfg(any(target_os = "macos", windows))]
159        {
160            os_outcome(subc_presence::prompt(
161                text,
162                Box::new(move |handle| publish(Arc::new(OsWithdraw(handle)))),
163            ))
164        }
165        #[cfg(not(any(target_os = "macos", windows, target_os = "linux")))]
166        {
167            let _ = (text, publish);
168            ProviderResult::Unavailable
169        }
170    }
171}
172
173/// Convert the operating-system prompt's result into the daemon's provider
174/// result. Only an explicit approval may become `Approved`. The real prompt
175/// needs a person, so tests can't reach this mapping through it; keeping it a
176/// separate function lets a test check that no other result grants a write.
177#[cfg(any(target_os = "macos", windows))]
178fn os_outcome(outcome: subc_presence::Outcome) -> ProviderResult {
179    match outcome {
180        subc_presence::Outcome::Approved => ProviderResult::Approved,
181        subc_presence::Outcome::Declined => ProviderResult::Declined,
182        subc_presence::Outcome::ProviderError => ProviderResult::Unavailable,
183        subc_presence::Outcome::NoPresence => ProviderResult::NoPresence,
184        subc_presence::Outcome::UnsupportedPlatform => ProviderResult::UnsupportedPlatform,
185        subc_presence::Outcome::LocalAuthenticationError {
186            failure,
187            domain,
188            code,
189        } => ProviderResult::LocalAuthenticationError {
190            failure: match failure {
191                subc_presence::LocalAuthenticationFailure::Declined => {
192                    OperatorOsErrorKind::Declined
193                }
194                subc_presence::LocalAuthenticationFailure::NoPresence => {
195                    OperatorOsErrorKind::NoPresence
196                }
197                subc_presence::LocalAuthenticationFailure::ProviderError => {
198                    OperatorOsErrorKind::ProviderError
199                }
200            },
201            domain,
202            code,
203        },
204    }
205}
206
207#[cfg(all(test, any(target_os = "macos", windows)))]
208mod os_outcome_tests {
209    use super::*;
210
211    #[test]
212    fn only_an_os_approval_becomes_approved() {
213        use subc_presence::Outcome as Os;
214        assert_eq!(os_outcome(Os::Approved), ProviderResult::Approved);
215        assert_eq!(os_outcome(Os::Declined), ProviderResult::Declined);
216        assert_eq!(os_outcome(Os::ProviderError), ProviderResult::Unavailable);
217        assert_eq!(os_outcome(Os::NoPresence), ProviderResult::NoPresence);
218        assert_eq!(
219            os_outcome(Os::UnsupportedPlatform),
220            ProviderResult::UnsupportedPlatform
221        );
222        assert_eq!(
223            os_outcome(Os::LocalAuthenticationError {
224                failure: subc_presence::LocalAuthenticationFailure::NoPresence,
225                domain: "com.apple.LocalAuthentication".into(),
226                code: -4,
227            }),
228            ProviderResult::LocalAuthenticationError {
229                failure: OperatorOsErrorKind::NoPresence,
230                domain: "com.apple.LocalAuthentication".into(),
231                code: -4,
232            }
233        );
234    }
235}
236
237#[derive(Clone, Debug)]
238pub(crate) struct Outcome {
239    pub code: &'static str,
240    pub reason: &'static str,
241    retry_after_ms: Option<u64>,
242    os_error_domain: Option<String>,
243    os_error_code: Option<isize>,
244}
245impl Outcome {
246    pub(crate) fn unavailable(reason: &'static str) -> Self {
247        Self {
248            code: error_codes::OPERATOR_PRESENCE_UNAVAILABLE,
249            reason,
250            retry_after_ms: None,
251            os_error_domain: None,
252            os_error_code: None,
253        }
254    }
255    fn declined(reason: &'static str) -> Self {
256        Self {
257            code: error_codes::OPERATOR_DECLINED,
258            reason,
259            retry_after_ms: None,
260            os_error_domain: None,
261            os_error_code: None,
262        }
263    }
264    fn with_retry_after_ms(mut self, retry_after_ms: u64) -> Self {
265        self.retry_after_ms = Some(retry_after_ms);
266        self
267    }
268    pub(crate) fn refusal(code: &'static str) -> Self {
269        Self {
270            code,
271            reason: "",
272            retry_after_ms: None,
273            os_error_domain: None,
274            os_error_code: None,
275        }
276    }
277    fn from_provider(result: ProviderResult) -> Self {
278        match result {
279            ProviderResult::Approved => Self {
280                code: "confirmed",
281                reason: "",
282                retry_after_ms: None,
283                os_error_domain: None,
284                os_error_code: None,
285            },
286            ProviderResult::Declined => Self::declined("person"),
287            ProviderResult::Unavailable => Self::unavailable("provider_error"),
288            ProviderResult::UnsupportedPlatform => Self::unavailable("unsupported_platform"),
289            ProviderResult::NoPresence => Self::unavailable("no_presence"),
290            ProviderResult::LocalAuthenticationError {
291                failure,
292                domain,
293                code,
294            } => {
295                let mut outcome = match failure {
296                    OperatorOsErrorKind::Declined => Self::declined("person"),
297                    OperatorOsErrorKind::NoPresence => Self::unavailable("no_presence"),
298                    OperatorOsErrorKind::ProviderError => Self::unavailable("provider_error"),
299                };
300                outcome.os_error_domain = Some(domain);
301                outcome.os_error_code = Some(code);
302                outcome
303            }
304        }
305    }
306    pub(crate) fn frame(self, request: &Frame) -> Frame {
307        let (ty, body) = if self.code == "confirmed" {
308            (
309                FrameType::Response,
310                serde_json::to_vec(&OperatorConfirmReply::confirmed()).expect("serializable reply"),
311            )
312        } else {
313            let mut error = ErrorBody::new(self.code, "operator confirmation refused");
314            if !self.reason.is_empty() {
315                let detail = match self.retry_after_ms {
316                    Some(retry_after_ms) => {
317                        serde_json::json!({"reason": self.reason, "retry_after_ms": retry_after_ms})
318                    }
319                    None => serde_json::json!({"reason": self.reason}),
320                };
321                error = error.with_detail(detail);
322            }
323            (
324                FrameType::Error,
325                serde_json::to_vec(&error).expect("serializable error"),
326            )
327        };
328        Frame::build_with_version(
329            request.header.ver,
330            ty,
331            Flags::new(false, Priority::Passive, false),
332            0,
333            0,
334            request.header.corr,
335            body,
336        )
337        .expect("bounded control reply")
338    }
339}
340
341fn retry_after_ms(remaining: Duration) -> u64 {
342    let whole_milliseconds = remaining.as_millis();
343    let rounded_milliseconds =
344        whole_milliseconds + u128::from(!remaining.subsec_nanos().is_multiple_of(1_000_000));
345    u64::try_from(rounded_milliseconds).expect("operator confirmation backoff fits in u64")
346}
347
348pub(crate) fn audit(
349    module_id: &str,
350    summary: &str,
351    principal: &str,
352    outcome: &Outcome,
353    wait: Duration,
354    prompt: Duration,
355    prompt_shown: bool,
356) {
357    tracing::info!(target: "subc_daemon::operator_confirm", module_id, summary = ?summary,
358        principal, outcome = outcome.code, reason = outcome.reason,
359        os_error_domain = ?outcome.os_error_domain, os_error_code = ?outcome.os_error_code,
360        wait_ms = wait.as_millis() as u64, prompt_ms = prompt.as_millis() as u64,
361        prompt_shown, "operator_confirm_audit");
362}
363
364fn principal_label(principal: &Principal) -> String {
365    match principal {
366        Principal::Direct => "direct".into(),
367        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
368        Principal::Unverified => String::new(),
369    }
370}
371
372fn valid_summary(summary: &str) -> bool {
373    let count = summary.chars().count();
374    (1..=200).contains(&count)
375        && !summary.starts_with(char::is_whitespace)
376        && !summary.ends_with(char::is_whitespace)
377        && !summary.chars().any(|c| {
378            c.is_control()
379                || matches!(c,
380            '\u{200b}'..='\u{200f}' | '\u{202a}'..='\u{202e}' | '\u{2028}' | '\u{2029}' |
381            '\u{2060}'..='\u{2069}' | '\u{feff}')
382        })
383}
384
385enum RequestKind {
386    Module(ModuleRouteKey),
387    Privileged {
388        answer: Mutex<Option<oneshot::Sender<Outcome>>>,
389        key: DeclineKey,
390    },
391}
392
393struct Request {
394    id: u64,
395    module_id: String,
396    summary: String,
397    principal: String,
398    text: String,
399    kind: RequestKind,
400    ctx: RouteCtx,
401    frame: Frame,
402    arrived: Instant,
403    delivery_deadline: Instant,
404}
405struct Active {
406    request: Arc<Request>,
407    started: Option<Instant>,
408    handle: Option<Arc<dyn OperatorWithdraw>>,
409    committed: bool,
410    withdrawn: Option<(Instant, &'static str)>,
411    withdraw_started: bool,
412}
413struct State {
414    provider: Arc<dyn OperatorProvider>,
415    limits: OperatorLimits,
416    queue: VecDeque<Arc<Request>>,
417    modules: HashMap<String, u64>,
418    backoff: HashMap<(String, String), Instant>,
419    cooldown_clock: CooldownClock,
420    /// Recent person declines of privileged requests, oldest first.
421    declines: HashMap<DeclineKey, VecDeque<Instant>>,
422    /// When each privileged caller's cooldown ends.
423    cooldowns: HashMap<DeclineKey, Instant>,
424    prompt: Option<Active>,
425    stuck: bool,
426    running: bool,
427    next_id: u64,
428    #[cfg(test)]
429    deliveries: usize,
430}
431
432pub(crate) struct OperatorConfirms {
433    state: Mutex<State>,
434    wake: Notify,
435}
436
437struct WithdrawOnDrop {
438    confirms: Arc<OperatorConfirms>,
439    id: u64,
440}
441
442impl Drop for WithdrawOnDrop {
443    fn drop(&mut self) {
444        self.confirms
445            .withdraw_matching(|request| request.id == self.id, "caller_cancelled");
446    }
447}
448impl std::fmt::Debug for OperatorConfirms {
449    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
450        f.debug_struct("OperatorConfirms").finish_non_exhaustive()
451    }
452}
453impl Default for OperatorConfirms {
454    fn default() -> Self {
455        Self {
456            state: Mutex::new(State {
457                provider: Arc::new(OsProvider),
458                limits: OperatorLimits::default(),
459                queue: VecDeque::new(),
460                modules: HashMap::new(),
461                backoff: HashMap::new(),
462                cooldown_clock: Arc::new(Instant::now),
463                declines: HashMap::new(),
464                cooldowns: HashMap::new(),
465                prompt: None,
466                stuck: false,
467                running: false,
468                next_id: 0,
469                #[cfg(test)]
470                deliveries: 0,
471            }),
472            wake: Notify::new(),
473        }
474    }
475}
476impl OperatorConfirms {
477    fn lock(&self) -> std::sync::MutexGuard<'_, State> {
478        self.state.lock().unwrap_or_else(|p| p.into_inner())
479    }
480    pub(crate) fn configure(&self, provider: Arc<dyn OperatorProvider>, limits: OperatorLimits) {
481        let mut state = self.lock();
482        assert!(
483            !state.running && state.modules.is_empty(),
484            "configure before serving"
485        );
486        state.provider = provider;
487        state.limits = limits;
488    }
489    pub(crate) fn configure_bootstrap(&self, config: &crate::bootstrap::BootstrapConfig) {
490        let provider: Arc<dyn OperatorProvider> = Arc::new(OsProvider);
491        #[cfg(feature = "test-support")]
492        let provider = test_provider::select(
493            provider,
494            config.operator_script.as_deref(),
495            config.operator_events.as_deref(),
496            std::env::current_exe().ok().as_deref(),
497        );
498        self.configure(
499            provider,
500            OperatorLimits {
501                queue_wait: config.operator_queue_wait,
502                prompt_timeout: config.operator_timeout,
503                stuck_grace: config.operator_stuck_grace,
504            },
505        );
506    }
507
508    #[cfg(feature = "test-support")]
509    pub(crate) fn configure_script(
510        &self,
511        script: &std::path::Path,
512        events: Option<&std::path::Path>,
513        limits: Option<OperatorLimits>,
514    ) {
515        let provider = test_provider::select(
516            Arc::new(OsProvider),
517            Some(script),
518            events,
519            test_provider::current_executable().as_deref(),
520        );
521        self.configure(provider, limits.unwrap_or_default());
522    }
523
524    /// Called while the forwarding read lock still protects this binding. All
525    /// registration/nonce lookups must have finished before entering this method.
526    pub(crate) fn admit(
527        self: &Arc<Self>,
528        ctx: &RouteCtx,
529        frame: Frame,
530        module_id: String,
531        nonce_proven: bool,
532        request: OperatorConfirmRequest,
533        binding: Option<&RouteBinding>,
534    ) -> Vec<Frame> {
535        let principal = binding
536            .map(|b| principal_label(&b.principal))
537            .unwrap_or_default();
538        let mut state = self.lock();
539        let now = Instant::now();
540        state.backoff.retain(|_, until| *until > now);
541        let refusal = if !nonce_proven
542            || binding.is_none_or(|b| matches!(b.principal, Principal::Unverified))
543        {
544            Some(Outcome::refusal(
545                error_codes::OPERATOR_REQUEST_NOT_PERMITTED,
546            ))
547        } else if !valid_summary(&request.summary) {
548            Some(Outcome::refusal(error_codes::OPERATOR_SUMMARY_INVALID))
549        } else if let Some(until) = state
550            .backoff
551            .get(&(module_id.clone(), principal.clone()))
552            .copied()
553        {
554            Some(Outcome::declined("backoff").with_retry_after_ms(retry_after_ms(until - now)))
555        } else if state.stuck {
556            Some(Outcome::unavailable("provider_stuck"))
557        } else if state.modules.contains_key(&module_id) {
558            Some(Outcome::unavailable("module_limit"))
559        } else if state.queue.len() >= 4 {
560            Some(Outcome::unavailable("queue_full"))
561        } else {
562            None
563        };
564        if let Some(outcome) = refusal {
565            audit(
566                &module_id,
567                &request.summary,
568                &principal,
569                &outcome,
570                Duration::ZERO,
571                Duration::ZERO,
572                false,
573            );
574            return vec![outcome.frame(&frame)];
575        }
576        let binding = binding.expect("permission checked");
577        let opener = match &binding.principal {
578            Principal::Direct => "a local program",
579            Principal::Reserved { module_id } => module_id,
580            Principal::Unverified => unreachable!(),
581        };
582        state.next_id += 1;
583        let pending = Arc::new(Request {
584            id: state.next_id,
585            text: format!(
586                "approve a request from {module_id}: {} (requested by {opener})",
587                request.summary
588            ),
589            module_id,
590            summary: request.summary,
591            principal,
592            kind: RequestKind::Module(ModuleRouteKey {
593                endpoint: binding.module_endpoint,
594                channel: binding.module_channel,
595            }),
596            ctx: ctx.clone(),
597            frame,
598            arrived: now,
599            delivery_deadline: now + state.limits.delivery_budget(),
600        });
601        state.modules.insert(pending.module_id.clone(), pending.id);
602        self.enqueue(&mut state, pending);
603        Vec::new()
604    }
605
606    /// Client-control approval shares the module confirmation queue and timers,
607    /// but returns a decision to dispatch instead of sending an operator reply.
608    /// A caller in its repeated-decline cooldown is refused before anything else
609    /// is checked, without a prompt.
610    pub(crate) async fn approve_privileged(
611        self: &Arc<Self>,
612        ctx: &RouteCtx,
613        frame: &Frame,
614        operation: &str,
615        text: String,
616        caller: String,
617        key: DeclineKey,
618    ) -> Outcome {
619        let (answer, receive) = oneshot::channel();
620        let id = {
621            let mut state = self.lock();
622            let now = (state.cooldown_clock)();
623            state.cooldowns.retain(|_, until| *until > now);
624            state.declines.retain(|_, declines| {
625                declines.retain(|at| now.saturating_duration_since(*at) < DECLINE_WINDOW);
626                !declines.is_empty()
627            });
628            let refusal = if state.cooldowns.contains_key(&key) {
629                Some(Outcome::declined("backoff"))
630            } else if state.stuck {
631                Some(Outcome::unavailable("provider_stuck"))
632            } else if state.queue.len() >= 4 {
633                Some(Outcome::unavailable("queue_full"))
634            } else {
635                None
636            };
637            if let Some(outcome) = refusal {
638                audit(
639                    operation,
640                    &text,
641                    &caller,
642                    &outcome,
643                    Duration::ZERO,
644                    Duration::ZERO,
645                    false,
646                );
647                return outcome;
648            }
649            state.next_id += 1;
650            let id = state.next_id;
651            let now = Instant::now();
652            let pending = Arc::new(Request {
653                id,
654                module_id: operation.into(),
655                summary: text.clone(),
656                principal: caller,
657                text,
658                kind: RequestKind::Privileged {
659                    answer: Mutex::new(Some(answer)),
660                    key,
661                },
662                ctx: ctx.clone(),
663                frame: frame.clone(),
664                arrived: now,
665                delivery_deadline: now + state.limits.delivery_budget(),
666            });
667            self.enqueue(&mut state, pending);
668            id
669        };
670        let _withdraw = WithdrawOnDrop {
671            confirms: Arc::clone(self),
672            id,
673        };
674        receive
675            .await
676            .unwrap_or_else(|_| Outcome::unavailable("provider_error"))
677    }
678
679    fn enqueue(self: &Arc<Self>, state: &mut State, pending: Arc<Request>) {
680        if state.prompt.is_none() && state.queue.is_empty() {
681            state.prompt = Some(Active {
682                request: pending,
683                started: None,
684                handle: None,
685                committed: false,
686                withdrawn: None,
687                withdraw_started: false,
688            });
689        } else {
690            state.queue.push_back(pending);
691        }
692        if !state.running {
693            state.running = true;
694            tokio::spawn(Arc::clone(self).run());
695        }
696        self.wake.notify_one();
697    }
698
699    fn commit(
700        self: &Arc<Self>,
701        state: &mut State,
702        request: Arc<Request>,
703        outcome: Outcome,
704        started: Option<Instant>,
705        shown: bool,
706    ) {
707        let now = Instant::now();
708        let wait = started.unwrap_or(now).duration_since(request.arrived);
709        let prompt = started.map(|at| now.duration_since(at)).unwrap_or_default();
710        if outcome.reason == "person" && matches!(request.kind, RequestKind::Module(_)) {
711            state.backoff.insert(
712                (request.module_id.clone(), request.principal.clone()),
713                now + Duration::from_secs(30),
714            );
715        }
716        audit(
717            &request.module_id,
718            &request.summary,
719            &request.principal,
720            &outcome,
721            wait,
722            prompt,
723            shown,
724        );
725        if let RequestKind::Privileged { answer, key } = &request.kind {
726            let person_declined = outcome.reason == "person";
727            if let Some(answer) = answer.lock().unwrap_or_else(|p| p.into_inner()).take() {
728                let _ = answer.send(outcome);
729            }
730            if person_declined {
731                self.count_person_decline(state, key);
732            }
733            return;
734        }
735        let this = Arc::clone(self);
736        #[cfg(test)]
737        {
738            state.deliveries += 1;
739        }
740        tokio::spawn(async move {
741            let answer = outcome.frame(&request.frame);
742            match timeout_at(request.delivery_deadline, request.ctx.egress.send(answer)).await {
743                Ok(Ok(())) => {}
744                result => {
745                    tracing::warn!(target: "subc_daemon::operator_confirm", module_id = request.module_id, ?result, "operator confirm answer dropped")
746                }
747            }
748            let mut state = this.lock();
749            #[cfg(test)]
750            {
751                state.deliveries -= 1;
752            }
753            if state.modules.get(&request.module_id) == Some(&request.id) {
754                state.modules.remove(&request.module_id);
755            }
756        });
757    }
758
759    /// Only a person's decline at the prompt reaches here: withdrawals, timeouts
760    /// and unavailability commit other reasons. The prompt that recorded the
761    /// limit-reaching decline has already ended, so starting the cooldown only
762    /// refuses this caller's requests still waiting in the queue.
763    fn count_person_decline(self: &Arc<Self>, state: &mut State, key: &DeclineKey) {
764        let now = (state.cooldown_clock)();
765        let declines = state.declines.entry(key.clone()).or_default();
766        declines.retain(|at| now.saturating_duration_since(*at) < DECLINE_WINDOW);
767        declines.push_back(now);
768        if declines.len() < DECLINE_LIMIT {
769            return;
770        }
771        // Counting restarts after the cooldown instead of carrying old declines.
772        state.declines.remove(key);
773        state.cooldowns.insert(key.clone(), now + DECLINE_COOLDOWN);
774        tracing::warn!(target: "subc_daemon::operator_confirm", caller = %key,
775            cooldown_secs = DECLINE_COOLDOWN.as_secs(),
776            "privileged requests refused without a prompt after repeated operator declines");
777        let mut index = 0;
778        while index < state.queue.len() {
779            if matches!(&state.queue[index].kind,
780                RequestKind::Privileged { key: queued, .. } if queued == key)
781            {
782                let request = state.queue.remove(index).expect("queue index");
783                self.commit(state, request, Outcome::declined("backoff"), None, false);
784            } else {
785                index += 1;
786            }
787        }
788    }
789
790    fn withdraw_matching(
791        self: &Arc<Self>,
792        predicate: impl Fn(&Request) -> bool,
793        reason: &'static str,
794    ) -> bool {
795        let mut state = self.lock();
796        let mut matched = false;
797        let mut index = 0;
798        while index < state.queue.len() {
799            if predicate(&state.queue[index]) {
800                let request = state.queue.remove(index).expect("queue index");
801                self.commit(&mut state, request, Outcome::declined(reason), None, false);
802                matched = true;
803            } else {
804                index += 1;
805            }
806        }
807        if let Some(active) = state
808            .prompt
809            .as_mut()
810            .filter(|a| !a.committed && predicate(&a.request))
811        {
812            active.committed = true;
813            active.withdrawn = Some((Instant::now(), reason));
814            let (request, started, shown) = (
815                Arc::clone(&active.request),
816                active.started,
817                active.handle.is_some(),
818            );
819            self.commit(
820                &mut state,
821                request,
822                Outcome::declined(reason),
823                started,
824                shown,
825            );
826            matched = true;
827        }
828        self.wake.notify_one();
829        matched
830    }
831    pub(crate) fn route_closed(self: &Arc<Self>, key: ModuleRouteKey) {
832        self.withdraw_matching(
833            |r| matches!(r.kind, RequestKind::Module(k) if k == key),
834            "route_closed",
835        );
836    }
837    pub(crate) fn module_closed(self: &Arc<Self>, endpoint: ModuleEndpointId) {
838        self.withdraw_matching(
839            |r| matches!(r.kind, RequestKind::Module(k) if k.endpoint == endpoint),
840            "module_closed",
841        );
842    }
843    pub(crate) fn connection_closed(self: &Arc<Self>, connection: crate::ConnectionId) {
844        self.withdraw_matching(
845            |r| {
846                matches!(r.kind, RequestKind::Privileged { .. })
847                    && r.ctx.connection_id == connection
848            },
849            "caller_cancelled",
850        );
851    }
852    pub(crate) fn cancel(self: &Arc<Self>, connection: crate::ConnectionId, corr: u64) -> bool {
853        self.withdraw_matching(
854            |r| r.ctx.connection_id == connection && r.frame.header.corr == corr,
855            "caller_cancelled",
856        )
857    }
858
859    fn publish(self: &Arc<Self>, id: u64, handle: Arc<dyn OperatorWithdraw>) {
860        let mut state = self.lock();
861        if let Some(active) = state.prompt.as_mut().filter(|a| a.request.id == id) {
862            active.handle = Some(Arc::clone(&handle));
863            if let Some((_, reason)) = active.withdrawn {
864                active.withdraw_started = true;
865                tokio::task::spawn_blocking(move || handle.withdraw(reason));
866            }
867        }
868        self.wake.notify_one();
869    }
870    fn returned(self: &Arc<Self>, id: u64, result: ProviderResult) {
871        let mut state = self.lock();
872        let Some(active) = state.prompt.take() else {
873            return;
874        };
875        assert_eq!(active.request.id, id);
876        // A fast late return can beat the prompt task's wake-up. The decision
877        // still requires a withdraw, even though the result is discarded.
878        if !active.withdraw_started {
879            if let (Some(handle), Some((_, reason))) = (active.handle.as_ref(), active.withdrawn) {
880                let handle = Arc::clone(handle);
881                tokio::task::spawn_blocking(move || handle.withdraw(reason));
882            }
883        }
884        if !active.committed {
885            self.commit(
886                &mut state,
887                active.request,
888                Outcome::from_provider(result),
889                active.started,
890                active.handle.is_some(),
891            );
892        } else {
893            tracing::debug!(target: "subc_daemon::operator_confirm", ?result, "late operator provider result discarded");
894        }
895        state.stuck = false;
896        self.wake.notify_one();
897    }
898
899    async fn run(self: Arc<Self>) {
900        loop {
901            let deadline = {
902                let mut state = self.lock();
903                let now = Instant::now();
904                let mut index = 0;
905                while index < state.queue.len() {
906                    if state.queue[index].arrived + state.limits.queue_wait <= now {
907                        let request = state.queue.remove(index).expect("queue index");
908                        self.commit(
909                            &mut state,
910                            request,
911                            Outcome::unavailable("queue_wait"),
912                            None,
913                            false,
914                        );
915                    } else {
916                        index += 1;
917                    }
918                }
919                let timeout = state.limits.prompt_timeout;
920                if let Some(active) = state
921                    .prompt
922                    .as_mut()
923                    .filter(|a| !a.committed && a.started.is_some_and(|at| at + timeout <= now))
924                {
925                    active.committed = true;
926                    active.withdrawn = Some((now, "timeout"));
927                    let (request, started, shown) = (
928                        Arc::clone(&active.request),
929                        active.started,
930                        active.handle.is_some(),
931                    );
932                    self.commit(
933                        &mut state,
934                        request,
935                        Outcome::unavailable("timeout"),
936                        started,
937                        shown,
938                    );
939                }
940                let grace = state.limits.stuck_grace;
941                if !state.stuck
942                    && state.prompt.as_ref().is_some_and(|a| {
943                        a.started.is_some() && a.withdrawn.is_some_and(|(at, _)| at + grace <= now)
944                    })
945                {
946                    state.stuck = true;
947                    tracing::error!(target: "subc_daemon::operator_confirm", "operator provider stuck after withdraw");
948                    while let Some(request) = state.queue.pop_front() {
949                        self.commit(
950                            &mut state,
951                            request,
952                            Outcome::unavailable("provider_stuck"),
953                            None,
954                            false,
955                        );
956                    }
957                }
958                if let Some(active) = state
959                    .prompt
960                    .as_mut()
961                    .filter(|a| !a.withdraw_started && a.withdrawn.is_some() && a.handle.is_some())
962                {
963                    active.withdraw_started = true;
964                    let handle = Arc::clone(active.handle.as_ref().expect("handle checked"));
965                    let reason = active.withdrawn.expect("withdraw checked").1;
966                    tokio::task::spawn_blocking(move || handle.withdraw(reason));
967                }
968                // A reserved prompt cancelled before setup needs no provider call.
969                if state
970                    .prompt
971                    .as_ref()
972                    .is_some_and(|a| a.started.is_none() && a.committed)
973                {
974                    state.prompt = None;
975                }
976                if state.prompt.is_none() {
977                    if let Some(request) = state.queue.pop_front() {
978                        state.prompt = Some(Active {
979                            request,
980                            started: None,
981                            handle: None,
982                            committed: false,
983                            withdrawn: None,
984                            withdraw_started: false,
985                        });
986                    }
987                }
988                if let Some(active) = state.prompt.as_mut().filter(|a| a.started.is_none()) {
989                    active.started = Some(now);
990                    let request = Arc::clone(&active.request);
991                    let provider = Arc::clone(&state.provider);
992                    let this = Arc::clone(&self);
993                    tokio::spawn(async move {
994                        let publish_to = Arc::clone(&this);
995                        let id = request.id;
996                        let result = tokio::task::spawn_blocking(move || {
997                            provider.prompt(
998                                &request.text,
999                                Box::new(move |handle| publish_to.publish(id, handle)),
1000                            )
1001                        })
1002                        .await
1003                        .unwrap_or(ProviderResult::Unavailable);
1004                        this.returned(id, result);
1005                    });
1006                }
1007                if state.prompt.is_none() && state.queue.is_empty() {
1008                    state.running = false;
1009                    return;
1010                }
1011                let mut deadline = state
1012                    .queue
1013                    .iter()
1014                    .map(|r| r.arrived + state.limits.queue_wait)
1015                    .min();
1016                if let Some(active) = &state.prompt {
1017                    let next = if !active.committed {
1018                        active.started.map(|at| at + timeout)
1019                    } else if !state.stuck {
1020                        active.withdrawn.map(|(at, _)| at + grace)
1021                    } else {
1022                        None
1023                    };
1024                    if let Some(next) = next {
1025                        deadline = Some(deadline.map_or(next, |d| d.min(next)));
1026                    }
1027                }
1028                deadline
1029            };
1030            if let Some(at) = deadline {
1031                tokio::select! { _ = self.wake.notified() => {}, _ = tokio::time::sleep_until(at) => {} }
1032            } else {
1033                self.wake.notified().await;
1034            }
1035        }
1036    }
1037}
1038
1039#[cfg(feature = "test-support")]
1040mod test_provider;
1041#[cfg(test)]
1042mod tests;