Skip to main content

subc_daemon/
operator_confirm.rs

1//! Operator confirmation policy and its blocking provider boundary.
2//!
3//! The forwarding lock precedes the state lock. Providers and deliveries never
4//! acquire the forwarding lock, and neither can hold up the policy timers.
5use std::{
6    collections::{HashMap, VecDeque},
7    path::PathBuf,
8    sync::{Arc, Mutex},
9    time::Duration,
10};
11
12use subc_protocol::{
13    error_codes,
14    session::{OperatorConfirmReply, OperatorConfirmRequest},
15    ErrorBody, Flags, FrameType, Principal, Priority,
16};
17use tokio::{
18    sync::{oneshot, Notify},
19    time::{timeout_at, Instant},
20};
21
22use crate::{
23    caller_identity::CallerIdentity,
24    forwarding::{ModuleEndpointId, ModuleRouteKey, RouteBinding},
25    router::RouteCtx,
26    ConnectionId, Frame,
27};
28
29mod shortcut;
30pub(crate) use shortcut::ShortcutContext;
31
32/// Person declines of privileged client requests that start a cooldown when
33/// they all fall within `DECLINE_WINDOW` of the newest one.
34const DECLINE_LIMIT: usize = 3;
35const DECLINE_WINDOW: Duration = Duration::from_secs(10 * 60);
36/// Measured from the decline that started the cooldown. Refusals during the
37/// cooldown are not declines by a person, so they never extend it.
38const DECLINE_COOLDOWN: Duration = Duration::from_secs(10 * 60);
39
40/// Monotonic time for the repeated-decline cooldown. Tests replace it so the
41/// ten-minute window and cooldown can be crossed without waiting.
42pub(crate) type CooldownClock = Arc<dyn Fn() -> Instant + Send + Sync>;
43
44/// Who repeated person declines of privileged requests are counted against.
45/// The executable path keeps a reconnecting program under its cooldown; a
46/// caller whose executable could not be resolved is counted per connection,
47/// because nothing else the daemon owns ties its connections together.
48#[derive(Clone, Debug, PartialEq, Eq, Hash)]
49pub(crate) enum DeclineKey {
50    Executable(PathBuf),
51    Connection(ConnectionId),
52}
53
54impl DeclineKey {
55    pub(crate) fn for_caller(caller: &CallerIdentity, connection: ConnectionId) -> Self {
56        match caller.executable() {
57            Some(executable) => Self::Executable(executable.to_path_buf()),
58            None => Self::Connection(connection),
59        }
60    }
61}
62
63impl std::fmt::Display for DeclineKey {
64    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
65        match self {
66            Self::Executable(path) => write!(f, "{}", path.display()),
67            Self::Connection(connection) => write!(
68                f,
69                "an unidentified local process on connection {}",
70                connection.get()
71            ),
72        }
73    }
74}
75
76/// A blocking provider publishes a withdraw handle as soon as one exists, then
77/// returns only when its prompt has ended. Publishing is allowed to be delayed:
78/// the daemon's timeout also covers setup and a delayed handle is still withdrawn.
79pub trait OperatorProvider: Send + Sync + 'static {
80    fn prompt(
81        &self,
82        text: &str,
83        publish: Box<dyn FnOnce(Arc<dyn OperatorWithdraw>) + Send>,
84    ) -> ProviderResult;
85}
86
87/// Called on a blocking thread, never while a daemon lock is held.
88pub trait OperatorWithdraw: Send + Sync + 'static {
89    fn withdraw(&self, reason: &str);
90}
91
92#[derive(Debug, Clone, PartialEq, Eq)]
93pub enum ProviderResult {
94    Approved,
95    Declined,
96    Unavailable,
97    ProviderError {
98        reason: &'static str,
99    },
100    UnsupportedPlatform,
101    NoPresence,
102    SetupRequired,
103    LocalAuthenticationError {
104        failure: OperatorOsErrorKind,
105        domain: String,
106        code: isize,
107    },
108}
109
110#[derive(Debug, Clone, Copy, PartialEq, Eq)]
111pub enum OperatorOsErrorKind {
112    Declined,
113    NoPresence,
114    ProviderError,
115}
116
117#[derive(Debug, Clone, Copy)]
118pub struct OperatorLimits {
119    pub queue_wait: Duration,
120    pub prompt_timeout: Duration,
121    pub stuck_grace: Duration,
122}
123
124impl Default for OperatorLimits {
125    fn default() -> Self {
126        Self {
127            queue_wait: Duration::from_secs(150),
128            prompt_timeout: Duration::from_secs(120),
129            stuck_grace: Duration::from_secs(10),
130        }
131    }
132}
133
134impl OperatorLimits {
135    fn delivery_budget(self) -> Duration {
136        self.queue_wait + self.prompt_timeout + self.stuck_grace + Duration::from_secs(10)
137    }
138}
139
140struct OsProvider;
141
142#[cfg(any(target_os = "macos", windows, target_os = "linux"))]
143struct OsWithdraw(Arc<dyn subc_presence::Withdraw>);
144#[cfg(any(target_os = "macos", windows, target_os = "linux"))]
145impl OperatorWithdraw for OsWithdraw {
146    fn withdraw(&self, _: &str) {
147        self.0.withdraw();
148    }
149}
150
151impl OperatorProvider for OsProvider {
152    fn prompt(
153        &self,
154        text: &str,
155        publish: Box<dyn FnOnce(Arc<dyn OperatorWithdraw>) + Send>,
156    ) -> ProviderResult {
157        #[cfg(any(target_os = "macos", windows, target_os = "linux"))]
158        {
159            os_outcome(subc_presence::prompt(
160                text,
161                Box::new(move |handle| publish(Arc::new(OsWithdraw(handle)))),
162            ))
163        }
164        #[cfg(not(any(target_os = "macos", windows, target_os = "linux")))]
165        {
166            let _ = (text, publish);
167            ProviderResult::Unavailable
168        }
169    }
170}
171
172/// Convert the operating-system prompt's result into the daemon's provider
173/// result. Only an explicit OS approval may become `Approved`. Keeping this
174/// separate from the platform bindings lets tests on every host verify that
175/// errors and unavailable providers never grant a write.
176#[cfg(any(target_os = "macos", windows, target_os = "linux"))]
177fn os_outcome(outcome: subc_presence::Outcome) -> ProviderResult {
178    match outcome {
179        subc_presence::Outcome::Approved => ProviderResult::Approved,
180        subc_presence::Outcome::Declined => ProviderResult::Declined,
181        subc_presence::Outcome::ProviderError => ProviderResult::Unavailable,
182        subc_presence::Outcome::ProviderErrorWithReason { reason } => {
183            ProviderResult::ProviderError { reason }
184        }
185        subc_presence::Outcome::NoPresence => ProviderResult::NoPresence,
186        subc_presence::Outcome::SetupRequired => ProviderResult::SetupRequired,
187        subc_presence::Outcome::UnsupportedPlatform => ProviderResult::UnsupportedPlatform,
188        subc_presence::Outcome::LocalAuthenticationError {
189            failure,
190            domain,
191            code,
192        } => ProviderResult::LocalAuthenticationError {
193            failure: match failure {
194                subc_presence::LocalAuthenticationFailure::Declined => {
195                    OperatorOsErrorKind::Declined
196                }
197                subc_presence::LocalAuthenticationFailure::NoPresence => {
198                    OperatorOsErrorKind::NoPresence
199                }
200                subc_presence::LocalAuthenticationFailure::ProviderError => {
201                    OperatorOsErrorKind::ProviderError
202                }
203            },
204            domain,
205            code,
206        },
207        // Future presence providers must fail closed until explicitly mapped.
208        _ => ProviderResult::Unavailable,
209    }
210}
211
212#[cfg(all(test, any(target_os = "macos", windows, target_os = "linux")))]
213mod os_outcome_tests {
214    use super::*;
215
216    #[test]
217    fn only_an_os_approval_becomes_approved() {
218        use subc_presence::Outcome as Os;
219        assert_eq!(os_outcome(Os::Approved), ProviderResult::Approved);
220        assert_eq!(os_outcome(Os::Declined), ProviderResult::Declined);
221        assert_eq!(os_outcome(Os::ProviderError), ProviderResult::Unavailable);
222        let root_error = os_outcome(Os::ProviderErrorWithReason {
223            reason: "root_user",
224        });
225        assert_eq!(
226            root_error,
227            ProviderResult::ProviderError {
228                reason: "root_user"
229            }
230        );
231        let refusal = Outcome::from_provider(root_error);
232        assert_eq!(refusal.code, error_codes::OPERATOR_PRESENCE_UNAVAILABLE);
233        assert_eq!(refusal.reason, "root_user");
234        assert_eq!(os_outcome(Os::NoPresence), ProviderResult::NoPresence);
235        assert_eq!(os_outcome(Os::SetupRequired), ProviderResult::SetupRequired);
236        let refusal = Outcome::from_provider(ProviderResult::SetupRequired);
237        assert_eq!(refusal.code, error_codes::OPERATOR_PRESENCE_UNAVAILABLE);
238        assert_eq!(refusal.reason, "operator_prompt_setup_required");
239        assert_eq!(
240            os_outcome(Os::UnsupportedPlatform),
241            ProviderResult::UnsupportedPlatform
242        );
243        assert_eq!(
244            os_outcome(Os::LocalAuthenticationError {
245                failure: subc_presence::LocalAuthenticationFailure::NoPresence,
246                domain: "com.apple.LocalAuthentication".into(),
247                code: -4,
248            }),
249            ProviderResult::LocalAuthenticationError {
250                failure: OperatorOsErrorKind::NoPresence,
251                domain: "com.apple.LocalAuthentication".into(),
252                code: -4,
253            }
254        );
255    }
256}
257
258#[derive(Clone, Debug)]
259pub(crate) struct Outcome {
260    pub code: &'static str,
261    pub reason: &'static str,
262    retry_after_ms: Option<u64>,
263    os_error_domain: Option<String>,
264    os_error_code: Option<isize>,
265}
266impl Outcome {
267    pub(crate) fn unavailable(reason: &'static str) -> Self {
268        Self {
269            code: error_codes::OPERATOR_PRESENCE_UNAVAILABLE,
270            reason,
271            retry_after_ms: None,
272            os_error_domain: None,
273            os_error_code: None,
274        }
275    }
276    fn declined(reason: &'static str) -> Self {
277        Self {
278            code: error_codes::OPERATOR_DECLINED,
279            reason,
280            retry_after_ms: None,
281            os_error_domain: None,
282            os_error_code: None,
283        }
284    }
285    fn with_retry_after_ms(mut self, retry_after_ms: u64) -> Self {
286        self.retry_after_ms = Some(retry_after_ms);
287        self
288    }
289    pub(crate) fn refusal(code: &'static str) -> Self {
290        Self {
291            code,
292            reason: "",
293            retry_after_ms: None,
294            os_error_domain: None,
295            os_error_code: None,
296        }
297    }
298    fn from_provider(result: ProviderResult) -> Self {
299        match result {
300            ProviderResult::Approved => Self {
301                code: "confirmed",
302                reason: "",
303                retry_after_ms: None,
304                os_error_domain: None,
305                os_error_code: None,
306            },
307            ProviderResult::Declined => Self::declined("person"),
308            ProviderResult::Unavailable => Self::unavailable("provider_error"),
309            ProviderResult::ProviderError { reason } => Self::unavailable(reason),
310            ProviderResult::UnsupportedPlatform => Self::unavailable("unsupported_platform"),
311            ProviderResult::NoPresence => Self::unavailable("no_presence"),
312            ProviderResult::SetupRequired => Self::unavailable("operator_prompt_setup_required"),
313            ProviderResult::LocalAuthenticationError {
314                failure,
315                domain,
316                code,
317            } => {
318                let mut outcome = match failure {
319                    OperatorOsErrorKind::Declined => Self::declined("person"),
320                    OperatorOsErrorKind::NoPresence => Self::unavailable("no_presence"),
321                    OperatorOsErrorKind::ProviderError => Self::unavailable("provider_error"),
322                };
323                outcome.os_error_domain = Some(domain);
324                outcome.os_error_code = Some(code);
325                outcome
326            }
327        }
328    }
329    pub(crate) fn frame(self, request: &Frame) -> Frame {
330        // A refusal is the daemon's own answer to the client's request, so
331        // its ERROR carries the daemon-origin bit; a confirmation is a plain
332        // response and keeps the control flags.
333        let (ty, body, daemon_origin) = if self.code == "confirmed" {
334            (
335                FrameType::Response,
336                serde_json::to_vec(&OperatorConfirmReply::confirmed()).expect("serializable reply"),
337                false,
338            )
339        } else {
340            let mut error = ErrorBody::new(self.code, "operator confirmation refused");
341            if !self.reason.is_empty() {
342                let detail = match self.retry_after_ms {
343                    Some(retry_after_ms) => {
344                        serde_json::json!({"reason": self.reason, "retry_after_ms": retry_after_ms})
345                    }
346                    None => serde_json::json!({"reason": self.reason}),
347                };
348                error = error.with_detail(detail);
349            }
350            (
351                FrameType::Error,
352                serde_json::to_vec(&error).expect("serializable error"),
353                true,
354            )
355        };
356        let flags = if daemon_origin {
357            crate::router::daemon_error_flags()
358        } else {
359            Flags::new(false, Priority::Passive, false)
360        };
361        Frame::build_with_version(
362            request.header.ver,
363            ty,
364            flags,
365            0,
366            0,
367            request.header.corr,
368            body,
369        )
370        .expect("bounded control reply")
371    }
372}
373
374fn retry_after_ms(remaining: Duration) -> u64 {
375    let whole_milliseconds = remaining.as_millis();
376    let rounded_milliseconds =
377        whole_milliseconds + u128::from(!remaining.subsec_nanos().is_multiple_of(1_000_000));
378    u64::try_from(rounded_milliseconds).expect("operator confirmation backoff fits in u64")
379}
380
381pub(crate) fn audit(
382    module_id: &str,
383    summary: &str,
384    principal: &str,
385    outcome: &Outcome,
386    wait: Duration,
387    prompt: Duration,
388    prompt_shown: bool,
389) {
390    tracing::info!(target: "subc_daemon::operator_confirm", module_id, summary = ?summary,
391        principal, outcome = outcome.code, reason = outcome.reason,
392        os_error_domain = ?outcome.os_error_domain, os_error_code = ?outcome.os_error_code,
393        wait_ms = wait.as_millis() as u64, prompt_ms = prompt.as_millis() as u64,
394        prompt_shown, "operator_confirm_audit");
395}
396
397fn principal_label(principal: &Principal) -> String {
398    match principal {
399        Principal::Direct => "direct".into(),
400        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
401        Principal::Unverified => String::new(),
402    }
403}
404
405pub(crate) fn valid_summary(summary: &str) -> bool {
406    let count = summary.chars().count();
407    (1..=200).contains(&count)
408        && !summary.starts_with(char::is_whitespace)
409        && !summary.ends_with(char::is_whitespace)
410        && !summary.chars().any(|c| {
411            c.is_control()
412                || matches!(c,
413            '\u{200b}'..='\u{200f}' | '\u{202a}'..='\u{202e}' | '\u{2028}' | '\u{2029}' |
414            '\u{2060}'..='\u{2069}' | '\u{feff}')
415        })
416}
417
418enum RequestKind {
419    Module(ModuleRouteKey),
420    Privileged {
421        answer: Mutex<Option<oneshot::Sender<Outcome>>>,
422        key: DeclineKey,
423    },
424}
425
426struct Request {
427    id: u64,
428    module_id: String,
429    summary: String,
430    principal: String,
431    text: String,
432    kind: RequestKind,
433    ctx: RouteCtx,
434    frame: Frame,
435    arrived: Instant,
436    delivery_deadline: Instant,
437    // A handoff reserves its queue position before its outcome append finishes.
438    prompt_ready: std::sync::atomic::AtomicBool,
439    shortcut_audit: bool,
440}
441struct Active {
442    request: Arc<Request>,
443    started: Option<Instant>,
444    handle: Option<Arc<dyn OperatorWithdraw>>,
445    committed: bool,
446    withdrawn: Option<(Instant, &'static str)>,
447    withdraw_started: bool,
448}
449struct State {
450    provider: Arc<dyn OperatorProvider>,
451    limits: OperatorLimits,
452    queue: VecDeque<Arc<Request>>,
453    modules: HashMap<String, u64>,
454    backoff: HashMap<(String, String), Instant>,
455    cooldown_clock: CooldownClock,
456    /// Recent person declines of privileged requests, oldest first.
457    declines: HashMap<DeclineKey, VecDeque<Instant>>,
458    /// When each privileged caller's cooldown ends.
459    cooldowns: HashMap<DeclineKey, Instant>,
460    prompt: Option<Active>,
461    stuck: bool,
462    running: bool,
463    next_id: u64,
464    shortcuts: HashMap<u64, shortcut::Pending>,
465    #[cfg(test)]
466    deliveries: usize,
467}
468
469pub(crate) struct OperatorConfirms {
470    state: Mutex<State>,
471    wake: Notify,
472}
473
474struct WithdrawOnDrop {
475    confirms: Arc<OperatorConfirms>,
476    id: u64,
477}
478
479impl Drop for WithdrawOnDrop {
480    fn drop(&mut self) {
481        self.confirms
482            .withdraw_matching(|request| request.id == self.id, "caller_cancelled");
483    }
484}
485impl std::fmt::Debug for OperatorConfirms {
486    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
487        f.debug_struct("OperatorConfirms").finish_non_exhaustive()
488    }
489}
490impl Default for OperatorConfirms {
491    fn default() -> Self {
492        Self {
493            state: Mutex::new(State {
494                provider: Arc::new(OsProvider),
495                limits: OperatorLimits::default(),
496                queue: VecDeque::new(),
497                modules: HashMap::new(),
498                backoff: HashMap::new(),
499                cooldown_clock: Arc::new(Instant::now),
500                declines: HashMap::new(),
501                cooldowns: HashMap::new(),
502                prompt: None,
503                stuck: false,
504                running: false,
505                next_id: 0,
506                shortcuts: HashMap::new(),
507                #[cfg(test)]
508                deliveries: 0,
509            }),
510            wake: Notify::new(),
511        }
512    }
513}
514impl OperatorConfirms {
515    fn lock(&self) -> std::sync::MutexGuard<'_, State> {
516        self.state.lock().unwrap_or_else(|p| p.into_inner())
517    }
518    pub(crate) fn configure(&self, provider: Arc<dyn OperatorProvider>, limits: OperatorLimits) {
519        let mut state = self.lock();
520        assert!(
521            !state.running && state.modules.is_empty(),
522            "configure before serving"
523        );
524        state.provider = provider;
525        state.limits = limits;
526    }
527    pub(crate) fn configure_bootstrap(&self, config: &crate::bootstrap::BootstrapConfig) {
528        let provider: Arc<dyn OperatorProvider> = Arc::new(OsProvider);
529        #[cfg(feature = "test-support")]
530        let provider = test_provider::select(
531            provider,
532            config.operator_script.as_deref(),
533            config.operator_events.as_deref(),
534            std::env::current_exe().ok().as_deref(),
535        );
536        self.configure(
537            provider,
538            OperatorLimits {
539                queue_wait: config.operator_queue_wait,
540                prompt_timeout: config.operator_timeout,
541                stuck_grace: config.operator_stuck_grace,
542            },
543        );
544    }
545
546    #[cfg(feature = "test-support")]
547    pub(crate) fn configure_script(
548        &self,
549        script: &std::path::Path,
550        events: Option<&std::path::Path>,
551        limits: Option<OperatorLimits>,
552    ) {
553        let provider = test_provider::select(
554            Arc::new(OsProvider),
555            Some(script),
556            events,
557            test_provider::current_executable().as_deref(),
558        );
559        self.configure(provider, limits.unwrap_or_default());
560    }
561
562    /// Called while the forwarding read lock still protects this binding. All
563    /// registration/nonce lookups must have finished before entering this method.
564    #[cfg(test)]
565    pub(crate) fn admit(
566        self: &Arc<Self>,
567        ctx: &RouteCtx,
568        frame: Frame,
569        module_id: String,
570        nonce_proven: bool,
571        request: OperatorConfirmRequest,
572        binding: Option<&RouteBinding>,
573    ) -> Vec<Frame> {
574        self.admit_with_shortcut(ctx, frame, module_id, nonce_proven, request, binding, None)
575    }
576
577    #[allow(clippy::too_many_arguments)]
578    pub(crate) fn admit_with_shortcut(
579        self: &Arc<Self>,
580        ctx: &RouteCtx,
581        frame: Frame,
582        module_id: String,
583        nonce_proven: bool,
584        request: OperatorConfirmRequest,
585        binding: Option<&RouteBinding>,
586        shortcut: Option<ShortcutContext>,
587    ) -> Vec<Frame> {
588        let principal = binding
589            .map(|b| principal_label(&b.principal))
590            .unwrap_or_default();
591        let mut state = self.lock();
592        let now = Instant::now();
593        state.backoff.retain(|_, until| *until > now);
594        let refusal = if !nonce_proven
595            || binding.is_none_or(|b| matches!(b.principal, Principal::Unverified))
596        {
597            Some(Outcome::refusal(
598                error_codes::OPERATOR_REQUEST_NOT_PERMITTED,
599            ))
600        } else if !valid_summary(&request.summary) {
601            Some(Outcome::refusal(error_codes::OPERATOR_SUMMARY_INVALID))
602        } else if let Some(until) = state
603            .backoff
604            .get(&(module_id.clone(), principal.clone()))
605            .copied()
606            .filter(|_| shortcut.is_none())
607        {
608            Some(Outcome::declined("backoff").with_retry_after_ms(retry_after_ms(until - now)))
609        } else if shortcut.is_none() && state.stuck {
610            Some(Outcome::unavailable("provider_stuck"))
611        } else if state.modules.contains_key(&module_id) {
612            Some(Outcome::unavailable("module_limit"))
613        } else if shortcut.is_none() && state.queue.len() >= 4 {
614            Some(Outcome::unavailable("queue_full"))
615        } else {
616            None
617        };
618        if let Some(outcome) = refusal {
619            audit(
620                &module_id,
621                &request.summary,
622                &principal,
623                &outcome,
624                Duration::ZERO,
625                Duration::ZERO,
626                false,
627            );
628            return vec![outcome.frame(&frame)];
629        }
630        let binding = binding.expect("permission checked");
631        let opener = match &binding.principal {
632            Principal::Direct => "a local program",
633            Principal::Reserved { module_id } => module_id,
634            Principal::Unverified => unreachable!(),
635        };
636        state.next_id += 1;
637        let pending = Arc::new(Request {
638            id: state.next_id,
639            text: format!(
640                "approve a request from {module_id}: {} (requested by {opener})",
641                request.summary
642            ),
643            module_id,
644            summary: request.summary,
645            principal,
646            kind: RequestKind::Module(ModuleRouteKey {
647                endpoint: binding.module_endpoint,
648                channel: binding.module_channel,
649            }),
650            ctx: ctx.clone(),
651            frame,
652            arrived: now,
653            delivery_deadline: now + state.limits.delivery_budget(),
654            prompt_ready: true.into(),
655            shortcut_audit: shortcut.is_some(),
656        });
657        state.modules.insert(pending.module_id.clone(), pending.id);
658        if let Some(shortcut) = shortcut {
659            self.start_shortcut(&mut state, pending, shortcut);
660        } else {
661            self.enqueue(&mut state, pending);
662        }
663        Vec::new()
664    }
665
666    /// Client-control approval shares the module confirmation queue and timers,
667    /// but returns a decision to dispatch instead of sending an operator reply.
668    /// A caller in its repeated-decline cooldown is refused before anything else
669    /// is checked, without a prompt.
670    #[allow(clippy::too_many_arguments)]
671    pub(crate) async fn approve_privileged_with_shortcut(
672        self: &Arc<Self>,
673        ctx: &RouteCtx,
674        frame: &Frame,
675        operation: &str,
676        text: String,
677        caller: String,
678        key: DeclineKey,
679        shortcut: Option<ShortcutContext>,
680    ) -> Outcome {
681        let (answer, receive) = oneshot::channel();
682        let id = {
683            let mut state = self.lock();
684            let now = (state.cooldown_clock)();
685            state.cooldowns.retain(|_, until| *until > now);
686            state.declines.retain(|_, declines| {
687                declines.retain(|at| now.saturating_duration_since(*at) < DECLINE_WINDOW);
688                !declines.is_empty()
689            });
690            let refusal = if state.cooldowns.contains_key(&key) {
691                Some(Outcome::declined("backoff"))
692            } else if shortcut.is_none() && state.stuck {
693                Some(Outcome::unavailable("provider_stuck"))
694            } else if shortcut.is_none() && state.queue.len() >= 4 {
695                Some(Outcome::unavailable("queue_full"))
696            } else {
697                None
698            };
699            if let Some(outcome) = refusal {
700                audit(
701                    operation,
702                    &text,
703                    &caller,
704                    &outcome,
705                    Duration::ZERO,
706                    Duration::ZERO,
707                    false,
708                );
709                return outcome;
710            }
711            state.next_id += 1;
712            let id = state.next_id;
713            let now = Instant::now();
714            let pending = Arc::new(Request {
715                id,
716                module_id: operation.into(),
717                summary: text.clone(),
718                principal: caller,
719                text,
720                kind: RequestKind::Privileged {
721                    answer: Mutex::new(Some(answer)),
722                    key,
723                },
724                ctx: ctx.clone(),
725                frame: frame.clone(),
726                arrived: now,
727                delivery_deadline: now + state.limits.delivery_budget(),
728                prompt_ready: true.into(),
729                shortcut_audit: shortcut.is_some(),
730            });
731            if let Some(shortcut) = shortcut {
732                self.start_shortcut(&mut state, pending, shortcut);
733            } else {
734                self.enqueue(&mut state, pending);
735            }
736            id
737        };
738        let _withdraw = WithdrawOnDrop {
739            confirms: Arc::clone(self),
740            id,
741        };
742        receive
743            .await
744            .unwrap_or_else(|_| Outcome::unavailable("provider_error"))
745    }
746
747    fn enqueue(self: &Arc<Self>, state: &mut State, pending: Arc<Request>) {
748        if state.prompt.is_none() && state.queue.is_empty() {
749            state.prompt = Some(Active {
750                request: pending,
751                started: None,
752                handle: None,
753                committed: false,
754                withdrawn: None,
755                withdraw_started: false,
756            });
757        } else {
758            state.queue.push_back(pending);
759        }
760        if !state.running {
761            state.running = true;
762            tokio::spawn(Arc::clone(self).run());
763        }
764        self.wake.notify_one();
765    }
766
767    fn commit(
768        self: &Arc<Self>,
769        state: &mut State,
770        request: Arc<Request>,
771        outcome: Outcome,
772        started: Option<Instant>,
773        shown: bool,
774    ) {
775        let now = Instant::now();
776        let wait = started.unwrap_or(now).duration_since(request.arrived);
777        let prompt = started.map(|at| now.duration_since(at)).unwrap_or_default();
778        if outcome.reason == "person" && matches!(request.kind, RequestKind::Module(_)) {
779            state.backoff.insert(
780                (request.module_id.clone(), request.principal.clone()),
781                now + Duration::from_secs(30),
782            );
783        }
784        audit(
785            &request.module_id,
786            if request.shortcut_audit {
787                "Operator shortcut request"
788            } else {
789                &request.summary
790            },
791            &request.principal,
792            &outcome,
793            wait,
794            prompt,
795            shown,
796        );
797        if let RequestKind::Privileged { answer, key } = &request.kind {
798            let person_declined = outcome.reason == "person";
799            if let Some(answer) = answer.lock().unwrap_or_else(|p| p.into_inner()).take() {
800                let _ = answer.send(outcome);
801            }
802            if person_declined {
803                self.count_person_decline(state, key);
804            }
805            return;
806        }
807        let this = Arc::clone(self);
808        #[cfg(test)]
809        {
810            state.deliveries += 1;
811        }
812        tokio::spawn(async move {
813            let answer = outcome.frame(&request.frame);
814            match timeout_at(request.delivery_deadline, request.ctx.egress.send(answer)).await {
815                Ok(Ok(())) => {}
816                result => {
817                    tracing::warn!(target: "subc_daemon::operator_confirm", module_id = request.module_id, ?result, "operator confirm answer dropped")
818                }
819            }
820            let mut state = this.lock();
821            #[cfg(test)]
822            {
823                state.deliveries -= 1;
824            }
825            if state.modules.get(&request.module_id) == Some(&request.id) {
826                state.modules.remove(&request.module_id);
827            }
828        });
829    }
830
831    /// Only a person's decline at the prompt reaches here: withdrawals, timeouts
832    /// and unavailability commit other reasons. The prompt that recorded the
833    /// limit-reaching decline has already ended, so starting the cooldown only
834    /// refuses this caller's requests still waiting in the queue.
835    fn count_person_decline(self: &Arc<Self>, state: &mut State, key: &DeclineKey) {
836        let now = (state.cooldown_clock)();
837        let declines = state.declines.entry(key.clone()).or_default();
838        declines.retain(|at| now.saturating_duration_since(*at) < DECLINE_WINDOW);
839        declines.push_back(now);
840        if declines.len() < DECLINE_LIMIT {
841            return;
842        }
843        // Counting restarts after the cooldown instead of carrying old declines.
844        state.declines.remove(key);
845        state.cooldowns.insert(key.clone(), now + DECLINE_COOLDOWN);
846        tracing::warn!(target: "subc_daemon::operator_confirm", caller = %key,
847            cooldown_secs = DECLINE_COOLDOWN.as_secs(),
848            "privileged requests refused without a prompt after repeated operator declines");
849        let mut index = 0;
850        while index < state.queue.len() {
851            if matches!(&state.queue[index].kind,
852                RequestKind::Privileged { key: queued, .. } if queued == key)
853            {
854                let request = state.queue.remove(index).expect("queue index");
855                self.commit(state, request, Outcome::declined("backoff"), None, false);
856            } else {
857                index += 1;
858            }
859        }
860    }
861
862    fn withdraw_matching(
863        self: &Arc<Self>,
864        predicate: impl Fn(&Request) -> bool,
865        reason: &'static str,
866    ) -> bool {
867        let mut state = self.lock();
868        let mut matched = false;
869        let ids: Vec<_> = state
870            .shortcuts
871            .iter()
872            .filter(|(_, pending)| predicate(&pending.request))
873            .map(|(id, _)| *id)
874            .collect();
875        for id in ids {
876            self.end_shortcut(&mut state, id, Outcome::declined(reason), "withdrawn");
877            matched = true;
878        }
879        let mut index = 0;
880        while index < state.queue.len() {
881            if predicate(&state.queue[index]) {
882                let request = state.queue.remove(index).expect("queue index");
883                self.commit(&mut state, request, Outcome::declined(reason), None, false);
884                matched = true;
885            } else {
886                index += 1;
887            }
888        }
889        if let Some(active) = state
890            .prompt
891            .as_mut()
892            .filter(|a| !a.committed && predicate(&a.request))
893        {
894            active.committed = true;
895            active.withdrawn = Some((Instant::now(), reason));
896            let (request, started, shown) = (
897                Arc::clone(&active.request),
898                active.started,
899                active.handle.is_some(),
900            );
901            self.commit(
902                &mut state,
903                request,
904                Outcome::declined(reason),
905                started,
906                shown,
907            );
908            matched = true;
909        }
910        self.wake.notify_one();
911        matched
912    }
913    pub(crate) fn route_closed(self: &Arc<Self>, key: ModuleRouteKey) {
914        self.withdraw_matching(
915            |r| matches!(r.kind, RequestKind::Module(k) if k == key),
916            "route_closed",
917        );
918    }
919    pub(crate) fn module_closed(self: &Arc<Self>, endpoint: ModuleEndpointId) {
920        self.withdraw_matching(
921            |r| matches!(r.kind, RequestKind::Module(k) if k.endpoint == endpoint),
922            "module_closed",
923        );
924    }
925    pub(crate) fn connection_closed(self: &Arc<Self>, connection: crate::ConnectionId) {
926        self.withdraw_matching(
927            |r| {
928                matches!(r.kind, RequestKind::Privileged { .. })
929                    && r.ctx.connection_id == connection
930            },
931            "caller_cancelled",
932        );
933    }
934    pub(crate) fn cancel(self: &Arc<Self>, connection: crate::ConnectionId, corr: u64) -> bool {
935        self.withdraw_matching(
936            |r| r.ctx.connection_id == connection && r.frame.header.corr == corr,
937            "caller_cancelled",
938        )
939    }
940
941    fn publish(self: &Arc<Self>, id: u64, handle: Arc<dyn OperatorWithdraw>) {
942        let mut state = self.lock();
943        if let Some(active) = state.prompt.as_mut().filter(|a| a.request.id == id) {
944            active.handle = Some(Arc::clone(&handle));
945            if let Some((_, reason)) = active.withdrawn {
946                active.withdraw_started = true;
947                tokio::task::spawn_blocking(move || handle.withdraw(reason));
948            }
949        }
950        self.wake.notify_one();
951    }
952    fn returned(self: &Arc<Self>, id: u64, result: ProviderResult) {
953        let mut state = self.lock();
954        let Some(active) = state.prompt.take() else {
955            return;
956        };
957        assert_eq!(active.request.id, id);
958        // A fast late return can beat the prompt task's wake-up. The decision
959        // still requires a withdraw, even though the result is discarded.
960        if !active.withdraw_started {
961            if let (Some(handle), Some((_, reason))) = (active.handle.as_ref(), active.withdrawn) {
962                let handle = Arc::clone(handle);
963                tokio::task::spawn_blocking(move || handle.withdraw(reason));
964            }
965        }
966        if !active.committed {
967            let outcome = if Instant::now() >= active.request.delivery_deadline {
968                Outcome::unavailable("timeout")
969            } else {
970                Outcome::from_provider(result)
971            };
972            self.commit(
973                &mut state,
974                active.request,
975                outcome,
976                active.started,
977                active.handle.is_some(),
978            );
979        } else {
980            tracing::debug!(target: "subc_daemon::operator_confirm", ?result, "late operator provider result discarded");
981        }
982        state.stuck = false;
983        self.wake.notify_one();
984    }
985
986    async fn run(self: Arc<Self>) {
987        loop {
988            let deadline = {
989                let mut state = self.lock();
990                let now = Instant::now();
991                let mut index = 0;
992                while index < state.queue.len() {
993                    if state.queue[index].arrived + state.limits.queue_wait <= now {
994                        let request = state.queue.remove(index).expect("queue index");
995                        self.commit(
996                            &mut state,
997                            request,
998                            Outcome::unavailable("queue_wait"),
999                            None,
1000                            false,
1001                        );
1002                    } else {
1003                        index += 1;
1004                    }
1005                }
1006                let timeout = state.limits.prompt_timeout;
1007                if let Some(active) = state.prompt.as_mut().filter(|a| {
1008                    !a.committed
1009                        && (a.request.delivery_deadline <= now
1010                            || a.started.is_some_and(|at| at + timeout <= now))
1011                }) {
1012                    active.committed = true;
1013                    active.withdrawn = Some((now, "timeout"));
1014                    let (request, started, shown) = (
1015                        Arc::clone(&active.request),
1016                        active.started,
1017                        active.handle.is_some(),
1018                    );
1019                    self.commit(
1020                        &mut state,
1021                        request,
1022                        Outcome::unavailable("timeout"),
1023                        started,
1024                        shown,
1025                    );
1026                }
1027                let grace = state.limits.stuck_grace;
1028                if !state.stuck
1029                    && state.prompt.as_ref().is_some_and(|a| {
1030                        a.started.is_some() && a.withdrawn.is_some_and(|(at, _)| at + grace <= now)
1031                    })
1032                {
1033                    state.stuck = true;
1034                    tracing::error!(target: "subc_daemon::operator_confirm", "operator provider stuck after withdraw");
1035                    while let Some(request) = state.queue.pop_front() {
1036                        self.commit(
1037                            &mut state,
1038                            request,
1039                            Outcome::unavailable("provider_stuck"),
1040                            None,
1041                            false,
1042                        );
1043                    }
1044                }
1045                if let Some(active) = state
1046                    .prompt
1047                    .as_mut()
1048                    .filter(|a| !a.withdraw_started && a.withdrawn.is_some() && a.handle.is_some())
1049                {
1050                    active.withdraw_started = true;
1051                    let handle = Arc::clone(active.handle.as_ref().expect("handle checked"));
1052                    let reason = active.withdrawn.expect("withdraw checked").1;
1053                    tokio::task::spawn_blocking(move || handle.withdraw(reason));
1054                }
1055                // A reserved prompt cancelled before setup needs no provider call.
1056                if state
1057                    .prompt
1058                    .as_ref()
1059                    .is_some_and(|a| a.started.is_none() && a.committed)
1060                {
1061                    state.prompt = None;
1062                }
1063                if state.prompt.is_none() {
1064                    if let Some(request) = state.queue.pop_front() {
1065                        state.prompt = Some(Active {
1066                            request,
1067                            started: None,
1068                            handle: None,
1069                            committed: false,
1070                            withdrawn: None,
1071                            withdraw_started: false,
1072                        });
1073                    }
1074                }
1075                if let Some(active) = state.prompt.as_mut().filter(|a| {
1076                    a.started.is_none()
1077                        && !a.committed
1078                        && a.request
1079                            .prompt_ready
1080                            .load(std::sync::atomic::Ordering::Acquire)
1081                }) {
1082                    active.started = Some(now);
1083                    let request = Arc::clone(&active.request);
1084                    let provider = Arc::clone(&state.provider);
1085                    let this = Arc::clone(&self);
1086                    tokio::spawn(async move {
1087                        let publish_to = Arc::clone(&this);
1088                        let id = request.id;
1089                        let result = tokio::task::spawn_blocking(move || {
1090                            provider.prompt(
1091                                &request.text,
1092                                Box::new(move |handle| publish_to.publish(id, handle)),
1093                            )
1094                        })
1095                        .await
1096                        .unwrap_or(ProviderResult::Unavailable);
1097                        this.returned(id, result);
1098                    });
1099                }
1100                if state.prompt.is_none() && state.queue.is_empty() {
1101                    state.running = false;
1102                    return;
1103                }
1104                let mut deadline = state
1105                    .queue
1106                    .iter()
1107                    .map(|r| r.arrived + state.limits.queue_wait)
1108                    .min();
1109                if let Some(active) = &state.prompt {
1110                    let next = if !active.committed {
1111                        Some(
1112                            active
1113                                .started
1114                                .map_or(active.request.delivery_deadline, |at| {
1115                                    (at + timeout).min(active.request.delivery_deadline)
1116                                }),
1117                        )
1118                    } else if !state.stuck {
1119                        active.withdrawn.map(|(at, _)| at + grace)
1120                    } else {
1121                        None
1122                    };
1123                    if let Some(next) = next {
1124                        deadline = Some(deadline.map_or(next, |d| d.min(next)));
1125                    }
1126                }
1127                deadline
1128            };
1129            if let Some(at) = deadline {
1130                tokio::select! { _ = self.wake.notified() => {}, _ = tokio::time::sleep_until(at) => {} }
1131            } else {
1132                self.wake.notified().await;
1133            }
1134        }
1135    }
1136}
1137
1138#[cfg(feature = "test-support")]
1139mod test_provider;
1140#[cfg(test)]
1141mod tests;