Skip to main content

subc_daemon/
operator_confirm.rs

1//! Operator confirmation policy and its blocking provider boundary.
2//!
3//! The forwarding lock precedes the state lock. Providers and deliveries never
4//! acquire the forwarding lock, and neither can hold up the policy timers.
5use std::{
6    collections::{HashMap, VecDeque},
7    path::PathBuf,
8    sync::{Arc, Mutex},
9    time::Duration,
10};
11
12use subc_protocol::{
13    error_codes,
14    session::{OperatorConfirmReply, OperatorConfirmRequest},
15    ErrorBody, Flags, FrameType, Principal, Priority,
16};
17use tokio::{
18    sync::{oneshot, Notify},
19    time::{timeout_at, Instant},
20};
21
22use crate::{
23    caller_identity::CallerIdentity,
24    forwarding::{ModuleEndpointId, ModuleRouteKey, RouteBinding},
25    router::RouteCtx,
26    ConnectionId, Frame,
27};
28
29mod shortcut;
30pub(crate) use shortcut::ShortcutContext;
31
32/// Person declines of privileged client requests that start a cooldown when
33/// they all fall within `DECLINE_WINDOW` of the newest one.
34const DECLINE_LIMIT: usize = 3;
35const DECLINE_WINDOW: Duration = Duration::from_secs(10 * 60);
36/// Measured from the decline that started the cooldown. Refusals during the
37/// cooldown are not declines by a person, so they never extend it.
38const DECLINE_COOLDOWN: Duration = Duration::from_secs(10 * 60);
39
40/// Monotonic time for the repeated-decline cooldown. Tests replace it so the
41/// ten-minute window and cooldown can be crossed without waiting.
42pub(crate) type CooldownClock = Arc<dyn Fn() -> Instant + Send + Sync>;
43
44/// Who repeated person declines of privileged requests are counted against.
45/// The executable path keeps a reconnecting program under its cooldown; a
46/// caller whose executable could not be resolved is counted per connection,
47/// because nothing else the daemon owns ties its connections together.
48#[derive(Clone, Debug, PartialEq, Eq, Hash)]
49pub(crate) enum DeclineKey {
50    Executable(PathBuf),
51    Connection(ConnectionId),
52}
53
54impl DeclineKey {
55    pub(crate) fn for_caller(caller: &CallerIdentity, connection: ConnectionId) -> Self {
56        match caller.executable() {
57            Some(executable) => Self::Executable(executable.to_path_buf()),
58            None => Self::Connection(connection),
59        }
60    }
61}
62
63impl std::fmt::Display for DeclineKey {
64    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
65        match self {
66            Self::Executable(path) => write!(f, "{}", path.display()),
67            Self::Connection(connection) => write!(
68                f,
69                "an unidentified local process on connection {}",
70                connection.get()
71            ),
72        }
73    }
74}
75
76/// A blocking provider publishes a withdraw handle as soon as one exists, then
77/// returns only when its prompt has ended. Publishing is allowed to be delayed:
78/// the daemon's timeout also covers setup and a delayed handle is still withdrawn.
79pub trait OperatorProvider: Send + Sync + 'static {
80    /// Whether this provider can attempt strict interactive configuration
81    /// presence. Readiness is not approval; the prompt result is authoritative.
82    fn config_gate_ready(&self) -> bool {
83        false
84    }
85    fn prompt(
86        &self,
87        text: &str,
88        publish: Box<dyn FnOnce(Arc<dyn OperatorWithdraw>) + Send>,
89    ) -> ProviderResult;
90
91    /// Configuration approval requires an interactive response. Providers that
92    /// distinguish policy grants override this instead of reusing legacy approval.
93    fn prompt_gate(
94        &self,
95        text: &str,
96        publish: Box<dyn FnOnce(Arc<dyn OperatorWithdraw>) + Send>,
97    ) -> ProviderResult {
98        self.prompt(text, publish)
99    }
100}
101
102/// Called on a blocking thread, never while a daemon lock is held.
103pub trait OperatorWithdraw: Send + Sync + 'static {
104    fn withdraw(&self, reason: &str);
105}
106
107#[derive(Debug, Clone, PartialEq, Eq)]
108pub enum ProviderResult {
109    Approved,
110    Declined,
111    Unavailable,
112    ProviderError {
113        reason: &'static str,
114    },
115    UnsupportedPlatform,
116    NoPresence,
117    SetupRequired,
118    LocalAuthenticationError {
119        failure: OperatorOsErrorKind,
120        domain: String,
121        code: isize,
122    },
123}
124
125#[derive(Debug, Clone, Copy, PartialEq, Eq)]
126pub enum OperatorOsErrorKind {
127    Declined,
128    NoPresence,
129    ProviderError,
130}
131
132#[derive(Debug, Clone, Copy)]
133pub struct OperatorLimits {
134    pub queue_wait: Duration,
135    pub prompt_timeout: Duration,
136    pub stuck_grace: Duration,
137}
138
139impl Default for OperatorLimits {
140    fn default() -> Self {
141        Self {
142            queue_wait: Duration::from_secs(150),
143            prompt_timeout: Duration::from_secs(120),
144            stuck_grace: Duration::from_secs(10),
145        }
146    }
147}
148
149impl OperatorLimits {
150    fn delivery_budget(self) -> Duration {
151        self.queue_wait + self.prompt_timeout + self.stuck_grace + Duration::from_secs(10)
152    }
153}
154
155struct OsProvider;
156
157#[cfg(any(target_os = "macos", windows, target_os = "linux"))]
158struct OsWithdraw(Arc<dyn subc_presence::Withdraw>);
159#[cfg(any(target_os = "macos", windows, target_os = "linux"))]
160impl OperatorWithdraw for OsWithdraw {
161    fn withdraw(&self, _: &str) {
162        self.0.withdraw();
163    }
164}
165
166impl OperatorProvider for OsProvider {
167    fn config_gate_ready(&self) -> bool {
168        cfg!(any(target_os = "macos", windows, target_os = "linux"))
169    }
170    fn prompt_gate(
171        &self,
172        text: &str,
173        publish: Box<dyn FnOnce(Arc<dyn OperatorWithdraw>) + Send>,
174    ) -> ProviderResult {
175        #[cfg(any(target_os = "macos", windows, target_os = "linux"))]
176        {
177            gate_os_outcome(subc_presence::prompt(
178                text,
179                Box::new(move |handle| publish(Arc::new(OsWithdraw(handle)))),
180            ))
181        }
182        #[cfg(not(any(target_os = "macos", windows, target_os = "linux")))]
183        {
184            let _ = (text, publish);
185            ProviderResult::Unavailable
186        }
187    }
188
189    fn prompt(
190        &self,
191        text: &str,
192        publish: Box<dyn FnOnce(Arc<dyn OperatorWithdraw>) + Send>,
193    ) -> ProviderResult {
194        #[cfg(any(target_os = "macos", windows, target_os = "linux"))]
195        {
196            os_outcome(subc_presence::prompt(
197                text,
198                Box::new(move |handle| publish(Arc::new(OsWithdraw(handle)))),
199            ))
200        }
201        #[cfg(not(any(target_os = "macos", windows, target_os = "linux")))]
202        {
203            let _ = (text, publish);
204            ProviderResult::Unavailable
205        }
206    }
207}
208
209/// Convert the operating-system prompt's result into the daemon's provider
210/// result. Legacy prompts accept an OS policy-rule grant even without an
211/// interactive prompt. Keeping this separate from the platform bindings lets
212/// tests on every host verify that errors and unavailable providers never
213/// grant a write.
214#[cfg(any(target_os = "macos", windows, target_os = "linux"))]
215fn os_outcome(outcome: subc_presence::Outcome) -> ProviderResult {
216    match outcome {
217        subc_presence::Outcome::Approved => ProviderResult::Approved,
218        subc_presence::Outcome::AuthorizedWithoutPrompt => ProviderResult::Approved,
219        subc_presence::Outcome::Declined => ProviderResult::Declined,
220        subc_presence::Outcome::ProviderError => ProviderResult::Unavailable,
221        subc_presence::Outcome::ProviderErrorWithReason { reason } => {
222            ProviderResult::ProviderError { reason }
223        }
224        subc_presence::Outcome::NoPresence => ProviderResult::NoPresence,
225        subc_presence::Outcome::SetupRequired => ProviderResult::SetupRequired,
226        subc_presence::Outcome::UnsupportedPlatform => ProviderResult::UnsupportedPlatform,
227        subc_presence::Outcome::LocalAuthenticationError {
228            failure,
229            domain,
230            code,
231        } => ProviderResult::LocalAuthenticationError {
232            failure: match failure {
233                subc_presence::LocalAuthenticationFailure::Declined => {
234                    OperatorOsErrorKind::Declined
235                }
236                subc_presence::LocalAuthenticationFailure::NoPresence => {
237                    OperatorOsErrorKind::NoPresence
238                }
239                subc_presence::LocalAuthenticationFailure::ProviderError => {
240                    OperatorOsErrorKind::ProviderError
241                }
242            },
243            domain,
244            code,
245        },
246        // Future presence providers must fail closed until explicitly mapped.
247        _ => ProviderResult::Unavailable,
248    }
249}
250
251#[cfg(any(target_os = "macos", windows, target_os = "linux"))]
252fn gate_os_outcome(outcome: subc_presence::Outcome) -> ProviderResult {
253    match outcome {
254        subc_presence::Outcome::AuthorizedWithoutPrompt => ProviderResult::Unavailable,
255        other => os_outcome(other),
256    }
257}
258
259#[cfg(all(test, any(target_os = "macos", windows, target_os = "linux")))]
260mod os_outcome_tests {
261    use super::*;
262
263    #[test]
264    fn config_gate_os_outcomes_require_interactive_confirmation() {
265        use crate::config_gate::PresenceDecision as Gate;
266        use subc_presence::{LocalAuthenticationFailure as Failure, Outcome as Os};
267        for (outcome, expected) in [
268            (Os::Approved, Gate::InteractiveApproval),
269            (Os::AuthorizedWithoutPrompt, Gate::Unavailable),
270            (Os::Declined, Gate::Declined),
271            (Os::NoPresence, Gate::Unavailable),
272            (Os::SetupRequired, Gate::Unavailable),
273            (Os::UnsupportedPlatform, Gate::Unavailable),
274            (Os::ProviderError, Gate::Unavailable),
275            (
276                Os::ProviderErrorWithReason { reason: "no_agent" },
277                Gate::Unavailable,
278            ),
279            (
280                Os::LocalAuthenticationError {
281                    failure: Failure::Declined,
282                    domain: "LA".into(),
283                    code: -2,
284                },
285                Gate::Declined,
286            ),
287            (
288                Os::LocalAuthenticationError {
289                    failure: Failure::NoPresence,
290                    domain: "LA".into(),
291                    code: -4,
292                },
293                Gate::Unavailable,
294            ),
295            (
296                Os::LocalAuthenticationError {
297                    failure: Failure::ProviderError,
298                    domain: "LA".into(),
299                    code: -1,
300                },
301                Gate::Unavailable,
302            ),
303        ] {
304            assert_eq!(
305                Gate::from_operator_outcome(&Outcome::from_provider(gate_os_outcome(outcome))),
306                expected
307            );
308        }
309        assert_eq!(
310            Gate::from_operator_outcome(&Outcome::declined("backoff")),
311            Gate::Declined
312        );
313        assert_eq!(
314            Gate::from_operator_outcome(&Outcome::unavailable("queue_full")),
315            Gate::Unavailable
316        );
317    }
318
319    #[test]
320    fn policy_authorization_still_approves_legacy_prompts() {
321        assert_eq!(
322            os_outcome(subc_presence::Outcome::AuthorizedWithoutPrompt),
323            ProviderResult::Approved
324        );
325    }
326
327    #[test]
328    fn only_an_os_approval_becomes_approved() {
329        use subc_presence::Outcome as Os;
330        assert_eq!(os_outcome(Os::Approved), ProviderResult::Approved);
331        assert_eq!(os_outcome(Os::Declined), ProviderResult::Declined);
332        assert_eq!(os_outcome(Os::ProviderError), ProviderResult::Unavailable);
333        let root_error = os_outcome(Os::ProviderErrorWithReason {
334            reason: "root_user",
335        });
336        assert_eq!(
337            root_error,
338            ProviderResult::ProviderError {
339                reason: "root_user"
340            }
341        );
342        let refusal = Outcome::from_provider(root_error);
343        assert_eq!(refusal.code, error_codes::OPERATOR_PRESENCE_UNAVAILABLE);
344        assert_eq!(refusal.reason, "root_user");
345        assert_eq!(os_outcome(Os::NoPresence), ProviderResult::NoPresence);
346        assert_eq!(os_outcome(Os::SetupRequired), ProviderResult::SetupRequired);
347        let refusal = Outcome::from_provider(ProviderResult::SetupRequired);
348        assert_eq!(refusal.code, error_codes::OPERATOR_PRESENCE_UNAVAILABLE);
349        assert_eq!(refusal.reason, "operator_prompt_setup_required");
350        assert_eq!(
351            os_outcome(Os::UnsupportedPlatform),
352            ProviderResult::UnsupportedPlatform
353        );
354        assert_eq!(
355            os_outcome(Os::LocalAuthenticationError {
356                failure: subc_presence::LocalAuthenticationFailure::NoPresence,
357                domain: "com.apple.LocalAuthentication".into(),
358                code: -4,
359            }),
360            ProviderResult::LocalAuthenticationError {
361                failure: OperatorOsErrorKind::NoPresence,
362                domain: "com.apple.LocalAuthentication".into(),
363                code: -4,
364            }
365        );
366    }
367}
368
369#[derive(Clone, Debug)]
370pub(crate) struct Outcome {
371    pub code: &'static str,
372    pub reason: &'static str,
373    retry_after_ms: Option<u64>,
374    os_error_domain: Option<String>,
375    os_error_code: Option<isize>,
376}
377impl Outcome {
378    pub(crate) fn unavailable(reason: &'static str) -> Self {
379        Self {
380            code: error_codes::OPERATOR_PRESENCE_UNAVAILABLE,
381            reason,
382            retry_after_ms: None,
383            os_error_domain: None,
384            os_error_code: None,
385        }
386    }
387    fn declined(reason: &'static str) -> Self {
388        Self {
389            code: error_codes::OPERATOR_DECLINED,
390            reason,
391            retry_after_ms: None,
392            os_error_domain: None,
393            os_error_code: None,
394        }
395    }
396    fn with_retry_after_ms(mut self, retry_after_ms: u64) -> Self {
397        self.retry_after_ms = Some(retry_after_ms);
398        self
399    }
400    pub(crate) fn refusal(code: &'static str) -> Self {
401        Self {
402            code,
403            reason: "",
404            retry_after_ms: None,
405            os_error_domain: None,
406            os_error_code: None,
407        }
408    }
409    fn from_provider(result: ProviderResult) -> Self {
410        match result {
411            ProviderResult::Approved => Self {
412                code: "confirmed",
413                reason: "",
414                retry_after_ms: None,
415                os_error_domain: None,
416                os_error_code: None,
417            },
418            ProviderResult::Declined => Self::declined("person"),
419            ProviderResult::Unavailable => Self::unavailable("provider_error"),
420            ProviderResult::ProviderError { reason } => Self::unavailable(reason),
421            ProviderResult::UnsupportedPlatform => Self::unavailable("unsupported_platform"),
422            ProviderResult::NoPresence => Self::unavailable("no_presence"),
423            ProviderResult::SetupRequired => Self::unavailable("operator_prompt_setup_required"),
424            ProviderResult::LocalAuthenticationError {
425                failure,
426                domain,
427                code,
428            } => {
429                let mut outcome = match failure {
430                    OperatorOsErrorKind::Declined => Self::declined("person"),
431                    OperatorOsErrorKind::NoPresence => Self::unavailable("no_presence"),
432                    OperatorOsErrorKind::ProviderError => Self::unavailable("provider_error"),
433                };
434                outcome.os_error_domain = Some(domain);
435                outcome.os_error_code = Some(code);
436                outcome
437            }
438        }
439    }
440    pub(crate) fn frame(self, request: &Frame) -> Frame {
441        // A refusal is the daemon's own answer to the client's request, so
442        // its ERROR carries the daemon-origin bit; a confirmation is a plain
443        // response and keeps the control flags.
444        let (ty, body, daemon_origin) = if self.code == "confirmed" {
445            (
446                FrameType::Response,
447                serde_json::to_vec(&OperatorConfirmReply::confirmed()).expect("serializable reply"),
448                false,
449            )
450        } else {
451            let mut error = ErrorBody::new(self.code, "operator confirmation refused");
452            if !self.reason.is_empty() {
453                let detail = match self.retry_after_ms {
454                    Some(retry_after_ms) => {
455                        serde_json::json!({"reason": self.reason, "retry_after_ms": retry_after_ms})
456                    }
457                    None => serde_json::json!({"reason": self.reason}),
458                };
459                error = error.with_detail(detail);
460            }
461            (
462                FrameType::Error,
463                serde_json::to_vec(&error).expect("serializable error"),
464                true,
465            )
466        };
467        let flags = if daemon_origin {
468            crate::router::daemon_error_flags()
469        } else {
470            Flags::new(false, Priority::Passive, false)
471        };
472        Frame::build_with_version(
473            request.header.ver,
474            ty,
475            flags,
476            0,
477            0,
478            request.header.corr,
479            body,
480        )
481        .expect("bounded control reply")
482    }
483}
484
485impl crate::config_gate::PresenceDecision {
486    pub(crate) fn from_operator_outcome(outcome: &Outcome) -> Self {
487        match outcome.code {
488            "confirmed" => Self::InteractiveApproval,
489            error_codes::OPERATOR_DECLINED => Self::Declined,
490            _ => Self::Unavailable,
491        }
492    }
493}
494
495fn retry_after_ms(remaining: Duration) -> u64 {
496    let whole_milliseconds = remaining.as_millis();
497    let rounded_milliseconds =
498        whole_milliseconds + u128::from(!remaining.subsec_nanos().is_multiple_of(1_000_000));
499    u64::try_from(rounded_milliseconds).expect("operator confirmation backoff fits in u64")
500}
501
502pub(crate) fn audit(
503    module_id: &str,
504    summary: &str,
505    principal: &str,
506    outcome: &Outcome,
507    wait: Duration,
508    prompt: Duration,
509    prompt_shown: bool,
510) {
511    tracing::info!(target: "subc_daemon::operator_confirm", module_id, summary = ?summary,
512        principal, outcome = outcome.code, reason = outcome.reason,
513        os_error_domain = ?outcome.os_error_domain, os_error_code = ?outcome.os_error_code,
514        wait_ms = wait.as_millis() as u64, prompt_ms = prompt.as_millis() as u64,
515        prompt_shown, "operator_confirm_audit");
516}
517
518fn principal_label(principal: &Principal) -> String {
519    match principal {
520        Principal::Direct => "direct".into(),
521        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
522        Principal::Unverified => String::new(),
523    }
524}
525
526pub(crate) fn valid_summary(summary: &str) -> bool {
527    let count = summary.chars().count();
528    (1..=200).contains(&count)
529        && !summary.starts_with(char::is_whitespace)
530        && !summary.ends_with(char::is_whitespace)
531        && !summary.chars().any(|c| {
532            c.is_control()
533                || matches!(c,
534            '\u{200b}'..='\u{200f}' | '\u{202a}'..='\u{202e}' | '\u{2028}' | '\u{2029}' |
535            '\u{2060}'..='\u{2069}' | '\u{feff}')
536        })
537}
538
539enum RequestKind {
540    Module(ModuleRouteKey),
541    Privileged {
542        answer: Mutex<Option<oneshot::Sender<Outcome>>>,
543        key: DeclineKey,
544    },
545}
546
547struct Request {
548    gate_presence: bool,
549    id: u64,
550    module_id: String,
551    summary: String,
552    principal: String,
553    text: String,
554    kind: RequestKind,
555    ctx: RouteCtx,
556    frame: Frame,
557    arrived: Instant,
558    delivery_deadline: Instant,
559    // A handoff reserves its queue position before its outcome append finishes.
560    prompt_ready: std::sync::atomic::AtomicBool,
561    shortcut_audit: bool,
562}
563struct Active {
564    request: Arc<Request>,
565    started: Option<Instant>,
566    handle: Option<Arc<dyn OperatorWithdraw>>,
567    committed: bool,
568    withdrawn: Option<(Instant, &'static str)>,
569    withdraw_started: bool,
570}
571struct State {
572    provider: Arc<dyn OperatorProvider>,
573    limits: OperatorLimits,
574    queue: VecDeque<Arc<Request>>,
575    modules: HashMap<String, u64>,
576    backoff: HashMap<(String, String), Instant>,
577    cooldown_clock: CooldownClock,
578    /// Recent person declines of privileged requests, oldest first.
579    declines: HashMap<DeclineKey, VecDeque<Instant>>,
580    /// When each privileged caller's cooldown ends.
581    cooldowns: HashMap<DeclineKey, Instant>,
582    prompt: Option<Active>,
583    stuck: bool,
584    running: bool,
585    next_id: u64,
586    shortcuts: HashMap<u64, shortcut::Pending>,
587    #[cfg(test)]
588    deliveries: usize,
589}
590
591pub(crate) struct OperatorConfirms {
592    state: Mutex<State>,
593    wake: Notify,
594}
595
596struct WithdrawOnDrop {
597    confirms: Arc<OperatorConfirms>,
598    id: u64,
599}
600
601impl Drop for WithdrawOnDrop {
602    fn drop(&mut self) {
603        self.confirms
604            .withdraw_matching(|request| request.id == self.id, "caller_cancelled");
605    }
606}
607impl std::fmt::Debug for OperatorConfirms {
608    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
609        f.debug_struct("OperatorConfirms").finish_non_exhaustive()
610    }
611}
612impl Default for OperatorConfirms {
613    fn default() -> Self {
614        Self {
615            state: Mutex::new(State {
616                provider: Arc::new(OsProvider),
617                limits: OperatorLimits::default(),
618                queue: VecDeque::new(),
619                modules: HashMap::new(),
620                backoff: HashMap::new(),
621                cooldown_clock: Arc::new(Instant::now),
622                declines: HashMap::new(),
623                cooldowns: HashMap::new(),
624                prompt: None,
625                stuck: false,
626                running: false,
627                next_id: 0,
628                shortcuts: HashMap::new(),
629                #[cfg(test)]
630                deliveries: 0,
631            }),
632            wake: Notify::new(),
633        }
634    }
635}
636impl OperatorConfirms {
637    pub(crate) fn config_gate_ready(&self) -> bool {
638        let state = self.lock();
639        !state.stuck && state.provider.config_gate_ready()
640    }
641    fn lock(&self) -> std::sync::MutexGuard<'_, State> {
642        self.state.lock().unwrap_or_else(|p| p.into_inner())
643    }
644    pub(crate) fn configure(&self, provider: Arc<dyn OperatorProvider>, limits: OperatorLimits) {
645        let mut state = self.lock();
646        assert!(
647            !state.running && state.modules.is_empty(),
648            "configure before serving"
649        );
650        state.provider = provider;
651        state.limits = limits;
652    }
653    pub(crate) fn configure_bootstrap(&self, config: &crate::bootstrap::BootstrapConfig) {
654        let provider: Arc<dyn OperatorProvider> = Arc::new(OsProvider);
655        #[cfg(feature = "test-support")]
656        let provider = test_provider::select(
657            provider,
658            config.operator_script.as_deref(),
659            config.operator_events.as_deref(),
660            std::env::current_exe().ok().as_deref(),
661        );
662        self.configure(
663            provider,
664            OperatorLimits {
665                queue_wait: config.operator_queue_wait,
666                prompt_timeout: config.operator_timeout,
667                stuck_grace: config.operator_stuck_grace,
668            },
669        );
670    }
671
672    #[cfg(feature = "test-support")]
673    pub(crate) fn configure_script(
674        &self,
675        script: &std::path::Path,
676        events: Option<&std::path::Path>,
677        limits: Option<OperatorLimits>,
678    ) {
679        let provider = test_provider::select(
680            Arc::new(OsProvider),
681            Some(script),
682            events,
683            test_provider::current_executable().as_deref(),
684        );
685        self.configure(provider, limits.unwrap_or_default());
686    }
687
688    /// Called while the forwarding read lock still protects this binding. All
689    /// registration/nonce lookups must have finished before entering this method.
690    #[cfg(test)]
691    pub(crate) fn admit(
692        self: &Arc<Self>,
693        ctx: &RouteCtx,
694        frame: Frame,
695        module_id: String,
696        nonce_proven: bool,
697        request: OperatorConfirmRequest,
698        binding: Option<&RouteBinding>,
699    ) -> Vec<Frame> {
700        self.admit_with_shortcut(ctx, frame, module_id, nonce_proven, request, binding, None)
701    }
702
703    #[allow(clippy::too_many_arguments)]
704    pub(crate) fn admit_with_shortcut(
705        self: &Arc<Self>,
706        ctx: &RouteCtx,
707        frame: Frame,
708        module_id: String,
709        nonce_proven: bool,
710        request: OperatorConfirmRequest,
711        binding: Option<&RouteBinding>,
712        shortcut: Option<ShortcutContext>,
713    ) -> Vec<Frame> {
714        let principal = binding
715            .map(|b| principal_label(&b.principal))
716            .unwrap_or_default();
717        let mut state = self.lock();
718        let now = Instant::now();
719        state.backoff.retain(|_, until| *until > now);
720        let refusal = if !nonce_proven
721            || binding.is_none_or(|b| matches!(b.principal, Principal::Unverified))
722        {
723            Some(Outcome::refusal(
724                error_codes::OPERATOR_REQUEST_NOT_PERMITTED,
725            ))
726        } else if !valid_summary(&request.summary) {
727            Some(Outcome::refusal(error_codes::OPERATOR_SUMMARY_INVALID))
728        } else if let Some(until) = state
729            .backoff
730            .get(&(module_id.clone(), principal.clone()))
731            .copied()
732            .filter(|_| shortcut.is_none())
733        {
734            Some(Outcome::declined("backoff").with_retry_after_ms(retry_after_ms(until - now)))
735        } else if shortcut.is_none() && state.stuck {
736            Some(Outcome::unavailable("provider_stuck"))
737        } else if state.modules.contains_key(&module_id) {
738            Some(Outcome::unavailable("module_limit"))
739        } else if shortcut.is_none() && state.queue.len() >= 4 {
740            Some(Outcome::unavailable("queue_full"))
741        } else {
742            None
743        };
744        if let Some(outcome) = refusal {
745            audit(
746                &module_id,
747                &request.summary,
748                &principal,
749                &outcome,
750                Duration::ZERO,
751                Duration::ZERO,
752                false,
753            );
754            return vec![outcome.frame(&frame)];
755        }
756        let binding = binding.expect("permission checked");
757        let opener = match &binding.principal {
758            Principal::Direct => "a local program",
759            Principal::Reserved { module_id } => module_id,
760            Principal::Unverified => unreachable!(),
761        };
762        state.next_id += 1;
763        let pending = Arc::new(Request {
764            id: state.next_id,
765            text: format!(
766                "approve a request from {module_id}: {} (requested by {opener})",
767                request.summary
768            ),
769            module_id,
770            summary: request.summary,
771            principal,
772            gate_presence: false,
773            kind: RequestKind::Module(ModuleRouteKey {
774                endpoint: binding.module_endpoint,
775                channel: binding.module_channel,
776            }),
777            ctx: ctx.clone(),
778            frame,
779            arrived: now,
780            delivery_deadline: now + state.limits.delivery_budget(),
781            prompt_ready: true.into(),
782            shortcut_audit: shortcut.is_some(),
783        });
784        state.modules.insert(pending.module_id.clone(), pending.id);
785        if let Some(shortcut) = shortcut {
786            self.start_shortcut(&mut state, pending, shortcut);
787        } else {
788            self.enqueue(&mut state, pending);
789        }
790        Vec::new()
791    }
792
793    /// Client-control approval shares the module confirmation queue and timers,
794    /// but returns a decision to dispatch instead of sending an operator reply.
795    /// A caller in its repeated-decline cooldown is refused before anything else
796    /// is checked, without a prompt.
797    #[allow(clippy::too_many_arguments)]
798    pub(crate) async fn approve_privileged_with_shortcut(
799        self: &Arc<Self>,
800        ctx: &RouteCtx,
801        frame: &Frame,
802        operation: &str,
803        text: String,
804        caller: String,
805        key: DeclineKey,
806        shortcut: Option<ShortcutContext>,
807    ) -> Outcome {
808        self.approve_client(ctx, frame, operation, text, caller, key, shortcut, false)
809            .await
810    }
811
812    pub(crate) async fn approve_config_gate(
813        self: &Arc<Self>,
814        ctx: &RouteCtx,
815        frame: &Frame,
816        text: String,
817        caller: String,
818        key: DeclineKey,
819    ) -> Outcome {
820        // Never accept an operator-app shortcut as config review.
821        self.approve_client(ctx, frame, "config.apply", text, caller, key, None, true)
822            .await
823    }
824
825    #[allow(clippy::too_many_arguments)]
826    async fn approve_client(
827        self: &Arc<Self>,
828        ctx: &RouteCtx,
829        frame: &Frame,
830        operation: &str,
831        text: String,
832        caller: String,
833        key: DeclineKey,
834        shortcut: Option<ShortcutContext>,
835        gate_presence: bool,
836    ) -> Outcome {
837        let (answer, receive) = oneshot::channel();
838        let id = {
839            let mut state = self.lock();
840            let now = (state.cooldown_clock)();
841            state.cooldowns.retain(|_, until| *until > now);
842            state.declines.retain(|_, declines| {
843                declines.retain(|at| now.saturating_duration_since(*at) < DECLINE_WINDOW);
844                !declines.is_empty()
845            });
846            let refusal = if state.cooldowns.contains_key(&key) {
847                Some(Outcome::declined("backoff"))
848            } else if shortcut.is_none() && state.stuck {
849                Some(Outcome::unavailable("provider_stuck"))
850            } else if shortcut.is_none() && state.queue.len() >= 4 {
851                Some(Outcome::unavailable("queue_full"))
852            } else {
853                None
854            };
855            if let Some(outcome) = refusal {
856                audit(
857                    operation,
858                    &text,
859                    &caller,
860                    &outcome,
861                    Duration::ZERO,
862                    Duration::ZERO,
863                    false,
864                );
865                return outcome;
866            }
867            state.next_id += 1;
868            let id = state.next_id;
869            let now = Instant::now();
870            let pending = Arc::new(Request {
871                id,
872                module_id: operation.into(),
873                summary: text.clone(),
874                principal: caller,
875                text,
876                gate_presence,
877                kind: RequestKind::Privileged {
878                    answer: Mutex::new(Some(answer)),
879                    key,
880                },
881                ctx: ctx.clone(),
882                frame: frame.clone(),
883                arrived: now,
884                delivery_deadline: now + state.limits.delivery_budget(),
885                prompt_ready: true.into(),
886                shortcut_audit: shortcut.is_some(),
887            });
888            if let Some(shortcut) = shortcut {
889                self.start_shortcut(&mut state, pending, shortcut);
890            } else {
891                self.enqueue(&mut state, pending);
892            }
893            id
894        };
895        let _withdraw = WithdrawOnDrop {
896            confirms: Arc::clone(self),
897            id,
898        };
899        receive
900            .await
901            .unwrap_or_else(|_| Outcome::unavailable("provider_error"))
902    }
903
904    fn enqueue(self: &Arc<Self>, state: &mut State, pending: Arc<Request>) {
905        if state.prompt.is_none() && state.queue.is_empty() {
906            state.prompt = Some(Active {
907                request: pending,
908                started: None,
909                handle: None,
910                committed: false,
911                withdrawn: None,
912                withdraw_started: false,
913            });
914        } else {
915            state.queue.push_back(pending);
916        }
917        if !state.running {
918            state.running = true;
919            tokio::spawn(Arc::clone(self).run());
920        }
921        self.wake.notify_one();
922    }
923
924    fn commit(
925        self: &Arc<Self>,
926        state: &mut State,
927        request: Arc<Request>,
928        outcome: Outcome,
929        started: Option<Instant>,
930        shown: bool,
931    ) {
932        let now = Instant::now();
933        let wait = started.unwrap_or(now).duration_since(request.arrived);
934        let prompt = started.map(|at| now.duration_since(at)).unwrap_or_default();
935        if outcome.reason == "person" && matches!(request.kind, RequestKind::Module(_)) {
936            state.backoff.insert(
937                (request.module_id.clone(), request.principal.clone()),
938                now + Duration::from_secs(30),
939            );
940        }
941        audit(
942            &request.module_id,
943            if request.shortcut_audit {
944                "Operator shortcut request"
945            } else {
946                &request.summary
947            },
948            &request.principal,
949            &outcome,
950            wait,
951            prompt,
952            shown,
953        );
954        if let RequestKind::Privileged { answer, key } = &request.kind {
955            let person_declined = outcome.reason == "person";
956            if let Some(answer) = answer.lock().unwrap_or_else(|p| p.into_inner()).take() {
957                let _ = answer.send(outcome);
958            }
959            if person_declined {
960                self.count_person_decline(state, key);
961            }
962            return;
963        }
964        let this = Arc::clone(self);
965        #[cfg(test)]
966        {
967            state.deliveries += 1;
968        }
969        tokio::spawn(async move {
970            let answer = outcome.frame(&request.frame);
971            match timeout_at(request.delivery_deadline, request.ctx.egress.send(answer)).await {
972                Ok(Ok(())) => {}
973                result => {
974                    tracing::warn!(target: "subc_daemon::operator_confirm", module_id = request.module_id, ?result, "operator confirm answer dropped")
975                }
976            }
977            let mut state = this.lock();
978            #[cfg(test)]
979            {
980                state.deliveries -= 1;
981            }
982            if state.modules.get(&request.module_id) == Some(&request.id) {
983                state.modules.remove(&request.module_id);
984            }
985        });
986    }
987
988    /// Only a person's decline at the prompt reaches here: withdrawals, timeouts
989    /// and unavailability commit other reasons. The prompt that recorded the
990    /// limit-reaching decline has already ended, so starting the cooldown only
991    /// refuses this caller's requests still waiting in the queue.
992    fn count_person_decline(self: &Arc<Self>, state: &mut State, key: &DeclineKey) {
993        let now = (state.cooldown_clock)();
994        let declines = state.declines.entry(key.clone()).or_default();
995        declines.retain(|at| now.saturating_duration_since(*at) < DECLINE_WINDOW);
996        declines.push_back(now);
997        if declines.len() < DECLINE_LIMIT {
998            return;
999        }
1000        // Counting restarts after the cooldown instead of carrying old declines.
1001        state.declines.remove(key);
1002        state.cooldowns.insert(key.clone(), now + DECLINE_COOLDOWN);
1003        tracing::warn!(target: "subc_daemon::operator_confirm", caller = %key,
1004            cooldown_secs = DECLINE_COOLDOWN.as_secs(),
1005            "privileged requests refused without a prompt after repeated operator declines");
1006        let mut index = 0;
1007        while index < state.queue.len() {
1008            if matches!(&state.queue[index].kind,
1009                RequestKind::Privileged { key: queued, .. } if queued == key)
1010            {
1011                let request = state.queue.remove(index).expect("queue index");
1012                self.commit(state, request, Outcome::declined("backoff"), None, false);
1013            } else {
1014                index += 1;
1015            }
1016        }
1017    }
1018
1019    fn withdraw_matching(
1020        self: &Arc<Self>,
1021        predicate: impl Fn(&Request) -> bool,
1022        reason: &'static str,
1023    ) -> bool {
1024        let mut state = self.lock();
1025        let mut matched = false;
1026        let ids: Vec<_> = state
1027            .shortcuts
1028            .iter()
1029            .filter(|(_, pending)| predicate(&pending.request))
1030            .map(|(id, _)| *id)
1031            .collect();
1032        for id in ids {
1033            self.end_shortcut(&mut state, id, Outcome::declined(reason), "withdrawn");
1034            matched = true;
1035        }
1036        let mut index = 0;
1037        while index < state.queue.len() {
1038            if predicate(&state.queue[index]) {
1039                let request = state.queue.remove(index).expect("queue index");
1040                self.commit(&mut state, request, Outcome::declined(reason), None, false);
1041                matched = true;
1042            } else {
1043                index += 1;
1044            }
1045        }
1046        if let Some(active) = state
1047            .prompt
1048            .as_mut()
1049            .filter(|a| !a.committed && predicate(&a.request))
1050        {
1051            active.committed = true;
1052            active.withdrawn = Some((Instant::now(), reason));
1053            let (request, started, shown) = (
1054                Arc::clone(&active.request),
1055                active.started,
1056                active.handle.is_some(),
1057            );
1058            self.commit(
1059                &mut state,
1060                request,
1061                Outcome::declined(reason),
1062                started,
1063                shown,
1064            );
1065            matched = true;
1066        }
1067        self.wake.notify_one();
1068        matched
1069    }
1070    pub(crate) fn route_closed(self: &Arc<Self>, key: ModuleRouteKey) {
1071        self.withdraw_matching(
1072            |r| matches!(r.kind, RequestKind::Module(k) if k == key),
1073            "route_closed",
1074        );
1075    }
1076    pub(crate) fn module_closed(self: &Arc<Self>, endpoint: ModuleEndpointId) {
1077        self.withdraw_matching(
1078            |r| matches!(r.kind, RequestKind::Module(k) if k.endpoint == endpoint),
1079            "module_closed",
1080        );
1081    }
1082    pub(crate) fn connection_closed(self: &Arc<Self>, connection: crate::ConnectionId) {
1083        self.withdraw_matching(
1084            |r| {
1085                matches!(r.kind, RequestKind::Privileged { .. })
1086                    && r.ctx.connection_id == connection
1087            },
1088            "caller_cancelled",
1089        );
1090    }
1091    pub(crate) fn cancel(self: &Arc<Self>, connection: crate::ConnectionId, corr: u64) -> bool {
1092        self.withdraw_matching(
1093            |r| r.ctx.connection_id == connection && r.frame.header.corr == corr,
1094            "caller_cancelled",
1095        )
1096    }
1097
1098    fn publish(self: &Arc<Self>, id: u64, handle: Arc<dyn OperatorWithdraw>) {
1099        let mut state = self.lock();
1100        if let Some(active) = state.prompt.as_mut().filter(|a| a.request.id == id) {
1101            active.handle = Some(Arc::clone(&handle));
1102            if let Some((_, reason)) = active.withdrawn {
1103                active.withdraw_started = true;
1104                tokio::task::spawn_blocking(move || handle.withdraw(reason));
1105            }
1106        }
1107        self.wake.notify_one();
1108    }
1109    fn returned(self: &Arc<Self>, id: u64, result: ProviderResult) {
1110        let mut state = self.lock();
1111        let Some(active) = state.prompt.take() else {
1112            return;
1113        };
1114        assert_eq!(active.request.id, id);
1115        // A fast late return can beat the prompt task's wake-up. The decision
1116        // still requires a withdraw, even though the result is discarded.
1117        if !active.withdraw_started {
1118            if let (Some(handle), Some((_, reason))) = (active.handle.as_ref(), active.withdrawn) {
1119                let handle = Arc::clone(handle);
1120                tokio::task::spawn_blocking(move || handle.withdraw(reason));
1121            }
1122        }
1123        if !active.committed {
1124            let outcome = if Instant::now() >= active.request.delivery_deadline {
1125                Outcome::unavailable("timeout")
1126            } else {
1127                Outcome::from_provider(result)
1128            };
1129            self.commit(
1130                &mut state,
1131                active.request,
1132                outcome,
1133                active.started,
1134                active.handle.is_some(),
1135            );
1136        } else {
1137            tracing::debug!(target: "subc_daemon::operator_confirm", ?result, "late operator provider result discarded");
1138        }
1139        state.stuck = false;
1140        self.wake.notify_one();
1141    }
1142
1143    async fn run(self: Arc<Self>) {
1144        loop {
1145            let deadline = {
1146                let mut state = self.lock();
1147                let now = Instant::now();
1148                let mut index = 0;
1149                while index < state.queue.len() {
1150                    if state.queue[index].arrived + state.limits.queue_wait <= now {
1151                        let request = state.queue.remove(index).expect("queue index");
1152                        self.commit(
1153                            &mut state,
1154                            request,
1155                            Outcome::unavailable("queue_wait"),
1156                            None,
1157                            false,
1158                        );
1159                    } else {
1160                        index += 1;
1161                    }
1162                }
1163                let timeout = state.limits.prompt_timeout;
1164                if let Some(active) = state.prompt.as_mut().filter(|a| {
1165                    !a.committed
1166                        && (a.request.delivery_deadline <= now
1167                            || a.started.is_some_and(|at| at + timeout <= now))
1168                }) {
1169                    active.committed = true;
1170                    active.withdrawn = Some((now, "timeout"));
1171                    let (request, started, shown) = (
1172                        Arc::clone(&active.request),
1173                        active.started,
1174                        active.handle.is_some(),
1175                    );
1176                    self.commit(
1177                        &mut state,
1178                        request,
1179                        Outcome::unavailable("timeout"),
1180                        started,
1181                        shown,
1182                    );
1183                }
1184                let grace = state.limits.stuck_grace;
1185                if !state.stuck
1186                    && state.prompt.as_ref().is_some_and(|a| {
1187                        a.started.is_some() && a.withdrawn.is_some_and(|(at, _)| at + grace <= now)
1188                    })
1189                {
1190                    state.stuck = true;
1191                    tracing::error!(target: "subc_daemon::operator_confirm", "operator provider stuck after withdraw");
1192                    while let Some(request) = state.queue.pop_front() {
1193                        self.commit(
1194                            &mut state,
1195                            request,
1196                            Outcome::unavailable("provider_stuck"),
1197                            None,
1198                            false,
1199                        );
1200                    }
1201                }
1202                if let Some(active) = state
1203                    .prompt
1204                    .as_mut()
1205                    .filter(|a| !a.withdraw_started && a.withdrawn.is_some() && a.handle.is_some())
1206                {
1207                    active.withdraw_started = true;
1208                    let handle = Arc::clone(active.handle.as_ref().expect("handle checked"));
1209                    let reason = active.withdrawn.expect("withdraw checked").1;
1210                    tokio::task::spawn_blocking(move || handle.withdraw(reason));
1211                }
1212                // A reserved prompt cancelled before setup needs no provider call.
1213                if state
1214                    .prompt
1215                    .as_ref()
1216                    .is_some_and(|a| a.started.is_none() && a.committed)
1217                {
1218                    state.prompt = None;
1219                }
1220                if state.prompt.is_none() {
1221                    if let Some(request) = state.queue.pop_front() {
1222                        state.prompt = Some(Active {
1223                            request,
1224                            started: None,
1225                            handle: None,
1226                            committed: false,
1227                            withdrawn: None,
1228                            withdraw_started: false,
1229                        });
1230                    }
1231                }
1232                if let Some(active) = state.prompt.as_mut().filter(|a| {
1233                    a.started.is_none()
1234                        && !a.committed
1235                        && a.request
1236                            .prompt_ready
1237                            .load(std::sync::atomic::Ordering::Acquire)
1238                }) {
1239                    active.started = Some(now);
1240                    let request = Arc::clone(&active.request);
1241                    let provider = Arc::clone(&state.provider);
1242                    let this = Arc::clone(&self);
1243                    tokio::spawn(async move {
1244                        let publish_to = Arc::clone(&this);
1245                        let id = request.id;
1246                        let result = tokio::task::spawn_blocking(move || {
1247                            let publish = Box::new(move |handle| publish_to.publish(id, handle));
1248                            if request.gate_presence {
1249                                provider.prompt_gate(&request.text, publish)
1250                            } else {
1251                                provider.prompt(&request.text, publish)
1252                            }
1253                        })
1254                        .await
1255                        .unwrap_or(ProviderResult::Unavailable);
1256                        this.returned(id, result);
1257                    });
1258                }
1259                if state.prompt.is_none() && state.queue.is_empty() {
1260                    state.running = false;
1261                    return;
1262                }
1263                let mut deadline = state
1264                    .queue
1265                    .iter()
1266                    .map(|r| r.arrived + state.limits.queue_wait)
1267                    .min();
1268                if let Some(active) = &state.prompt {
1269                    let next = if !active.committed {
1270                        Some(
1271                            active
1272                                .started
1273                                .map_or(active.request.delivery_deadline, |at| {
1274                                    (at + timeout).min(active.request.delivery_deadline)
1275                                }),
1276                        )
1277                    } else if !state.stuck {
1278                        active.withdrawn.map(|(at, _)| at + grace)
1279                    } else {
1280                        None
1281                    };
1282                    if let Some(next) = next {
1283                        deadline = Some(deadline.map_or(next, |d| d.min(next)));
1284                    }
1285                }
1286                deadline
1287            };
1288            if let Some(at) = deadline {
1289                tokio::select! { _ = self.wake.notified() => {}, _ = tokio::time::sleep_until(at) => {} }
1290            } else {
1291                self.wake.notified().await;
1292            }
1293        }
1294    }
1295}
1296
1297#[cfg(feature = "test-support")]
1298mod test_provider;
1299#[cfg(test)]
1300mod tests;