Skip to main content

open_net/api/ws/
request_operation.rs

1//! Logical requests rebuild opaque protocol messages for every bounded attempt.
2use super::cancellation::{lock_recover, CancelHookGuard};
3use super::*;
4use crate::api::retry::{RetryDecision, RetryEvent, RetryReport, RetrySetting, RetryStopReason};
5use crate::error::{ErrorKind, ErrorStage};
6use crate::module::ws_client::native_pending::PendingLease;
7use crate::module::ws_client::timeout_budget::TimeoutBudget;
8use crate::module::ws_client::{
9    native_task_observer::TaskEventReservation, session_runtime::SessionRuntime,
10};
11use crate::retry_core::RetryState;
12use crate::{EnqueueError, Metadata, NetError, Result};
13use std::fmt;
14use std::future::Future;
15use std::sync::{Arc, Mutex};
16use std::time::{Duration, Instant};
17use tokio::sync::oneshot;
18use tokio_util::sync::CancellationToken;
19
20type AttemptFactory = dyn Fn(&RequestAttemptContext) -> Result<Message> + Send + Sync;
21
22/// Immutable identity supplied to a synchronous message factory. The application
23/// must encode the fresh wire ID and stable key in its own protocol.
24#[derive(Clone)]
25pub struct RequestAttemptContext {
26    operation_id: OperationId,
27    attempt: u32,
28    request_id: RequestId,
29    idempotency_key: Arc<str>,
30}
31impl RequestAttemptContext {
32    pub fn operation_id(&self) -> OperationId {
33        self.operation_id
34    }
35    pub fn attempt(&self) -> u32 {
36        self.attempt
37    }
38    pub fn request_id(&self) -> &RequestId {
39        &self.request_id
40    }
41    pub fn idempotency_key(&self) -> &str {
42        &self.idempotency_key
43    }
44}
45impl fmt::Debug for RequestAttemptContext {
46    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
47        f.debug_struct("RequestAttemptContext")
48            .field("operation_id", &self.operation_id)
49            .field("attempt", &self.attempt)
50            .field("request_id", &self.request_id)
51            .field("idempotency_key_bytes", &self.idempotency_key.len())
52            .finish()
53    }
54}
55
56/// Limits for one logical request. Total time starts when enqueue/send is first
57/// polled, and includes admission, construction, reconnection, backoff and all attempts.
58#[derive(Clone)]
59pub struct WsRequestOperationOptions {
60    pub priority: Priority,
61    pub admission_timeout: Option<Duration>,
62    pub write_timeout: Duration,
63    pub response_timeout: Duration,
64    pub response_timeout_origin: ResponseTimeoutOrigin,
65    pub attempt_timeout: Option<Duration>,
66    pub total_timeout: Option<Duration>,
67    pub deadline: Option<Instant>,
68    pub disconnected: DisconnectedPolicy,
69    pub cancellation: Option<CancellationGroup>,
70    pub metadata: Metadata,
71    pub retry_setting: RetrySetting<WsRequestRetryPolicy>,
72}
73impl fmt::Debug for WsRequestOperationOptions {
74    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
75        f.debug_struct("WsRequestOperationOptions")
76            .field("priority", &self.priority)
77            .field("admission_timeout", &self.admission_timeout)
78            .field("write_timeout", &self.write_timeout)
79            .field("response_timeout", &self.response_timeout)
80            .field("response_timeout_origin", &self.response_timeout_origin)
81            .field("attempt_timeout", &self.attempt_timeout)
82            .field("total_timeout", &self.total_timeout)
83            .field("deadline", &self.deadline)
84            .field("disconnected", &self.disconnected)
85            .field("cancellation", &self.cancellation)
86            .field("metadata_entries", &self.metadata.len())
87            .field("retry_setting", &self.retry_setting)
88            .finish()
89    }
90}
91impl Default for WsRequestOperationOptions {
92    fn default() -> Self {
93        Self {
94            priority: Priority::Normal,
95            admission_timeout: None,
96            write_timeout: Duration::from_secs(10),
97            response_timeout: Duration::from_secs(30),
98            response_timeout_origin: ResponseTimeoutOrigin::Written,
99            attempt_timeout: None,
100            total_timeout: None,
101            deadline: None,
102            disconnected: DisconnectedPolicy::Reject,
103            cancellation: None,
104            metadata: Metadata::new(),
105            retry_setting: RetrySetting::Inherit,
106        }
107    }
108}
109impl WsRequestOperationOptions {
110    pub fn validate(&self) -> Result<()> {
111        for (name, duration) in [
112            ("admission_timeout", self.admission_timeout),
113            ("attempt_timeout", self.attempt_timeout),
114            ("total_timeout", self.total_timeout),
115            ("write_timeout", Some(self.write_timeout)),
116            ("response_timeout", Some(self.response_timeout)),
117        ] {
118            if let Some(duration) = duration {
119                super::config::deadline(duration, name, true)?;
120            }
121        }
122        validate_metadata(&self.metadata)
123    }
124}
125
126/// Builder retaining the protocol factory until its operation terminates.
127#[must_use]
128pub struct WsRequestOperationBuilder {
129    client: RequestClient,
130    key: Arc<str>,
131    factory: Arc<AttemptFactory>,
132    options: WsRequestOperationOptions,
133    started: Option<Instant>,
134}
135impl fmt::Debug for WsRequestOperationBuilder {
136    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
137        f.debug_struct("WsRequestOperationBuilder")
138            .field("client", &self.client)
139            .field("key_bytes", &self.key.len())
140            .field("options", &self.options)
141            .finish_non_exhaustive()
142    }
143}
144impl RequestClient {
145    /// Creates a logical operation. The factory must encode both context IDs;
146    /// response routing must extract the current attempt's wire request ID.
147    pub fn request_with_factory<F>(
148        &self,
149        key: impl Into<String>,
150        factory: F,
151    ) -> Result<WsRequestOperationBuilder>
152    where
153        F: Fn(&RequestAttemptContext) -> Result<Message> + Send + Sync + 'static,
154    {
155        let key = key.into();
156        if key.len() > 1024 || key.trim().is_empty() {
157            return Err(NetError::input(
158                "idempotency_key",
159                "must contain a non-whitespace character and be at most 1024 UTF-8 bytes",
160            ));
161        }
162        Ok(WsRequestOperationBuilder {
163            client: self.clone(),
164            key: Arc::from(key),
165            factory: Arc::new(factory),
166            options: WsRequestOperationOptions::default(),
167            started: None,
168        })
169    }
170}
171impl WsRequestOperationBuilder {
172    /// Replace the operation policy without merging fields or resetting submission time.
173    pub fn retry_setting(mut self, setting: RetrySetting<WsRequestRetryPolicy>) -> Self {
174        self.options.retry_setting = setting;
175        self
176    }
177    pub fn options(mut self, options: WsRequestOperationOptions) -> Self {
178        self.options = options;
179        self.started = None;
180        self
181    }
182    pub async fn send(self) -> Result<Response> {
183        self.enqueue()
184            .await
185            .map_err(EnqueueError::into_error)?
186            .response()
187            .await
188    }
189    /// Reserve one pending-operation slot and one terminal event. Individual
190    /// attempts share that slot, including backoff, so max_pending=1 can retry.
191    pub async fn enqueue(
192        mut self,
193    ) -> std::result::Result<WsRequestOperationReceipt, EnqueueError<Self>> {
194        let started = *self.started.get_or_insert_with(Instant::now);
195        match self.admit(started).await {
196            Ok(receipt) => Ok(receipt),
197            Err(error) => {
198                let error =
199                    if error.kind() == ErrorKind::TimedOut && error.ws_timeout_phase().is_none() {
200                        let total = earlier(
201                            self.options.deadline,
202                            add(started, self.options.total_timeout).ok().flatten(),
203                        );
204                        let admission = add(started, self.options.admission_timeout).ok().flatten();
205                        error.with_ws_timeout_phase(
206                            if total.is_some() && earlier(total, admission) == total {
207                                WsTimeoutPhase::Total
208                            } else {
209                                WsTimeoutPhase::Admission
210                            },
211                        )
212                    } else {
213                        error
214                    };
215                let report = RetryReport {
216                    elapsed: started.elapsed(),
217                    stop_reason: Some(error_stop_reason(&error)),
218                    ..RetryReport::default()
219                };
220                Err(EnqueueError::new(self, error.with_retry_report(report)))
221            }
222        }
223    }
224    async fn admit(&self, started: Instant) -> Result<WsRequestOperationReceipt> {
225        #[cfg(feature = "quality-doctor")]
226        let mut quality_scope = crate::module::ws_client::quality::QualityScope::new(
227            self.client
228                .runtime
229                .quality_begin(crate::api::quality::QualityOperationKind::WsRequestOperation),
230        );
231        #[cfg(feature = "quality-doctor")]
232        let admission = quality_scope
233            .probe
234            .as_ref()
235            .map(|probe| probe.phase(crate::api::quality::PhaseKind::Admission));
236        self.options.validate()?;
237        let runtime = &self.client.runtime;
238        let deadline = earlier(
239            self.options.deadline,
240            add(started, self.options.total_timeout)?,
241        );
242        let admission_deadline = earlier(deadline, add(started, self.options.admission_timeout)?);
243        let cancellation = CancellationToken::new();
244        // Reject captures the current physical generation before any bounded wait.
245        let admission_lease = runtime.lease(self.options.disconnected)?;
246        let source = TaskSource::RequestOperation(Arc::new(self.options.metadata.clone()));
247        let snapshot = runtime.tasks.snapshot()?;
248        let reservation = admission_scope(
249            &admission_lease,
250            runtime,
251            &self.options,
252            &cancellation,
253            admission_deadline,
254            snapshot.reserve(
255                source.payload_bytes()?,
256                MessageLane::Normal,
257                &cancellation,
258                admission_deadline,
259            ),
260        )
261        .await?;
262        let lease = admission_scope(
263            &admission_lease,
264            runtime,
265            &self.options,
266            &cancellation,
267            admission_deadline,
268            runtime.pending.reserve_slot(),
269        )
270        .await?;
271        check_scope(runtime, &self.options, &cancellation, admission_deadline)?;
272        if admission_lease.is_cancelled() {
273            return Err(NetError::from(ErrorKind::NotConnected));
274        }
275        let policy = match &self.options.retry_setting {
276            RetrySetting::Inherit => runtime.config.default_request_retry_policy.clone(),
277            RetrySetting::Disabled => WsRequestRetryPolicy::no_retry(),
278            RetrySetting::Policy(policy) => policy.clone(),
279        };
280        let (sender, receiver) = oneshot::channel();
281        #[cfg(feature = "quality-doctor")]
282        drop(admission);
283        let control = Arc::new(LogicalControl {
284            #[cfg(feature = "quality-doctor")]
285            quality: quality_scope.probe.take(),
286            id: runtime.allocate_operation()?,
287            runtime: runtime.clone(),
288            started,
289            deadline,
290            cancellation,
291            connection_lease: admission_lease.clone(),
292            policy: policy.clone(),
293            options: self.options.clone(),
294            source,
295            registration_gate: Mutex::new(()),
296            state: Mutex::new(LogicalState {
297                retry: RetryState::new(policy.schedule().clone()),
298                current: None,
299                terminal: false,
300                elapsed: None,
301                delivery: DeliveryEvidence::NotStarted,
302                phase: OperationPhase::WaitingForCapacity,
303                connection: None,
304                response: Some(sender),
305                reservation: Some(reservation),
306                lease: Some(lease),
307                hook: None,
308            }),
309        });
310        #[cfg(feature = "quality-doctor")]
311        if let Some(probe) = &control.quality {
312            probe.operation(control.id.as_u64());
313        }
314        if let Some(group) = &self.options.cancellation {
315            let weak = Arc::downgrade(&control);
316            let hook = group.inner.bind_cancel_hook(Arc::new(move || {
317                if let Some(control) = weak.upgrade() {
318                    control.finish(
319                        Err(NetError::from(ErrorKind::Cancelled)),
320                        RetryStopReason::Cancelled,
321                    );
322                }
323            }))?;
324            lock_recover(&control.state).hook = Some(hook);
325        }
326        let handle = WsRequestOperationHandle {
327            control: control.clone(),
328        };
329        check_scope(
330            runtime,
331            &self.options,
332            &control.cancellation,
333            admission_deadline,
334        )?;
335        if admission_lease.is_cancelled() {
336            return Err(NetError::from(ErrorKind::NotConnected));
337        }
338        let guard = RunnerGuard(control.clone());
339        let factory = self.factory.clone();
340        let key = self.key.clone();
341        runtime.spawn_request(async move {
342            let _guard = guard;
343            run(control, factory, key, admission_deadline).await;
344        })?;
345        Ok(WsRequestOperationReceipt {
346            handle,
347            response: receiver,
348        })
349    }
350}
351
352/// Cloneable control for the stable operation, independent of individual registrations.
353#[derive(Clone)]
354pub struct WsRequestOperationHandle {
355    control: Arc<LogicalControl>,
356}
357impl WsRequestOperationHandle {
358    /// Returns the stable `WsRequestOperation` reference for this logical request.
359    ///
360    /// The observation is created during logical-operation admission and is shared
361    /// by all clones of this handle. It spans retries, backoff, final result
362    /// selection, and observed delivery boundaries. Individual request
363    /// registrations have separate `WsRequest` reports whose parent field links
364    /// to this reference when their link events are retained. Bounded retention
365    /// and event loss can leave either side of that relationship unavailable.
366    ///
367    /// The final response or error selected by this logical operation carries
368    /// this logical reference when one exists. To inspect one registration's
369    /// observation, use that registration's [`super::RequestHandle::quality_ref`].
370    /// `None` means the logical observation was not allocated because collection
371    /// was inactive, filtered out, unavailable, or over its diagnostic budget; it
372    /// does not prevent retries or change the operation's result.
373    ///
374    /// This accessor neither starts collection nor waits for completion. Resolve
375    /// the reference using the owning engine's `QualityDoctor::lookup`; retaining
376    /// it does not pin history or keep the engine's workers alive.
377    #[cfg(feature = "quality-doctor")]
378    pub fn quality_ref(&self) -> Option<crate::api::quality::QualityReportRef> {
379        self.control
380            .quality
381            .as_ref()
382            .map(crate::module::quality::QualityProbe::reference)
383    }
384    pub fn id(&self) -> OperationId {
385        self.control.id
386    }
387    /// Snapshot only: None before registration, during backoff, and after completion.
388    pub fn current_registration(&self) -> Option<RequestRegistration> {
389        lock_recover(&self.control.state)
390            .current
391            .as_ref()
392            .map(|handle| handle.registration().clone())
393    }
394    pub fn retry_report(&self) -> RetryReport {
395        self.control.report()
396    }
397    pub fn cancel(&self) -> Result<TerminationOutcome> {
398        Ok(self.control.finish(
399            Err(NetError::from(ErrorKind::Cancelled)),
400            RetryStopReason::Cancelled,
401        ))
402    }
403}
404impl fmt::Debug for WsRequestOperationHandle {
405    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
406        f.debug_struct("WsRequestOperationHandle")
407            .field("id", &self.id())
408            .field("retry_report", &self.retry_report())
409            .finish()
410    }
411}
412#[must_use]
413#[derive(Debug)]
414pub struct WsRequestOperationReceipt {
415    handle: WsRequestOperationHandle,
416    response: oneshot::Receiver<Result<Response>>,
417}
418impl WsRequestOperationReceipt {
419    pub fn handle(&self) -> &WsRequestOperationHandle {
420        &self.handle
421    }
422    /// Dropping the receipt/future stops waiting and does not cancel the operation.
423    pub async fn response(self) -> Result<Response> {
424        self.response.await.map_err(|_| {
425            NetError::from(ErrorKind::EngineDropped).with_retry_report(self.handle.retry_report())
426        })?
427    }
428}
429
430struct LogicalControl {
431    #[cfg(feature = "quality-doctor")]
432    quality: Option<crate::module::quality::QualityProbe>,
433    id: OperationId,
434    runtime: Arc<SessionRuntime>,
435    started: Instant,
436    deadline: Option<Instant>,
437    cancellation: CancellationToken,
438    /// Reject retains the physical generation captured at submission. A
439    /// reconnect cannot renew this authority between factory or retry phases.
440    connection_lease: CancellationToken,
441    policy: WsRequestRetryPolicy,
442    options: WsRequestOperationOptions,
443    source: TaskSource,
444    registration_gate: Mutex<()>,
445    state: Mutex<LogicalState>,
446}
447struct LogicalState {
448    retry: RetryState,
449    current: Option<RequestHandle>,
450    terminal: bool,
451    elapsed: Option<Duration>,
452    delivery: DeliveryEvidence,
453    phase: OperationPhase,
454    connection: Option<ConnectionId>,
455    response: Option<oneshot::Sender<Result<Response>>>,
456    reservation: Option<TaskEventReservation>,
457    lease: Option<Arc<PendingLease>>,
458    hook: Option<CancelHookGuard>,
459}
460impl LogicalControl {
461    fn report(&self) -> RetryReport {
462        let state = lock_recover(&self.state);
463        let mut report = state.retry.report(
464            state
465                .elapsed
466                .map_or_else(|| self.started.elapsed(), |elapsed| elapsed),
467        );
468        if let Some(current) = &state.current {
469            report.transport_sends = report
470                .transport_sends
471                .saturating_add(current.retry_report().transport_sends);
472        }
473        report
474    }
475    fn notify(&self, event: RetryEvent) {
476        if let Some(observer) = self.policy.observer() {
477            observer.notify(event);
478        }
479    }
480    fn check(&self) -> Result<()> {
481        self.check_result(false)
482    }
483    fn check_result(&self, selected: bool) -> Result<()> {
484        check_result_scope(
485            &self.runtime,
486            &self.options,
487            &self.cancellation,
488            self.deadline,
489            || selected,
490        )
491        .map_err(|error| {
492            if error.kind() == ErrorKind::TimedOut {
493                error.with_ws_timeout_phase(WsTimeoutPhase::Total)
494            } else {
495                error
496            }
497        })
498    }
499    /// Only unfinished work needs a live generation. Already selected outcomes
500    /// retain the pending table's result, subject to cancellation and total time.
501    fn check_connection(&self) -> Result<()> {
502        self.check()?;
503        if self.connection_lease.is_cancelled() {
504            return Err(NetError::from(ErrorKind::NotConnected));
505        }
506        Ok(())
507    }
508    fn clear_current(&self) -> Option<RequestHandle> {
509        let mut state = lock_recover(&self.state);
510        let current = state.current.take();
511        if let Some(current) = &current {
512            // Retire before sampling so the writer cannot start between the
513            // transport counter snapshot and removal of this registration.
514            let _ = current.cancel();
515            absorb(&mut state, current);
516        }
517        current
518    }
519    fn finish(&self, result: Result<Response>, reason: RetryStopReason) -> TerminationOutcome {
520        let registration_gate = lock_recover(&self.registration_gate);
521        let (current, sender, reservation, lease, hook, report, event, outcome) = {
522            let mut state = lock_recover(&self.state);
523            if state.terminal {
524                return TerminationOutcome::AlreadyFinished;
525            }
526            let current = state.current.take();
527            // Retire the internal single-attempt control before sampling sends;
528            // this closes the race with a concurrent first transport write.
529            if let Some(current) = &current {
530                let _ = current.cancel();
531                absorb(&mut state, current);
532            }
533            if let Ok(response) = &result {
534                state.delivery = DeliveryEvidence::ResponseConfirmed;
535                state.connection = Some(response.connection_id());
536                state.phase = OperationPhase::AwaitingResponse;
537            }
538            state.terminal = true;
539            state.retry.finish(reason);
540            let elapsed = self.started.elapsed();
541            state.elapsed = Some(elapsed);
542            let report = state.retry.report(elapsed);
543            let outcome = match state.delivery {
544                DeliveryEvidence::NotStarted => TerminationOutcome::TerminatedBeforeWrite,
545                DeliveryEvidence::Unknown => TerminationOutcome::DeliveryUnknown,
546                _ => TerminationOutcome::TerminatedAfterWrite,
547            };
548            let event = TaskEvent {
549                client_id: self.runtime.client_id,
550                session_id: self.runtime.id,
551                operation_id: self.id,
552                connection_id: state.connection,
553                source: self.source.clone(),
554                lane: MessageLane::Normal,
555                phase: state.phase,
556                delivery: state.delivery,
557                cause: end_cause(&result, reason),
558                result: result
559                    .as_ref()
560                    .map(|_| TaskSuccess::ResponseReceived)
561                    .map_err(|error| error.clone().with_retry_report(report.clone())),
562            };
563            (
564                current,
565                state.response.take(),
566                state.reservation.take(),
567                state.lease.take(),
568                state.hook.take(),
569                report,
570                event,
571                outcome,
572            )
573        };
574        drop(registration_gate);
575        self.cancellation.cancel();
576        if let Some(current) = current {
577            let _ = current.cancel();
578        }
579        if let Some(lease) = lease {
580            lease.release();
581        }
582        drop(hook);
583        let result = result
584            .map(|response| response.with_retry_report(report.clone()))
585            .map_err(|error| error.with_retry_report(report.clone()));
586        #[cfg(feature = "quality-doctor")]
587        let result = {
588            let reference = self
589                .quality
590                .as_ref()
591                .map(crate::module::quality::QualityProbe::reference);
592            if let Some(probe) = &self.quality {
593                use crate::api::quality::{PhaseKind, QualityOutcome};
594                probe.mark(PhaseKind::ResultSelected);
595                probe.mark(PhaseKind::ResourcesReleased);
596                match &result {
597                    Ok(response) => {
598                        probe.connection(response.connection_id().as_u64());
599                        probe.result(QualityOutcome::Success);
600                    }
601                    Err(error) => probe.error(error),
602                }
603            }
604            result
605                .map(|response| response.with_quality_ref(reference))
606                .map_err(|error| match reference {
607                    Some(reference) => error.with_quality_ref(reference),
608                    None => error,
609                })
610        };
611        if let Some(reservation) = reservation {
612            if let Err(error) = reservation.publish(event) {
613                crate::log_e!(crate::LogType::WSC; "ws.request_operation.publish", "error", format!("{error:?}"));
614            }
615        }
616        self.notify(RetryEvent::Completed(report));
617        if let Some(sender) = sender {
618            let _ = sender.send(result);
619        }
620        #[cfg(feature = "quality-doctor")]
621        if let Some(probe) = &self.quality {
622            probe.mark(crate::api::quality::PhaseKind::DeliveryFinished);
623            probe.finish();
624        }
625        outcome
626    }
627}
628fn absorb(state: &mut LogicalState, current: &RequestHandle) {
629    let sends = current.retry_report().transport_sends;
630    for _ in 0..sends {
631        let _ = state.retry.record_transport_send();
632    }
633    if let Ok(snapshot) = current.state() {
634        if delivery_rank(snapshot.delivery) > delivery_rank(state.delivery) {
635            state.delivery = snapshot.delivery;
636        }
637        if snapshot.phase != OperationPhase::Finished {
638            state.phase = snapshot.phase;
639        }
640        if let Some(Err(error)) = snapshot.result {
641            state.connection = error.context().connection_id.or(state.connection);
642            state.phase = match error.context().stage {
643                Some(ErrorStage::Write) => OperationPhase::Writing,
644                Some(ErrorStage::Response) => OperationPhase::AwaitingResponse,
645                Some(ErrorStage::Admission) => OperationPhase::WaitingForCapacity,
646                _ => state.phase,
647            };
648        }
649    }
650}
651fn delivery_rank(value: DeliveryEvidence) -> u8 {
652    match value {
653        DeliveryEvidence::NotStarted => 0,
654        DeliveryEvidence::Unknown => 1,
655        DeliveryEvidence::Written => 2,
656        DeliveryEvidence::ResponseConfirmed => 3,
657    }
658}
659struct RunnerGuard(Arc<LogicalControl>);
660impl Drop for RunnerGuard {
661    fn drop(&mut self) {
662        self.0.finish(
663            Err(NetError::from(ErrorKind::EngineDropped)),
664            RetryStopReason::Shutdown,
665        );
666    }
667}
668async fn run(
669    control: Arc<LogicalControl>,
670    factory: Arc<AttemptFactory>,
671    key: Arc<str>,
672    admission_deadline: Option<Instant>,
673) {
674    // Keep the previous winner until the next attempt actually begins. Natural
675    // retirement during backoff must not turn that winner into Closed.
676    let mut previous_result = None;
677    loop {
678        if let Err(error) = control.check_connection() {
679            if let Some((result, selected)) = previous_result.take() {
680                finish_after_attempt_error(&control, result, selected, error);
681            } else {
682                finish_error(&control, error);
683            }
684            return;
685        }
686        drop(previous_result.take());
687        let attempt = {
688            let mut state = lock_recover(&control.state);
689            if state.terminal {
690                return;
691            }
692            match state.retry.begin_attempt() {
693                Ok(attempt) => attempt,
694                Err(_) => {
695                    drop(state);
696                    finish_error(&control, NetError::from(ErrorKind::InvalidPolicy));
697                    return;
698                }
699            }
700        };
701        let attempt_started = Instant::now();
702        #[cfg(feature = "quality-doctor")]
703        if let Some(probe) = &control.quality {
704            probe.attempt(attempt as u64);
705        }
706        control.notify(RetryEvent::AttemptStarted { attempt });
707        let result = attempt_once(
708            &control,
709            &factory,
710            key.clone(),
711            attempt,
712            attempt_started,
713            if attempt == 1 {
714                admission_deadline
715            } else {
716                None
717            },
718        )
719        .await;
720        let selected = lock_recover(&control.state)
721            .current
722            .as_ref()
723            .is_some_and(RequestHandle::has_selected_result);
724        if let Some(current) = control.clear_current() {
725            let _ = current.cancel();
726        }
727        if lock_recover(&control.state).terminal {
728            return;
729        }
730        if let Err(error) = control.check_result(selected) {
731            control.notify(RetryEvent::AttemptFinished {
732                attempt,
733                final_attempt: true,
734            });
735            finish_error(&control, error);
736            return;
737        }
738        let mut decision = classify(&control, &result);
739        if let Err(error) = control.check_result(selected) {
740            control.notify(RetryEvent::AttemptFinished {
741                attempt,
742                final_attempt: true,
743            });
744            finish_error(&control, error);
745            return;
746        }
747        if selected && control.runtime.preserves_selected_result()
748            && matches!(decision, Ok(RetryStep::Retry(_)))
749        {
750            decision = Ok(RetryStep::Stop(RetryStopReason::NotRetryable));
751        }
752        let delay = match decision {
753            Ok(RetryStep::Retry(delay)) => delay,
754            Ok(RetryStep::Stop(reason)) => {
755                control.notify(RetryEvent::AttemptFinished {
756                    attempt,
757                    final_attempt: true,
758                });
759                if let Err(error) = control.check_result(selected) {
760                    finish_error(&control, error);
761                } else {
762                    control.finish(result, reason);
763                }
764                return;
765            }
766            Err(error) => {
767                control.notify(RetryEvent::AttemptFinished {
768                    attempt,
769                    final_attempt: true,
770                });
771                control.finish(Err(error), RetryStopReason::PolicyFailed);
772                return;
773            }
774        };
775        control.notify(RetryEvent::AttemptFinished {
776            attempt,
777            final_attempt: false,
778        });
779        if let Err(error) = control.check() {
780            finish_after_attempt_error(&control, result, selected, error);
781            return;
782        }
783        lock_recover(&control.state).phase = OperationPhase::Queued;
784        control.notify(RetryEvent::RetryScheduled { attempt, delay });
785        let ready = match Instant::now().checked_add(delay) {
786            Some(ready) => ready,
787            None => {
788                finish_error(&control, NetError::from(ErrorKind::InvalidPolicy));
789                return;
790            }
791        };
792        let sleep = control.runtime.wait_deadline(Some(ready));
793        #[cfg(feature = "quality-doctor")]
794        let backoff = control
795            .quality
796            .as_ref()
797            .map(|probe| probe.phase(crate::api::quality::PhaseKind::Backoff));
798        let backoff_result = admission_scope(
799            &control.connection_lease,
800            &control.runtime,
801            &control.options,
802            &control.cancellation,
803            control.deadline,
804            sleep,
805        )
806        .await;
807        #[cfg(feature = "quality-doctor")]
808        drop(backoff);
809        if let Err(error) = backoff_result {
810            let error = if error.kind() == ErrorKind::TimedOut {
811                error.with_ws_timeout_phase(WsTimeoutPhase::Total)
812            } else {
813                error
814            };
815            finish_after_attempt_error(&control, result, selected, error);
816            return;
817        }
818        previous_result = Some((result, selected));
819    }
820}
821async fn attempt_once(
822    control: &Arc<LogicalControl>,
823    factory: &Arc<AttemptFactory>,
824    key: Arc<str>,
825    attempt: u32,
826    started: Instant,
827    admission_deadline: Option<Instant>,
828) -> Result<Response> {
829    check_attempt_budget(control, started)?;
830    control.check_connection()?;
831    let attempt_budget = TimeoutBudget::earlier(
832        control
833            .deadline
834            .map(|at| TimeoutBudget::new(at, WsTimeoutPhase::Total)),
835        add(started, control.options.attempt_timeout)?
836            .map(|at| TimeoutBudget::new(at, WsTimeoutPhase::Attempt)),
837    );
838    let attempt_deadline = attempt_budget.map(|budget| budget.at);
839    let context = RequestAttemptContext {
840        operation_id: control.id,
841        attempt,
842        request_id: RequestId::random()?,
843        idempotency_key: key,
844    };
845    #[cfg(feature = "quality-doctor")]
846    let build = control
847        .quality
848        .as_ref()
849        .map(|probe| probe.phase(crate::api::quality::PhaseKind::RequestBuild));
850    let built = super::retry::isolate_classifier(|| factory(&context));
851    #[cfg(feature = "quality-doctor")]
852    drop(build);
853    // A synchronous factory cannot be preempted, so arbitrate its elapsed time
854    // before accepting either its success or its returned construction error.
855    check_attempt_budget(control, started)?;
856    control.check_connection()?;
857    let message = built?.map_err(|error| error.with_stage(ErrorStage::RequestBuild))?;
858    let request =
859        Request::new(context.request_id, message).with_metadata(control.options.metadata.clone());
860    let options = RequestOptions {
861        send: SendOptions {
862            priority: control.options.priority,
863            write_timeout: control.options.write_timeout,
864            deadline: attempt_deadline,
865            disconnected: control.options.disconnected,
866            cancellation: control.options.cancellation.clone(),
867            retry_setting: RetrySetting::Disabled,
868            metadata: control.options.metadata.clone(),
869            ..SendOptions::default()
870        },
871        response_timeout: control.options.response_timeout,
872        response_timeout_origin: control.options.response_timeout_origin,
873        registration_deadline: admission_deadline,
874    };
875    let lease = lock_recover(&control.state)
876        .lease
877        .clone()
878        .ok_or_else(|| NetError::from(ErrorKind::Cancelled))?;
879    let registration_check = || {
880        control.check_connection()?;
881        if lock_recover(&control.state).terminal {
882            return Err(NetError::from(ErrorKind::Cancelled));
883        }
884        Ok(())
885    };
886    let on_registered = |handle: &RequestHandle| {
887        let mut state = lock_recover(&control.state);
888        if state.terminal {
889            return Err(NetError::from(ErrorKind::Cancelled));
890        }
891        state.current = Some(handle.clone());
892        state.phase = OperationPhase::WaitingForCapacity;
893        Ok(())
894    };
895    let mut admission = scope_attempt(
896        &control.runtime,
897        &control.options,
898        &control.cancellation,
899        control.deadline,
900        TimeoutBudget::earlier(
901            attempt_budget,
902            admission_deadline.map(|at| TimeoutBudget::new(at, WsTimeoutPhase::Admission)),
903        ),
904        None,
905        dispatch::prepare_operation_attempt(
906            &control.runtime,
907            &request,
908            &options,
909            control.started,
910            lease,
911            dispatch::AttemptRegistration {
912                absolute_budget: attempt_budget,
913                connection_lease: &control.connection_lease,
914                gate: &control.registration_gate,
915                check: &registration_check,
916                install: &on_registered,
917            },
918        ),
919    )
920    .await
921    .map_err(|error| {
922        let error = preserve_timeout_phase(error);
923        if error.context().stage.is_none() {
924            error.with_stage(ErrorStage::Admission)
925        } else {
926            error
927        }
928    })?;
929    let (handle, response) = admission
930        .request
931        .take()
932        .ok_or_else(|| NetError::from(ErrorKind::Internal))?;
933    #[cfg(feature = "quality-doctor")]
934    if let Some(probe) = &control.quality {
935        handle.quality_parent(probe.reference());
936    }
937    {
938        let mut state = lock_recover(&control.state);
939        if state.terminal {
940            return Err(NetError::from(ErrorKind::Cancelled));
941        }
942        state.current = Some(handle.clone());
943        state.phase = OperationPhase::Queued;
944    }
945    admission
946        .commit()
947        .map_err(preserve_timeout_phase)?;
948    let result = scope_attempt(
949        &control.runtime,
950        &control.options,
951        &control.cancellation,
952        control.deadline,
953        attempt_budget,
954        Some(&handle),
955        async {
956            response
957                .await
958                .map_err(|_| NetError::from(ErrorKind::EngineDropped))?
959        },
960    )
961    .await;
962    result.map_err(|error| {
963        let error = preserve_timeout_phase(error);
964        if error.kind() == ErrorKind::TimedOut && error.context().stage.is_none() {
965            error.with_stage(ErrorStage::Response)
966        } else {
967            error
968        }
969    })
970}
971
972fn check_attempt_budget(control: &LogicalControl, started: Instant) -> Result<()> {
973    control.check()?;
974    if add(started, control.options.attempt_timeout)?
975        .is_some_and(|deadline| deadline <= Instant::now())
976    {
977        return Err(
978            NetError::from(ErrorKind::TimedOut).with_ws_timeout_phase(WsTimeoutPhase::Attempt)
979        );
980    }
981    Ok(())
982}
983
984fn preserve_timeout_phase(error: NetError) -> NetError {
985    match super::retry::timeout_phase(&error) {
986        Some(phase) => error.with_ws_timeout_phase(phase),
987        None => error,
988    }
989}
990enum RetryStep {
991    Retry(Duration),
992    Stop(RetryStopReason),
993}
994fn classify(control: &LogicalControl, result: &Result<Response>) -> Result<RetryStep> {
995    if matches!(control.options.retry_setting, RetrySetting::Disabled) {
996        return Ok(RetryStep::Stop(RetryStopReason::Disabled));
997    }
998    if let Err(error) = result {
999        if super::retry::terminal_error(error) {
1000            let reason = if error.kind() == ErrorKind::CallbackPanicked {
1001                RetryStopReason::PolicyFailed
1002            } else {
1003                RetryStopReason::NotRetryable
1004            };
1005            return Ok(RetryStep::Stop(reason));
1006        }
1007    }
1008    let retry = lock_recover(&control.state).retry.context(
1009        control.started.elapsed(),
1010        control
1011            .deadline
1012            .map(|deadline| deadline.saturating_duration_since(Instant::now())),
1013    );
1014    let decision = control.policy.classify(&WsRequestRetryContext {
1015        outcome: result.as_ref(),
1016        retry: &retry,
1017    })?;
1018    let reason = if decision == RetryDecision::Stop {
1019        Some(RetryStopReason::ClassifierStopped)
1020    } else if decision == RetryDecision::Default
1021        && !result
1022            .as_ref()
1023            .err()
1024            .is_some_and(|error| control.policy.should_retry_error(error))
1025    {
1026        Some(if result.is_ok() {
1027            RetryStopReason::Completed
1028        } else {
1029            RetryStopReason::NotRetryable
1030        })
1031    } else if !lock_recover(&control.state).retry.can_retry() {
1032        Some(RetryStopReason::AttemptsExhausted)
1033    } else if !control.policy.allows_replay() {
1034        Some(RetryStopReason::UnsafeToReplay)
1035    } else {
1036        None
1037    };
1038    let mut state = lock_recover(&control.state);
1039    if let Some(reason) = reason {
1040        return Ok(RetryStep::Stop(reason));
1041    }
1042    state
1043        .retry
1044        .next_delay()
1045        .map(|delay| match delay {
1046            Some(delay) => RetryStep::Retry(delay),
1047            None => RetryStep::Stop(RetryStopReason::AttemptsExhausted),
1048        })
1049        .map_err(|_| NetError::from(ErrorKind::InvalidPolicy))
1050}
1051fn error_stop_reason(error: &NetError) -> RetryStopReason {
1052    match error.kind() {
1053        ErrorKind::Cancelled => RetryStopReason::Cancelled,
1054        ErrorKind::TimedOut => RetryStopReason::TimedOut,
1055        ErrorKind::Closed | ErrorKind::EngineDropped => RetryStopReason::Shutdown,
1056        ErrorKind::CallbackPanicked | ErrorKind::InvalidPolicy => RetryStopReason::PolicyFailed,
1057        _ => RetryStopReason::NotRetryable,
1058    }
1059}
1060fn finish_error(control: &LogicalControl, error: NetError) {
1061    let reason = error_stop_reason(&error);
1062    control.finish(Err(error), reason);
1063}
1064
1065fn finish_after_attempt_error(
1066    control: &LogicalControl,
1067    result: Result<Response>,
1068    selected: bool,
1069    error: NetError,
1070) {
1071    if selected && control.runtime.preserves_selected_result() {
1072        match control.check_result(true) {
1073            Ok(()) => {
1074                // Every caller has already decided to retry. Natural retirement
1075                // prevents that retry even when the retained result is a response.
1076                control.finish(result, RetryStopReason::NotRetryable);
1077            }
1078            Err(error) => finish_error(control, error),
1079        }
1080    } else {
1081        finish_error(control, error);
1082    }
1083}
1084fn end_cause(result: &Result<Response>, reason: RetryStopReason) -> TaskEndCause {
1085    if result.is_ok() {
1086        return TaskEndCause::Completed;
1087    }
1088    match reason {
1089        RetryStopReason::Cancelled => TaskEndCause::Cancelled,
1090        RetryStopReason::TimedOut => TaskEndCause::Expired,
1091        RetryStopReason::Shutdown => TaskEndCause::Shutdown,
1092        _ => match result.as_ref().err().map(NetError::kind) {
1093            Some(ErrorKind::TimedOut) => TaskEndCause::Expired,
1094            Some(ErrorKind::Cancelled) => TaskEndCause::Cancelled,
1095            Some(ErrorKind::Closed | ErrorKind::NotConnected) => TaskEndCause::Disconnected,
1096            _ => TaskEndCause::Failed,
1097        },
1098    }
1099}
1100fn earlier(left: Option<Instant>, right: Option<Instant>) -> Option<Instant> {
1101    match (left, right) {
1102        (Some(a), Some(b)) => Some(a.min(b)),
1103        (a, b) => a.or(b),
1104    }
1105}
1106fn add(start: Instant, duration: Option<Duration>) -> Result<Option<Instant>> {
1107    duration
1108        .map(|duration| {
1109            start
1110                .checked_add(duration)
1111                .ok_or_else(|| NetError::config("timeout", "deadline is not representable"))
1112        })
1113        .transpose()
1114}
1115fn check_scope(
1116    runtime: &SessionRuntime,
1117    options: &WsRequestOperationOptions,
1118    cancel: &CancellationToken,
1119    deadline: Option<Instant>,
1120) -> Result<()> {
1121    check_result_scope(runtime, options, cancel, deadline, || false)
1122}
1123
1124fn check_result_scope(
1125    runtime: &SessionRuntime,
1126    options: &WsRequestOperationOptions,
1127    cancel: &CancellationToken,
1128    deadline: Option<Instant>,
1129    selected: impl FnOnce() -> bool,
1130) -> Result<()> {
1131    if cancel.is_cancelled()
1132        || options
1133            .cancellation
1134            .as_ref()
1135            .is_some_and(CancellationGroup::is_cancelled)
1136    {
1137        return Err(NetError::from(ErrorKind::Cancelled));
1138    }
1139    // Read the request authority only after observing closure. A false snapshot
1140    // taken earlier could miss a result selected just before natural retirement.
1141    if runtime.is_closed() && !(selected() && runtime.preserves_selected_result()) {
1142        return Err(NetError::from(ErrorKind::Closed));
1143    }
1144    if deadline.is_some_and(|deadline| Instant::now() >= deadline) {
1145        return Err(NetError::from(ErrorKind::TimedOut));
1146    }
1147    Ok(())
1148}
1149/// Keep unfinished admission and backoff bound to the original Reject connection.
1150async fn admission_scope<T>(
1151    lease: &CancellationToken,
1152    runtime: &SessionRuntime,
1153    options: &WsRequestOperationOptions,
1154    cancel: &CancellationToken,
1155    deadline: Option<Instant>,
1156    future: impl Future<Output = Result<T>>,
1157) -> Result<T> {
1158    if options.disconnected == DisconnectedPolicy::WaitForReconnect {
1159        return scope(runtime, options, cancel, deadline, future).await;
1160    }
1161    check_scope(runtime, options, cancel, deadline)?;
1162    if lease.is_cancelled() {
1163        return Err(NetError::from(ErrorKind::NotConnected));
1164    }
1165    let result = tokio::select! {
1166        biased;
1167        _ = lease.cancelled() => {
1168            check_scope(runtime, options, cancel, deadline)?;
1169            Err(NetError::from(ErrorKind::NotConnected))
1170        },
1171        result = scope(runtime, options, cancel, deadline, future) => result,
1172    }?;
1173    check_scope(runtime, options, cancel, deadline)?;
1174    if lease.is_cancelled() {
1175        return Err(NetError::from(ErrorKind::NotConnected));
1176    }
1177    Ok(result)
1178}
1179
1180async fn scope<T>(
1181    runtime: &SessionRuntime,
1182    options: &WsRequestOperationOptions,
1183    cancel: &CancellationToken,
1184    deadline: Option<Instant>,
1185    future: impl Future<Output = Result<T>>,
1186) -> Result<T> {
1187    scope_with_request(runtime, options, cancel, deadline, None, future).await
1188}
1189
1190/// A natural session end cannot replace a pending table winner. Its producer
1191/// may still be releasing resources before publishing to the response channel.
1192async fn result_closure(runtime: &SessionRuntime, request: Option<&RequestHandle>) {
1193    let closed = runtime.closed_token();
1194    let session_end = runtime.lifecycle.completion_token();
1195    tokio::select! {
1196        _ = closed.cancelled() => {},
1197        _ = session_end.cancelled() => {},
1198    }
1199    if request.is_some_and(RequestHandle::has_selected_result)
1200        && runtime.preserves_selected_result()
1201    {
1202        // Explicit retirement has its own signal in the surrounding scope.
1203        std::future::pending::<()>().await;
1204    }
1205}
1206
1207async fn scope_with_request<T>(
1208    runtime: &SessionRuntime,
1209    options: &WsRequestOperationOptions,
1210    cancel: &CancellationToken,
1211    deadline: Option<Instant>,
1212    request: Option<&RequestHandle>,
1213    future: impl Future<Output = Result<T>>,
1214) -> Result<T> {
1215    check_result_scope(
1216        runtime,
1217        options,
1218        cancel,
1219        deadline,
1220        || request.is_some_and(RequestHandle::has_selected_result),
1221    )?;
1222    let forced_closed = runtime.forced_closed_token();
1223    let session_cancel = runtime.lifecycle.cancel_token();
1224    let group = async {
1225        match &options.cancellation {
1226            Some(group) => group.cancelled().await,
1227            None => std::future::pending().await,
1228        }
1229    };
1230    tokio::select! {
1231        biased;
1232        _ = cancel.cancelled() => Err(NetError::from(ErrorKind::Cancelled)),
1233        _ = group => Err(NetError::from(ErrorKind::Cancelled)),
1234        _ = runtime.shutdown.cancelled() => Err(NetError::from(ErrorKind::Closed)),
1235        _ = forced_closed.cancelled() => Err(NetError::from(ErrorKind::Closed)),
1236        _ = session_cancel.cancelled() => Err(NetError::from(ErrorKind::Closed)),
1237        _ = result_closure(runtime, request) => Err(NetError::from(ErrorKind::Closed)),
1238        result = runtime.wait_deadline(deadline) => result.and_then(|()| Err(NetError::from(ErrorKind::TimedOut))),
1239        result = future => result,
1240    }
1241}
1242
1243/// An outcome already selected by the pending table or writer wins over a late
1244/// poll of the per-attempt timer. Cancellation and the logical total budget still
1245/// arbitrate before and after consuming that outcome.
1246async fn scope_attempt<T>(
1247    runtime: &SessionRuntime,
1248    options: &WsRequestOperationOptions,
1249    cancel: &CancellationToken,
1250    total_deadline: Option<Instant>,
1251    budget: Option<TimeoutBudget>,
1252    request: Option<&RequestHandle>,
1253    future: impl Future<Output = Result<T>>,
1254) -> Result<T> {
1255    let check = || {
1256        let selected = || request.is_some_and(RequestHandle::has_selected_result);
1257        #[cfg(test)]
1258        if let Some(request) = request {
1259            runtime.before_request_result_check_for_test(request)?;
1260        }
1261        check_result_scope(
1262            runtime,
1263            options,
1264            cancel,
1265            total_deadline,
1266            selected,
1267        ).map_err(|error| {
1268            if error.kind() == ErrorKind::TimedOut {
1269                error.with_ws_timeout_phase(WsTimeoutPhase::Total)
1270            } else {
1271                error
1272            }
1273        })
1274    };
1275    check()?;
1276    let forced_closed = runtime.forced_closed_token();
1277    let session_cancel = runtime.lifecycle.cancel_token();
1278    let group = async {
1279        match &options.cancellation {
1280            Some(group) => group.cancelled().await,
1281            None => std::future::pending().await,
1282        }
1283    };
1284    tokio::pin!(future);
1285    let result = tokio::select! {
1286        biased;
1287        _ = cancel.cancelled() => Err(NetError::from(ErrorKind::Cancelled)),
1288        _ = group => Err(NetError::from(ErrorKind::Cancelled)),
1289        _ = runtime.shutdown.cancelled() => Err(NetError::from(ErrorKind::Closed)),
1290        _ = forced_closed.cancelled() => Err(NetError::from(ErrorKind::Closed)),
1291        _ = session_cancel.cancelled() => Err(NetError::from(ErrorKind::Closed)),
1292        _ = result_closure(runtime, request) => Err(NetError::from(ErrorKind::Closed)),
1293        result = runtime.wait_deadline(total_deadline) => result.and_then(|()|
1294            Err(NetError::from(ErrorKind::TimedOut).with_ws_timeout_phase(WsTimeoutPhase::Total))),
1295        result = &mut future => result,
1296        result = runtime.wait_deadline(budget.map(|budget| budget.at)) => {
1297            result?;
1298            if let Some(request) = request {
1299                // The channel may still be pending after its outcome was selected.
1300                // Expiration competes with that selection under the same P/C
1301                // authority; a previously chosen result cannot be replaced here.
1302                request.expire_operation_attempt();
1303                scope_with_request(runtime, options, cancel, total_deadline, Some(request), &mut future).await
1304            } else {
1305                Err(budget.map_or_else(|| NetError::from(ErrorKind::TimedOut), TimeoutBudget::timeout_error))
1306            }
1307        },
1308    };
1309    check()?;
1310    result
1311}
1312
1313#[cfg(test)]
1314#[path = "request_operation_tests.rs"]
1315mod tests;
1316
1317#[cfg(test)]
1318#[path = "request_operation_generation_tests.rs"]
1319mod generation_tests;
1320
1321#[cfg(test)]
1322#[path = "request_operation_selection_tests.rs"]
1323mod selection_tests;