Skip to main content

open_net/api/ws/
request_operation.rs

1//! Logical requests rebuild opaque protocol messages for every bounded attempt.
2use super::cancellation::{lock_recover, CancelHookGuard};
3use super::*;
4use crate::api::retry::{RetryDecision, RetryEvent, RetryReport, RetrySetting, RetryStopReason};
5use crate::error::{ErrorKind, ErrorStage};
6use crate::module::ws_client::native_pending::PendingLease;
7use crate::module::ws_client::timeout_budget::TimeoutBudget;
8use crate::module::ws_client::{
9    native_task_observer::TaskEventReservation, session_runtime::SessionRuntime,
10};
11use crate::retry_core::RetryState;
12use crate::{EnqueueError, Metadata, NetError, Result};
13use std::fmt;
14use std::future::Future;
15use std::sync::{Arc, Mutex};
16use std::time::{Duration, Instant};
17use tokio::sync::oneshot;
18use tokio_util::sync::CancellationToken;
19
20type AttemptFactory = dyn Fn(&RequestAttemptContext) -> Result<Message> + Send + Sync;
21
22/// Immutable identity supplied to a synchronous message factory. The application
23/// must encode the fresh wire ID and stable key in its own protocol.
24#[derive(Clone)]
25pub struct RequestAttemptContext {
26    operation_id: OperationId,
27    attempt: u32,
28    request_id: RequestId,
29    idempotency_key: Arc<str>,
30}
31impl RequestAttemptContext {
32    pub fn operation_id(&self) -> OperationId {
33        self.operation_id
34    }
35    pub fn attempt(&self) -> u32 {
36        self.attempt
37    }
38    pub fn request_id(&self) -> &RequestId {
39        &self.request_id
40    }
41    pub fn idempotency_key(&self) -> &str {
42        &self.idempotency_key
43    }
44}
45impl fmt::Debug for RequestAttemptContext {
46    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
47        f.debug_struct("RequestAttemptContext")
48            .field("operation_id", &self.operation_id)
49            .field("attempt", &self.attempt)
50            .field("request_id", &self.request_id)
51            .field("idempotency_key_bytes", &self.idempotency_key.len())
52            .finish()
53    }
54}
55
56/// Limits for one logical request. Total time starts when enqueue/send is first
57/// polled, and includes admission, construction, reconnection, backoff and all attempts.
58#[derive(Clone)]
59pub struct WsRequestOperationOptions {
60    pub priority: Priority,
61    pub admission_timeout: Option<Duration>,
62    pub write_timeout: Duration,
63    pub response_timeout: Duration,
64    pub response_timeout_origin: ResponseTimeoutOrigin,
65    pub attempt_timeout: Option<Duration>,
66    pub total_timeout: Option<Duration>,
67    pub deadline: Option<Instant>,
68    pub disconnected: DisconnectedPolicy,
69    pub cancellation: Option<CancellationGroup>,
70    pub metadata: Metadata,
71    pub retry_setting: RetrySetting<WsRequestRetryPolicy>,
72}
73impl fmt::Debug for WsRequestOperationOptions {
74    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
75        f.debug_struct("WsRequestOperationOptions")
76            .field("priority", &self.priority)
77            .field("admission_timeout", &self.admission_timeout)
78            .field("write_timeout", &self.write_timeout)
79            .field("response_timeout", &self.response_timeout)
80            .field("response_timeout_origin", &self.response_timeout_origin)
81            .field("attempt_timeout", &self.attempt_timeout)
82            .field("total_timeout", &self.total_timeout)
83            .field("deadline", &self.deadline)
84            .field("disconnected", &self.disconnected)
85            .field("cancellation", &self.cancellation)
86            .field("metadata_entries", &self.metadata.len())
87            .field("retry_setting", &self.retry_setting)
88            .finish()
89    }
90}
91impl Default for WsRequestOperationOptions {
92    fn default() -> Self {
93        Self {
94            priority: Priority::Normal,
95            admission_timeout: None,
96            write_timeout: Duration::from_secs(10),
97            response_timeout: Duration::from_secs(30),
98            response_timeout_origin: ResponseTimeoutOrigin::Written,
99            attempt_timeout: None,
100            total_timeout: None,
101            deadline: None,
102            disconnected: DisconnectedPolicy::Reject,
103            cancellation: None,
104            metadata: Metadata::new(),
105            retry_setting: RetrySetting::Inherit,
106        }
107    }
108}
109impl WsRequestOperationOptions {
110    pub fn validate(&self) -> Result<()> {
111        for (name, duration) in [
112            ("admission_timeout", self.admission_timeout),
113            ("attempt_timeout", self.attempt_timeout),
114            ("total_timeout", self.total_timeout),
115            ("write_timeout", Some(self.write_timeout)),
116            ("response_timeout", Some(self.response_timeout)),
117        ] {
118            if let Some(duration) = duration {
119                super::config::deadline(duration, name, true)?;
120            }
121        }
122        validate_metadata(&self.metadata)
123    }
124}
125
126/// Builder retaining the protocol factory until its operation terminates.
127#[must_use]
128pub struct WsRequestOperationBuilder {
129    client: RequestClient,
130    key: Arc<str>,
131    factory: Arc<AttemptFactory>,
132    options: WsRequestOperationOptions,
133    started: Option<Instant>,
134}
135impl fmt::Debug for WsRequestOperationBuilder {
136    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
137        f.debug_struct("WsRequestOperationBuilder")
138            .field("client", &self.client)
139            .field("key_bytes", &self.key.len())
140            .field("options", &self.options)
141            .finish_non_exhaustive()
142    }
143}
144impl RequestClient {
145    /// Creates a logical operation. The factory must encode both context IDs;
146    /// response routing must extract the current attempt's wire request ID.
147    pub fn request_with_factory<F>(
148        &self,
149        key: impl Into<String>,
150        factory: F,
151    ) -> Result<WsRequestOperationBuilder>
152    where
153        F: Fn(&RequestAttemptContext) -> Result<Message> + Send + Sync + 'static,
154    {
155        let key = key.into();
156        if key.len() > 1024 || key.trim().is_empty() {
157            return Err(NetError::input(
158                "idempotency_key",
159                "must contain a non-whitespace character and be at most 1024 UTF-8 bytes",
160            ));
161        }
162        Ok(WsRequestOperationBuilder {
163            client: self.clone(),
164            key: Arc::from(key),
165            factory: Arc::new(factory),
166            options: WsRequestOperationOptions::default(),
167            started: None,
168        })
169    }
170}
171impl WsRequestOperationBuilder {
172    /// Replace the operation policy without merging fields or resetting submission time.
173    pub fn retry_setting(mut self, setting: RetrySetting<WsRequestRetryPolicy>) -> Self {
174        self.options.retry_setting = setting;
175        self
176    }
177    pub fn options(mut self, options: WsRequestOperationOptions) -> Self {
178        self.options = options;
179        self.started = None;
180        self
181    }
182    pub async fn send(self) -> Result<Response> {
183        self.enqueue()
184            .await
185            .map_err(EnqueueError::into_error)?
186            .response()
187            .await
188    }
189    /// Reserve one pending-operation slot and one terminal event. Individual
190    /// attempts share that slot, including backoff, so max_pending=1 can retry.
191    pub async fn enqueue(
192        mut self,
193    ) -> std::result::Result<WsRequestOperationReceipt, EnqueueError<Self>> {
194        let started = *self.started.get_or_insert_with(Instant::now);
195        match self.admit(started).await {
196            Ok(receipt) => Ok(receipt),
197            Err(error) => {
198                let error =
199                    if error.kind() == ErrorKind::TimedOut && error.ws_timeout_phase().is_none() {
200                        let total = earlier(
201                            self.options.deadline,
202                            add(started, self.options.total_timeout).ok().flatten(),
203                        );
204                        let admission = add(started, self.options.admission_timeout).ok().flatten();
205                        error.with_ws_timeout_phase(
206                            if total.is_some() && earlier(total, admission) == total {
207                                WsTimeoutPhase::Total
208                            } else {
209                                WsTimeoutPhase::Admission
210                            },
211                        )
212                    } else {
213                        error
214                    };
215                let report = RetryReport {
216                    elapsed: started.elapsed(),
217                    stop_reason: Some(error_stop_reason(&error)),
218                    ..RetryReport::default()
219                };
220                Err(EnqueueError::new(self, error.with_retry_report(report)))
221            }
222        }
223    }
224    async fn admit(&self, started: Instant) -> Result<WsRequestOperationReceipt> {
225        self.options.validate()?;
226        let runtime = &self.client.runtime;
227        let deadline = earlier(
228            self.options.deadline,
229            add(started, self.options.total_timeout)?,
230        );
231        let admission_deadline = earlier(deadline, add(started, self.options.admission_timeout)?);
232        let cancellation = CancellationToken::new();
233        // Reject captures the current physical generation before any bounded wait.
234        let admission_lease = runtime.lease(self.options.disconnected)?;
235        let source = TaskSource::RequestOperation(Arc::new(self.options.metadata.clone()));
236        let snapshot = runtime.tasks.snapshot()?;
237        let reservation = admission_scope(
238            &admission_lease,
239            runtime,
240            &self.options,
241            &cancellation,
242            admission_deadline,
243            snapshot.reserve(
244                source.payload_bytes()?,
245                MessageLane::Normal,
246                &cancellation,
247                admission_deadline,
248            ),
249        )
250        .await?;
251        let lease = admission_scope(
252            &admission_lease,
253            runtime,
254            &self.options,
255            &cancellation,
256            admission_deadline,
257            runtime.pending.reserve_slot(),
258        )
259        .await?;
260        check_scope(runtime, &self.options, &cancellation, admission_deadline)?;
261        if admission_lease.is_cancelled() {
262            return Err(NetError::from(ErrorKind::NotConnected));
263        }
264        let policy = match &self.options.retry_setting {
265            RetrySetting::Inherit => runtime.config.default_request_retry_policy.clone(),
266            RetrySetting::Disabled => WsRequestRetryPolicy::no_retry(),
267            RetrySetting::Policy(policy) => policy.clone(),
268        };
269        let (sender, receiver) = oneshot::channel();
270        let control = Arc::new(LogicalControl {
271            id: runtime.allocate_operation()?,
272            runtime: runtime.clone(),
273            started,
274            deadline,
275            cancellation,
276            connection_lease: admission_lease.clone(),
277            policy: policy.clone(),
278            options: self.options.clone(),
279            source,
280            registration_gate: Mutex::new(()),
281            state: Mutex::new(LogicalState {
282                retry: RetryState::new(policy.schedule().clone()),
283                current: None,
284                terminal: false,
285                elapsed: None,
286                delivery: DeliveryEvidence::NotStarted,
287                phase: OperationPhase::WaitingForCapacity,
288                connection: None,
289                response: Some(sender),
290                reservation: Some(reservation),
291                lease: Some(lease),
292                hook: None,
293            }),
294        });
295        if let Some(group) = &self.options.cancellation {
296            let weak = Arc::downgrade(&control);
297            let hook = group.inner.bind_cancel_hook(Arc::new(move || {
298                if let Some(control) = weak.upgrade() {
299                    control.finish(
300                        Err(NetError::from(ErrorKind::Cancelled)),
301                        RetryStopReason::Cancelled,
302                    );
303                }
304            }))?;
305            lock_recover(&control.state).hook = Some(hook);
306        }
307        let handle = WsRequestOperationHandle {
308            control: control.clone(),
309        };
310        check_scope(
311            runtime,
312            &self.options,
313            &control.cancellation,
314            admission_deadline,
315        )?;
316        if admission_lease.is_cancelled() {
317            return Err(NetError::from(ErrorKind::NotConnected));
318        }
319        let guard = RunnerGuard(control.clone());
320        let factory = self.factory.clone();
321        let key = self.key.clone();
322        runtime.spawn_request(async move {
323            let _guard = guard;
324            run(control, factory, key, admission_deadline).await;
325        })?;
326        Ok(WsRequestOperationReceipt {
327            handle,
328            response: receiver,
329        })
330    }
331}
332
333/// Cloneable control for the stable operation, independent of individual registrations.
334#[derive(Clone)]
335pub struct WsRequestOperationHandle {
336    control: Arc<LogicalControl>,
337}
338impl WsRequestOperationHandle {
339    pub fn id(&self) -> OperationId {
340        self.control.id
341    }
342    /// Snapshot only: None before registration, during backoff, and after completion.
343    pub fn current_registration(&self) -> Option<RequestRegistration> {
344        lock_recover(&self.control.state)
345            .current
346            .as_ref()
347            .map(|handle| handle.registration().clone())
348    }
349    pub fn retry_report(&self) -> RetryReport {
350        self.control.report()
351    }
352    pub fn cancel(&self) -> Result<TerminationOutcome> {
353        Ok(self.control.finish(
354            Err(NetError::from(ErrorKind::Cancelled)),
355            RetryStopReason::Cancelled,
356        ))
357    }
358}
359impl fmt::Debug for WsRequestOperationHandle {
360    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
361        f.debug_struct("WsRequestOperationHandle")
362            .field("id", &self.id())
363            .field("retry_report", &self.retry_report())
364            .finish()
365    }
366}
367#[must_use]
368#[derive(Debug)]
369pub struct WsRequestOperationReceipt {
370    handle: WsRequestOperationHandle,
371    response: oneshot::Receiver<Result<Response>>,
372}
373impl WsRequestOperationReceipt {
374    pub fn handle(&self) -> &WsRequestOperationHandle {
375        &self.handle
376    }
377    /// Dropping the receipt/future stops waiting and does not cancel the operation.
378    pub async fn response(self) -> Result<Response> {
379        self.response.await.map_err(|_| {
380            NetError::from(ErrorKind::EngineDropped).with_retry_report(self.handle.retry_report())
381        })?
382    }
383}
384
385struct LogicalControl {
386    id: OperationId,
387    runtime: Arc<SessionRuntime>,
388    started: Instant,
389    deadline: Option<Instant>,
390    cancellation: CancellationToken,
391    /// Reject retains the physical generation captured at submission. A
392    /// reconnect cannot renew this authority between factory or retry phases.
393    connection_lease: CancellationToken,
394    policy: WsRequestRetryPolicy,
395    options: WsRequestOperationOptions,
396    source: TaskSource,
397    registration_gate: Mutex<()>,
398    state: Mutex<LogicalState>,
399}
400struct LogicalState {
401    retry: RetryState,
402    current: Option<RequestHandle>,
403    terminal: bool,
404    elapsed: Option<Duration>,
405    delivery: DeliveryEvidence,
406    phase: OperationPhase,
407    connection: Option<ConnectionId>,
408    response: Option<oneshot::Sender<Result<Response>>>,
409    reservation: Option<TaskEventReservation>,
410    lease: Option<Arc<PendingLease>>,
411    hook: Option<CancelHookGuard>,
412}
413impl LogicalControl {
414    fn report(&self) -> RetryReport {
415        let state = lock_recover(&self.state);
416        let mut report = state.retry.report(
417            state
418                .elapsed
419                .map_or_else(|| self.started.elapsed(), |elapsed| elapsed),
420        );
421        if let Some(current) = &state.current {
422            report.transport_sends = report
423                .transport_sends
424                .saturating_add(current.retry_report().transport_sends);
425        }
426        report
427    }
428    fn notify(&self, event: RetryEvent) {
429        if let Some(observer) = self.policy.observer() {
430            observer.notify(event);
431        }
432    }
433    fn check(&self) -> Result<()> {
434        self.check_result(false)
435    }
436    fn check_result(&self, selected: bool) -> Result<()> {
437        check_result_scope(
438            &self.runtime,
439            &self.options,
440            &self.cancellation,
441            self.deadline,
442            || selected,
443        )
444        .map_err(|error| {
445            if error.kind() == ErrorKind::TimedOut {
446                error.with_ws_timeout_phase(WsTimeoutPhase::Total)
447            } else {
448                error
449            }
450        })
451    }
452    /// Only unfinished work needs a live generation. Already selected outcomes
453    /// retain the pending table's result, subject to cancellation and total time.
454    fn check_connection(&self) -> Result<()> {
455        self.check()?;
456        if self.connection_lease.is_cancelled() {
457            return Err(NetError::from(ErrorKind::NotConnected));
458        }
459        Ok(())
460    }
461    fn clear_current(&self) -> Option<RequestHandle> {
462        let mut state = lock_recover(&self.state);
463        let current = state.current.take();
464        if let Some(current) = &current {
465            // Retire before sampling so the writer cannot start between the
466            // transport counter snapshot and removal of this registration.
467            let _ = current.cancel();
468            absorb(&mut state, current);
469        }
470        current
471    }
472    fn finish(&self, result: Result<Response>, reason: RetryStopReason) -> TerminationOutcome {
473        let registration_gate = lock_recover(&self.registration_gate);
474        let (current, sender, reservation, lease, hook, report, event, outcome) = {
475            let mut state = lock_recover(&self.state);
476            if state.terminal {
477                return TerminationOutcome::AlreadyFinished;
478            }
479            let current = state.current.take();
480            // Retire the internal single-attempt control before sampling sends;
481            // this closes the race with a concurrent first transport write.
482            if let Some(current) = &current {
483                let _ = current.cancel();
484                absorb(&mut state, current);
485            }
486            if let Ok(response) = &result {
487                state.delivery = DeliveryEvidence::ResponseConfirmed;
488                state.connection = Some(response.connection_id());
489                state.phase = OperationPhase::AwaitingResponse;
490            }
491            state.terminal = true;
492            state.retry.finish(reason);
493            let elapsed = self.started.elapsed();
494            state.elapsed = Some(elapsed);
495            let report = state.retry.report(elapsed);
496            let outcome = match state.delivery {
497                DeliveryEvidence::NotStarted => TerminationOutcome::TerminatedBeforeWrite,
498                DeliveryEvidence::Unknown => TerminationOutcome::DeliveryUnknown,
499                _ => TerminationOutcome::TerminatedAfterWrite,
500            };
501            let event = TaskEvent {
502                client_id: self.runtime.client_id,
503                session_id: self.runtime.id,
504                operation_id: self.id,
505                connection_id: state.connection,
506                source: self.source.clone(),
507                lane: MessageLane::Normal,
508                phase: state.phase,
509                delivery: state.delivery,
510                cause: end_cause(&result, reason),
511                result: result
512                    .as_ref()
513                    .map(|_| TaskSuccess::ResponseReceived)
514                    .map_err(|error| error.clone().with_retry_report(report.clone())),
515            };
516            (
517                current,
518                state.response.take(),
519                state.reservation.take(),
520                state.lease.take(),
521                state.hook.take(),
522                report,
523                event,
524                outcome,
525            )
526        };
527        drop(registration_gate);
528        self.cancellation.cancel();
529        if let Some(current) = current {
530            let _ = current.cancel();
531        }
532        if let Some(lease) = lease {
533            lease.release();
534        }
535        drop(hook);
536        let result = result
537            .map(|response| response.with_retry_report(report.clone()))
538            .map_err(|error| error.with_retry_report(report.clone()));
539        if let Some(reservation) = reservation {
540            if let Err(error) = reservation.publish(event) {
541                crate::log_e!(crate::LogType::WSC; "ws.request_operation.publish", "error", format!("{error:?}"));
542            }
543        }
544        self.notify(RetryEvent::Completed(report));
545        if let Some(sender) = sender {
546            let _ = sender.send(result);
547        }
548        outcome
549    }
550}
551fn absorb(state: &mut LogicalState, current: &RequestHandle) {
552    let sends = current.retry_report().transport_sends;
553    for _ in 0..sends {
554        let _ = state.retry.record_transport_send();
555    }
556    if let Ok(snapshot) = current.state() {
557        if delivery_rank(snapshot.delivery) > delivery_rank(state.delivery) {
558            state.delivery = snapshot.delivery;
559        }
560        if snapshot.phase != OperationPhase::Finished {
561            state.phase = snapshot.phase;
562        }
563        if let Some(Err(error)) = snapshot.result {
564            state.connection = error.context().connection_id.or(state.connection);
565            state.phase = match error.context().stage {
566                Some(ErrorStage::Write) => OperationPhase::Writing,
567                Some(ErrorStage::Response) => OperationPhase::AwaitingResponse,
568                Some(ErrorStage::Admission) => OperationPhase::WaitingForCapacity,
569                _ => state.phase,
570            };
571        }
572    }
573}
574fn delivery_rank(value: DeliveryEvidence) -> u8 {
575    match value {
576        DeliveryEvidence::NotStarted => 0,
577        DeliveryEvidence::Unknown => 1,
578        DeliveryEvidence::Written => 2,
579        DeliveryEvidence::ResponseConfirmed => 3,
580    }
581}
582struct RunnerGuard(Arc<LogicalControl>);
583impl Drop for RunnerGuard {
584    fn drop(&mut self) {
585        self.0.finish(
586            Err(NetError::from(ErrorKind::EngineDropped)),
587            RetryStopReason::Shutdown,
588        );
589    }
590}
591async fn run(
592    control: Arc<LogicalControl>,
593    factory: Arc<AttemptFactory>,
594    key: Arc<str>,
595    admission_deadline: Option<Instant>,
596) {
597    // Keep the previous winner until the next attempt actually begins. Natural
598    // retirement during backoff must not turn that winner into Closed.
599    let mut previous_result = None;
600    loop {
601        if let Err(error) = control.check_connection() {
602            if let Some((result, selected)) = previous_result.take() {
603                finish_after_attempt_error(&control, result, selected, error);
604            } else {
605                finish_error(&control, error);
606            }
607            return;
608        }
609        drop(previous_result.take());
610        let attempt = {
611            let mut state = lock_recover(&control.state);
612            if state.terminal {
613                return;
614            }
615            match state.retry.begin_attempt() {
616                Ok(attempt) => attempt,
617                Err(_) => {
618                    drop(state);
619                    finish_error(&control, NetError::from(ErrorKind::InvalidPolicy));
620                    return;
621                }
622            }
623        };
624        let attempt_started = Instant::now();
625        control.notify(RetryEvent::AttemptStarted { attempt });
626        let result = attempt_once(
627            &control,
628            &factory,
629            key.clone(),
630            attempt,
631            attempt_started,
632            if attempt == 1 {
633                admission_deadline
634            } else {
635                None
636            },
637        )
638        .await;
639        let selected = lock_recover(&control.state)
640            .current
641            .as_ref()
642            .is_some_and(RequestHandle::has_selected_result);
643        if let Some(current) = control.clear_current() {
644            let _ = current.cancel();
645        }
646        if lock_recover(&control.state).terminal {
647            return;
648        }
649        if let Err(error) = control.check_result(selected) {
650            control.notify(RetryEvent::AttemptFinished {
651                attempt,
652                final_attempt: true,
653            });
654            finish_error(&control, error);
655            return;
656        }
657        let mut decision = classify(&control, &result);
658        if let Err(error) = control.check_result(selected) {
659            control.notify(RetryEvent::AttemptFinished {
660                attempt,
661                final_attempt: true,
662            });
663            finish_error(&control, error);
664            return;
665        }
666        if selected && control.runtime.preserves_selected_result()
667            && matches!(decision, Ok(RetryStep::Retry(_)))
668        {
669            decision = Ok(RetryStep::Stop(RetryStopReason::NotRetryable));
670        }
671        let delay = match decision {
672            Ok(RetryStep::Retry(delay)) => delay,
673            Ok(RetryStep::Stop(reason)) => {
674                control.notify(RetryEvent::AttemptFinished {
675                    attempt,
676                    final_attempt: true,
677                });
678                if let Err(error) = control.check_result(selected) {
679                    finish_error(&control, error);
680                } else {
681                    control.finish(result, reason);
682                }
683                return;
684            }
685            Err(error) => {
686                control.notify(RetryEvent::AttemptFinished {
687                    attempt,
688                    final_attempt: true,
689                });
690                control.finish(Err(error), RetryStopReason::PolicyFailed);
691                return;
692            }
693        };
694        control.notify(RetryEvent::AttemptFinished {
695            attempt,
696            final_attempt: false,
697        });
698        if let Err(error) = control.check() {
699            finish_after_attempt_error(&control, result, selected, error);
700            return;
701        }
702        lock_recover(&control.state).phase = OperationPhase::Queued;
703        control.notify(RetryEvent::RetryScheduled { attempt, delay });
704        let ready = match Instant::now().checked_add(delay) {
705            Some(ready) => ready,
706            None => {
707                finish_error(&control, NetError::from(ErrorKind::InvalidPolicy));
708                return;
709            }
710        };
711        let sleep = control.runtime.wait_deadline(Some(ready));
712        if let Err(error) = admission_scope(
713            &control.connection_lease,
714            &control.runtime,
715            &control.options,
716            &control.cancellation,
717            control.deadline,
718            sleep,
719        )
720        .await
721        {
722            let error = if error.kind() == ErrorKind::TimedOut {
723                error.with_ws_timeout_phase(WsTimeoutPhase::Total)
724            } else {
725                error
726            };
727            finish_after_attempt_error(&control, result, selected, error);
728            return;
729        }
730        previous_result = Some((result, selected));
731    }
732}
733async fn attempt_once(
734    control: &Arc<LogicalControl>,
735    factory: &Arc<AttemptFactory>,
736    key: Arc<str>,
737    attempt: u32,
738    started: Instant,
739    admission_deadline: Option<Instant>,
740) -> Result<Response> {
741    check_attempt_budget(control, started)?;
742    control.check_connection()?;
743    let attempt_budget = TimeoutBudget::earlier(
744        control
745            .deadline
746            .map(|at| TimeoutBudget::new(at, WsTimeoutPhase::Total)),
747        add(started, control.options.attempt_timeout)?
748            .map(|at| TimeoutBudget::new(at, WsTimeoutPhase::Attempt)),
749    );
750    let attempt_deadline = attempt_budget.map(|budget| budget.at);
751    let context = RequestAttemptContext {
752        operation_id: control.id,
753        attempt,
754        request_id: RequestId::random()?,
755        idempotency_key: key,
756    };
757    let built = super::retry::isolate_classifier(|| factory(&context));
758    // A synchronous factory cannot be preempted, so arbitrate its elapsed time
759    // before accepting either its success or its returned construction error.
760    check_attempt_budget(control, started)?;
761    control.check_connection()?;
762    let message = built?.map_err(|error| error.with_stage(ErrorStage::RequestBuild))?;
763    let request =
764        Request::new(context.request_id, message).with_metadata(control.options.metadata.clone());
765    let options = RequestOptions {
766        send: SendOptions {
767            priority: control.options.priority,
768            write_timeout: control.options.write_timeout,
769            deadline: attempt_deadline,
770            disconnected: control.options.disconnected,
771            cancellation: control.options.cancellation.clone(),
772            retry_setting: RetrySetting::Disabled,
773            metadata: control.options.metadata.clone(),
774            ..SendOptions::default()
775        },
776        response_timeout: control.options.response_timeout,
777        response_timeout_origin: control.options.response_timeout_origin,
778        registration_deadline: admission_deadline,
779    };
780    let lease = lock_recover(&control.state)
781        .lease
782        .clone()
783        .ok_or_else(|| NetError::from(ErrorKind::Cancelled))?;
784    let registration_check = || {
785        control.check_connection()?;
786        if lock_recover(&control.state).terminal {
787            return Err(NetError::from(ErrorKind::Cancelled));
788        }
789        Ok(())
790    };
791    let on_registered = |handle: &RequestHandle| {
792        let mut state = lock_recover(&control.state);
793        if state.terminal {
794            return Err(NetError::from(ErrorKind::Cancelled));
795        }
796        state.current = Some(handle.clone());
797        state.phase = OperationPhase::WaitingForCapacity;
798        Ok(())
799    };
800    let mut admission = scope_attempt(
801        &control.runtime,
802        &control.options,
803        &control.cancellation,
804        control.deadline,
805        TimeoutBudget::earlier(
806            attempt_budget,
807            admission_deadline.map(|at| TimeoutBudget::new(at, WsTimeoutPhase::Admission)),
808        ),
809        None,
810        dispatch::prepare_operation_attempt(
811            &control.runtime,
812            &request,
813            &options,
814            control.started,
815            lease,
816            dispatch::AttemptRegistration {
817                absolute_budget: attempt_budget,
818                connection_lease: &control.connection_lease,
819                gate: &control.registration_gate,
820                check: &registration_check,
821                install: &on_registered,
822            },
823        ),
824    )
825    .await
826    .map_err(|error| {
827        let error = preserve_timeout_phase(error);
828        if error.context().stage.is_none() {
829            error.with_stage(ErrorStage::Admission)
830        } else {
831            error
832        }
833    })?;
834    let (handle, response) = admission
835        .request
836        .take()
837        .ok_or_else(|| NetError::from(ErrorKind::Internal))?;
838    {
839        let mut state = lock_recover(&control.state);
840        if state.terminal {
841            return Err(NetError::from(ErrorKind::Cancelled));
842        }
843        state.current = Some(handle.clone());
844        state.phase = OperationPhase::Queued;
845    }
846    admission
847        .commit()
848        .map_err(|error| preserve_timeout_phase(error))?;
849    let result = scope_attempt(
850        &control.runtime,
851        &control.options,
852        &control.cancellation,
853        control.deadline,
854        attempt_budget,
855        Some(&handle),
856        async {
857            response
858                .await
859                .map_err(|_| NetError::from(ErrorKind::EngineDropped))?
860        },
861    )
862    .await;
863    result.map_err(|error| {
864        let error = preserve_timeout_phase(error);
865        if error.kind() == ErrorKind::TimedOut && error.context().stage.is_none() {
866            error.with_stage(ErrorStage::Response)
867        } else {
868            error
869        }
870    })
871}
872
873fn check_attempt_budget(control: &LogicalControl, started: Instant) -> Result<()> {
874    control.check()?;
875    if add(started, control.options.attempt_timeout)?
876        .is_some_and(|deadline| deadline <= Instant::now())
877    {
878        return Err(
879            NetError::from(ErrorKind::TimedOut).with_ws_timeout_phase(WsTimeoutPhase::Attempt)
880        );
881    }
882    Ok(())
883}
884
885fn preserve_timeout_phase(error: NetError) -> NetError {
886    match super::retry::timeout_phase(&error) {
887        Some(phase) => error.with_ws_timeout_phase(phase),
888        None => error,
889    }
890}
891enum RetryStep {
892    Retry(Duration),
893    Stop(RetryStopReason),
894}
895fn classify(control: &LogicalControl, result: &Result<Response>) -> Result<RetryStep> {
896    if matches!(control.options.retry_setting, RetrySetting::Disabled) {
897        return Ok(RetryStep::Stop(RetryStopReason::Disabled));
898    }
899    if let Err(error) = result {
900        if super::retry::terminal_error(error) {
901            let reason = if error.kind() == ErrorKind::CallbackPanicked {
902                RetryStopReason::PolicyFailed
903            } else {
904                RetryStopReason::NotRetryable
905            };
906            return Ok(RetryStep::Stop(reason));
907        }
908    }
909    let retry = lock_recover(&control.state).retry.context(
910        control.started.elapsed(),
911        control
912            .deadline
913            .map(|deadline| deadline.saturating_duration_since(Instant::now())),
914    );
915    let decision = control.policy.classify(&WsRequestRetryContext {
916        outcome: result.as_ref(),
917        retry: &retry,
918    })?;
919    let reason = if decision == RetryDecision::Stop {
920        Some(RetryStopReason::ClassifierStopped)
921    } else if decision == RetryDecision::Default
922        && !result
923            .as_ref()
924            .err()
925            .is_some_and(|error| control.policy.should_retry_error(error))
926    {
927        Some(if result.is_ok() {
928            RetryStopReason::Completed
929        } else {
930            RetryStopReason::NotRetryable
931        })
932    } else if !lock_recover(&control.state).retry.can_retry() {
933        Some(RetryStopReason::AttemptsExhausted)
934    } else if !control.policy.allows_replay() {
935        Some(RetryStopReason::UnsafeToReplay)
936    } else {
937        None
938    };
939    let mut state = lock_recover(&control.state);
940    if let Some(reason) = reason {
941        return Ok(RetryStep::Stop(reason));
942    }
943    state
944        .retry
945        .next_delay()
946        .map(|delay| match delay {
947            Some(delay) => RetryStep::Retry(delay),
948            None => RetryStep::Stop(RetryStopReason::AttemptsExhausted),
949        })
950        .map_err(|_| NetError::from(ErrorKind::InvalidPolicy))
951}
952fn error_stop_reason(error: &NetError) -> RetryStopReason {
953    match error.kind() {
954        ErrorKind::Cancelled => RetryStopReason::Cancelled,
955        ErrorKind::TimedOut => RetryStopReason::TimedOut,
956        ErrorKind::Closed | ErrorKind::EngineDropped => RetryStopReason::Shutdown,
957        ErrorKind::CallbackPanicked | ErrorKind::InvalidPolicy => RetryStopReason::PolicyFailed,
958        _ => RetryStopReason::NotRetryable,
959    }
960}
961fn finish_error(control: &LogicalControl, error: NetError) {
962    let reason = error_stop_reason(&error);
963    control.finish(Err(error), reason);
964}
965
966fn finish_after_attempt_error(
967    control: &LogicalControl,
968    result: Result<Response>,
969    selected: bool,
970    error: NetError,
971) {
972    if selected && control.runtime.preserves_selected_result() {
973        match control.check_result(true) {
974            Ok(()) => {
975                // Every caller has already decided to retry. Natural retirement
976                // prevents that retry even when the retained result is a response.
977                control.finish(result, RetryStopReason::NotRetryable);
978            }
979            Err(error) => finish_error(control, error),
980        }
981    } else {
982        finish_error(control, error);
983    }
984}
985fn end_cause(result: &Result<Response>, reason: RetryStopReason) -> TaskEndCause {
986    if result.is_ok() {
987        return TaskEndCause::Completed;
988    }
989    match reason {
990        RetryStopReason::Cancelled => TaskEndCause::Cancelled,
991        RetryStopReason::TimedOut => TaskEndCause::Expired,
992        RetryStopReason::Shutdown => TaskEndCause::Shutdown,
993        _ => match result.as_ref().err().map(NetError::kind) {
994            Some(ErrorKind::TimedOut) => TaskEndCause::Expired,
995            Some(ErrorKind::Cancelled) => TaskEndCause::Cancelled,
996            Some(ErrorKind::Closed | ErrorKind::NotConnected) => TaskEndCause::Disconnected,
997            _ => TaskEndCause::Failed,
998        },
999    }
1000}
1001fn earlier(left: Option<Instant>, right: Option<Instant>) -> Option<Instant> {
1002    match (left, right) {
1003        (Some(a), Some(b)) => Some(a.min(b)),
1004        (a, b) => a.or(b),
1005    }
1006}
1007fn add(start: Instant, duration: Option<Duration>) -> Result<Option<Instant>> {
1008    duration
1009        .map(|duration| {
1010            start
1011                .checked_add(duration)
1012                .ok_or_else(|| NetError::config("timeout", "deadline is not representable"))
1013        })
1014        .transpose()
1015}
1016fn check_scope(
1017    runtime: &SessionRuntime,
1018    options: &WsRequestOperationOptions,
1019    cancel: &CancellationToken,
1020    deadline: Option<Instant>,
1021) -> Result<()> {
1022    check_result_scope(runtime, options, cancel, deadline, || false)
1023}
1024
1025fn check_result_scope(
1026    runtime: &SessionRuntime,
1027    options: &WsRequestOperationOptions,
1028    cancel: &CancellationToken,
1029    deadline: Option<Instant>,
1030    selected: impl FnOnce() -> bool,
1031) -> Result<()> {
1032    if cancel.is_cancelled()
1033        || options
1034            .cancellation
1035            .as_ref()
1036            .is_some_and(CancellationGroup::is_cancelled)
1037    {
1038        return Err(NetError::from(ErrorKind::Cancelled));
1039    }
1040    // Read the request authority only after observing closure. A false snapshot
1041    // taken earlier could miss a result selected just before natural retirement.
1042    if runtime.is_closed() && !(selected() && runtime.preserves_selected_result()) {
1043        return Err(NetError::from(ErrorKind::Closed));
1044    }
1045    if deadline.is_some_and(|deadline| Instant::now() >= deadline) {
1046        return Err(NetError::from(ErrorKind::TimedOut));
1047    }
1048    Ok(())
1049}
1050/// Keep unfinished admission and backoff bound to the original Reject connection.
1051async fn admission_scope<T>(
1052    lease: &CancellationToken,
1053    runtime: &SessionRuntime,
1054    options: &WsRequestOperationOptions,
1055    cancel: &CancellationToken,
1056    deadline: Option<Instant>,
1057    future: impl Future<Output = Result<T>>,
1058) -> Result<T> {
1059    if options.disconnected == DisconnectedPolicy::WaitForReconnect {
1060        return scope(runtime, options, cancel, deadline, future).await;
1061    }
1062    check_scope(runtime, options, cancel, deadline)?;
1063    if lease.is_cancelled() {
1064        return Err(NetError::from(ErrorKind::NotConnected));
1065    }
1066    let result = tokio::select! {
1067        biased;
1068        _ = lease.cancelled() => {
1069            check_scope(runtime, options, cancel, deadline)?;
1070            Err(NetError::from(ErrorKind::NotConnected))
1071        },
1072        result = scope(runtime, options, cancel, deadline, future) => result,
1073    }?;
1074    check_scope(runtime, options, cancel, deadline)?;
1075    if lease.is_cancelled() {
1076        return Err(NetError::from(ErrorKind::NotConnected));
1077    }
1078    Ok(result)
1079}
1080
1081async fn scope<T>(
1082    runtime: &SessionRuntime,
1083    options: &WsRequestOperationOptions,
1084    cancel: &CancellationToken,
1085    deadline: Option<Instant>,
1086    future: impl Future<Output = Result<T>>,
1087) -> Result<T> {
1088    scope_with_request(runtime, options, cancel, deadline, None, future).await
1089}
1090
1091/// A natural session end cannot replace a pending table winner. Its producer
1092/// may still be releasing resources before publishing to the response channel.
1093async fn result_closure(runtime: &SessionRuntime, request: Option<&RequestHandle>) {
1094    let closed = runtime.closed_token();
1095    let session_end = runtime.lifecycle.completion_token();
1096    tokio::select! {
1097        _ = closed.cancelled() => {},
1098        _ = session_end.cancelled() => {},
1099    }
1100    if request.is_some_and(RequestHandle::has_selected_result)
1101        && runtime.preserves_selected_result()
1102    {
1103        // Explicit retirement has its own signal in the surrounding scope.
1104        std::future::pending::<()>().await;
1105    }
1106}
1107
1108async fn scope_with_request<T>(
1109    runtime: &SessionRuntime,
1110    options: &WsRequestOperationOptions,
1111    cancel: &CancellationToken,
1112    deadline: Option<Instant>,
1113    request: Option<&RequestHandle>,
1114    future: impl Future<Output = Result<T>>,
1115) -> Result<T> {
1116    check_result_scope(
1117        runtime,
1118        options,
1119        cancel,
1120        deadline,
1121        || request.is_some_and(RequestHandle::has_selected_result),
1122    )?;
1123    let forced_closed = runtime.forced_closed_token();
1124    let session_cancel = runtime.lifecycle.cancel_token();
1125    let group = async {
1126        match &options.cancellation {
1127            Some(group) => group.cancelled().await,
1128            None => std::future::pending().await,
1129        }
1130    };
1131    tokio::select! {
1132        biased;
1133        _ = cancel.cancelled() => Err(NetError::from(ErrorKind::Cancelled)),
1134        _ = group => Err(NetError::from(ErrorKind::Cancelled)),
1135        _ = runtime.shutdown.cancelled() => Err(NetError::from(ErrorKind::Closed)),
1136        _ = forced_closed.cancelled() => Err(NetError::from(ErrorKind::Closed)),
1137        _ = session_cancel.cancelled() => Err(NetError::from(ErrorKind::Closed)),
1138        _ = result_closure(runtime, request) => Err(NetError::from(ErrorKind::Closed)),
1139        result = runtime.wait_deadline(deadline) => result.and_then(|()| Err(NetError::from(ErrorKind::TimedOut))),
1140        result = future => result,
1141    }
1142}
1143
1144/// An outcome already selected by the pending table or writer wins over a late
1145/// poll of the per-attempt timer. Cancellation and the logical total budget still
1146/// arbitrate before and after consuming that outcome.
1147async fn scope_attempt<T>(
1148    runtime: &SessionRuntime,
1149    options: &WsRequestOperationOptions,
1150    cancel: &CancellationToken,
1151    total_deadline: Option<Instant>,
1152    budget: Option<TimeoutBudget>,
1153    request: Option<&RequestHandle>,
1154    future: impl Future<Output = Result<T>>,
1155) -> Result<T> {
1156    let check = || {
1157        let selected = || request.is_some_and(RequestHandle::has_selected_result);
1158        #[cfg(test)]
1159        if let Some(request) = request {
1160            runtime.before_request_result_check_for_test(request)?;
1161        }
1162        check_result_scope(
1163            runtime,
1164            options,
1165            cancel,
1166            total_deadline,
1167            selected,
1168        ).map_err(|error| {
1169            if error.kind() == ErrorKind::TimedOut {
1170                error.with_ws_timeout_phase(WsTimeoutPhase::Total)
1171            } else {
1172                error
1173            }
1174        })
1175    };
1176    check()?;
1177    let forced_closed = runtime.forced_closed_token();
1178    let session_cancel = runtime.lifecycle.cancel_token();
1179    let group = async {
1180        match &options.cancellation {
1181            Some(group) => group.cancelled().await,
1182            None => std::future::pending().await,
1183        }
1184    };
1185    tokio::pin!(future);
1186    let result = tokio::select! {
1187        biased;
1188        _ = cancel.cancelled() => Err(NetError::from(ErrorKind::Cancelled)),
1189        _ = group => Err(NetError::from(ErrorKind::Cancelled)),
1190        _ = runtime.shutdown.cancelled() => Err(NetError::from(ErrorKind::Closed)),
1191        _ = forced_closed.cancelled() => Err(NetError::from(ErrorKind::Closed)),
1192        _ = session_cancel.cancelled() => Err(NetError::from(ErrorKind::Closed)),
1193        _ = result_closure(runtime, request) => Err(NetError::from(ErrorKind::Closed)),
1194        result = runtime.wait_deadline(total_deadline) => result.and_then(|()|
1195            Err(NetError::from(ErrorKind::TimedOut).with_ws_timeout_phase(WsTimeoutPhase::Total))),
1196        result = &mut future => result,
1197        result = runtime.wait_deadline(budget.map(|budget| budget.at)) => {
1198            result?;
1199            if let Some(request) = request {
1200                // The channel may still be pending after its outcome was selected.
1201                // Expiration competes with that selection under the same P/C
1202                // authority; a previously chosen result cannot be replaced here.
1203                request.expire_operation_attempt();
1204                scope_with_request(runtime, options, cancel, total_deadline, Some(request), &mut future).await
1205            } else {
1206                Err(budget.map_or_else(|| NetError::from(ErrorKind::TimedOut), TimeoutBudget::timeout_error))
1207            }
1208        },
1209    };
1210    check()?;
1211    result
1212}
1213
1214#[cfg(test)]
1215#[path = "request_operation_tests.rs"]
1216mod tests;
1217
1218#[cfg(test)]
1219#[path = "request_operation_generation_tests.rs"]
1220mod generation_tests;
1221
1222#[cfg(test)]
1223#[path = "request_operation_selection_tests.rs"]
1224mod selection_tests;