1use super::cancellation::{lock_recover, CancelHookGuard};
3use super::*;
4use crate::api::retry::{RetryDecision, RetryEvent, RetryReport, RetrySetting, RetryStopReason};
5use crate::error::{ErrorKind, ErrorStage};
6use crate::module::ws_client::native_pending::PendingLease;
7use crate::module::ws_client::timeout_budget::TimeoutBudget;
8use crate::module::ws_client::{
9 native_task_observer::TaskEventReservation, session_runtime::SessionRuntime,
10};
11use crate::retry_core::RetryState;
12use crate::{EnqueueError, Metadata, NetError, Result};
13use std::fmt;
14use std::future::Future;
15use std::sync::{Arc, Mutex};
16use std::time::{Duration, Instant};
17use tokio::sync::oneshot;
18use tokio_util::sync::CancellationToken;
19
20type AttemptFactory = dyn Fn(&RequestAttemptContext) -> Result<Message> + Send + Sync;
21
22#[derive(Clone)]
25pub struct RequestAttemptContext {
26 operation_id: OperationId,
27 attempt: u32,
28 request_id: RequestId,
29 idempotency_key: Arc<str>,
30}
31impl RequestAttemptContext {
32 pub fn operation_id(&self) -> OperationId {
33 self.operation_id
34 }
35 pub fn attempt(&self) -> u32 {
36 self.attempt
37 }
38 pub fn request_id(&self) -> &RequestId {
39 &self.request_id
40 }
41 pub fn idempotency_key(&self) -> &str {
42 &self.idempotency_key
43 }
44}
45impl fmt::Debug for RequestAttemptContext {
46 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
47 f.debug_struct("RequestAttemptContext")
48 .field("operation_id", &self.operation_id)
49 .field("attempt", &self.attempt)
50 .field("request_id", &self.request_id)
51 .field("idempotency_key_bytes", &self.idempotency_key.len())
52 .finish()
53 }
54}
55
56#[derive(Clone)]
59pub struct WsRequestOperationOptions {
60 pub priority: Priority,
61 pub admission_timeout: Option<Duration>,
62 pub write_timeout: Duration,
63 pub response_timeout: Duration,
64 pub response_timeout_origin: ResponseTimeoutOrigin,
65 pub attempt_timeout: Option<Duration>,
66 pub total_timeout: Option<Duration>,
67 pub deadline: Option<Instant>,
68 pub disconnected: DisconnectedPolicy,
69 pub cancellation: Option<CancellationGroup>,
70 pub metadata: Metadata,
71 pub retry_setting: RetrySetting<WsRequestRetryPolicy>,
72}
73impl fmt::Debug for WsRequestOperationOptions {
74 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
75 f.debug_struct("WsRequestOperationOptions")
76 .field("priority", &self.priority)
77 .field("admission_timeout", &self.admission_timeout)
78 .field("write_timeout", &self.write_timeout)
79 .field("response_timeout", &self.response_timeout)
80 .field("response_timeout_origin", &self.response_timeout_origin)
81 .field("attempt_timeout", &self.attempt_timeout)
82 .field("total_timeout", &self.total_timeout)
83 .field("deadline", &self.deadline)
84 .field("disconnected", &self.disconnected)
85 .field("cancellation", &self.cancellation)
86 .field("metadata_entries", &self.metadata.len())
87 .field("retry_setting", &self.retry_setting)
88 .finish()
89 }
90}
91impl Default for WsRequestOperationOptions {
92 fn default() -> Self {
93 Self {
94 priority: Priority::Normal,
95 admission_timeout: None,
96 write_timeout: Duration::from_secs(10),
97 response_timeout: Duration::from_secs(30),
98 response_timeout_origin: ResponseTimeoutOrigin::Written,
99 attempt_timeout: None,
100 total_timeout: None,
101 deadline: None,
102 disconnected: DisconnectedPolicy::Reject,
103 cancellation: None,
104 metadata: Metadata::new(),
105 retry_setting: RetrySetting::Inherit,
106 }
107 }
108}
109impl WsRequestOperationOptions {
110 pub fn validate(&self) -> Result<()> {
111 for (name, duration) in [
112 ("admission_timeout", self.admission_timeout),
113 ("attempt_timeout", self.attempt_timeout),
114 ("total_timeout", self.total_timeout),
115 ("write_timeout", Some(self.write_timeout)),
116 ("response_timeout", Some(self.response_timeout)),
117 ] {
118 if let Some(duration) = duration {
119 super::config::deadline(duration, name, true)?;
120 }
121 }
122 validate_metadata(&self.metadata)
123 }
124}
125
126#[must_use]
128pub struct WsRequestOperationBuilder {
129 client: RequestClient,
130 key: Arc<str>,
131 factory: Arc<AttemptFactory>,
132 options: WsRequestOperationOptions,
133 started: Option<Instant>,
134}
135impl fmt::Debug for WsRequestOperationBuilder {
136 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
137 f.debug_struct("WsRequestOperationBuilder")
138 .field("client", &self.client)
139 .field("key_bytes", &self.key.len())
140 .field("options", &self.options)
141 .finish_non_exhaustive()
142 }
143}
144impl RequestClient {
145 pub fn request_with_factory<F>(
148 &self,
149 key: impl Into<String>,
150 factory: F,
151 ) -> Result<WsRequestOperationBuilder>
152 where
153 F: Fn(&RequestAttemptContext) -> Result<Message> + Send + Sync + 'static,
154 {
155 let key = key.into();
156 if key.len() > 1024 || key.trim().is_empty() {
157 return Err(NetError::input(
158 "idempotency_key",
159 "must contain a non-whitespace character and be at most 1024 UTF-8 bytes",
160 ));
161 }
162 Ok(WsRequestOperationBuilder {
163 client: self.clone(),
164 key: Arc::from(key),
165 factory: Arc::new(factory),
166 options: WsRequestOperationOptions::default(),
167 started: None,
168 })
169 }
170}
171impl WsRequestOperationBuilder {
172 pub fn retry_setting(mut self, setting: RetrySetting<WsRequestRetryPolicy>) -> Self {
174 self.options.retry_setting = setting;
175 self
176 }
177 pub fn options(mut self, options: WsRequestOperationOptions) -> Self {
178 self.options = options;
179 self.started = None;
180 self
181 }
182 pub async fn send(self) -> Result<Response> {
183 self.enqueue()
184 .await
185 .map_err(EnqueueError::into_error)?
186 .response()
187 .await
188 }
189 pub async fn enqueue(
192 mut self,
193 ) -> std::result::Result<WsRequestOperationReceipt, EnqueueError<Self>> {
194 let started = *self.started.get_or_insert_with(Instant::now);
195 match self.admit(started).await {
196 Ok(receipt) => Ok(receipt),
197 Err(error) => {
198 let error =
199 if error.kind() == ErrorKind::TimedOut && error.ws_timeout_phase().is_none() {
200 let total = earlier(
201 self.options.deadline,
202 add(started, self.options.total_timeout).ok().flatten(),
203 );
204 let admission = add(started, self.options.admission_timeout).ok().flatten();
205 error.with_ws_timeout_phase(
206 if total.is_some() && earlier(total, admission) == total {
207 WsTimeoutPhase::Total
208 } else {
209 WsTimeoutPhase::Admission
210 },
211 )
212 } else {
213 error
214 };
215 let report = RetryReport {
216 elapsed: started.elapsed(),
217 stop_reason: Some(error_stop_reason(&error)),
218 ..RetryReport::default()
219 };
220 Err(EnqueueError::new(self, error.with_retry_report(report)))
221 }
222 }
223 }
224 async fn admit(&self, started: Instant) -> Result<WsRequestOperationReceipt> {
225 #[cfg(feature = "quality-doctor")]
226 let mut quality_scope = crate::module::ws_client::quality::QualityScope::new(
227 self.client
228 .runtime
229 .quality_begin(crate::api::quality::QualityOperationKind::WsRequestOperation),
230 );
231 #[cfg(feature = "quality-doctor")]
232 let admission = quality_scope
233 .probe
234 .as_ref()
235 .map(|probe| probe.phase(crate::api::quality::PhaseKind::Admission));
236 self.options.validate()?;
237 let runtime = &self.client.runtime;
238 let deadline = earlier(
239 self.options.deadline,
240 add(started, self.options.total_timeout)?,
241 );
242 let admission_deadline = earlier(deadline, add(started, self.options.admission_timeout)?);
243 let cancellation = CancellationToken::new();
244 let admission_lease = runtime.lease(self.options.disconnected)?;
246 let source = TaskSource::RequestOperation(Arc::new(self.options.metadata.clone()));
247 let snapshot = runtime.tasks.snapshot()?;
248 let reservation = admission_scope(
249 &admission_lease,
250 runtime,
251 &self.options,
252 &cancellation,
253 admission_deadline,
254 snapshot.reserve(
255 source.payload_bytes()?,
256 MessageLane::Normal,
257 &cancellation,
258 admission_deadline,
259 ),
260 )
261 .await?;
262 let lease = admission_scope(
263 &admission_lease,
264 runtime,
265 &self.options,
266 &cancellation,
267 admission_deadline,
268 runtime.pending.reserve_slot(),
269 )
270 .await?;
271 check_scope(runtime, &self.options, &cancellation, admission_deadline)?;
272 if admission_lease.is_cancelled() {
273 return Err(NetError::from(ErrorKind::NotConnected));
274 }
275 let policy = match &self.options.retry_setting {
276 RetrySetting::Inherit => runtime.config.default_request_retry_policy.clone(),
277 RetrySetting::Disabled => WsRequestRetryPolicy::no_retry(),
278 RetrySetting::Policy(policy) => policy.clone(),
279 };
280 let (sender, receiver) = oneshot::channel();
281 #[cfg(feature = "quality-doctor")]
282 drop(admission);
283 let control = Arc::new(LogicalControl {
284 #[cfg(feature = "quality-doctor")]
285 quality: quality_scope.probe.take(),
286 id: runtime.allocate_operation()?,
287 runtime: runtime.clone(),
288 started,
289 deadline,
290 cancellation,
291 connection_lease: admission_lease.clone(),
292 policy: policy.clone(),
293 options: self.options.clone(),
294 source,
295 registration_gate: Mutex::new(()),
296 state: Mutex::new(LogicalState {
297 retry: RetryState::new(policy.schedule().clone()),
298 current: None,
299 terminal: false,
300 elapsed: None,
301 delivery: DeliveryEvidence::NotStarted,
302 phase: OperationPhase::WaitingForCapacity,
303 connection: None,
304 response: Some(sender),
305 reservation: Some(reservation),
306 lease: Some(lease),
307 hook: None,
308 }),
309 });
310 #[cfg(feature = "quality-doctor")]
311 if let Some(probe) = &control.quality {
312 probe.operation(control.id.as_u64());
313 }
314 if let Some(group) = &self.options.cancellation {
315 let weak = Arc::downgrade(&control);
316 let hook = group.inner.bind_cancel_hook(Arc::new(move || {
317 if let Some(control) = weak.upgrade() {
318 control.finish(
319 Err(NetError::from(ErrorKind::Cancelled)),
320 RetryStopReason::Cancelled,
321 );
322 }
323 }))?;
324 lock_recover(&control.state).hook = Some(hook);
325 }
326 let handle = WsRequestOperationHandle {
327 control: control.clone(),
328 };
329 check_scope(
330 runtime,
331 &self.options,
332 &control.cancellation,
333 admission_deadline,
334 )?;
335 if admission_lease.is_cancelled() {
336 return Err(NetError::from(ErrorKind::NotConnected));
337 }
338 let guard = RunnerGuard(control.clone());
339 let factory = self.factory.clone();
340 let key = self.key.clone();
341 runtime.spawn_request(async move {
342 let _guard = guard;
343 run(control, factory, key, admission_deadline).await;
344 })?;
345 Ok(WsRequestOperationReceipt {
346 handle,
347 response: receiver,
348 })
349 }
350}
351
352#[derive(Clone)]
354pub struct WsRequestOperationHandle {
355 control: Arc<LogicalControl>,
356}
357impl WsRequestOperationHandle {
358 #[cfg(feature = "quality-doctor")]
378 pub fn quality_ref(&self) -> Option<crate::api::quality::QualityReportRef> {
379 self.control
380 .quality
381 .as_ref()
382 .map(crate::module::quality::QualityProbe::reference)
383 }
384 pub fn id(&self) -> OperationId {
385 self.control.id
386 }
387 pub fn current_registration(&self) -> Option<RequestRegistration> {
389 lock_recover(&self.control.state)
390 .current
391 .as_ref()
392 .map(|handle| handle.registration().clone())
393 }
394 pub fn retry_report(&self) -> RetryReport {
395 self.control.report()
396 }
397 pub fn cancel(&self) -> Result<TerminationOutcome> {
398 Ok(self.control.finish(
399 Err(NetError::from(ErrorKind::Cancelled)),
400 RetryStopReason::Cancelled,
401 ))
402 }
403}
404impl fmt::Debug for WsRequestOperationHandle {
405 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
406 f.debug_struct("WsRequestOperationHandle")
407 .field("id", &self.id())
408 .field("retry_report", &self.retry_report())
409 .finish()
410 }
411}
412#[must_use]
413#[derive(Debug)]
414pub struct WsRequestOperationReceipt {
415 handle: WsRequestOperationHandle,
416 response: oneshot::Receiver<Result<Response>>,
417}
418impl WsRequestOperationReceipt {
419 pub fn handle(&self) -> &WsRequestOperationHandle {
420 &self.handle
421 }
422 pub async fn response(self) -> Result<Response> {
424 self.response.await.map_err(|_| {
425 NetError::from(ErrorKind::EngineDropped).with_retry_report(self.handle.retry_report())
426 })?
427 }
428}
429
430struct LogicalControl {
431 #[cfg(feature = "quality-doctor")]
432 quality: Option<crate::module::quality::QualityProbe>,
433 id: OperationId,
434 runtime: Arc<SessionRuntime>,
435 started: Instant,
436 deadline: Option<Instant>,
437 cancellation: CancellationToken,
438 connection_lease: CancellationToken,
441 policy: WsRequestRetryPolicy,
442 options: WsRequestOperationOptions,
443 source: TaskSource,
444 registration_gate: Mutex<()>,
445 state: Mutex<LogicalState>,
446}
447struct LogicalState {
448 retry: RetryState,
449 current: Option<RequestHandle>,
450 terminal: bool,
451 elapsed: Option<Duration>,
452 delivery: DeliveryEvidence,
453 phase: OperationPhase,
454 connection: Option<ConnectionId>,
455 response: Option<oneshot::Sender<Result<Response>>>,
456 reservation: Option<TaskEventReservation>,
457 lease: Option<Arc<PendingLease>>,
458 hook: Option<CancelHookGuard>,
459}
460impl LogicalControl {
461 fn report(&self) -> RetryReport {
462 let state = lock_recover(&self.state);
463 let mut report = state.retry.report(
464 state
465 .elapsed
466 .map_or_else(|| self.started.elapsed(), |elapsed| elapsed),
467 );
468 if let Some(current) = &state.current {
469 report.transport_sends = report
470 .transport_sends
471 .saturating_add(current.retry_report().transport_sends);
472 }
473 report
474 }
475 fn notify(&self, event: RetryEvent) {
476 if let Some(observer) = self.policy.observer() {
477 observer.notify(event);
478 }
479 }
480 fn check(&self) -> Result<()> {
481 self.check_result(false)
482 }
483 fn check_result(&self, selected: bool) -> Result<()> {
484 check_result_scope(
485 &self.runtime,
486 &self.options,
487 &self.cancellation,
488 self.deadline,
489 || selected,
490 )
491 .map_err(|error| {
492 if error.kind() == ErrorKind::TimedOut {
493 error.with_ws_timeout_phase(WsTimeoutPhase::Total)
494 } else {
495 error
496 }
497 })
498 }
499 fn check_connection(&self) -> Result<()> {
502 self.check()?;
503 if self.connection_lease.is_cancelled() {
504 return Err(NetError::from(ErrorKind::NotConnected));
505 }
506 Ok(())
507 }
508 fn clear_current(&self) -> Option<RequestHandle> {
509 let mut state = lock_recover(&self.state);
510 let current = state.current.take();
511 if let Some(current) = ¤t {
512 let _ = current.cancel();
515 absorb(&mut state, current);
516 }
517 current
518 }
519 fn finish(&self, result: Result<Response>, reason: RetryStopReason) -> TerminationOutcome {
520 let registration_gate = lock_recover(&self.registration_gate);
521 let (current, sender, reservation, lease, hook, report, event, outcome) = {
522 let mut state = lock_recover(&self.state);
523 if state.terminal {
524 return TerminationOutcome::AlreadyFinished;
525 }
526 let current = state.current.take();
527 if let Some(current) = ¤t {
530 let _ = current.cancel();
531 absorb(&mut state, current);
532 }
533 if let Ok(response) = &result {
534 state.delivery = DeliveryEvidence::ResponseConfirmed;
535 state.connection = Some(response.connection_id());
536 state.phase = OperationPhase::AwaitingResponse;
537 }
538 state.terminal = true;
539 state.retry.finish(reason);
540 let elapsed = self.started.elapsed();
541 state.elapsed = Some(elapsed);
542 let report = state.retry.report(elapsed);
543 let outcome = match state.delivery {
544 DeliveryEvidence::NotStarted => TerminationOutcome::TerminatedBeforeWrite,
545 DeliveryEvidence::Unknown => TerminationOutcome::DeliveryUnknown,
546 _ => TerminationOutcome::TerminatedAfterWrite,
547 };
548 let event = TaskEvent {
549 client_id: self.runtime.client_id,
550 session_id: self.runtime.id,
551 operation_id: self.id,
552 connection_id: state.connection,
553 source: self.source.clone(),
554 lane: MessageLane::Normal,
555 phase: state.phase,
556 delivery: state.delivery,
557 cause: end_cause(&result, reason),
558 result: result
559 .as_ref()
560 .map(|_| TaskSuccess::ResponseReceived)
561 .map_err(|error| error.clone().with_retry_report(report.clone())),
562 };
563 (
564 current,
565 state.response.take(),
566 state.reservation.take(),
567 state.lease.take(),
568 state.hook.take(),
569 report,
570 event,
571 outcome,
572 )
573 };
574 drop(registration_gate);
575 self.cancellation.cancel();
576 if let Some(current) = current {
577 let _ = current.cancel();
578 }
579 if let Some(lease) = lease {
580 lease.release();
581 }
582 drop(hook);
583 let result = result
584 .map(|response| response.with_retry_report(report.clone()))
585 .map_err(|error| error.with_retry_report(report.clone()));
586 #[cfg(feature = "quality-doctor")]
587 let result = {
588 let reference = self
589 .quality
590 .as_ref()
591 .map(crate::module::quality::QualityProbe::reference);
592 if let Some(probe) = &self.quality {
593 use crate::api::quality::{PhaseKind, QualityOutcome};
594 probe.mark(PhaseKind::ResultSelected);
595 probe.mark(PhaseKind::ResourcesReleased);
596 match &result {
597 Ok(response) => {
598 probe.connection(response.connection_id().as_u64());
599 probe.result(QualityOutcome::Success);
600 }
601 Err(error) => probe.error(error),
602 }
603 }
604 result
605 .map(|response| response.with_quality_ref(reference))
606 .map_err(|error| match reference {
607 Some(reference) => error.with_quality_ref(reference),
608 None => error,
609 })
610 };
611 if let Some(reservation) = reservation {
612 if let Err(error) = reservation.publish(event) {
613 crate::log_e!(crate::LogType::WSC; "ws.request_operation.publish", "error", format!("{error:?}"));
614 }
615 }
616 self.notify(RetryEvent::Completed(report));
617 if let Some(sender) = sender {
618 let _ = sender.send(result);
619 }
620 #[cfg(feature = "quality-doctor")]
621 if let Some(probe) = &self.quality {
622 probe.mark(crate::api::quality::PhaseKind::DeliveryFinished);
623 probe.finish();
624 }
625 outcome
626 }
627}
628fn absorb(state: &mut LogicalState, current: &RequestHandle) {
629 let sends = current.retry_report().transport_sends;
630 for _ in 0..sends {
631 let _ = state.retry.record_transport_send();
632 }
633 if let Ok(snapshot) = current.state() {
634 if delivery_rank(snapshot.delivery) > delivery_rank(state.delivery) {
635 state.delivery = snapshot.delivery;
636 }
637 if snapshot.phase != OperationPhase::Finished {
638 state.phase = snapshot.phase;
639 }
640 if let Some(Err(error)) = snapshot.result {
641 state.connection = error.context().connection_id.or(state.connection);
642 state.phase = match error.context().stage {
643 Some(ErrorStage::Write) => OperationPhase::Writing,
644 Some(ErrorStage::Response) => OperationPhase::AwaitingResponse,
645 Some(ErrorStage::Admission) => OperationPhase::WaitingForCapacity,
646 _ => state.phase,
647 };
648 }
649 }
650}
651fn delivery_rank(value: DeliveryEvidence) -> u8 {
652 match value {
653 DeliveryEvidence::NotStarted => 0,
654 DeliveryEvidence::Unknown => 1,
655 DeliveryEvidence::Written => 2,
656 DeliveryEvidence::ResponseConfirmed => 3,
657 }
658}
659struct RunnerGuard(Arc<LogicalControl>);
660impl Drop for RunnerGuard {
661 fn drop(&mut self) {
662 self.0.finish(
663 Err(NetError::from(ErrorKind::EngineDropped)),
664 RetryStopReason::Shutdown,
665 );
666 }
667}
668async fn run(
669 control: Arc<LogicalControl>,
670 factory: Arc<AttemptFactory>,
671 key: Arc<str>,
672 admission_deadline: Option<Instant>,
673) {
674 let mut previous_result = None;
677 loop {
678 if let Err(error) = control.check_connection() {
679 if let Some((result, selected)) = previous_result.take() {
680 finish_after_attempt_error(&control, result, selected, error);
681 } else {
682 finish_error(&control, error);
683 }
684 return;
685 }
686 drop(previous_result.take());
687 let attempt = {
688 let mut state = lock_recover(&control.state);
689 if state.terminal {
690 return;
691 }
692 match state.retry.begin_attempt() {
693 Ok(attempt) => attempt,
694 Err(_) => {
695 drop(state);
696 finish_error(&control, NetError::from(ErrorKind::InvalidPolicy));
697 return;
698 }
699 }
700 };
701 let attempt_started = Instant::now();
702 #[cfg(feature = "quality-doctor")]
703 if let Some(probe) = &control.quality {
704 probe.attempt(attempt as u64);
705 }
706 control.notify(RetryEvent::AttemptStarted { attempt });
707 let result = attempt_once(
708 &control,
709 &factory,
710 key.clone(),
711 attempt,
712 attempt_started,
713 if attempt == 1 {
714 admission_deadline
715 } else {
716 None
717 },
718 )
719 .await;
720 let selected = lock_recover(&control.state)
721 .current
722 .as_ref()
723 .is_some_and(RequestHandle::has_selected_result);
724 if let Some(current) = control.clear_current() {
725 let _ = current.cancel();
726 }
727 if lock_recover(&control.state).terminal {
728 return;
729 }
730 if let Err(error) = control.check_result(selected) {
731 control.notify(RetryEvent::AttemptFinished {
732 attempt,
733 final_attempt: true,
734 });
735 finish_error(&control, error);
736 return;
737 }
738 let mut decision = classify(&control, &result);
739 if let Err(error) = control.check_result(selected) {
740 control.notify(RetryEvent::AttemptFinished {
741 attempt,
742 final_attempt: true,
743 });
744 finish_error(&control, error);
745 return;
746 }
747 if selected && control.runtime.preserves_selected_result()
748 && matches!(decision, Ok(RetryStep::Retry(_)))
749 {
750 decision = Ok(RetryStep::Stop(RetryStopReason::NotRetryable));
751 }
752 let delay = match decision {
753 Ok(RetryStep::Retry(delay)) => delay,
754 Ok(RetryStep::Stop(reason)) => {
755 control.notify(RetryEvent::AttemptFinished {
756 attempt,
757 final_attempt: true,
758 });
759 if let Err(error) = control.check_result(selected) {
760 finish_error(&control, error);
761 } else {
762 control.finish(result, reason);
763 }
764 return;
765 }
766 Err(error) => {
767 control.notify(RetryEvent::AttemptFinished {
768 attempt,
769 final_attempt: true,
770 });
771 control.finish(Err(error), RetryStopReason::PolicyFailed);
772 return;
773 }
774 };
775 control.notify(RetryEvent::AttemptFinished {
776 attempt,
777 final_attempt: false,
778 });
779 if let Err(error) = control.check() {
780 finish_after_attempt_error(&control, result, selected, error);
781 return;
782 }
783 lock_recover(&control.state).phase = OperationPhase::Queued;
784 control.notify(RetryEvent::RetryScheduled { attempt, delay });
785 let ready = match Instant::now().checked_add(delay) {
786 Some(ready) => ready,
787 None => {
788 finish_error(&control, NetError::from(ErrorKind::InvalidPolicy));
789 return;
790 }
791 };
792 let sleep = control.runtime.wait_deadline(Some(ready));
793 #[cfg(feature = "quality-doctor")]
794 let backoff = control
795 .quality
796 .as_ref()
797 .map(|probe| probe.phase(crate::api::quality::PhaseKind::Backoff));
798 let backoff_result = admission_scope(
799 &control.connection_lease,
800 &control.runtime,
801 &control.options,
802 &control.cancellation,
803 control.deadline,
804 sleep,
805 )
806 .await;
807 #[cfg(feature = "quality-doctor")]
808 drop(backoff);
809 if let Err(error) = backoff_result {
810 let error = if error.kind() == ErrorKind::TimedOut {
811 error.with_ws_timeout_phase(WsTimeoutPhase::Total)
812 } else {
813 error
814 };
815 finish_after_attempt_error(&control, result, selected, error);
816 return;
817 }
818 previous_result = Some((result, selected));
819 }
820}
821async fn attempt_once(
822 control: &Arc<LogicalControl>,
823 factory: &Arc<AttemptFactory>,
824 key: Arc<str>,
825 attempt: u32,
826 started: Instant,
827 admission_deadline: Option<Instant>,
828) -> Result<Response> {
829 check_attempt_budget(control, started)?;
830 control.check_connection()?;
831 let attempt_budget = TimeoutBudget::earlier(
832 control
833 .deadline
834 .map(|at| TimeoutBudget::new(at, WsTimeoutPhase::Total)),
835 add(started, control.options.attempt_timeout)?
836 .map(|at| TimeoutBudget::new(at, WsTimeoutPhase::Attempt)),
837 );
838 let attempt_deadline = attempt_budget.map(|budget| budget.at);
839 let context = RequestAttemptContext {
840 operation_id: control.id,
841 attempt,
842 request_id: RequestId::random()?,
843 idempotency_key: key,
844 };
845 #[cfg(feature = "quality-doctor")]
846 let build = control
847 .quality
848 .as_ref()
849 .map(|probe| probe.phase(crate::api::quality::PhaseKind::RequestBuild));
850 let built = super::retry::isolate_classifier(|| factory(&context));
851 #[cfg(feature = "quality-doctor")]
852 drop(build);
853 check_attempt_budget(control, started)?;
856 control.check_connection()?;
857 let message = built?.map_err(|error| error.with_stage(ErrorStage::RequestBuild))?;
858 let request =
859 Request::new(context.request_id, message).with_metadata(control.options.metadata.clone());
860 let options = RequestOptions {
861 send: SendOptions {
862 priority: control.options.priority,
863 write_timeout: control.options.write_timeout,
864 deadline: attempt_deadline,
865 disconnected: control.options.disconnected,
866 cancellation: control.options.cancellation.clone(),
867 retry_setting: RetrySetting::Disabled,
868 metadata: control.options.metadata.clone(),
869 ..SendOptions::default()
870 },
871 response_timeout: control.options.response_timeout,
872 response_timeout_origin: control.options.response_timeout_origin,
873 registration_deadline: admission_deadline,
874 };
875 let lease = lock_recover(&control.state)
876 .lease
877 .clone()
878 .ok_or_else(|| NetError::from(ErrorKind::Cancelled))?;
879 let registration_check = || {
880 control.check_connection()?;
881 if lock_recover(&control.state).terminal {
882 return Err(NetError::from(ErrorKind::Cancelled));
883 }
884 Ok(())
885 };
886 let on_registered = |handle: &RequestHandle| {
887 let mut state = lock_recover(&control.state);
888 if state.terminal {
889 return Err(NetError::from(ErrorKind::Cancelled));
890 }
891 state.current = Some(handle.clone());
892 state.phase = OperationPhase::WaitingForCapacity;
893 Ok(())
894 };
895 let mut admission = scope_attempt(
896 &control.runtime,
897 &control.options,
898 &control.cancellation,
899 control.deadline,
900 TimeoutBudget::earlier(
901 attempt_budget,
902 admission_deadline.map(|at| TimeoutBudget::new(at, WsTimeoutPhase::Admission)),
903 ),
904 None,
905 dispatch::prepare_operation_attempt(
906 &control.runtime,
907 &request,
908 &options,
909 control.started,
910 lease,
911 dispatch::AttemptRegistration {
912 absolute_budget: attempt_budget,
913 connection_lease: &control.connection_lease,
914 gate: &control.registration_gate,
915 check: ®istration_check,
916 install: &on_registered,
917 },
918 ),
919 )
920 .await
921 .map_err(|error| {
922 let error = preserve_timeout_phase(error);
923 if error.context().stage.is_none() {
924 error.with_stage(ErrorStage::Admission)
925 } else {
926 error
927 }
928 })?;
929 let (handle, response) = admission
930 .request
931 .take()
932 .ok_or_else(|| NetError::from(ErrorKind::Internal))?;
933 #[cfg(feature = "quality-doctor")]
934 if let Some(probe) = &control.quality {
935 handle.quality_parent(probe.reference());
936 }
937 {
938 let mut state = lock_recover(&control.state);
939 if state.terminal {
940 return Err(NetError::from(ErrorKind::Cancelled));
941 }
942 state.current = Some(handle.clone());
943 state.phase = OperationPhase::Queued;
944 }
945 admission
946 .commit()
947 .map_err(preserve_timeout_phase)?;
948 let result = scope_attempt(
949 &control.runtime,
950 &control.options,
951 &control.cancellation,
952 control.deadline,
953 attempt_budget,
954 Some(&handle),
955 async {
956 response
957 .await
958 .map_err(|_| NetError::from(ErrorKind::EngineDropped))?
959 },
960 )
961 .await;
962 result.map_err(|error| {
963 let error = preserve_timeout_phase(error);
964 if error.kind() == ErrorKind::TimedOut && error.context().stage.is_none() {
965 error.with_stage(ErrorStage::Response)
966 } else {
967 error
968 }
969 })
970}
971
972fn check_attempt_budget(control: &LogicalControl, started: Instant) -> Result<()> {
973 control.check()?;
974 if add(started, control.options.attempt_timeout)?
975 .is_some_and(|deadline| deadline <= Instant::now())
976 {
977 return Err(
978 NetError::from(ErrorKind::TimedOut).with_ws_timeout_phase(WsTimeoutPhase::Attempt)
979 );
980 }
981 Ok(())
982}
983
984fn preserve_timeout_phase(error: NetError) -> NetError {
985 match super::retry::timeout_phase(&error) {
986 Some(phase) => error.with_ws_timeout_phase(phase),
987 None => error,
988 }
989}
990enum RetryStep {
991 Retry(Duration),
992 Stop(RetryStopReason),
993}
994fn classify(control: &LogicalControl, result: &Result<Response>) -> Result<RetryStep> {
995 if matches!(control.options.retry_setting, RetrySetting::Disabled) {
996 return Ok(RetryStep::Stop(RetryStopReason::Disabled));
997 }
998 if let Err(error) = result {
999 if super::retry::terminal_error(error) {
1000 let reason = if error.kind() == ErrorKind::CallbackPanicked {
1001 RetryStopReason::PolicyFailed
1002 } else {
1003 RetryStopReason::NotRetryable
1004 };
1005 return Ok(RetryStep::Stop(reason));
1006 }
1007 }
1008 let retry = lock_recover(&control.state).retry.context(
1009 control.started.elapsed(),
1010 control
1011 .deadline
1012 .map(|deadline| deadline.saturating_duration_since(Instant::now())),
1013 );
1014 let decision = control.policy.classify(&WsRequestRetryContext {
1015 outcome: result.as_ref(),
1016 retry: &retry,
1017 })?;
1018 let reason = if decision == RetryDecision::Stop {
1019 Some(RetryStopReason::ClassifierStopped)
1020 } else if decision == RetryDecision::Default
1021 && !result
1022 .as_ref()
1023 .err()
1024 .is_some_and(|error| control.policy.should_retry_error(error))
1025 {
1026 Some(if result.is_ok() {
1027 RetryStopReason::Completed
1028 } else {
1029 RetryStopReason::NotRetryable
1030 })
1031 } else if !lock_recover(&control.state).retry.can_retry() {
1032 Some(RetryStopReason::AttemptsExhausted)
1033 } else if !control.policy.allows_replay() {
1034 Some(RetryStopReason::UnsafeToReplay)
1035 } else {
1036 None
1037 };
1038 let mut state = lock_recover(&control.state);
1039 if let Some(reason) = reason {
1040 return Ok(RetryStep::Stop(reason));
1041 }
1042 state
1043 .retry
1044 .next_delay()
1045 .map(|delay| match delay {
1046 Some(delay) => RetryStep::Retry(delay),
1047 None => RetryStep::Stop(RetryStopReason::AttemptsExhausted),
1048 })
1049 .map_err(|_| NetError::from(ErrorKind::InvalidPolicy))
1050}
1051fn error_stop_reason(error: &NetError) -> RetryStopReason {
1052 match error.kind() {
1053 ErrorKind::Cancelled => RetryStopReason::Cancelled,
1054 ErrorKind::TimedOut => RetryStopReason::TimedOut,
1055 ErrorKind::Closed | ErrorKind::EngineDropped => RetryStopReason::Shutdown,
1056 ErrorKind::CallbackPanicked | ErrorKind::InvalidPolicy => RetryStopReason::PolicyFailed,
1057 _ => RetryStopReason::NotRetryable,
1058 }
1059}
1060fn finish_error(control: &LogicalControl, error: NetError) {
1061 let reason = error_stop_reason(&error);
1062 control.finish(Err(error), reason);
1063}
1064
1065fn finish_after_attempt_error(
1066 control: &LogicalControl,
1067 result: Result<Response>,
1068 selected: bool,
1069 error: NetError,
1070) {
1071 if selected && control.runtime.preserves_selected_result() {
1072 match control.check_result(true) {
1073 Ok(()) => {
1074 control.finish(result, RetryStopReason::NotRetryable);
1077 }
1078 Err(error) => finish_error(control, error),
1079 }
1080 } else {
1081 finish_error(control, error);
1082 }
1083}
1084fn end_cause(result: &Result<Response>, reason: RetryStopReason) -> TaskEndCause {
1085 if result.is_ok() {
1086 return TaskEndCause::Completed;
1087 }
1088 match reason {
1089 RetryStopReason::Cancelled => TaskEndCause::Cancelled,
1090 RetryStopReason::TimedOut => TaskEndCause::Expired,
1091 RetryStopReason::Shutdown => TaskEndCause::Shutdown,
1092 _ => match result.as_ref().err().map(NetError::kind) {
1093 Some(ErrorKind::TimedOut) => TaskEndCause::Expired,
1094 Some(ErrorKind::Cancelled) => TaskEndCause::Cancelled,
1095 Some(ErrorKind::Closed | ErrorKind::NotConnected) => TaskEndCause::Disconnected,
1096 _ => TaskEndCause::Failed,
1097 },
1098 }
1099}
1100fn earlier(left: Option<Instant>, right: Option<Instant>) -> Option<Instant> {
1101 match (left, right) {
1102 (Some(a), Some(b)) => Some(a.min(b)),
1103 (a, b) => a.or(b),
1104 }
1105}
1106fn add(start: Instant, duration: Option<Duration>) -> Result<Option<Instant>> {
1107 duration
1108 .map(|duration| {
1109 start
1110 .checked_add(duration)
1111 .ok_or_else(|| NetError::config("timeout", "deadline is not representable"))
1112 })
1113 .transpose()
1114}
1115fn check_scope(
1116 runtime: &SessionRuntime,
1117 options: &WsRequestOperationOptions,
1118 cancel: &CancellationToken,
1119 deadline: Option<Instant>,
1120) -> Result<()> {
1121 check_result_scope(runtime, options, cancel, deadline, || false)
1122}
1123
1124fn check_result_scope(
1125 runtime: &SessionRuntime,
1126 options: &WsRequestOperationOptions,
1127 cancel: &CancellationToken,
1128 deadline: Option<Instant>,
1129 selected: impl FnOnce() -> bool,
1130) -> Result<()> {
1131 if cancel.is_cancelled()
1132 || options
1133 .cancellation
1134 .as_ref()
1135 .is_some_and(CancellationGroup::is_cancelled)
1136 {
1137 return Err(NetError::from(ErrorKind::Cancelled));
1138 }
1139 if runtime.is_closed() && !(selected() && runtime.preserves_selected_result()) {
1142 return Err(NetError::from(ErrorKind::Closed));
1143 }
1144 if deadline.is_some_and(|deadline| Instant::now() >= deadline) {
1145 return Err(NetError::from(ErrorKind::TimedOut));
1146 }
1147 Ok(())
1148}
1149async fn admission_scope<T>(
1151 lease: &CancellationToken,
1152 runtime: &SessionRuntime,
1153 options: &WsRequestOperationOptions,
1154 cancel: &CancellationToken,
1155 deadline: Option<Instant>,
1156 future: impl Future<Output = Result<T>>,
1157) -> Result<T> {
1158 if options.disconnected == DisconnectedPolicy::WaitForReconnect {
1159 return scope(runtime, options, cancel, deadline, future).await;
1160 }
1161 check_scope(runtime, options, cancel, deadline)?;
1162 if lease.is_cancelled() {
1163 return Err(NetError::from(ErrorKind::NotConnected));
1164 }
1165 let result = tokio::select! {
1166 biased;
1167 _ = lease.cancelled() => {
1168 check_scope(runtime, options, cancel, deadline)?;
1169 Err(NetError::from(ErrorKind::NotConnected))
1170 },
1171 result = scope(runtime, options, cancel, deadline, future) => result,
1172 }?;
1173 check_scope(runtime, options, cancel, deadline)?;
1174 if lease.is_cancelled() {
1175 return Err(NetError::from(ErrorKind::NotConnected));
1176 }
1177 Ok(result)
1178}
1179
1180async fn scope<T>(
1181 runtime: &SessionRuntime,
1182 options: &WsRequestOperationOptions,
1183 cancel: &CancellationToken,
1184 deadline: Option<Instant>,
1185 future: impl Future<Output = Result<T>>,
1186) -> Result<T> {
1187 scope_with_request(runtime, options, cancel, deadline, None, future).await
1188}
1189
1190async fn result_closure(runtime: &SessionRuntime, request: Option<&RequestHandle>) {
1193 let closed = runtime.closed_token();
1194 let session_end = runtime.lifecycle.completion_token();
1195 tokio::select! {
1196 _ = closed.cancelled() => {},
1197 _ = session_end.cancelled() => {},
1198 }
1199 if request.is_some_and(RequestHandle::has_selected_result)
1200 && runtime.preserves_selected_result()
1201 {
1202 std::future::pending::<()>().await;
1204 }
1205}
1206
1207async fn scope_with_request<T>(
1208 runtime: &SessionRuntime,
1209 options: &WsRequestOperationOptions,
1210 cancel: &CancellationToken,
1211 deadline: Option<Instant>,
1212 request: Option<&RequestHandle>,
1213 future: impl Future<Output = Result<T>>,
1214) -> Result<T> {
1215 check_result_scope(
1216 runtime,
1217 options,
1218 cancel,
1219 deadline,
1220 || request.is_some_and(RequestHandle::has_selected_result),
1221 )?;
1222 let forced_closed = runtime.forced_closed_token();
1223 let session_cancel = runtime.lifecycle.cancel_token();
1224 let group = async {
1225 match &options.cancellation {
1226 Some(group) => group.cancelled().await,
1227 None => std::future::pending().await,
1228 }
1229 };
1230 tokio::select! {
1231 biased;
1232 _ = cancel.cancelled() => Err(NetError::from(ErrorKind::Cancelled)),
1233 _ = group => Err(NetError::from(ErrorKind::Cancelled)),
1234 _ = runtime.shutdown.cancelled() => Err(NetError::from(ErrorKind::Closed)),
1235 _ = forced_closed.cancelled() => Err(NetError::from(ErrorKind::Closed)),
1236 _ = session_cancel.cancelled() => Err(NetError::from(ErrorKind::Closed)),
1237 _ = result_closure(runtime, request) => Err(NetError::from(ErrorKind::Closed)),
1238 result = runtime.wait_deadline(deadline) => result.and_then(|()| Err(NetError::from(ErrorKind::TimedOut))),
1239 result = future => result,
1240 }
1241}
1242
1243async fn scope_attempt<T>(
1247 runtime: &SessionRuntime,
1248 options: &WsRequestOperationOptions,
1249 cancel: &CancellationToken,
1250 total_deadline: Option<Instant>,
1251 budget: Option<TimeoutBudget>,
1252 request: Option<&RequestHandle>,
1253 future: impl Future<Output = Result<T>>,
1254) -> Result<T> {
1255 let check = || {
1256 let selected = || request.is_some_and(RequestHandle::has_selected_result);
1257 #[cfg(test)]
1258 if let Some(request) = request {
1259 runtime.before_request_result_check_for_test(request)?;
1260 }
1261 check_result_scope(
1262 runtime,
1263 options,
1264 cancel,
1265 total_deadline,
1266 selected,
1267 ).map_err(|error| {
1268 if error.kind() == ErrorKind::TimedOut {
1269 error.with_ws_timeout_phase(WsTimeoutPhase::Total)
1270 } else {
1271 error
1272 }
1273 })
1274 };
1275 check()?;
1276 let forced_closed = runtime.forced_closed_token();
1277 let session_cancel = runtime.lifecycle.cancel_token();
1278 let group = async {
1279 match &options.cancellation {
1280 Some(group) => group.cancelled().await,
1281 None => std::future::pending().await,
1282 }
1283 };
1284 tokio::pin!(future);
1285 let result = tokio::select! {
1286 biased;
1287 _ = cancel.cancelled() => Err(NetError::from(ErrorKind::Cancelled)),
1288 _ = group => Err(NetError::from(ErrorKind::Cancelled)),
1289 _ = runtime.shutdown.cancelled() => Err(NetError::from(ErrorKind::Closed)),
1290 _ = forced_closed.cancelled() => Err(NetError::from(ErrorKind::Closed)),
1291 _ = session_cancel.cancelled() => Err(NetError::from(ErrorKind::Closed)),
1292 _ = result_closure(runtime, request) => Err(NetError::from(ErrorKind::Closed)),
1293 result = runtime.wait_deadline(total_deadline) => result.and_then(|()|
1294 Err(NetError::from(ErrorKind::TimedOut).with_ws_timeout_phase(WsTimeoutPhase::Total))),
1295 result = &mut future => result,
1296 result = runtime.wait_deadline(budget.map(|budget| budget.at)) => {
1297 result?;
1298 if let Some(request) = request {
1299 request.expire_operation_attempt();
1303 scope_with_request(runtime, options, cancel, total_deadline, Some(request), &mut future).await
1304 } else {
1305 Err(budget.map_or_else(|| NetError::from(ErrorKind::TimedOut), TimeoutBudget::timeout_error))
1306 }
1307 },
1308 };
1309 check()?;
1310 result
1311}
1312
1313#[cfg(test)]
1314#[path = "request_operation_tests.rs"]
1315mod tests;
1316
1317#[cfg(test)]
1318#[path = "request_operation_generation_tests.rs"]
1319mod generation_tests;
1320
1321#[cfg(test)]
1322#[path = "request_operation_selection_tests.rs"]
1323mod selection_tests;