1use std::{
2 cell, fmt, future::Future, marker, mem, ops, pin, pin::Pin, rc::Rc, task::Context, task::Poll,
3 task::Waker,
4};
5
6use crate::readiness::{Generation, ReadyScope, RunCheck, ScopeStore};
7use crate::{BindCtx, Service, ServiceFactory, ready_call::ReadyThenCall};
8
9pub struct Ctx<'a, Svc: ?Sized, St = ()> {
20 idx: u32,
21 st: &'a St,
22 waiters: &'a WaitersRef,
23 scope: &'a ReadyScope,
24 in_call: bool,
26 _t: marker::PhantomData<Rc<Svc>>,
27}
28
29#[derive(Debug)]
30pub(crate) struct WaitersRef {
31 scope: ReadyScope,
32 inner: Rc<WaitersInner>,
33}
34
35#[derive(Debug)]
36pub(crate) struct WaitersInner {
37 pub(crate) scopes: ScopeStore,
38 cur: cell::Cell<u32>,
39 running: cell::Cell<bool>,
40 shutdown: cell::Cell<bool>,
41 wakers: cell::UnsafeCell<Vec<u32>>,
42 scratch: cell::RefCell<Vec<Waker>>,
44 indexes: cell::UnsafeCell<slab::Slab<Option<Waker>>>,
45 calls: cell::UnsafeCell<slab::Slab<Option<Waker>>>,
47}
48
49impl WaitersRef {
50 pub(crate) fn new() -> Self {
51 let mut waiters = slab::Slab::with_capacity(4);
52 waiters.insert(None);
53 let inner = Rc::new(WaitersInner {
54 scopes: ScopeStore::default(),
55 cur: cell::Cell::new(u32::MAX),
56 running: cell::Cell::new(false),
57 shutdown: cell::Cell::new(false),
58 indexes: cell::UnsafeCell::new(waiters),
59 calls: cell::UnsafeCell::new(slab::Slab::new()),
60 wakers: cell::UnsafeCell::new(Vec::default()),
61 scratch: cell::RefCell::default(),
62 });
63 WaitersRef {
64 scope: ReadyScope::root(inner.clone()),
65 inner,
66 }
67 }
68
69 #[allow(clippy::mut_from_ref)]
70 pub(crate) fn get(&self) -> &mut slab::Slab<Option<Waker>> {
71 unsafe { &mut *self.indexes.get() }
74 }
75
76 #[allow(clippy::mut_from_ref)]
77 fn get_calls(&self) -> &mut slab::Slab<Option<Waker>> {
78 unsafe { &mut *self.calls.get() }
81 }
82
83 #[allow(clippy::mut_from_ref)]
84 pub(crate) fn get_wakers(&self) -> &mut Vec<u32> {
85 unsafe { &mut *self.wakers.get() }
88 }
89
90 fn queue_wake(&self, idx: u32) {
91 let wakers = self.get_wakers();
92 if !wakers.contains(&idx) {
93 wakers.push(idx);
94 }
95 }
96
97 pub(crate) fn insert(&self) -> u32 {
98 self.get().insert(None) as u32
99 }
100
101 pub(crate) fn remove(&self, idx: u32) {
102 self.get().remove(idx as usize);
103
104 if self.cur.get() == idx {
105 self.notify();
106 }
107 }
108
109 pub(crate) fn notify(&self) {
110 let mut wakers = self.scratch.take();
112 let indexes = self.get();
113 for idx in self.get_wakers().drain(..) {
114 if let Some(item) = indexes.get_mut(idx as usize)
115 && let Some(waker) = item.take()
116 {
117 wakers.push(waker);
118 }
119 }
120 self.take_calls(&mut wakers);
121
122 self.cur.set(u32::MAX);
123 self.wake_all(wakers);
124 }
125
126 fn take_calls(&self, wakers: &mut Vec<Waker>) {
127 let calls = self.get_calls();
128 if !calls.is_empty() {
129 wakers.extend(calls.iter_mut().filter_map(|(_, waker)| waker.take()));
130 }
131 }
132
133 fn wake_all(&self, mut wakers: Vec<Waker>) {
134 for waker in wakers.drain(..) {
135 waker.wake();
136 }
137 let mut scratch = self.scratch.borrow_mut();
139 if wakers.capacity() > scratch.capacity() {
140 *scratch = wakers;
141 }
142 }
143
144 fn wake_checks(&self) {
151 let cur = self.cur.get();
152 if cur == u32::MAX {
153 return;
154 }
155 let mut wakers = self.scratch.take();
156 if let Some(waker) = self.get().get_mut(cur as usize).and_then(Option::take) {
158 wakers.push(waker.clone());
159 if let Some(slot @ None) = self.get().get_mut(cur as usize) {
160 *slot = Some(waker);
161 }
162 }
163 self.take_calls(&mut wakers);
164 self.wake_all(wakers);
165 }
166
167 fn store_waker(
173 &self,
174 slab: fn(&Self) -> &mut slab::Slab<Option<Waker>>,
175 key: u32,
176 waker: &Waker,
177 ) -> bool {
178 match slab(self).get(key as usize) {
179 None => return false,
180 Some(Some(w)) if w.will_wake(waker) => return true,
181 Some(_) => {}
182 }
183 let waker = waker.clone();
184 let old = match slab(self).get_mut(key as usize) {
185 Some(slot) => slot.replace(waker),
186 None => Some(waker),
188 };
189 drop(old);
190 true
191 }
192
193 fn remove_call(&self, key: &mut Option<u32>) {
194 if let Some(key) = key.take() {
195 self.get_calls().remove(key as usize);
196 }
197 }
198
199 fn run_call<F, R>(&self, idx: u32, key: &mut Option<u32>, cx: &mut Context<'_>, f: F) -> Poll<R>
204 where
205 F: FnOnce(&mut Context<'_>) -> Poll<R>,
206 {
207 let cur = self.cur.get();
208 if cur == u32::MAX || cur == idx {
209 self.remove_call(key);
210 return self.run(idx, cx, f);
211 }
212
213 let result = f(cx);
214 if result.is_pending() {
215 if !key.is_some_and(|key| self.store_waker(Self::get_calls, key, cx.waker())) {
216 let waker = cx.waker().clone();
217 *key = Some(self.get_calls().insert(Some(waker)) as u32);
218 }
219 } else {
220 self.remove_call(key);
221 self.wake_checks();
222 }
223 result
224 }
225
226 pub(crate) fn run<F, R>(&self, idx: u32, cx: &mut Context<'_>, f: F) -> Poll<R>
227 where
228 F: FnOnce(&mut Context<'_>) -> Poll<R>,
229 {
230 self.store_waker(Self::get, idx, cx.waker());
233
234 let cur = self.cur.get();
235 if cur == u32::MAX {
236 self.cur.set(idx);
237 } else if cur != idx {
238 self.queue_wake(idx);
240 return Poll::Pending;
241 }
242
243 if self.running.replace(true) {
245 return f(cx);
246 }
247 let guard = RunGuard(self);
248 let result = f(cx);
249 mem::forget(guard);
250 if result.is_pending() {
251 self.queue_wake(idx);
252 } else {
253 self.notify();
254 }
255 self.running.set(false);
256 result
257 }
258
259 pub(crate) fn shutdown(&self) {
260 self.shutdown.set(true);
261 }
262
263 pub(crate) fn scope(&self) -> &ReadyScope {
264 &self.scope
265 }
266
267 pub(crate) fn register(&self) -> WaiterRegistration<'_> {
268 WaiterRegistration {
269 idx: self.insert(),
270 waiters: self,
271 }
272 }
273
274 pub(crate) fn is_shutdown(&self) -> bool {
275 self.shutdown.get()
276 }
277}
278
279impl ops::Deref for WaitersRef {
280 type Target = WaitersInner;
281
282 fn deref(&self) -> &Self::Target {
283 &self.inner
284 }
285}
286
287impl Drop for WaitersRef {
288 fn drop(&mut self) {
289 let indexes = mem::take(self.get());
291 let calls = mem::take(self.get_calls());
292 let scratch = self.scratch.take();
293 self.get_wakers().clear();
294 drop((indexes, calls, scratch));
295 }
296}
297
298struct RunGuard<'a>(&'a WaitersRef);
301
302impl Drop for RunGuard<'_> {
303 fn drop(&mut self) {
304 self.0.running.set(false);
305 self.0.notify();
306 }
307}
308
309pub(crate) struct WaiterRegistration<'a> {
310 pub(crate) idx: u32,
311 waiters: &'a WaitersRef,
312}
313
314impl Drop for WaiterRegistration<'_> {
315 fn drop(&mut self) {
316 self.waiters.remove(self.idx);
317 }
318}
319
320impl<'a, Svc, St> Ctx<'a, Svc, St> {
321 pub(crate) fn new(idx: u32, waiters: &'a WaitersRef, st: &'a St) -> Self {
322 Self {
323 idx,
324 waiters,
325 st,
326 scope: &waiters.scope,
327 in_call: false,
328 _t: marker::PhantomData,
329 }
330 }
331
332 pub(crate) fn scope(&self) -> &'a ReadyScope {
333 self.scope
334 }
335
336 pub(crate) fn cast<S>(self) -> Ctx<'a, S, St> {
337 self.with_scope(self.scope)
338 }
339
340 pub(crate) fn with_scope<S>(self, scope: &'a ReadyScope) -> Ctx<'a, S, St> {
341 Ctx {
342 idx: self.idx,
343 st: self.st,
344 waiters: self.waiters,
345 scope,
346 in_call: self.in_call,
347 _t: marker::PhantomData,
348 }
349 }
350
351 pub(crate) fn into_call(mut self) -> Self {
353 self.in_call = true;
354 self
355 }
356
357 #[inline]
358 pub fn id(&self) -> u32 {
360 self.idx
361 }
362
363 #[inline]
364 pub fn st(&self) -> &'a St {
366 self.st
367 }
368
369 #[inline]
381 pub async fn create<Sf, Req>(&self, factory: &Sf) -> Result<Sf::Service, Sf::InitError>
382 where
383 Sf: ServiceFactory<St, Req>,
384 {
385 let service = factory.create(self.st).await?;
386 service.bind(BindCtx::new(self.scope));
387 self.scope.reset_root();
388 Ok(service)
389 }
390
391 pub fn ready<'b, S, Req>(
396 &self,
397 svc: &'b S,
398 ) -> impl Future<Output = Result<(), S::Error>> + use<'a, 'b, Svc, St, S, Req>
399 where
400 S: Service<St, Req>,
401 {
402 self.ready_call(svc)
403 }
404
405 pub(crate) fn ready_generation<'b, S, Req>(
406 &self,
407 svc: &'b S,
408 ) -> impl Future<Output = Result<Generation, S::Error>> + use<'a, 'b, Svc, St, S, Req>
409 where
410 S: Service<St, Req>,
411 {
412 self.ready_call(svc)
413 }
414
415 fn ready_call<'b, S, Req, O>(
416 &self,
417 svc: &'b S,
418 ) -> impl Future<Output = Result<O, S::Error>> + use<'a, 'b, Svc, St, S, Req, O>
419 where
420 S: Service<St, Req>,
421 O: From<Generation>,
422 {
423 let ctx = *self;
424 ReadyCall {
425 check: RunCheck::new(self.scope, move || svc.ready(ctx.cast())),
426 waiters: self.waiters,
427 idx: self.idx,
428 in_call: self.in_call,
429 key: None,
430 active: false,
431 _t: marker::PhantomData,
432 }
433 }
434
435 #[inline]
436 pub fn call<S, Req>(
473 &self,
474 svc: &'a S,
475 req: Req,
476 ) -> impl Future<Output = Result<S::Res, S::Error>>
477 where
478 S: Service<St, Req>,
479 {
480 ReadyThenCall::new(
481 (svc, self.cast::<S>()),
482 req,
483 |&(svc, ctx): &(&'a S, Ctx<'a, S, St>)| {
484 (!ctx.scope.is_ready()).then(|| ctx.ready_cached(svc))
485 },
486 |(svc, ctx): (&'a S, Ctx<'a, S, St>), req| svc.call(req, ctx.into_call()),
487 )
488 }
489
490 pub(crate) async fn ready_cached<S, Req>(self, svc: &'a S) -> Result<(), S::Error>
491 where
492 S: Service<St, Req>,
493 {
494 while !self.scope.is_ready() {
495 let result = {
496 let observer = self.scope.observer();
497 let mut ready = pin::pin!(self.ready_generation(svc));
498 std::future::poll_fn(|cx| {
499 observer.register(cx.waker());
500 if self.scope.is_ready() {
502 Poll::Ready(Ok(None))
503 } else {
504 ready.as_mut().poll(cx).map(|result| result.map(Some))
505 }
506 })
507 .await
508 };
509 let generation = result?;
510 if self.scope.is_ready()
512 || generation.is_some_and(|generation| self.scope.is_current(generation))
513 {
514 return Ok(());
515 }
516 }
517 Ok(())
518 }
519
520 #[inline]
521 pub fn shutdown<'b, S, Req>(
523 &self,
524 svc: &'b S,
525 ) -> impl Future<Output = ()> + use<'a, 'b, Svc, St, S, Req>
526 where
527 S: Service<St, Req>,
528 {
529 svc.shutdown(self.cast())
530 }
531
532 #[inline]
533 pub fn map_state<NewSt>(&'a self, st: &'a NewSt) -> Ctx<'a, Self, NewSt> {
535 Ctx {
536 st,
537 idx: self.idx,
538 waiters: self.waiters,
539 scope: self.scope,
540 in_call: self.in_call,
541 _t: marker::PhantomData,
542 }
543 }
544}
545
546impl<S, St> Copy for Ctx<'_, S, St> {}
547
548impl<S, St> Clone for Ctx<'_, S, St> {
549 #[inline]
550 fn clone(&self) -> Self {
551 *self
552 }
553}
554
555impl<S, St> ops::Deref for Ctx<'_, S, St> {
556 type Target = St;
557
558 #[inline]
559 fn deref(&self) -> &St {
560 self.st
561 }
562}
563
564impl<S, St> fmt::Debug for Ctx<'_, S, St> {
565 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
566 f.debug_struct("Ctx")
567 .field("idx", &self.idx)
568 .field("waiters", &self.waiters.get().len())
569 .finish()
570 }
571}
572
573pin_project_lite::pin_project! {
575 struct ReadyCall<'a, F, Fut, O> {
576 #[pin]
577 check: RunCheck<'a, F, Fut>,
578 waiters: &'a WaitersRef,
579 idx: u32,
580 in_call: bool,
581 key: Option<u32>,
583 active: bool,
585 _t: marker::PhantomData<O>,
586 }
587
588 impl<F, Fut, O> PinnedDrop for ReadyCall<'_, F, Fut, O> {
589 fn drop(this: Pin<&mut Self>) {
590 let this = this.project();
591 if this.key.is_some() {
592 this.waiters.remove_call(this.key);
593 this.waiters.wake_checks();
594 } else if *this.active && this.waiters.cur.get() == *this.idx {
595 this.waiters.notify();
596 }
597 }
598 }
599}
600
601impl<F, Fut, E, O> Future for ReadyCall<'_, F, Fut, O>
602where
603 F: FnMut() -> Fut,
604 Fut: Future<Output = Result<(), E>>,
605 O: From<Generation>,
606{
607 type Output = Result<O, E>;
608
609 fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
610 let mut this = self.project();
611 let mut restarted = false;
612 loop {
613 *this.active = true;
614 let mut check = this.check.as_mut();
615 let f = |cx: &mut Context<'_>| check.as_mut().poll(cx);
616 let result = if *this.in_call {
617 this.waiters.run_call(*this.idx, this.key, cx, f)
618 } else {
619 this.waiters.run(*this.idx, cx, f)
620 };
621 let Poll::Ready(result) = result else {
622 return Poll::Pending;
623 };
624 *this.active = false;
625 match result {
626 Ok(generation) if !this.check.scope().is_current(generation) => {}
628 result => return Poll::Ready(result.map(O::from)),
629 }
630 if restarted {
632 cx.waker().wake_by_ref();
633 return Poll::Pending;
634 }
635 restarted = true;
636 }
637 }
638}
639
640#[cfg(test)]
641#[path = "../tests/unit/ctx.rs"]
642mod tests;