Skip to main content

ntex_util/services/
inflight.rs

1//! Middleware for limiting concurrent service calls.
2use ntex_service::{BindCtx, Ctx, Middleware, ReadyBinding, Service};
3
4use super::counter::Counter;
5
6/// Middleware that limits the number of concurrent calls to a service.
7///
8/// Readiness remains pending while every slot is in use. The default limit is
9/// 15 concurrent calls.
10#[derive(Copy, Clone, Debug)]
11pub struct InFlight {
12    max_inflight: usize,
13}
14
15impl InFlight {
16    /// Creates middleware with the specified concurrency limit.
17    ///
18    /// A limit of zero keeps the service permanently unavailable.
19    pub fn new(max: usize) -> Self {
20        Self { max_inflight: max }
21    }
22}
23
24impl Default for InFlight {
25    fn default() -> Self {
26        Self::new(15)
27    }
28}
29
30impl<S, St> Middleware<S, St> for InFlight {
31    type Service = InFlightService<S>;
32
33    fn create(&self, _: &St, service: S) -> Self::Service {
34        InFlightService::new(self.max_inflight, service)
35    }
36}
37
38#[derive(Debug)]
39/// Service wrapper that enforces a concurrent-call limit.
40///
41/// Capacity changes invalidate this service's readiness. The wrapped service
42/// has an independent readiness cache and must notify when its readiness changes.
43pub struct InFlightService<S> {
44    count: Counter,
45    service: ReadyBinding<S>,
46}
47
48impl<S> InFlightService<S> {
49    /// Wraps `service` with the specified concurrency limit.
50    ///
51    /// A limit of zero keeps the service permanently unavailable.
52    pub fn new(max: usize, service: S) -> Self {
53        Self {
54            service: ReadyBinding::new(service),
55            count: Counter::new(max),
56        }
57    }
58}
59
60impl<S, St, Req> Service<St, Req> for InFlightService<S>
61where
62    S: Service<St, Req>,
63{
64    type Res = S::Res;
65    type Error = S::Error;
66
67    fn bind(&self, ctx: BindCtx<'_>) {
68        self.count.bind(&ctx);
69        ctx.bind::<_, St, Req>(&self.service);
70    }
71
72    #[inline]
73    async fn ready(&self, ctx: Ctx<'_, Self, St>) -> Result<(), S::Error> {
74        if self.count.is_available() {
75            ctx.ready(&self.service).await
76        } else {
77            crate::future::join(self.count.available(), ctx.ready(&self.service))
78                .await
79                .1?;
80            // the inner readiness can be stale after waiting for a free slot
81            ctx.ready(&self.service).await
82        }
83    }
84
85    #[inline]
86    async fn call(&self, req: Req, ctx: Ctx<'_, Self, St>) -> Result<S::Res, S::Error> {
87        if !self.count.is_available() {
88            // Bypassed calls can wait independently of the shared readiness check.
89            self.count.clone().available().await;
90        }
91        let _guard = self.count.get();
92        ctx.call(&self.service, req).await
93    }
94
95    ntex_service::forward_shutdown!(St, service);
96}
97
98#[cfg(test)]
99mod tests {
100    use std::{cell::Cell, cell::RefCell, future::Future, rc::Rc, task::Poll, time::Duration};
101
102    use async_channel as mpmc;
103    use ntex_service::{Pipeline, ReadyNotifier, apply, fn_factory};
104
105    use super::*;
106    use crate::{channel::oneshot, future::OnDropFn, future::lazy};
107
108    struct SleepService(mpmc::Receiver<()>);
109
110    impl Service<(), ()> for SleepService {
111        type Res = ();
112        type Error = ();
113
114        async fn call(&self, _r: (), _: Ctx<'_, Self>) -> Result<(), ()> {
115            let _ = self.0.recv().await;
116            Ok(())
117        }
118    }
119
120    /// STATEMENT: A single-slot in-flight limiter queues calls and remains unready while occupied.
121    ///
122    /// HOW:
123    /// - Spawn three channel-blocked calls and check readiness while capacity is exhausted.
124    /// - Release them individually and verify all three completions before shutdown.
125    #[ntex::test]
126    async fn test_service() {
127        let (tx, rx) = mpmc::unbounded();
128        let counter = Rc::new(Cell::new(0));
129
130        let srv = Pipeline::new((), InFlightService::new(1, SleepService(rx)));
131        assert_eq!(lazy(|cx| srv.poll_ready(cx)).await, Poll::Ready(Ok(())));
132
133        let counter2 = counter.clone();
134        let fut = srv.call_static(());
135        ntex::rt::spawn(async move {
136            let _ = fut.await;
137            counter2.set(counter2.get() + 1);
138        });
139        crate::time::sleep(Duration::from_millis(25)).await;
140        assert_eq!(lazy(|cx| srv.poll_ready(cx)).await, Poll::Pending);
141
142        let counter2 = counter.clone();
143        let fut = srv.call_static(());
144        ntex::rt::spawn(async move {
145            let _ = fut.await;
146            counter2.set(counter2.get() + 1);
147        });
148        crate::time::sleep(Duration::from_millis(25)).await;
149        assert_eq!(lazy(|cx| srv.poll_ready(cx)).await, Poll::Pending);
150
151        let counter2 = counter.clone();
152        let fut = srv.call_static(());
153        let (stx, srx) = oneshot::channel::<()>();
154        ntex::rt::spawn(async move {
155            let _ = fut.await;
156            counter2.set(counter2.get() + 1);
157            let _ = stx.send(());
158        });
159        crate::time::sleep(Duration::from_millis(25)).await;
160        assert_eq!(lazy(|cx| srv.poll_ready(cx)).await, Poll::Pending);
161
162        let _ = tx.send(()).await;
163        crate::time::sleep(Duration::from_millis(25)).await;
164        assert_eq!(lazy(|cx| srv.poll_ready(cx)).await, Poll::Pending);
165        assert_eq!(counter.get(), 1);
166
167        let _ = tx.send(()).await;
168        crate::time::sleep(Duration::from_millis(25)).await;
169        assert_eq!(lazy(|cx| srv.poll_ready(cx)).await, Poll::Pending);
170
171        let _ = tx.send(()).await;
172        let _ = srx.recv().await;
173        assert_eq!(counter.get(), 3);
174        srv.shutdown().await;
175    }
176
177    /// STATEMENT: In-flight middleware blocks readiness until its active call completes.
178    ///
179    /// HOW:
180    /// - Check default and debug configuration, then create a single-slot middleware pipeline.
181    /// - Start a blocked call and verify readiness recovers only after releasing it.
182    #[ntex::test]
183    async fn test_middleware() {
184        assert_eq!(InFlight::default().max_inflight, 15);
185        assert_eq!(
186            format!("{:?}", InFlight::new(1)),
187            "InFlight { max_inflight: 1 }"
188        );
189
190        let (tx, rx) = mpmc::unbounded();
191        let rx = RefCell::new(Some(rx));
192        let sf = apply(
193            InFlight::new(1),
194            fn_factory(move |(): &()| {
195                let rx = rx.borrow_mut().take().unwrap();
196                async move { Ok::<_, ()>(SleepService(rx)) }
197            }),
198        );
199
200        let srv = sf.pipeline(()).await.unwrap();
201        assert_eq!(lazy(|cx| srv.poll_ready(cx)).await, Poll::Ready(Ok(())));
202
203        let srv2 = srv.bind();
204        ntex::rt::spawn(async move {
205            let _ = srv2.call(()).await;
206        });
207        crate::time::sleep(Duration::from_millis(25)).await;
208        assert_eq!(lazy(|cx| srv.poll_ready(cx)).await, Poll::Pending);
209
210        let _ = tx.send(()).await;
211        crate::time::sleep(Duration::from_millis(25)).await;
212        assert_eq!(lazy(|cx| srv.poll_ready(cx)).await, Poll::Ready(Ok(())));
213    }
214
215    /// STATEMENT: A factory-created in-flight limiter restores readiness after releasing capacity.
216    ///
217    /// HOW:
218    /// - Create a single-slot middleware service and confirm its configuration.
219    /// - Observe ready, pending, and ready states around a channel-blocked request.
220    #[ntex::test]
221    async fn test_middleware2() {
222        assert_eq!(InFlight::default().max_inflight, 15);
223        assert_eq!(
224            format!("{:?}", InFlight::new(1)),
225            "InFlight { max_inflight: 1 }"
226        );
227
228        let (tx, rx) = mpmc::unbounded();
229        let rx = RefCell::new(Some(rx));
230        let sf = apply(
231            InFlight::new(1),
232            fn_factory(move |(): &()| {
233                let rx = rx.borrow_mut().take().unwrap();
234                async move { Ok::<_, ()>(SleepService(rx)) }
235            }),
236        );
237
238        let srv = sf.pipeline(()).await.unwrap();
239        assert_eq!(lazy(|cx| srv.poll_ready(cx)).await, Poll::Ready(Ok(())));
240
241        let srv2 = srv.bind();
242        ntex::rt::spawn(async move {
243            let _ = srv2.call(()).await;
244        });
245        crate::time::sleep(Duration::from_millis(25)).await;
246        assert_eq!(lazy(|cx| srv.poll_ready(cx)).await, Poll::Pending);
247
248        let _ = tx.send(()).await;
249        crate::time::sleep(Duration::from_millis(25)).await;
250        assert_eq!(lazy(|cx| srv.poll_ready(cx)).await, Poll::Ready(Ok(())));
251    }
252
253    #[derive(Default)]
254    struct ProbeState {
255        active: Cell<usize>,
256        max: Cell<usize>,
257        checks: Cell<usize>,
258        waker: crate::task::LocalWaker,
259        changed: ReadyNotifier,
260    }
261
262    /// Inner service with its own concurrency limit
263    struct Probe(Rc<ProbeState>, usize, mpmc::Receiver<()>);
264
265    impl Service<(), ()> for Probe {
266        type Res = ();
267        type Error = ();
268
269        fn bind(&self, ctx: BindCtx<'_>) {
270            ctx.subscribe(&self.0.changed);
271        }
272
273        async fn ready(&self, _: Ctx<'_, Self>) -> Result<(), ()> {
274            self.0.checks.set(self.0.checks.get() + 1);
275            std::future::poll_fn(|cx| {
276                if self.0.active.get() < self.1 {
277                    Poll::Ready(Ok(()))
278                } else {
279                    self.0.waker.register(cx.waker());
280                    Poll::Pending
281                }
282            })
283            .await
284        }
285
286        async fn call(&self, (): (), _: Ctx<'_, Self>) -> Result<(), ()> {
287            let st = &self.0;
288            st.active.set(st.active.get() + 1);
289            let _release = OnDropFn::new(|| {
290                let active = st.active.get();
291                st.active.set(active - 1);
292                let _wake = OnDropFn::new(|| st.waker.wake());
293                if active == self.1 {
294                    st.changed.notify();
295                }
296            });
297            if st.active.get() == self.1 {
298                st.changed.notify();
299            }
300            st.max.set(st.max.get().max(st.active.get()));
301            let _ = self.2.recv().await;
302            Ok(())
303        }
304    }
305
306    /// STATEMENT: Inner readiness stays cached across calls until its notifier invalidates it.
307    ///
308    /// HOW:
309    /// - Make repeated calls with and without explicit readiness checks and count inner checks.
310    /// - Notify a readiness change and verify exactly one additional check.
311    #[ntex::test]
312    async fn test_inner_readiness_cached_until_notification() {
313        let (tx, rx) = mpmc::unbounded();
314        let st = Rc::new(ProbeState::default());
315        let srv = Pipeline::new((), InFlightService::new(4, Probe(st.clone(), 4, rx)));
316        for _ in 0..3 {
317            let _ = tx.send(()).await;
318            srv.call(()).await.unwrap();
319        }
320        assert_eq!(st.checks.get(), 1);
321
322        for _ in 0..3 {
323            let _ = tx.send(()).await;
324            srv.ready().await.unwrap();
325            srv.call(()).await.unwrap();
326        }
327        assert_eq!(st.checks.get(), 1);
328
329        st.changed.notify();
330        srv.ready().await.unwrap();
331        assert_eq!(st.checks.get(), 2);
332    }
333
334    /// Calls the inner service without checking its readiness, the inner
335    /// service has to enforce its limits
336    struct NoWait<S>(S);
337
338    impl<S: Service<(), (), Res = (), Error = ()>> Service<(), ()> for NoWait<S> {
339        type Res = ();
340        type Error = ();
341
342        ntex_service::forward_bind!((), 0);
343
344        #[allow(
345            deprecated,
346            reason = "exercise limiter enforcement under the legacy bypass"
347        )]
348        async fn call(&self, req: (), ctx: Ctx<'_, Self>) -> Result<(), ()> {
349            ctx.call_nowait(&self.0, req).await
350        }
351    }
352
353    /// Returns the max number of concurrent calls of the inner service
354    async fn run_concurrent(max: usize, inner: usize) -> usize {
355        let (tx, rx) = mpmc::unbounded();
356        let st = Rc::new(ProbeState::default());
357        let srv = Pipeline::new(
358            (),
359            NoWait(InFlightService::new(max, Probe(st.clone(), inner, rx))),
360        );
361        let mut futs = Vec::new();
362        for _ in 0..4 {
363            futs.push(ntex::rt::spawn(srv.call_static(())));
364        }
365        crate::time::sleep(Duration::from_millis(20)).await;
366        for _ in 0..4 {
367            let _ = tx.send(()).await;
368        }
369        for f in futs {
370            let _ = f.await;
371        }
372        st.max.get()
373    }
374
375    /// STATEMENT: Bypassing outer readiness still enforces both limiter and inner capacity.
376    ///
377    /// HOW:
378    /// - Run four competing calls with several outer and inner limit combinations.
379    /// - Verify peak concurrency equals the smaller configured limit.
380    #[ntex::test]
381    async fn test_limits_without_readiness_check() {
382        assert_eq!(run_concurrent(1, 4).await, 1, "inflight limit");
383        assert_eq!(run_concurrent(2, 1).await, 1, "inner limit");
384        assert_eq!(run_concurrent(2, 4).await, 2, "inflight limit 2");
385    }
386
387    /// STATEMENT: A zero in-flight limit blocks calls even through the legacy readiness bypass.
388    ///
389    /// HOW:
390    /// - Poll readiness and a normal call against a zero-limit service.
391    /// - Poll a bypassed call and verify all operations remain pending.
392    #[ntex::test]
393    async fn test_zero_limit() {
394        let (_tx, rx) = mpmc::unbounded();
395        let srv = Pipeline::new((), InFlightService::new(0, SleepService(rx.clone())));
396        assert_eq!(lazy(|cx| srv.poll_ready(cx)).await, Poll::Pending);
397        let mut call = Box::pin(srv.call(()));
398        assert_eq!(lazy(|cx| call.as_mut().poll(cx)).await, Poll::Pending);
399
400        let srv = Pipeline::new((), NoWait(InFlightService::new(0, SleepService(rx))));
401        let mut call = Box::pin(srv.call(()));
402        assert_eq!(lazy(|cx| call.as_mut().poll(cx)).await, Poll::Pending);
403    }
404}