pingora_timeout/
fast_timeout.rs

1// Copyright 2026 Cloudflare, Inc.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! The fast and more complicated version of pingora-timeout
16//!
17//! The following optimizations are applied:
18//! - The timeouts lazily initialize their timer when the Future is pending for the first time.
19//! - There is no global lock for creating and cancelling timeouts.
20//! - Timeout timers are rounded to the next 10ms tick and timers are shared across all timeouts with the same deadline.
21//!
22//! In order for this to work, a standalone thread is created to arm the timers, which has some
23//! overhead. As a general rule, the benefits of this don't outweigh the overhead unless
24//! there are more than about 100 timeout() calls/sec in the system. Use regular tokio timeout or
25//! [super::tokio_timeout] in the low usage case.
26
27use super::timer::*;
28use super::*;
29use once_cell::sync::Lazy;
30use std::sync::atomic::{AtomicU64, Ordering};
31use std::sync::Arc;
32
33static TIMER_MANAGER: Lazy<Arc<TimerManager>> = Lazy::new(|| {
34    let tm = Arc::new(TimerManager::new());
35    check_clock_thread(&tm);
36    tm
37});
38
39/// Default duration above which [`fast_timeout()`] falls back to Tokio's timeout.
40pub const DEFAULT_FAST_TIMEOUT_TO_TOKIO_THRESHOLD: Duration = Duration::from_secs(15 * 60);
41const DEFAULT_FAST_TIMEOUT_TO_TOKIO_THRESHOLD_SECS: u64 =
42    DEFAULT_FAST_TIMEOUT_TO_TOKIO_THRESHOLD.as_secs();
43const FAST_TIMEOUT_TO_TOKIO_DISABLED: u64 = u64::MAX;
44
45static FAST_TIMEOUT_TO_TOKIO_THRESHOLD_SECS: AtomicU64 =
46    AtomicU64::new(DEFAULT_FAST_TIMEOUT_TO_TOKIO_THRESHOLD_SECS);
47
48fn check_clock_thread(tm: &Arc<TimerManager>) {
49    if tm.should_i_start_clock() {
50        std::thread::Builder::new()
51            .name("Timer thread".into())
52            .spawn(|| TIMER_MANAGER.clock_thread())
53            .unwrap();
54    }
55}
56
57fn should_use_tokio_timeout(duration: Duration) -> bool {
58    let threshold_secs = FAST_TIMEOUT_TO_TOKIO_THRESHOLD_SECS.load(Ordering::Relaxed);
59    threshold_secs != FAST_TIMEOUT_TO_TOKIO_DISABLED && duration.as_secs() > threshold_secs
60}
61
62/// Set the duration above which [`fast_timeout()`] falls back to Tokio's timeout.
63///
64/// When set to `None`, [`fast_timeout()`] never falls back to Tokio's timeout.
65///
66/// Long Pingora fast-timeout timers are kept in [`TimerManager`] until
67/// their deadline expires. Tokio's timeout has cancellation cleanup, so it is a
68/// better fit for long durations that are often cancelled before expiry.
69pub fn set_fast_timeout_to_tokio_threshold(threshold: Option<Duration>) {
70    FAST_TIMEOUT_TO_TOKIO_THRESHOLD_SECS.store(
71        threshold.map_or(FAST_TIMEOUT_TO_TOKIO_DISABLED, |threshold| {
72            threshold.as_secs()
73        }),
74        Ordering::Relaxed,
75    );
76}
77
78/// Return the duration above which [`fast_timeout()`] falls back to Tokio's timeout.
79pub fn fast_timeout_to_tokio_threshold() -> Option<Duration> {
80    match FAST_TIMEOUT_TO_TOKIO_THRESHOLD_SECS.load(Ordering::Relaxed) {
81        FAST_TIMEOUT_TO_TOKIO_DISABLED => None,
82        threshold_secs => Some(Duration::from_secs(threshold_secs)),
83    }
84}
85
86/// The timeout generated by [fast_timeout()].
87///
88/// Users don't need to interact with this object.
89pub struct FastTimeout(Duration);
90
91impl ToTimeout for FastTimeout {
92    fn timeout(&self) -> Pin<Box<dyn Future<Output = ()> + Send + Sync>> {
93        Box::pin(TIMER_MANAGER.register_timer(self.0).poll())
94    }
95
96    fn create(d: Duration) -> Self {
97        FastTimeout(d)
98    }
99}
100
101enum TimeoutKind {
102    Fast,
103    Tokio,
104}
105
106/// The timeout selector generated by [fast_timeout()].
107///
108/// Users don't need to interact with this object.
109pub struct FastOrTokioTimeout {
110    duration: Duration,
111    kind: TimeoutKind,
112}
113
114impl FastOrTokioTimeout {
115    fn use_tokio(&self) -> bool {
116        matches!(self.kind, TimeoutKind::Tokio)
117    }
118}
119
120impl ToTimeout for FastOrTokioTimeout {
121    fn timeout(&self) -> Pin<Box<dyn Future<Output = ()> + Send + Sync>> {
122        match self.kind {
123            TimeoutKind::Fast => FastTimeout::create(self.duration).timeout(),
124            TimeoutKind::Tokio => TokioTimeout::create(self.duration).timeout(),
125        }
126    }
127
128    fn create(duration: Duration) -> Self {
129        Self {
130            duration,
131            kind: if should_use_tokio_timeout(duration) {
132                TimeoutKind::Tokio
133            } else {
134                TimeoutKind::Fast
135            },
136        }
137    }
138}
139
140/// Similar to [tokio::time::timeout] but more efficient.
141pub fn fast_timeout<T>(duration: Duration, future: T) -> Timeout<T, FastOrTokioTimeout>
142where
143    T: Future,
144{
145    let callback = FastOrTokioTimeout::create(duration);
146    if !callback.use_tokio() {
147        check_clock_thread(&TIMER_MANAGER);
148    }
149
150    Timeout::new_with_callback(future, callback)
151}
152
153/// Similar to [tokio::time::sleep] but more efficient.
154pub async fn fast_sleep(duration: Duration) {
155    check_clock_thread(&TIMER_MANAGER);
156    TIMER_MANAGER.register_timer(duration).poll().await
157}
158
159/// Pause the timer for fork()
160///
161/// Because RwLock across fork() is undefined behavior, this function makes sure that no one
162/// holds any locks.
163///
164/// This function should be called right before fork().
165pub fn pause_for_fork() {
166    TIMER_MANAGER.pause_for_fork();
167}
168
169/// Unpause the timer after fork()
170///
171/// This function should be called right after fork().
172pub fn unpause() {
173    TIMER_MANAGER.unpause();
174}
175
176#[cfg(test)]
177mod tests {
178    use super::*;
179
180    #[tokio::test]
181    async fn test_timeout() {
182        let fut = tokio_sleep(Duration::from_secs(1000));
183        let to = fast_timeout(Duration::from_secs(1), fut);
184        assert!(to.await.is_err())
185    }
186
187    #[tokio::test]
188    async fn test_instantly_return() {
189        let fut = async { 1 };
190        let to = fast_timeout(Duration::from_secs(1), fut);
191        assert_eq!(to.await.unwrap(), 1)
192    }
193
194    #[tokio::test]
195    async fn test_delayed_return() {
196        let fut = async {
197            tokio_sleep(Duration::from_secs(1)).await;
198            1
199        };
200        let to = fast_timeout(Duration::from_secs(60), fut);
201        assert_eq!(to.await.unwrap(), 1)
202    }
203
204    #[tokio::test]
205    async fn test_sleep() {
206        let fut = async {
207            fast_sleep(Duration::from_secs(1)).await;
208            1
209        };
210        let to = fast_timeout(Duration::from_secs(60), fut);
211        assert_eq!(to.await.unwrap(), 1)
212    }
213
214    #[test]
215    fn test_default_fast_timeout_to_tokio_threshold() {
216        assert_eq!(
217            Some(Duration::from_secs(15 * 60)),
218            fast_timeout_to_tokio_threshold()
219        );
220    }
221
222    #[test]
223    fn test_fast_timeout_uses_fast_below_threshold() {
224        let callback = FastOrTokioTimeout::create(Duration::from_secs(1));
225        assert!(!callback.use_tokio());
226    }
227
228    #[test]
229    fn test_fast_timeout_uses_fast_at_threshold() {
230        let callback = FastOrTokioTimeout::create(fast_timeout_to_tokio_threshold().unwrap());
231        assert!(!callback.use_tokio());
232    }
233
234    #[test]
235    fn test_fast_timeout_uses_tokio_above_threshold() {
236        let callback = FastOrTokioTimeout::create(
237            fast_timeout_to_tokio_threshold().unwrap() + Duration::from_secs(1),
238        );
239        assert!(callback.use_tokio());
240    }
241}
pingora_timeout/fast_timeout.rs

pingora_timeout/
fast_timeout.rs
