Skip to main content

nitr_core/runtime/
pool.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2// This file is part of Nitr.
3// See https://nitrweb.com/ for more information
4// Copyright (C) 2024-present Jose Quintana <joseluisq.net>
5
6//! A fixed pool of independent Lua runtimes checked out per request.
7
8use std::ops::{Deref, DerefMut};
9use std::sync::Arc;
10
11use std::time::Duration;
12use tracing::Instrument as _;
13
14use crate::error::Result;
15use crate::runtime::Runtime;
16
17/// Builds a replacement Lua state, identical to the ones the pool was
18/// created with. Used to recycle a state a request left unfit for reuse.
19///
20/// Construction is synchronous (the configuration script is not re-run —
21/// its snapshot is injected instead), so a rebuild can happen off the
22/// request path on a blocking worker.
23pub type RebuildFn = dyn Fn() -> Result<Runtime> + Send + Sync;
24
25/// A fixed pool of [`Runtime`]s over an MPMC channel.
26///
27/// A request checks a runtime out with [`get()`](Self::get) and uses it
28/// exclusively; the returned [`RuntimeGuard`] sends it back on drop. When all
29/// runtimes are busy, `get()` waits fairly (FIFO) — the channel is the
30/// backpressure mechanism — and
31/// [`get_timeout()`](Self::get_timeout) bounds that wait so an overloaded
32/// server sheds load instead of queueing without limit.
33#[derive(Clone)]
34pub struct RuntimePool {
35    tx: async_channel::Sender<Runtime>,
36    rx: async_channel::Receiver<Runtime>,
37    /// How a poisoned state is replaced. Without one, a damaged state is
38    /// returned to the pool rather than shrinking capacity.
39    rebuild: Option<Arc<RebuildFn>>,
40    /// Data the embedder keeps with this pool and swaps with it: a request
41    /// that reads it and then checks a state out sees one build of both.
42    companion: Option<Arc<dyn std::any::Any + Send + Sync>>,
43}
44
45impl std::fmt::Debug for RuntimePool {
46    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
47        f.debug_struct("RuntimePool")
48            .field("size", &self.size())
49            .field("available", &self.rx.len())
50            .field("recycling", &self.rebuild.is_some())
51            .finish()
52    }
53}
54
55impl RuntimePool {
56    /// Creates a pool holding the given runtimes.
57    pub fn new(runtimes: Vec<Runtime>) -> Self {
58        Self::build(runtimes, None)
59    }
60
61    /// Creates a pool that recycles damaged states: when a request leaves a
62    /// state poisoned (memory limit hit, panic), it is dropped and `rebuild`
63    /// produces a fresh one so capacity is restored.
64    pub fn with_rebuild<F>(runtimes: Vec<Runtime>, rebuild: F) -> Self
65    where
66        F: Fn() -> Result<Runtime> + Send + Sync + 'static,
67    {
68        Self::build(runtimes, Some(Arc::new(rebuild)))
69    }
70
71    fn build(runtimes: Vec<Runtime>, rebuild: Option<Arc<RebuildFn>>) -> Self {
72        let (tx, rx) = async_channel::bounded(runtimes.len().max(1));
73        for rt in runtimes {
74            // Cannot fail: the channel capacity equals the number of runtimes.
75            let _ = tx.try_send(rt);
76        }
77        Self {
78            tx,
79            rx,
80            rebuild,
81            companion: None,
82        }
83    }
84
85    /// Keeps `companion` with this pool, readable with
86    /// [`companion()`](Self::companion).
87    #[must_use]
88    pub fn with_companion<T: std::any::Any + Send + Sync>(mut self, companion: T) -> Self {
89        self.companion = Some(Arc::new(companion));
90        self
91    }
92
93    /// The companion set with [`with_companion()`](Self::with_companion),
94    /// when it is a `T`.
95    pub fn companion<T: std::any::Any + Send + Sync>(&self) -> Option<&T> {
96        self.companion.as_deref()?.downcast_ref()
97    }
98
99    /// Number of runtimes owned by the pool.
100    pub fn size(&self) -> usize {
101        self.tx.capacity().unwrap_or(0)
102    }
103
104    /// Number of runtimes currently idle and available for checkout.
105    pub fn available(&self) -> usize {
106        self.rx.len()
107    }
108
109    /// Checks a runtime out of the pool, waiting until one is available.
110    pub async fn get(&self) -> RuntimeGuard {
111        // Invariant, not a fallible path: the pool owns both channel ends
112        // for its whole lifetime, so recv can only fail after the pool —
113        // and every guard borrowing from it — is gone.
114        #[allow(clippy::expect_used)]
115        let rt = self
116            .rx
117            .recv()
118            .await
119            .expect("runtime pool channel cannot be closed while the pool is alive");
120        self.guard(rt)
121    }
122
123    /// Checks a runtime out, giving up after `wait`.
124    ///
125    /// Returns `None` when no state became available in time, so the caller
126    /// can shed the request (503) instead of letting it queue behind an
127    /// overloaded pool. A zero duration disables the bound and waits like
128    /// [`get()`](Self::get).
129    pub async fn get_timeout(&self, wait: Duration) -> Option<RuntimeGuard> {
130        // The `pool_checkout` span: how long this request waited for a
131        // state and whether it got one. DEBUG so the per-request
132        // decomposition is opt-in via the level filter.
133        let span = tracing::debug_span!(
134            "pool_checkout",
135            wait_ms = tracing::field::Empty,
136            outcome = tracing::field::Empty,
137        );
138        // The clock is read only when the span is enabled: a disabled span
139        // records nothing, and two `clock_gettime` calls per request are
140        // measurable at the level this path runs at (`get_timeout_hit` vs
141        // `pool_checkout` in crates/nitr/benches/runtime.rs).
142        let started = (!span.is_disabled()).then(std::time::Instant::now);
143        let got = async {
144            if wait.is_zero() {
145                Some(self.get().await)
146            } else if let Ok(rt) = self.rx.try_recv() {
147                // The fast path (a state is already idle) skips the timer.
148                Some(self.guard(rt))
149            } else {
150                match tokio::time::timeout(wait, self.rx.recv()).await {
151                    Ok(Ok(rt)) => Some(self.guard(rt)),
152                    // The channel cannot close while the pool is alive; a
153                    // timeout is the only real outcome here.
154                    Ok(Err(_)) | Err(_) => None,
155                }
156            }
157        }
158        .instrument(span.clone())
159        .await;
160        if let Some(started) = started {
161            span.record("wait_ms", started.elapsed().as_millis() as u64);
162            span.record("outcome", if got.is_some() { "hit" } else { "shed" });
163        }
164        got
165    }
166
167    fn guard(&self, rt: Runtime) -> RuntimeGuard {
168        RuntimeGuard {
169            rt: Some(rt),
170            tx: self.tx.clone(),
171            rebuild: self.rebuild.clone(),
172        }
173    }
174}
175
176/// RAII guard over a checked-out [`Runtime`]; returns it to the pool on drop.
177///
178/// A state the request left poisoned is not returned: it is dropped and, when
179/// the pool was built with [`RuntimePool::with_rebuild`], replaced by a fresh
180/// one so the pool keeps its size.
181pub struct RuntimeGuard {
182    rt: Option<Runtime>,
183    tx: async_channel::Sender<Runtime>,
184    rebuild: Option<Arc<RebuildFn>>,
185}
186
187impl std::fmt::Debug for RuntimeGuard {
188    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
189        f.debug_struct("RuntimeGuard")
190            .field("runtime", &self.rt)
191            .finish_non_exhaustive()
192    }
193}
194
195impl Deref for RuntimeGuard {
196    type Target = Runtime;
197    fn deref(&self) -> &Self::Target {
198        // Invariant: `rt` is only taken in `Drop`, after the last deref.
199        #[allow(clippy::expect_used)]
200        self.rt.as_ref().expect("runtime present until drop")
201    }
202}
203
204impl DerefMut for RuntimeGuard {
205    fn deref_mut(&mut self) -> &mut Self::Target {
206        // Invariant: `rt` is only taken in `Drop`, after the last deref.
207        #[allow(clippy::expect_used)]
208        self.rt.as_mut().expect("runtime present until drop")
209    }
210}
211
212impl Drop for RuntimeGuard {
213    fn drop(&mut self) {
214        let Some(rt) = self.rt.take() else {
215            return;
216        };
217        // A guard dropped mid-unwind was in use when something panicked, so
218        // the state is presumed damaged even though nothing marked it: the
219        // panic escaped before any code could. Conservative by design —
220        // recycling a state that was in fact fine costs one rebuild.
221        if !rt.is_poisoned() && !std::thread::panicking() {
222            // Cannot fail: capacity equals the number of runtimes and the
223            // receiver lives as long as the pool.
224            let _ = self.tx.try_send(rt);
225            return;
226        }
227
228        let Some(rebuild) = self.rebuild.clone() else {
229            // No rebuild hook: returning a damaged state is still better
230            // than permanently shrinking the pool.
231            tracing::warn!("a damaged Lua state was returned to a pool without recycling");
232            let _ = self.tx.try_send(rt);
233            return;
234        };
235
236        // Drop the damaged state *before* building its replacement so the
237        // two heaps do not coexist, then rebuild off the request path: the
238        // caller has already been answered and the pool runs one slot short
239        // until the replacement lands.
240        drop(rt);
241        let tx = self.tx.clone();
242        let handle = tokio::task::spawn_blocking(move || match rebuild() {
243            Ok(fresh) => {
244                let _ = tx.try_send(fresh);
245                tracing::info!(outcome = "rebuilt", "recycled a damaged Lua state");
246            }
247            // Capacity is lost until the next reload. Loud, because a pool
248            // that silently shrinks looks like a mysterious slowdown.
249            Err(err) => tracing::error!("failed to recycle a damaged Lua state: {err}"),
250        });
251        // A rebuild that *panics* never reaches either arm above; only the
252        // `JoinHandle` sees it. Observe it so that failure path is exactly
253        // as loud as an `Err` — the slot is just as lost.
254        tokio::spawn(async move {
255            if let Err(err) = handle.await {
256                tracing::error!("the Lua state rebuild task panicked: {err}");
257            }
258        });
259    }
260}
261
262#[cfg(test)]
263mod tests {
264    use super::*;
265    use crate::runtime::{Runtime, RuntimeOpts};
266
267    fn runtime() -> Runtime {
268        Runtime::new_with(RuntimeOpts {
269            libs: mlua::StdLib::STRING,
270            memory_limit: 8 * 1024 * 1024,
271            dev_mode: false,
272            exec_timeout: None,
273            package_dir: None,
274            extra_package_dirs: Vec::new(),
275        })
276        .expect("runtime")
277    }
278
279    #[tokio::test]
280    async fn checkout_returns_the_state_on_drop() {
281        let pool = RuntimePool::new(vec![runtime()]);
282        assert_eq!(pool.size(), 1);
283        {
284            let _guard = pool.get().await;
285            assert_eq!(pool.available(), 0);
286        }
287        assert_eq!(pool.available(), 1);
288    }
289
290    #[test]
291    fn a_companion_travels_with_the_pool_and_its_clones() {
292        let pool = RuntimePool::new(vec![runtime()]).with_companion(7u32);
293        assert_eq!(pool.clone().companion::<u32>(), Some(&7));
294        assert_eq!(pool.companion::<String>(), None);
295        assert_eq!(RuntimePool::new(vec![runtime()]).companion::<u32>(), None);
296    }
297
298    #[tokio::test]
299    async fn get_timeout_sheds_when_the_pool_is_busy() {
300        let pool = RuntimePool::new(vec![runtime()]);
301        let held = pool.get().await;
302
303        // Nothing is available: the wait budget expires and the caller can
304        // shed instead of queueing.
305        assert!(pool.get_timeout(Duration::from_millis(50)).await.is_none());
306
307        // Once the state comes back, the same call succeeds immediately.
308        drop(held);
309        assert!(pool.get_timeout(Duration::from_millis(50)).await.is_some());
310    }
311
312    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
313    async fn poisoned_states_are_rebuilt_not_reused() {
314        let pool = RuntimePool::with_rebuild(vec![runtime()], || Ok(runtime()));
315        {
316            let mut guard = pool.get().await;
317            guard.poison();
318        }
319        // The replacement arrives from a blocking task; wait for capacity to
320        // come back rather than assuming an ordering.
321        let guard = tokio::time::timeout(Duration::from_secs(5), pool.get())
322            .await
323            .expect("a replacement state must be built");
324        assert!(!guard.is_poisoned());
325    }
326}