nitr_core/runtime/pool.rs
1// SPDX-License-Identifier: MIT OR Apache-2.0
2// This file is part of Nitr.
3// See https://nitrweb.com/ for more information
4// Copyright (C) 2024-present Jose Quintana <joseluisq.net>
5
6//! A fixed pool of independent Lua runtimes checked out per request.
7
8use std::ops::{Deref, DerefMut};
9use std::sync::Arc;
10
11use std::time::Duration;
12use tracing::Instrument as _;
13
14use crate::error::Result;
15use crate::runtime::Runtime;
16
17/// Builds a replacement Lua state, identical to the ones the pool was
18/// created with. Used to recycle a state a request left unfit for reuse.
19///
20/// Construction is synchronous (the configuration script is not re-run —
21/// its snapshot is injected instead), so a rebuild can happen off the
22/// request path on a blocking worker.
23pub type RebuildFn = dyn Fn() -> Result<Runtime> + Send + Sync;
24
25/// A fixed pool of [`Runtime`]s over an MPMC channel.
26///
27/// A request checks a runtime out with [`get()`](Self::get) and uses it
28/// exclusively; the returned [`RuntimeGuard`] sends it back on drop. When all
29/// runtimes are busy, `get()` waits fairly (FIFO) — the channel is the
30/// backpressure mechanism — and
31/// [`get_timeout()`](Self::get_timeout) bounds that wait so an overloaded
32/// server sheds load instead of queueing without limit.
33#[derive(Clone)]
34pub struct RuntimePool {
35 tx: async_channel::Sender<Runtime>,
36 rx: async_channel::Receiver<Runtime>,
37 /// How a poisoned state is replaced. Without one, a damaged state is
38 /// returned to the pool rather than shrinking capacity.
39 rebuild: Option<Arc<RebuildFn>>,
40 /// Data the embedder keeps with this pool and swaps with it: a request
41 /// that reads it and then checks a state out sees one build of both.
42 companion: Option<Arc<dyn std::any::Any + Send + Sync>>,
43}
44
45impl std::fmt::Debug for RuntimePool {
46 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
47 f.debug_struct("RuntimePool")
48 .field("size", &self.size())
49 .field("available", &self.rx.len())
50 .field("recycling", &self.rebuild.is_some())
51 .finish()
52 }
53}
54
55impl RuntimePool {
56 /// Creates a pool holding the given runtimes.
57 pub fn new(runtimes: Vec<Runtime>) -> Self {
58 Self::build(runtimes, None)
59 }
60
61 /// Creates a pool that recycles damaged states: when a request leaves a
62 /// state poisoned (memory limit hit, panic), it is dropped and `rebuild`
63 /// produces a fresh one so capacity is restored.
64 pub fn with_rebuild<F>(runtimes: Vec<Runtime>, rebuild: F) -> Self
65 where
66 F: Fn() -> Result<Runtime> + Send + Sync + 'static,
67 {
68 Self::build(runtimes, Some(Arc::new(rebuild)))
69 }
70
71 fn build(runtimes: Vec<Runtime>, rebuild: Option<Arc<RebuildFn>>) -> Self {
72 let (tx, rx) = async_channel::bounded(runtimes.len().max(1));
73 for rt in runtimes {
74 // Cannot fail: the channel capacity equals the number of runtimes.
75 let _ = tx.try_send(rt);
76 }
77 Self {
78 tx,
79 rx,
80 rebuild,
81 companion: None,
82 }
83 }
84
85 /// Keeps `companion` with this pool, readable with
86 /// [`companion()`](Self::companion).
87 #[must_use]
88 pub fn with_companion<T: std::any::Any + Send + Sync>(mut self, companion: T) -> Self {
89 self.companion = Some(Arc::new(companion));
90 self
91 }
92
93 /// The companion set with [`with_companion()`](Self::with_companion),
94 /// when it is a `T`.
95 pub fn companion<T: std::any::Any + Send + Sync>(&self) -> Option<&T> {
96 self.companion.as_deref()?.downcast_ref()
97 }
98
99 /// Number of runtimes owned by the pool.
100 pub fn size(&self) -> usize {
101 self.tx.capacity().unwrap_or(0)
102 }
103
104 /// Number of runtimes currently idle and available for checkout.
105 pub fn available(&self) -> usize {
106 self.rx.len()
107 }
108
109 /// Checks a runtime out of the pool, waiting until one is available.
110 pub async fn get(&self) -> RuntimeGuard {
111 // Invariant, not a fallible path: the pool owns both channel ends
112 // for its whole lifetime, so recv can only fail after the pool —
113 // and every guard borrowing from it — is gone.
114 #[allow(clippy::expect_used)]
115 let rt = self
116 .rx
117 .recv()
118 .await
119 .expect("runtime pool channel cannot be closed while the pool is alive");
120 self.guard(rt)
121 }
122
123 /// Checks a runtime out, giving up after `wait`.
124 ///
125 /// Returns `None` when no state became available in time, so the caller
126 /// can shed the request (503) instead of letting it queue behind an
127 /// overloaded pool. A zero duration disables the bound and waits like
128 /// [`get()`](Self::get).
129 pub async fn get_timeout(&self, wait: Duration) -> Option<RuntimeGuard> {
130 // The `pool_checkout` span: how long this request waited for a
131 // state and whether it got one. DEBUG so the per-request
132 // decomposition is opt-in via the level filter.
133 let span = tracing::debug_span!(
134 "pool_checkout",
135 wait_ms = tracing::field::Empty,
136 outcome = tracing::field::Empty,
137 );
138 // The clock is read only when the span is enabled: a disabled span
139 // records nothing, and two `clock_gettime` calls per request are
140 // measurable at the level this path runs at (`get_timeout_hit` vs
141 // `pool_checkout` in crates/nitr/benches/runtime.rs).
142 let started = (!span.is_disabled()).then(std::time::Instant::now);
143 let got = async {
144 if wait.is_zero() {
145 Some(self.get().await)
146 } else if let Ok(rt) = self.rx.try_recv() {
147 // The fast path (a state is already idle) skips the timer.
148 Some(self.guard(rt))
149 } else {
150 match tokio::time::timeout(wait, self.rx.recv()).await {
151 Ok(Ok(rt)) => Some(self.guard(rt)),
152 // The channel cannot close while the pool is alive; a
153 // timeout is the only real outcome here.
154 Ok(Err(_)) | Err(_) => None,
155 }
156 }
157 }
158 .instrument(span.clone())
159 .await;
160 if let Some(started) = started {
161 span.record("wait_ms", started.elapsed().as_millis() as u64);
162 span.record("outcome", if got.is_some() { "hit" } else { "shed" });
163 }
164 got
165 }
166
167 fn guard(&self, rt: Runtime) -> RuntimeGuard {
168 RuntimeGuard {
169 rt: Some(rt),
170 tx: self.tx.clone(),
171 rebuild: self.rebuild.clone(),
172 }
173 }
174}
175
176/// RAII guard over a checked-out [`Runtime`]; returns it to the pool on drop.
177///
178/// A state the request left poisoned is not returned: it is dropped and, when
179/// the pool was built with [`RuntimePool::with_rebuild`], replaced by a fresh
180/// one so the pool keeps its size.
181pub struct RuntimeGuard {
182 rt: Option<Runtime>,
183 tx: async_channel::Sender<Runtime>,
184 rebuild: Option<Arc<RebuildFn>>,
185}
186
187impl std::fmt::Debug for RuntimeGuard {
188 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
189 f.debug_struct("RuntimeGuard")
190 .field("runtime", &self.rt)
191 .finish_non_exhaustive()
192 }
193}
194
195impl Deref for RuntimeGuard {
196 type Target = Runtime;
197 fn deref(&self) -> &Self::Target {
198 // Invariant: `rt` is only taken in `Drop`, after the last deref.
199 #[allow(clippy::expect_used)]
200 self.rt.as_ref().expect("runtime present until drop")
201 }
202}
203
204impl DerefMut for RuntimeGuard {
205 fn deref_mut(&mut self) -> &mut Self::Target {
206 // Invariant: `rt` is only taken in `Drop`, after the last deref.
207 #[allow(clippy::expect_used)]
208 self.rt.as_mut().expect("runtime present until drop")
209 }
210}
211
212impl Drop for RuntimeGuard {
213 fn drop(&mut self) {
214 let Some(rt) = self.rt.take() else {
215 return;
216 };
217 // A guard dropped mid-unwind was in use when something panicked, so
218 // the state is presumed damaged even though nothing marked it: the
219 // panic escaped before any code could. Conservative by design —
220 // recycling a state that was in fact fine costs one rebuild.
221 if !rt.is_poisoned() && !std::thread::panicking() {
222 // Cannot fail: capacity equals the number of runtimes and the
223 // receiver lives as long as the pool.
224 let _ = self.tx.try_send(rt);
225 return;
226 }
227
228 let Some(rebuild) = self.rebuild.clone() else {
229 // No rebuild hook: returning a damaged state is still better
230 // than permanently shrinking the pool.
231 tracing::warn!("a damaged Lua state was returned to a pool without recycling");
232 let _ = self.tx.try_send(rt);
233 return;
234 };
235
236 // Drop the damaged state *before* building its replacement so the
237 // two heaps do not coexist, then rebuild off the request path: the
238 // caller has already been answered and the pool runs one slot short
239 // until the replacement lands.
240 drop(rt);
241 let tx = self.tx.clone();
242 let handle = tokio::task::spawn_blocking(move || match rebuild() {
243 Ok(fresh) => {
244 let _ = tx.try_send(fresh);
245 tracing::info!(outcome = "rebuilt", "recycled a damaged Lua state");
246 }
247 // Capacity is lost until the next reload. Loud, because a pool
248 // that silently shrinks looks like a mysterious slowdown.
249 Err(err) => tracing::error!("failed to recycle a damaged Lua state: {err}"),
250 });
251 // A rebuild that *panics* never reaches either arm above; only the
252 // `JoinHandle` sees it. Observe it so that failure path is exactly
253 // as loud as an `Err` — the slot is just as lost.
254 tokio::spawn(async move {
255 if let Err(err) = handle.await {
256 tracing::error!("the Lua state rebuild task panicked: {err}");
257 }
258 });
259 }
260}
261
262#[cfg(test)]
263mod tests {
264 use super::*;
265 use crate::runtime::{Runtime, RuntimeOpts};
266
267 fn runtime() -> Runtime {
268 Runtime::new_with(RuntimeOpts {
269 libs: mlua::StdLib::STRING,
270 memory_limit: 8 * 1024 * 1024,
271 dev_mode: false,
272 exec_timeout: None,
273 package_dir: None,
274 extra_package_dirs: Vec::new(),
275 })
276 .expect("runtime")
277 }
278
279 #[tokio::test]
280 async fn checkout_returns_the_state_on_drop() {
281 let pool = RuntimePool::new(vec![runtime()]);
282 assert_eq!(pool.size(), 1);
283 {
284 let _guard = pool.get().await;
285 assert_eq!(pool.available(), 0);
286 }
287 assert_eq!(pool.available(), 1);
288 }
289
290 #[test]
291 fn a_companion_travels_with_the_pool_and_its_clones() {
292 let pool = RuntimePool::new(vec![runtime()]).with_companion(7u32);
293 assert_eq!(pool.clone().companion::<u32>(), Some(&7));
294 assert_eq!(pool.companion::<String>(), None);
295 assert_eq!(RuntimePool::new(vec![runtime()]).companion::<u32>(), None);
296 }
297
298 #[tokio::test]
299 async fn get_timeout_sheds_when_the_pool_is_busy() {
300 let pool = RuntimePool::new(vec![runtime()]);
301 let held = pool.get().await;
302
303 // Nothing is available: the wait budget expires and the caller can
304 // shed instead of queueing.
305 assert!(pool.get_timeout(Duration::from_millis(50)).await.is_none());
306
307 // Once the state comes back, the same call succeeds immediately.
308 drop(held);
309 assert!(pool.get_timeout(Duration::from_millis(50)).await.is_some());
310 }
311
312 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
313 async fn poisoned_states_are_rebuilt_not_reused() {
314 let pool = RuntimePool::with_rebuild(vec![runtime()], || Ok(runtime()));
315 {
316 let mut guard = pool.get().await;
317 guard.poison();
318 }
319 // The replacement arrives from a blocking task; wait for capacity to
320 // come back rather than assuming an ordering.
321 let guard = tokio::time::timeout(Duration::from_secs(5), pool.get())
322 .await
323 .expect("a replacement state must be built");
324 assert!(!guard.is_poisoned());
325 }
326}