use std::{
future::Future,
sync::{
atomic::{AtomicBool, Ordering},
Arc, Mutex,
},
thread,
time::{Duration, Instant},
};
use wasmtime::{Engine, ResourceLimiter, StoreLimits, StoreLimitsBuilder};
pub(crate) const EPOCH_TICK_INTERVAL: Duration = Duration::from_millis(100);
pub const DEFAULT_MEMORY_BYTES: usize = 512 * 1024 * 1024;
pub const DEFAULT_TABLE_ELEMENTS: usize = 100_000;
pub const DEFAULT_INSTANCES: usize = 1_000;
pub(crate) const WALL_DEADLINE_GRACE: Duration = Duration::from_secs(2);
const _: () = assert!(
WALL_DEADLINE_GRACE.as_millis() > EPOCH_TICK_INTERVAL.as_millis(),
"the wall bound must leave the epoch at least one tick to fire first"
);
pub const DEFAULT_DEADLINE_SECONDS: u64 = 600;
pub const HOOK_DEFAULT_DEADLINE_SECONDS: u64 = 30;
const _: () = assert!(
HOOK_DEFAULT_DEADLINE_SECONDS < DEFAULT_DEADLINE_SECONDS,
"a hook call must not be able to stall a session for the capsule-wide budget"
);
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct ExecutionLimits {
pub memory_bytes: usize,
pub table_elements: usize,
pub instances: usize,
pub deadline_seconds: u64,
}
impl Default for ExecutionLimits {
fn default() -> Self {
Self {
memory_bytes: DEFAULT_MEMORY_BYTES,
table_elements: DEFAULT_TABLE_ELEMENTS,
instances: DEFAULT_INSTANCES,
deadline_seconds: DEFAULT_DEADLINE_SECONDS,
}
}
}
impl ExecutionLimits {
#[must_use]
pub fn resolve(declared: Option<&murmur_artifact::ResourceLimits>) -> Self {
let defaults = Self::default();
let Some(declared) = declared else {
return defaults;
};
Self {
memory_bytes: declared.memory_bytes.unwrap_or(defaults.memory_bytes),
table_elements: declared.table_elements.unwrap_or(defaults.table_elements),
instances: declared.instances.unwrap_or(defaults.instances),
deadline_seconds: declared
.deadline_seconds
.unwrap_or(defaults.deadline_seconds),
}
}
#[must_use]
pub fn for_hook_calls(self, declared_deadline_seconds: Option<u64>) -> Self {
Self {
deadline_seconds: declared_deadline_seconds.unwrap_or(HOOK_DEFAULT_DEADLINE_SECONDS),
..self
}
}
#[must_use]
pub(crate) fn for_hook_entry(self, entry_deadline_seconds: Option<u64>) -> Self {
Self {
deadline_seconds: entry_deadline_seconds.unwrap_or(self.deadline_seconds),
..self
}
}
pub(crate) fn deadline_ticks(&self) -> u64 {
let tick_ms = EPOCH_TICK_INTERVAL.as_millis() as u64;
(self.deadline_seconds.saturating_mul(1_000) / tick_ms).max(1)
}
pub(crate) fn wall_bound(&self) -> Duration {
Duration::from_secs(self.deadline_seconds).saturating_add(WALL_DEADLINE_GRACE)
}
pub(crate) fn limiter(&self) -> ExecutionLimiter {
ExecutionLimiter::new(*self)
}
}
pub(crate) struct ExecutionLimiter {
inner: StoreLimits,
limits: ExecutionLimits,
denial: Option<String>,
}
impl ExecutionLimiter {
fn new(limits: ExecutionLimits) -> Self {
let inner = StoreLimitsBuilder::new()
.memory_size(limits.memory_bytes)
.table_elements(limits.table_elements)
.instances(limits.instances)
.trap_on_grow_failure(false)
.build();
Self {
inner,
limits,
denial: None,
}
}
pub(crate) fn limits(&self) -> ExecutionLimits {
self.limits
}
pub(crate) fn denial(&self) -> Option<&str> {
self.denial.as_deref()
}
}
impl ResourceLimiter for ExecutionLimiter {
fn memory_growing(
&mut self,
current: usize,
desired: usize,
maximum: Option<usize>,
) -> wasmtime::Result<bool> {
if self.inner.memory_growing(current, desired, maximum)? {
return Ok(true);
}
let message = format!(
"linear memory growth to {desired} bytes exceeds \
capabilities.limits.memory_bytes ({})",
self.limits.memory_bytes
);
self.denial = Some(message.clone());
Err(wasmtime::Error::msg(message))
}
fn table_growing(
&mut self,
current: usize,
desired: usize,
maximum: Option<usize>,
) -> wasmtime::Result<bool> {
if self.inner.table_growing(current, desired, maximum)? {
return Ok(true);
}
let message = format!(
"table growth to {desired} elements exceeds \
capabilities.limits.table_elements ({})",
self.limits.table_elements
);
self.denial = Some(message.clone());
Err(wasmtime::Error::msg(message))
}
fn instances(&self) -> usize {
self.inner.instances()
}
fn tables(&self) -> usize {
self.inner.tables()
}
fn memories(&self) -> usize {
self.inner.memories()
}
}
pub(crate) struct EpochTicker {
stop: Arc<AtomicBool>,
}
impl EpochTicker {
pub(crate) fn spawn(engine: &Engine) -> Self {
let weak = engine.weak();
let stop = Arc::new(AtomicBool::new(false));
let thread_stop = Arc::clone(&stop);
thread::spawn(move || {
while !thread_stop.load(Ordering::Relaxed) {
thread::sleep(EPOCH_TICK_INTERVAL);
match weak.upgrade() {
Some(engine) => engine.increment_epoch(),
None => break,
}
}
});
Self { stop }
}
}
impl Drop for EpochTicker {
fn drop(&mut self) {
self.stop.store(true, Ordering::Relaxed);
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct WallDeadline {
pub(crate) elapsed: Duration,
}
pub(crate) async fn bound_call<F: Future>(
bound: Duration,
call: F,
) -> Result<F::Output, WallDeadline> {
let started = Instant::now();
tokio::time::timeout(bound, call)
.await
.map_err(|_| WallDeadline {
elapsed: started.elapsed(),
})
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub(crate) enum ComponentRole {
Capsule,
Driver,
Tool,
Hook,
ProcessDriver,
}
impl ComponentRole {
pub(crate) fn wire_name(self) -> &'static str {
match self {
Self::Capsule => "capsule",
Self::Driver => "driver",
Self::Tool => "tool",
Self::Hook => "hook",
Self::ProcessDriver => "process_driver",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub(crate) struct CallDeadline {
pub(crate) component: String,
pub(crate) role: ComponentRole,
pub(crate) call: String,
pub(crate) deadline_seconds: u64,
pub(crate) elapsed_ms: u64,
}
#[derive(Debug, Clone, Default)]
pub(crate) struct CallDeadlines(Arc<Mutex<Vec<CallDeadline>>>);
impl CallDeadlines {
pub(crate) async fn bound<F: Future>(
&self,
component: &str,
role: ComponentRole,
call: &str,
limits: &ExecutionLimits,
future: F,
) -> Result<F::Output, WallDeadline> {
let deadline = (limits.deadline_seconds, limits.wall_bound());
self.bound_within(component, role, call, deadline, future)
.await
}
async fn bound_within<F: Future>(
&self,
component: &str,
role: ComponentRole,
call: &str,
(deadline_seconds, bound): (u64, Duration),
future: F,
) -> Result<F::Output, WallDeadline> {
let bounded = bound_call(bound, future).await;
if let Err(cut) = &bounded {
self.lock().push(CallDeadline {
component: component.to_string(),
role,
call: call.to_string(),
deadline_seconds,
elapsed_ms: u64::try_from(cut.elapsed.as_millis()).unwrap_or(u64::MAX),
});
}
bounded
}
pub(crate) fn record(&self, cut: CallDeadline) {
self.lock().push(cut);
}
pub(crate) fn take(&self) -> Vec<CallDeadline> {
std::mem::take(&mut *self.lock())
}
#[cfg(test)]
pub(crate) fn push_for_test(&self, cut: CallDeadline) {
self.lock().push(cut);
}
fn lock(&self) -> std::sync::MutexGuard<'_, Vec<CallDeadline>> {
self.0
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
}
}
pub(crate) fn wall_deadline_message(subject: &str, seconds: u64) -> String {
format!(
"{subject} waited on the host past its {seconds}s execution deadline \
(capabilities.limits.deadline_seconds) and was stopped"
)
}
pub(crate) fn wall_deadline_hook_message(
hook: &str,
entry: &str,
fn_name: &str,
seconds: u64,
) -> String {
format!(
"hook '{hook}' {fn_name} waited on the host past its {seconds}s execution deadline \
(capabilities.limits.deadline_seconds on the artifacts: entry '{entry}') and was stopped"
)
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum GuestFailure {
DeadlineExceeded { seconds: u64 },
ResourceLimit { message: String },
Other,
}
pub(crate) fn classify_guest_failure(
error: &wasmtime::Error,
limiter: &ExecutionLimiter,
) -> GuestFailure {
if let Some(message) = limiter.denial() {
return GuestFailure::ResourceLimit {
message: message.to_string(),
};
}
if matches!(
error.downcast_ref::<wasmtime::Trap>(),
Some(wasmtime::Trap::Interrupt)
) {
return GuestFailure::DeadlineExceeded {
seconds: limiter.limits().deadline_seconds,
};
}
GuestFailure::Other
}
impl GuestFailure {
pub(crate) fn message(&self, subject: &str, error: &wasmtime::Error) -> String {
match self {
Self::DeadlineExceeded { seconds } => format!(
"{subject} exceeded its {seconds}s execution deadline \
(capabilities.limits.deadline_seconds) and was interrupted"
),
Self::ResourceLimit { message } => {
format!("{subject} exceeded its configured resource limits: {message}")
}
Self::Other => format!("{subject} trapped: {error}"),
}
}
pub(crate) fn hook_message(
&self,
hook: &str,
entry: &str,
fn_name: &str,
error: &wasmtime::Error,
) -> String {
let subject = format!("hook '{hook}' {fn_name}");
match self {
Self::DeadlineExceeded { seconds } => format!(
"{subject} exceeded its {seconds}s execution deadline \
(capabilities.limits.deadline_seconds on the artifacts: entry '{entry}') and was \
interrupted"
),
_ => self.message(&subject, error),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn resolve_uses_defaults_when_manifest_is_silent() {
assert_eq!(ExecutionLimits::resolve(None), ExecutionLimits::default());
}
#[test]
fn resolve_fills_each_omitted_field_independently() {
let declared = murmur_artifact::ResourceLimits {
memory_bytes: Some(1024),
table_elements: None,
instances: None,
deadline_seconds: Some(5),
};
let limits = ExecutionLimits::resolve(Some(&declared));
assert_eq!(limits.memory_bytes, 1024);
assert_eq!(limits.deadline_seconds, 5);
assert_eq!(limits.table_elements, DEFAULT_TABLE_ELEMENTS);
assert_eq!(limits.instances, DEFAULT_INSTANCES);
}
#[test]
fn hook_calls_get_the_lower_deadline_default_when_none_is_declared() {
let capsule = ExecutionLimits::resolve(None);
let hook = capsule.for_hook_calls(None);
assert_eq!(capsule.deadline_seconds, DEFAULT_DEADLINE_SECONDS);
assert_eq!(hook.deadline_seconds, HOOK_DEFAULT_DEADLINE_SECONDS);
assert_eq!(hook.memory_bytes, capsule.memory_bytes);
assert_eq!(hook.table_elements, capsule.table_elements);
assert_eq!(hook.instances, capsule.instances);
}
#[test]
fn an_explicit_deadline_is_honored_for_hook_calls() {
let declared = murmur_artifact::ResourceLimits {
memory_bytes: None,
table_elements: None,
instances: None,
deadline_seconds: Some(DEFAULT_DEADLINE_SECONDS),
};
let capsule = ExecutionLimits::resolve(Some(&declared));
assert_eq!(
capsule
.for_hook_calls(declared.deadline_seconds)
.deadline_seconds,
DEFAULT_DEADLINE_SECONDS
);
assert_eq!(capsule.for_hook_calls(Some(5)).deadline_seconds, 5);
}
#[test]
fn deadline_ticks_converts_seconds_and_never_yields_zero() {
let limits = ExecutionLimits {
deadline_seconds: 1,
..ExecutionLimits::default()
};
assert_eq!(limits.deadline_ticks(), 10);
assert_eq!(ExecutionLimits::default().deadline_ticks(), 6_000);
}
#[test]
fn classify_reports_resource_limit_after_a_denial() {
let limits = ExecutionLimits {
memory_bytes: 1024,
..ExecutionLimits::default()
};
let mut limiter = limits.limiter();
let error = limiter
.memory_growing(0, 4096, None)
.expect_err("growth past memory_bytes must be denied");
match classify_guest_failure(&error, &limiter) {
GuestFailure::ResourceLimit { message } => {
assert!(message.contains("capabilities.limits.memory_bytes"));
}
other => panic!("expected ResourceLimit, got {other:?}"),
}
}
#[test]
fn classify_reports_deadline_for_an_interrupt_trap() {
let limits = ExecutionLimits {
deadline_seconds: 7,
..ExecutionLimits::default()
};
let limiter = limits.limiter();
let error = wasmtime::Error::from(wasmtime::Trap::Interrupt);
assert_eq!(
classify_guest_failure(&error, &limiter),
GuestFailure::DeadlineExceeded { seconds: 7 }
);
}
#[test]
fn a_hook_deadline_names_the_key_on_the_hooks_entry() {
let error = wasmtime::Error::from(wasmtime::Trap::Interrupt);
let failure = GuestFailure::DeadlineExceeded { seconds: 2 };
assert_eq!(
failure.hook_message("compact", "local/compact", "on-compaction", &error),
"hook 'compact' on-compaction exceeded its 2s execution deadline \
(capabilities.limits.deadline_seconds on the artifacts: entry 'local/compact') and \
was interrupted"
);
assert_eq!(
failure.message("tool 'echo'", &error),
"tool 'echo' exceeded its 2s execution deadline (capabilities.limits.deadline_seconds) \
and was interrupted",
"every other guest keeps the capsule-wide key"
);
assert_eq!(
GuestFailure::Other.hook_message("h", "local/h", "on-stage", &error),
GuestFailure::Other.message("hook 'h' on-stage", &error)
);
}
#[test]
fn classify_reports_other_for_an_unrelated_trap() {
let limiter = ExecutionLimits::default().limiter();
let error = wasmtime::Error::from(wasmtime::Trap::UnreachableCodeReached);
let failure = classify_guest_failure(&error, &limiter);
assert_eq!(failure, GuestFailure::Other);
assert!(failure
.message("tool 'echo'", &error)
.starts_with("tool 'echo' trapped: "));
}
#[test]
fn the_wall_bound_is_the_deadline_plus_a_positive_grace() {
assert!(WALL_DEADLINE_GRACE > Duration::ZERO);
let limits = ExecutionLimits {
deadline_seconds: 3,
..ExecutionLimits::default()
};
assert_eq!(
limits.wall_bound(),
Duration::from_secs(3) + WALL_DEADLINE_GRACE
);
}
#[tokio::test]
async fn bound_call_returns_a_finished_futures_output() {
assert_eq!(
bound_call(Duration::from_millis(50), async { 7 }).await,
Ok(7)
);
}
#[tokio::test]
async fn bound_call_stops_a_pending_future_after_the_bound() {
let bound = Duration::from_millis(30);
let cut = bound_call(bound, std::future::pending::<()>())
.await
.expect_err("a future that never finishes must be stopped");
assert!(cut.elapsed >= bound, "stopped after {:?}", cut.elapsed);
}
#[tokio::test]
async fn the_latch_records_one_entry_per_firing() {
let deadlines = CallDeadlines::default();
let bound = (3, Duration::from_millis(30));
let finished = deadlines
.bound_within("echo", ComponentRole::Tool, "run", bound, async { 1 })
.await;
assert_eq!(finished, Ok(1));
assert!(deadlines.take().is_empty());
let cut = deadlines
.bound_within(
"stall",
ComponentRole::Driver,
"run",
bound,
std::future::pending::<()>(),
)
.await;
assert!(cut.is_err());
let taken = deadlines.take();
assert_eq!(taken.len(), 1);
assert_eq!(taken[0].component, "stall");
assert_eq!(taken[0].role.wire_name(), "driver");
assert_eq!(taken[0].call, "run");
assert_eq!(taken[0].deadline_seconds, 3);
assert!(taken[0].elapsed_ms >= 30);
assert!(deadlines.take().is_empty(), "a take clears the latch");
}
#[test]
fn wall_deadline_messages_name_the_key_that_set_the_deadline() {
assert_eq!(
wall_deadline_message("tool 'stall'", 3),
"tool 'stall' waited on the host past its 3s execution deadline \
(capabilities.limits.deadline_seconds) and was stopped"
);
assert_eq!(
wall_deadline_hook_message("sleep", "local/sleep", "on-session-start", 2),
"hook 'sleep' on-session-start waited on the host past its 2s execution deadline \
(capabilities.limits.deadline_seconds on the artifacts: entry 'local/sleep') and was \
stopped"
);
}
#[test]
fn limiter_allows_growth_within_the_cap() {
let limits = ExecutionLimits {
memory_bytes: 4096,
..ExecutionLimits::default()
};
let mut limiter = limits.limiter();
assert!(limiter.memory_growing(0, 4096, None).unwrap());
assert!(limiter.denial().is_none());
}
}