Skip to main content

rivetkit_core/actor/
context.rs

1use std::collections::{BTreeMap, BTreeSet};
2use std::future::Future;
3use std::sync::Weak;
4#[cfg(any(test, feature = "test-support"))]
5use std::sync::atomic::AtomicI64;
6use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering};
7use std::sync::{Arc, OnceLock};
8use std::time::Duration;
9
10use crate::time::Instant;
11
12use anyhow::{Context as AnyhowContext, Result};
13use futures::future::BoxFuture;
14use parking_lot::{Mutex, RwLock};
15use rivet_envoy_client::handle::EnvoyHandle;
16use rivet_envoy_client::tunnel::HibernatingWebSocketMetadata;
17use rivet_error::ActorSpecifier;
18use scc::HashMap as SccHashMap;
19use scc::HashSet as SccHashSet;
20use tokio::runtime::Handle;
21use tokio::sync::{Mutex as AsyncMutex, Notify, OnceCell, broadcast, mpsc, oneshot};
22use tokio::task::JoinHandle;
23use tokio_util::sync::CancellationToken;
24
25#[cfg(not(feature = "wasm-runtime"))]
26type LocalAlarmTask = JoinHandle<()>;
27#[cfg(feature = "wasm-runtime")]
28type LocalAlarmTask = futures::future::AbortHandle;
29
30use crate::ActorConfig;
31#[cfg(feature = "sqlite-local")]
32use crate::actor::actor_runtime_socket::{
33	ActorRuntimeSocketEndpoint, ActorRuntimeSocketEndpointInfo,
34};
35use crate::actor::connection::{
36	ConnHandle, ConnHandles, HibernatableConnectionMetadata, PendingHibernationChanges,
37	hibernatable_id_from_slice,
38};
39use crate::actor::diagnostics::ActorDiagnostics;
40use crate::actor::internal_storage;
41use crate::actor::kv::LegacyActorKv;
42use crate::actor::lifecycle_hooks::Reply;
43use crate::actor::messages::{ActorEvent, Request, StateDelta, WorkflowKvWrite};
44use crate::actor::metrics::ActorMetrics;
45use crate::actor::queue::{QueueInspectorUpdateCallback, QueueMetadata, QueueWaitActivityCallback};
46use crate::actor::schedule::{InternalKeepAwakeCallback, LocalAlarmCallback};
47use crate::actor::sleep::{CanSleep, SleepState};
48use crate::actor::state::{PendingSave, PersistedActor, RequestSaveOpts};
49use crate::actor::task::LifecycleEvent;
50use crate::actor::task_types::UserTaskKind;
51use crate::actor::work_registry::{ActorWorkKind, CountGuard, RegionGuard};
52use crate::error::{ActorLifecycle as ActorLifecycleError, ActorRuntime};
53use crate::inspector::{Inspector, InspectorSnapshot};
54use crate::sqlite::SqliteDb;
55use crate::telemetry::{
56	ActorInvocationTelemetry, ActorInvocationTraceContext, ActorTelemetryIdentity,
57	IncomingTraceContext, OutboundCallInvocation, WorkflowRunInvocation, WorkflowRunOutcome,
58	WorkflowStepSpan,
59};
60use crate::types::{ActorKey, ConnId, ListOpts, format_actor_key};
61
62/// Shared actor runtime context.
63///
64/// This public surface is the foreign-runtime contract for `rivetkit-core`.
65/// Native Rust, NAPI-backed TypeScript, and future V8 runtimes should be able
66/// to drive actor behavior through `ActorFactory` plus the methods exposed here
67/// and on the returned runtime objects like `SqliteDb`, schedule APIs,
68/// queue APIs, `ConnHandle`, and `WebSocket`.
69#[derive(Clone)]
70pub struct ActorContext(
71	pub(crate) Arc<ActorContextInner>,
72	/// Telemetry of the invocation this handle serves, `None` outside an invocation.
73	pub(crate) Option<ActorInvocationTelemetry>,
74);
75
76#[derive(Clone)]
77pub struct ActorKv {
78	sql: SqliteDb,
79}
80
81pub(crate) struct ActorContextInner {
82	pub(super) legacy_kv: LegacyActorKv,
83	user_kv: ActorKv,
84	sql: SqliteDb,
85	#[cfg(feature = "sqlite-local")]
86	actor_runtime_socket: ActorRuntimeSocketEndpoint,
87	// Forced-sync: actor state snapshots are exposed through synchronous
88	// accessors and are never held across `.await`.
89	pub(super) current_state: RwLock<Vec<u8>>,
90	pub(super) persisted: RwLock<PersistedActor>,
91	pub(super) last_pushed_alarm: RwLock<Option<i64>>,
92	pub(super) run_wake_at: RwLock<Option<i64>>,
93	pub(super) run_wake_revision: AtomicU64,
94	pub(super) state_save_interval: Duration,
95	pub(super) state_dirty: AtomicBool,
96	pub(super) state_revision: AtomicU64,
97	pub(super) save_request_revision: AtomicU64,
98	pub(super) save_completed_revision: AtomicU64,
99	pub(super) save_completion: Notify,
100	pub(super) save_requested: AtomicBool,
101	pub(super) save_requested_immediate: AtomicBool,
102	// Forced-sync: debounce bookkeeping is updated from sync save-request paths.
103	pub(super) save_requested_within_deadline: Mutex<Option<crate::time::Instant>>,
104	pub(super) last_save_at: Mutex<Option<crate::time::Instant>>,
105	pub(super) pending_save: Mutex<Option<PendingSave>>,
106	pub(super) tracked_persist: Mutex<Option<JoinHandle<()>>>,
107	pub(super) save_guard: Arc<AsyncMutex<()>>,
108	pub(super) state_transaction_epoch: AtomicU64,
109	pub(super) in_flight_state_writes: AtomicUsize,
110	pub(super) state_write_completion: Notify,
111	pub(super) on_state_change_in_flight: AtomicUsize,
112	pub(super) on_state_change_idle: Notify,
113	// Forced-sync: hooks are registered and cloned from synchronous runtime
114	// wiring slots before use.
115	pub(super) request_save_hooks: RwLock<Vec<Arc<dyn Fn(RequestSaveOpts) + Send + Sync>>>,
116	// Forced-sync: schedule runtime handles and callbacks are synchronous
117	// wiring slots cloned before actor/envoy I/O.
118	pub(super) schedule_generation: Mutex<Option<u32>>,
119	pub(super) schedule_envoy_handle: Mutex<Option<EnvoyHandle>>,
120	pub(super) client_endpoint: OnceLock<String>,
121	pub(super) client_token: OnceLock<String>,
122	pub(super) client_namespace: OnceLock<String>,
123	pub(super) client_pool_name: OnceLock<String>,
124	pub(super) schedule_internal_keep_awake: Mutex<Option<InternalKeepAwakeCallback>>,
125	pub(super) schedule_local_alarm_callback: Mutex<Option<LocalAlarmCallback>>,
126	// Forced-sync: the local alarm timer is aborted from sync paths.
127	pub(super) schedule_local_alarm_task: Mutex<Option<LocalAlarmTask>>,
128	// Forced-sync: receivers are pushed/taken from sync paths and awaited after
129	// being moved out of the lock.
130	pub(super) schedule_pending_alarm_writes: Mutex<Vec<oneshot::Receiver<()>>>,
131	pub(super) schedule_local_alarm_epoch: AtomicU64,
132	pub(super) schedule_alarm_push_epoch: AtomicU64,
133	pub(super) schedule_alarm_dispatch_enabled: AtomicBool,
134	pub(super) schedule_dirty_since_push: AtomicBool,
135	pub(super) schedule_mutation_lock: AsyncMutex<()>,
136	pub(super) schedule_running: SccHashSet<String>,
137	pub(super) schedule_history_insert_count: AtomicUsize,
138	pub(super) max_schedules: u32,
139	#[cfg(any(test, feature = "test-support"))]
140	pub(super) schedule_now_override: AtomicI64,
141	#[cfg(test)]
142	pub(super) schedule_driver_alarm_cancel_count: AtomicUsize,
143	#[cfg(test)]
144	pub(super) schedule_sync_alarm_failures: AtomicUsize,
145	// Forced-sync: queue config is read from sync public methods before blocking
146	// on async queue work.
147	pub(super) queue_config: Mutex<ActorConfig>,
148	pub(super) queue_abort_signal: Mutex<CancellationToken>,
149	pub(super) queue_initialize: OnceCell<()>,
150	pub(super) queue_metadata: AsyncMutex<QueueMetadata>,
151	pub(super) queue_receive_lock: AsyncMutex<()>,
152	pub(super) queue_completion_waiters: SccHashMap<u64, oneshot::Sender<Option<Vec<u8>>>>,
153	pub(super) queue_notify: Notify,
154	pub(super) active_queue_wait_count: AtomicU32,
155	// Forced-sync: callbacks are registered and cloned from synchronous hooks.
156	pub(super) queue_wait_activity_callback: Mutex<Option<QueueWaitActivityCallback>>,
157	pub(super) queue_inspector_update_callback: Mutex<Option<QueueInspectorUpdateCallback>>,
158	// Forced-sync: connection operations expose sync accessors or clone handles
159	// before awaiting; connection_disconnect_state serializes disconnect
160	// bookkeeping with pending hibernation snapshots.
161	pub(super) connection_config: RwLock<ActorConfig>,
162	pub(super) connections: RwLock<BTreeMap<ConnId, ConnHandle>>,
163	pub(super) pending_hibernation_updates: RwLock<BTreeSet<ConnId>>,
164	pub(super) pending_hibernation_removals: RwLock<BTreeSet<ConnId>>,
165	pub(super) connection_disconnect_state: Mutex<()>,
166	pub(super) sleep: SleepState,
167	activity: ActivityState,
168	sleep_requested: AtomicBool,
169	destroy_requested: AtomicBool,
170	destroy_completed: AtomicBool,
171	destroy_completion_notify: Notify,
172	abort_signal: Mutex<CancellationToken>,
173	shutdown_deadline: CancellationToken,
174	// Forced-sync: runtime wiring slots are configured through synchronous
175	// lifecycle setup and cloned before sending events.
176	inspector: RwLock<Option<Inspector>>,
177	inspector_attach_count: RwLock<Option<Arc<AtomicU32>>>,
178	inspector_overlay_tx: RwLock<Option<broadcast::Sender<Arc<Vec<u8>>>>>,
179	actor_events: RwLock<Option<mpsc::UnboundedSender<ActorEvent>>>,
180	pub(super) lifecycle_events: RwLock<Option<mpsc::UnboundedSender<LifecycleEvent>>>,
181	hibernated_connection_liveness_override: RwLock<Option<BTreeSet<(Vec<u8>, Vec<u8>)>>>,
182	pub(super) metrics: ActorMetrics,
183	diagnostics: ActorDiagnostics,
184	telemetry_identity: Arc<ActorTelemetryIdentity>,
185	actor_id: String,
186	name: String,
187	key: ActorKey,
188	region: String,
189}
190
191#[derive(Debug, Default)]
192pub(crate) struct ActivityState {
193	dirty: AtomicBool,
194}
195
196impl ActivityState {
197	fn mark_dirty(&self) -> bool {
198		!self.dirty.swap(true, Ordering::AcqRel)
199	}
200
201	fn take_dirty(&self) -> bool {
202		self.dirty.swap(false, Ordering::AcqRel)
203	}
204}
205
206impl ActorKv {
207	pub async fn get(&self, key: &[u8]) -> Result<Option<Vec<u8>>> {
208		let mut values = self.batch_get(&[key]).await?;
209		Ok(values.pop().flatten())
210	}
211
212	pub async fn put(&self, key: &[u8], value: &[u8]) -> Result<()> {
213		self.batch_put(&[(key, value)]).await
214	}
215
216	pub async fn delete(&self, key: &[u8]) -> Result<()> {
217		self.batch_delete(&[key]).await
218	}
219
220	pub async fn batch_get(&self, keys: &[&[u8]]) -> Result<Vec<Option<Vec<u8>>>> {
221		internal_storage::user_kv_batch_get(&self.sql, keys).await
222	}
223
224	pub async fn batch_put(&self, entries: &[(&[u8], &[u8])]) -> Result<()> {
225		internal_storage::user_kv_batch_put(&self.sql, entries).await
226	}
227
228	pub async fn batch_delete(&self, keys: &[&[u8]]) -> Result<()> {
229		internal_storage::user_kv_batch_delete(&self.sql, keys).await
230	}
231
232	pub async fn delete_range(&self, start: &[u8], end: &[u8]) -> Result<()> {
233		internal_storage::user_kv_delete_range(&self.sql, start, end).await
234	}
235
236	pub async fn list_prefix(
237		&self,
238		prefix: &[u8],
239		opts: ListOpts,
240	) -> Result<Vec<(Vec<u8>, Vec<u8>)>> {
241		internal_storage::user_kv_list_prefix(&self.sql, prefix, opts).await
242	}
243
244	pub async fn list_range(
245		&self,
246		start: &[u8],
247		end: &[u8],
248		opts: ListOpts,
249	) -> Result<Vec<(Vec<u8>, Vec<u8>)>> {
250		internal_storage::user_kv_list_range(&self.sql, start, end, opts).await
251	}
252}
253
254impl ActorContext {
255	/// Returns a handle bound to `telemetry`, so schedules and SQLite work done
256	/// through it are attributed to that invocation.
257	pub fn with_invocation_telemetry(
258		mut self,
259		telemetry: Option<ActorInvocationTelemetry>,
260	) -> Self {
261		self.1 = telemetry;
262		self
263	}
264
265	/// Returns a handle for the same invocation whose spans parent to the
266	/// application span the host runtime has active, given as W3C
267	/// `traceparent` and `tracestate`. A handle that serves no invocation is
268	/// returned unchanged, because it opens no spans.
269	#[doc(hidden)]
270	pub fn with_application_span(
271		&self,
272		traceparent: Option<&str>,
273		tracestate: Option<&str>,
274	) -> Self {
275		Self(
276			self.0.clone(),
277			self.1
278				.as_ref()
279				.map(|telemetry| telemetry.with_application_span(traceparent, tracestate)),
280		)
281	}
282
283	pub(crate) fn invocation_telemetry(&self) -> Option<&ActorInvocationTelemetry> {
284		self.1.as_ref()
285	}
286
287	/// Returns the SQLite handle bound to this handle's invocation.
288	pub fn invocation_sql(&self) -> SqliteDb {
289		self.0.sql.clone().with_invocation_telemetry(self.1.clone())
290	}
291
292	/// Opens the span covering one call out to another actor, or nothing when
293	/// this handle serves no invocation or tracing is disabled.
294	///
295	/// `actor_name` and `action_name` name the callee. Both come from the
296	/// caller's own registry rather than from a remote peer, so neither is a
297	/// cardinality surface.
298	#[doc(hidden)]
299	pub fn start_call_span(
300		&self,
301		actor_name: &str,
302		action_name: &str,
303	) -> Option<OutboundCallInvocation> {
304		self.1
305			.as_ref()?
306			.start_outbound_call(actor_name, action_name)
307	}
308
309	/// Opens one workflow run as its own invocation, linked to the span the
310	/// previous run persisted.
311	#[doc(hidden)]
312	pub async fn start_workflow_span(&self) -> WorkflowRunInvocation {
313		let previous = if tracing::enabled!(target: "rivetkit::telemetry", tracing::Level::INFO) {
314			internal_storage::load_workflow_trace(&self.0.sql)
315				.await
316				.unwrap_or_else(|error| {
317					tracing::warn!(
318						actor_id = %self.actor_id(),
319						?error,
320						"failed to load the previous workflow run span, so this run will not link to it"
321					);
322					IncomingTraceContext::default()
323				})
324		} else {
325			IncomingTraceContext::default()
326		};
327		WorkflowRunInvocation::start(self, previous)
328	}
329
330	/// Closes a workflow run and persists its span and ray for the next run.
331	#[doc(hidden)]
332	pub async fn finish_workflow_span(
333		&self,
334		run: WorkflowRunInvocation,
335		outcome: WorkflowRunOutcome,
336	) {
337		let Some(trace_context) = run.finish(outcome) else {
338			return;
339		};
340		if let Err(error) =
341			internal_storage::persist_workflow_trace(&self.0.sql, trace_context).await
342		{
343			tracing::warn!(
344				actor_id = %self.actor_id(),
345				?error,
346				"failed to persist the workflow run span, so the next run will not link to it"
347			);
348		}
349	}
350
351	/// Opens the span for one step attempt, or nothing when this handle serves
352	/// no invocation or tracing is off.
353	#[doc(hidden)]
354	pub fn start_workflow_step_span(
355		&self,
356		step_name: &str,
357		attempt: u32,
358	) -> Option<WorkflowStepSpan> {
359		WorkflowStepSpan::start(self, step_name, attempt)
360	}
361
362	/// Returns correlation for the invocation this handle serves, absent when
363	/// the handle is not bound to one or tracing is disabled.
364	pub fn invocation_trace_context(&self) -> Option<ActorInvocationTraceContext> {
365		self.1.as_ref()?.trace_context()
366	}
367
368	/// Returns whether two handles belong to the same running actor generation.
369	#[doc(hidden)]
370	pub fn is_same_instance(&self, other: &Self) -> bool {
371		Arc::ptr_eq(&self.0, &other.0)
372	}
373
374	#[cfg(test)]
375	pub(crate) fn new(
376		actor_id: impl Into<String>,
377		name: impl Into<String>,
378		key: ActorKey,
379		region: impl Into<String>,
380	) -> Self {
381		tests::new_with_kv(actor_id, name, key, region, LegacyActorKv::new_in_memory())
382	}
383
384	#[cfg(test)]
385	pub(crate) fn new_with_kv(
386		actor_id: impl Into<String>,
387		name: impl Into<String>,
388		key: ActorKey,
389		region: impl Into<String>,
390		legacy_kv: LegacyActorKv,
391	) -> Self {
392		tests::new_with_kv(actor_id, name, key, region, legacy_kv)
393	}
394
395	#[cfg(test)]
396	pub(crate) fn new_for_state_tests(legacy_kv: LegacyActorKv, config: ActorConfig) -> Self {
397		let ctx = tests::new_with_kv("state-test", "state-test", Vec::new(), "local", legacy_kv);
398		ctx.configure_connection_runtime(config.clone());
399		ctx.configure_queue(config);
400		ctx
401	}
402
403	pub(crate) fn build(
404		actor_id: String,
405		name: String,
406		key: ActorKey,
407		region: String,
408		_generation: Option<u32>,
409		_envoy_key: String,
410		config: ActorConfig,
411		legacy_kv: LegacyActorKv,
412		sql: SqliteDb,
413	) -> Self {
414		#[cfg(feature = "sqlite-local")]
415		let mut sql = sql;
416		#[cfg(feature = "sqlite-local")]
417		sql.set_profiling_config(config.sqlite_profiling.clone());
418		let metrics =
419			ActorMetrics::new_with_sqlite_profiling(name.clone(), config.sqlite_profiling.clone());
420		#[cfg(feature = "sqlite-local")]
421		sql.set_vfs_metrics(Arc::new(metrics.clone()));
422		let diagnostics = ActorDiagnostics::new(actor_id.clone());
423		#[cfg(feature = "sqlite-local")]
424		let actor_runtime_socket =
425			ActorRuntimeSocketEndpoint::new(config.enable_actor_runtime_socket, sql.clone());
426		let state_save_interval = config.state_save_interval;
427		let max_schedules = config.max_schedules;
428		let abort_signal = CancellationToken::new();
429		let shutdown_deadline = CancellationToken::new();
430		let sleep = SleepState::new(config.clone());
431		let user_kv = ActorKv { sql: sql.clone() };
432		let inner = Arc::new(ActorContextInner {
433			legacy_kv,
434			user_kv,
435			sql,
436			#[cfg(feature = "sqlite-local")]
437			actor_runtime_socket,
438			current_state: RwLock::new(Vec::new()),
439			persisted: RwLock::new(PersistedActor::default()),
440			last_pushed_alarm: RwLock::new(None),
441			run_wake_at: RwLock::new(None),
442			run_wake_revision: AtomicU64::new(0),
443			state_save_interval,
444			state_dirty: AtomicBool::new(false),
445			state_revision: AtomicU64::new(0),
446			save_request_revision: AtomicU64::new(0),
447			save_completed_revision: AtomicU64::new(0),
448			save_completion: Notify::new(),
449			save_requested: AtomicBool::new(false),
450			save_requested_immediate: AtomicBool::new(false),
451			save_requested_within_deadline: Mutex::new(None),
452			last_save_at: Mutex::new(None),
453			pending_save: Mutex::new(None),
454			tracked_persist: Mutex::new(None),
455			save_guard: Arc::new(AsyncMutex::new(())),
456			state_transaction_epoch: AtomicU64::new(0),
457			in_flight_state_writes: AtomicUsize::new(0),
458			state_write_completion: Notify::new(),
459			on_state_change_in_flight: AtomicUsize::new(0),
460			on_state_change_idle: Notify::new(),
461			request_save_hooks: RwLock::new(Vec::new()),
462			schedule_generation: Mutex::new(None),
463			schedule_envoy_handle: Mutex::new(None),
464			client_endpoint: OnceLock::new(),
465			client_token: OnceLock::new(),
466			client_namespace: OnceLock::new(),
467			client_pool_name: OnceLock::new(),
468			schedule_internal_keep_awake: Mutex::new(None),
469			schedule_local_alarm_callback: Mutex::new(None),
470			schedule_local_alarm_task: Mutex::new(None),
471			schedule_pending_alarm_writes: Mutex::new(Vec::new()),
472			schedule_local_alarm_epoch: AtomicU64::new(0),
473			schedule_alarm_push_epoch: AtomicU64::new(0),
474			schedule_alarm_dispatch_enabled: AtomicBool::new(true),
475			// A fresh actor context has no in-process record of a successful
476			// envoy alarm push yet, so the first sync must always push.
477			schedule_dirty_since_push: AtomicBool::new(true),
478			schedule_mutation_lock: AsyncMutex::new(()),
479			schedule_running: SccHashSet::new(),
480			schedule_history_insert_count: AtomicUsize::new(0),
481			max_schedules,
482			#[cfg(any(test, feature = "test-support"))]
483			schedule_now_override: AtomicI64::new(i64::MIN),
484			#[cfg(test)]
485			schedule_driver_alarm_cancel_count: AtomicUsize::new(0),
486			#[cfg(test)]
487			schedule_sync_alarm_failures: AtomicUsize::new(0),
488			queue_config: Mutex::new(config.clone()),
489			queue_abort_signal: Mutex::new(abort_signal.clone()),
490			queue_initialize: OnceCell::new(),
491			queue_metadata: AsyncMutex::new(QueueMetadata::default()),
492			queue_receive_lock: AsyncMutex::new(()),
493			queue_completion_waiters: SccHashMap::new(),
494			queue_notify: Notify::new(),
495			active_queue_wait_count: AtomicU32::new(0),
496			queue_wait_activity_callback: Mutex::new(None),
497			queue_inspector_update_callback: Mutex::new(None),
498			connection_config: RwLock::new(config),
499			connections: RwLock::new(BTreeMap::new()),
500			pending_hibernation_updates: RwLock::new(BTreeSet::new()),
501			pending_hibernation_removals: RwLock::new(BTreeSet::new()),
502			connection_disconnect_state: Mutex::new(()),
503			sleep,
504			activity: ActivityState::default(),
505			sleep_requested: AtomicBool::new(false),
506			destroy_requested: AtomicBool::new(false),
507			destroy_completed: AtomicBool::new(false),
508			destroy_completion_notify: Notify::new(),
509			abort_signal: Mutex::new(abort_signal),
510			shutdown_deadline,
511			inspector: RwLock::new(None),
512			inspector_attach_count: RwLock::new(None),
513			inspector_overlay_tx: RwLock::new(None),
514			actor_events: RwLock::new(None),
515			lifecycle_events: RwLock::new(None),
516			hibernated_connection_liveness_override: RwLock::new(None),
517			metrics,
518			diagnostics,
519			telemetry_identity: Arc::new(ActorTelemetryIdentity {
520				actor_id: actor_id.clone(),
521				actor_name: name.clone(),
522				actor_key: format_actor_key(&key),
523			}),
524			actor_id,
525			name,
526			key,
527			region,
528		});
529		let ctx = Self(inner, None);
530		ctx.configure_sleep_hooks();
531		ctx
532	}
533
534	#[deprecated(
535		note = "Actor KV is deprecated. Use embedded SQLite (`sql()`) or actor state instead."
536	)]
537	pub async fn kv_batch_get(&self, keys: &[&[u8]]) -> Result<Vec<Option<Vec<u8>>>> {
538		self.0.user_kv.batch_get(keys).await
539	}
540
541	#[deprecated(
542		note = "Actor KV is deprecated. Use embedded SQLite (`sql()`) or actor state instead."
543	)]
544	pub async fn kv_batch_put(&self, entries: &[(&[u8], &[u8])]) -> Result<()> {
545		self.0.user_kv.batch_put(entries).await
546	}
547
548	#[deprecated(
549		note = "Actor KV is deprecated. Use embedded SQLite (`sql()`) or actor state instead."
550	)]
551	pub async fn kv_batch_delete(&self, keys: &[&[u8]]) -> Result<()> {
552		self.0.user_kv.batch_delete(keys).await
553	}
554
555	#[deprecated(
556		note = "Actor KV is deprecated. Use embedded SQLite (`sql()`) or actor state instead."
557	)]
558	pub async fn kv_delete_range(&self, start: &[u8], end: &[u8]) -> Result<()> {
559		self.0.user_kv.delete_range(start, end).await
560	}
561
562	#[deprecated(
563		note = "Actor KV is deprecated. Use embedded SQLite (`sql()`) or actor state instead."
564	)]
565	pub async fn kv_list_prefix(
566		&self,
567		prefix: &[u8],
568		opts: ListOpts,
569	) -> Result<Vec<(Vec<u8>, Vec<u8>)>> {
570		self.0.user_kv.list_prefix(prefix, opts).await
571	}
572
573	#[deprecated(
574		note = "Actor KV is deprecated. Use embedded SQLite (`sql()`) or actor state instead."
575	)]
576	pub async fn kv_list_range(
577		&self,
578		start: &[u8],
579		end: &[u8],
580		opts: ListOpts,
581	) -> Result<Vec<(Vec<u8>, Vec<u8>)>> {
582		self.0.user_kv.list_range(start, end, opts).await
583	}
584
585	#[deprecated(
586		note = "Actor KV is deprecated. Use embedded SQLite (`sql()`) or actor state instead."
587	)]
588	pub fn kv(&self) -> &ActorKv {
589		&self.0.user_kv
590	}
591
592	/// Legacy actor KV access used only by the one-time KV-to-SQLite importer and
593	/// the temporary inspector-token mirror consumed by the dashboard.
594	pub(crate) fn legacy_kv(&self) -> &LegacyActorKv {
595		&self.0.legacy_kv
596	}
597
598	pub fn sql(&self) -> &SqliteDb {
599		&self.0.sql
600	}
601
602	#[cfg(feature = "sqlite-local")]
603	pub async fn provision_actor_runtime_socket(&self) -> Result<ActorRuntimeSocketEndpointInfo> {
604		self.0.actor_runtime_socket.provision().await
605	}
606
607	#[cfg(feature = "sqlite-local")]
608	pub(crate) async fn shutdown_actor_runtime_socket(&self) {
609		self.0.actor_runtime_socket.shutdown().await;
610	}
611
612	pub async fn db_exec(&self, sql: &str) -> Result<Vec<u8>> {
613		self.0.sql.exec_rows_cbor(sql).await
614	}
615
616	pub async fn db_query(&self, sql: &str, params: Option<&[u8]>) -> Result<Vec<u8>> {
617		self.0.sql.query_rows_cbor(sql, params).await
618	}
619
620	pub async fn db_execute(&self, sql: &str, params: Option<&[u8]>) -> Result<Vec<u8>> {
621		self.0.sql.execute_rows_cbor(sql, params).await
622	}
623
624	pub async fn db_run(&self, sql: &str, params: Option<&[u8]>) -> Result<()> {
625		self.0.sql.run_cbor(sql, params).await?;
626		Ok(())
627	}
628
629	pub fn set_alarm(&self, timestamp_ms: Option<i64>) -> Result<()> {
630		self.set_schedule_alarm(timestamp_ms)
631	}
632
633	/// Resync persisted alarms with the runtime's alarm transport.
634	///
635	/// Foreign-runtime adapters should call this during startup after loading
636	/// any persisted schedule state and before accepting user callbacks that rely
637	/// on future alarms being armed.
638	pub async fn init_alarms(&self) {
639		if let Err(error) = self.recover_interrupted_schedule_history().await {
640			tracing::error!(?error, "failed to recover interrupted schedule history");
641		}
642		self.sync_future_alarm_logged().await;
643	}
644
645	pub fn queue(&self) -> &Self {
646		self
647	}
648
649	pub fn sleep(&self) -> Result<()> {
650		// `started` is cleared when the lifecycle state machine transitions
651		// into SleepGrace / DestroyGrace, so `started=false` covers both
652		// "never started" and "already shutting down". Distinguish with the
653		// request flags for an accurate diagnostic.
654		if !self.0.sleep.lifecycle_started.load(Ordering::SeqCst) {
655			let already_stopping = self.0.sleep_requested.load(Ordering::SeqCst)
656				|| self.0.destroy_requested.load(Ordering::SeqCst);
657			return if already_stopping {
658				Err(ActorLifecycleError::Stopping.build()).context("actor is already shutting down")
659			} else {
660				Err(ActorLifecycleError::Starting.build())
661					.context("cannot request sleep before actor startup completes")
662			};
663		}
664		if self.0.sleep_requested.swap(true, Ordering::SeqCst) {
665			return Ok(());
666		}
667		self.cancel_sleep_timer();
668		if Handle::try_current().is_ok() {
669			let ctx = self.clone();
670			let tracked = self.track_shutdown_task(async move {
671				ctx.record_user_task_started(UserTaskKind::SleepFinalize);
672				let started_at = Instant::now();
673				ctx.request_sleep_from_envoy();
674				ctx.record_user_task_finished(UserTaskKind::SleepFinalize, started_at.elapsed());
675			});
676			if tracked {
677				return Ok(());
678			}
679		}
680
681		self.request_sleep_from_envoy();
682		Ok(())
683	}
684
685	pub fn destroy(&self) -> Result<()> {
686		self.request_stop(None)
687	}
688
689	/// Request a stop with an error attached. The envoy reports the stop to the
690	/// engine with `StopCode::Error` and the message, and the engine answers a
691	/// crash by putting the actor back to sleep rather than destroying it. The
692	/// local teardown therefore takes the sleep path too, so `onSleep` runs,
693	/// hibernatable connections are preserved, the persisted alarm stays armed
694	/// for the next generation, and incoming work is not rejected with
695	/// `Destroying` for an actor that is about to resume.
696	pub fn stop_with_error(&self, message: impl Into<String>) -> Result<()> {
697		self.request_stop(Some(truncate_stop_error_message(message.into())))
698	}
699
700	fn request_stop(&self, error: Option<String>) -> Result<()> {
701		// See `sleep` for why the request flags disambiguate `started=false`.
702		// destroy() is allowed after sleep() has been requested because
703		// destroy is a stronger signal that escalates an in-flight sleep.
704		if !self.0.sleep.lifecycle_started.load(Ordering::SeqCst)
705			&& !self.0.sleep_requested.load(Ordering::SeqCst)
706			&& !self.0.destroy_requested.load(Ordering::SeqCst)
707		{
708			return Err(ActorLifecycleError::Starting.build())
709				.context("cannot request stop before actor startup completes");
710		}
711
712		if let Some(error) = error {
713			// An errored stop is a crash report, not a destroy. A destroy that
714			// already won owns the teardown, so leave it alone.
715			if self.0.destroy_requested.load(Ordering::SeqCst) {
716				return Err(ActorLifecycleError::Stopping.build())
717					.context("destroy already requested for this generation");
718			}
719			// Record the error even when a sleep is already in flight. The
720			// envoy attaches it to the actor regardless of whether the intent
721			// itself is a duplicate, so the eventual `Stopped` still carries
722			// `StopCode::Error` and the engine still records the crash.
723			let queued_error = self.0.sleep.stop_error.lock().replace(error);
724			if !self.0.sleep_requested.swap(true, Ordering::SeqCst) {
725				self.mark_errored_stop_requested();
726			}
727			// An errored stop is already queued and has not reached the envoy
728			// yet. It picks up the error recorded above, so sending a second
729			// intent would only race an error-less `stop_actor` against it:
730			// `request_stop_from_envoy` consumes the single error slot with
731			// `take`, and the loser reports the crash as a deliberate destroy.
732			// A sleep that is already in flight leaves the slot empty, so the
733			// `sleep()` -> `stop_with_error()` upgrade still reports here.
734			if queued_error.is_some() {
735				return Ok(());
736			}
737		} else {
738			if self.0.destroy_requested.swap(true, Ordering::SeqCst) {
739				return Err(ActorLifecycleError::Stopping.build())
740					.context("destroy already requested for this generation");
741			}
742			// A destroy supersedes an errored stop that has not reached the
743			// envoy yet. Without clearing the slot, whichever request runs
744			// first consumes the error and reports this destroy as a sleep
745			// intent, leaving the actor alive.
746			*self.0.sleep.stop_error.lock() = None;
747			// Reuse the shared teardown sequence used by the registry shutdown
748			// path so future changes to `mark_destroy_requested` cannot drift.
749			// `destroy_requested` is already true from the swap above. The
750			// redundant `store(true)` inside is harmless.
751			#[cfg(not(feature = "wasm-runtime"))]
752			self.mark_destroy_requested();
753			#[cfg(feature = "wasm-runtime")]
754			self.mark_destroy_requested_without_spawn();
755		}
756
757		let ctx = self.clone();
758		if Handle::try_current().is_ok() {
759			let tracked = self.track_shutdown_task(async move {
760				ctx.record_user_task_started(UserTaskKind::DestroyRequest);
761				let started_at = Instant::now();
762				ctx.request_stop_from_envoy();
763				ctx.record_user_task_finished(UserTaskKind::DestroyRequest, started_at.elapsed());
764			});
765			if tracked {
766				return Ok(());
767			}
768		}
769
770		self.request_stop_from_envoy();
771		Ok(())
772	}
773
774	pub fn mark_destroy_requested(&self) {
775		self.cancel_sleep_timer();
776		self.flush_on_shutdown();
777		self.0.destroy_requested.store(true, Ordering::SeqCst);
778		self.0.destroy_completed.store(false, Ordering::SeqCst);
779	}
780
781	/// Teardown bookkeeping for an errored stop. Mirrors
782	/// `mark_destroy_requested` minus the destroy flags, since the actor is
783	/// heading for the sleep path. The state flush still runs so a crash
784	/// persists whatever the actor had before its teardown.
785	fn mark_errored_stop_requested(&self) {
786		self.cancel_sleep_timer();
787		#[cfg(not(feature = "wasm-runtime"))]
788		self.flush_on_shutdown();
789	}
790
791	#[cfg(feature = "wasm-runtime")]
792	fn mark_destroy_requested_without_spawn(&self) {
793		self.cancel_sleep_timer();
794		self.0.destroy_requested.store(true, Ordering::SeqCst);
795		self.0.destroy_completed.store(false, Ordering::SeqCst);
796	}
797
798	#[doc(hidden)]
799	pub fn cancel_actor_abort_signal(&self) {
800		self.0.abort_signal.lock().cancel();
801	}
802
803	pub(crate) fn reset_abort_signal_for_start(&self) {
804		let mut abort_signal = self.0.abort_signal.lock();
805		if !abort_signal.is_cancelled() {
806			return;
807		}
808
809		// Sleep or destroy cancels the generation abort signal to break actor
810		// scoped waits. A restarted actor needs a fresh signal so the next
811		// generation can wait normally.
812		let next_signal = CancellationToken::new();
813		*abort_signal = next_signal.clone();
814		*self.0.queue_abort_signal.lock() = next_signal;
815	}
816
817	#[doc(hidden)]
818	pub fn actor_abort_signal(&self) -> CancellationToken {
819		self.0.abort_signal.lock().clone()
820	}
821
822	#[doc(hidden)]
823	pub fn actor_aborted(&self) -> bool {
824		self.0.abort_signal.lock().is_cancelled()
825	}
826
827	/// Fires when the shutdown grace deadline has elapsed and core is forcing
828	/// cleanup. Foreign-runtime adapters should abort any in-flight shutdown
829	/// work (for example `onSleep` / `onDestroy`) when this token is cancelled
830	/// so resources like SQLite are not torn down mid-operation.
831	#[doc(hidden)]
832	pub fn shutdown_deadline_token(&self) -> CancellationToken {
833		self.0.shutdown_deadline.clone()
834	}
835
836	#[doc(hidden)]
837	pub fn cancel_shutdown_deadline(&self) {
838		self.0.shutdown_deadline.cancel();
839	}
840
841	/// Deprecated no-op. Use `keep_awake` to hold the actor awake for the
842	/// duration of a future, or `wait_until` to keep work alive across the
843	/// sleep grace period. Retained only for NAPI bridge compatibility.
844	#[deprecated(note = "no-op: use `keep_awake` or `wait_until` instead")]
845	pub fn set_prevent_sleep(&self, _enabled: bool) {}
846
847	#[deprecated(note = "no-op: always returns false")]
848	pub fn prevent_sleep(&self) -> bool {
849		false
850	}
851
852	/// Runs `future` to completion after the current reply, without blocking
853	/// it. Work started from an invocation keeps that invocation's span open
854	/// until it settles, so its SQLite operations and logs stay attributed to
855	/// the request that started them.
856	#[cfg(not(feature = "wasm-runtime"))]
857	pub fn wait_until(&self, future: impl Future<Output = ()> + Send + 'static) {
858		let invocation = self
859			.1
860			.as_ref()
861			.and_then(crate::ActorInvocationTelemetry::hold_open);
862		self.spawn_work(ActorWorkKind::WaitUntil, async move {
863			future.await;
864			drop(invocation);
865		});
866	}
867
868	#[cfg(not(feature = "wasm-runtime"))]
869	pub fn register_task(&self, future: impl Future<Output = ()> + Send + 'static) {
870		self.spawn_work(ActorWorkKind::RegisteredTask, future);
871	}
872
873	#[cfg(feature = "wasm-runtime")]
874	pub fn wait_until(&self, future: impl Future<Output = ()> + 'static) {
875		let invocation = self
876			.1
877			.as_ref()
878			.and_then(crate::ActorInvocationTelemetry::hold_open);
879		self.spawn_work(ActorWorkKind::WaitUntil, async move {
880			future.await;
881			drop(invocation);
882		});
883	}
884
885	#[cfg(feature = "wasm-runtime")]
886	pub fn register_task(&self, future: impl Future<Output = ()> + 'static) {
887		self.spawn_work(ActorWorkKind::RegisteredTask, future);
888	}
889
890	pub async fn keep_awake<F>(&self, future: F) -> F::Output
891	where
892		F: Future,
893	{
894		self.track_work(ActorWorkKind::KeepAwake, future).await
895	}
896
897	pub fn keep_awake_region(&self) -> KeepAwakeRegion {
898		KeepAwakeRegion {
899			region: Some(self.begin_work_region(ActorWorkKind::KeepAwake)),
900		}
901	}
902
903	pub async fn internal_keep_awake<F>(&self, future: F) -> F::Output
904	where
905		F: Future,
906	{
907		self.track_work(ActorWorkKind::InternalKeepAwake, future)
908			.await
909	}
910
911	pub fn keep_awake_count(&self) -> usize {
912		self.sleep_keep_awake_count()
913	}
914
915	pub fn internal_keep_awake_count(&self) -> usize {
916		self.sleep_internal_keep_awake_count()
917	}
918
919	pub async fn track_work<F>(&self, kind: ActorWorkKind, future: F) -> F::Output
920	where
921		F: Future,
922	{
923		let _region = self.begin_work_region(kind);
924		future.await
925	}
926
927	#[cfg(not(feature = "wasm-runtime"))]
928	pub fn spawn_work<F>(&self, kind: ActorWorkKind, future: F)
929	where
930		F: Future<Output = ()> + Send + 'static,
931	{
932		self.spawn_work_inner(kind, future);
933	}
934
935	#[cfg(feature = "wasm-runtime")]
936	pub fn spawn_work<F>(&self, kind: ActorWorkKind, future: F)
937	where
938		F: Future<Output = ()> + 'static,
939	{
940		self.spawn_work_inner(kind, future);
941	}
942
943	pub fn begin_work_region(&self, kind: ActorWorkKind) -> ActorWorkRegion {
944		ActorWorkRegion {
945			guard: Some(ActorWorkGuard::new(self.clone(), kind)),
946		}
947	}
948
949	pub fn actor_id(&self) -> &str {
950		&self.0.actor_id
951	}
952
953	pub fn name(&self) -> &str {
954		&self.0.name
955	}
956
957	pub fn key(&self) -> &ActorKey {
958		&self.0.key
959	}
960
961	pub(crate) fn actor_specifier(&self) -> Option<ActorSpecifier> {
962		Some(
963			ActorSpecifier::new(self.actor_id().to_owned(), self.sleep_generation()? as u64)
964				.with_key(format_actor_key(self.key())),
965		)
966	}
967
968	pub(crate) fn attach_actor_to_error(&self, error: anyhow::Error) -> anyhow::Error {
969		match self.actor_specifier() {
970			Some(actor) => error.context(actor),
971			None => error,
972		}
973	}
974
975	pub fn region(&self) -> &str {
976		&self.0.region
977	}
978
979	pub fn has_state(&self) -> bool {
980		self.0.connection_config.read().has_state
981	}
982
983	#[doc(hidden)]
984	pub fn record_startup_create_state(&self, duration: Duration) {
985		self.0.metrics.observe_create_state(duration);
986	}
987
988	#[doc(hidden)]
989	pub fn record_startup_create_vars(&self, duration: Duration) {
990		self.0.metrics.observe_create_vars(duration);
991	}
992
993	pub fn broadcast(&self, name: &str, args: &[u8]) {
994		for connection in self.conns() {
995			if connection.is_subscribed(name) {
996				connection.send(name, args);
997			}
998		}
999	}
1000
1001	/// Returns a lock-backed iterator over live connections.
1002	///
1003	/// Do not hold the returned iterator across `.await`. It keeps a read lock
1004	/// on the connection map until dropped, which blocks connection writers.
1005	#[must_use]
1006	pub fn conns(&self) -> ConnHandles<'_> {
1007		self.iter_connections()
1008	}
1009
1010	pub fn client_endpoint(&self) -> Option<&str> {
1011		self.0.client_endpoint.get().map(String::as_str)
1012	}
1013
1014	pub fn client_token(&self) -> Option<&str> {
1015		self.0.client_token.get().map(String::as_str)
1016	}
1017
1018	pub fn client_namespace(&self) -> Option<&str> {
1019		self.0.client_namespace.get().map(String::as_str)
1020	}
1021
1022	pub fn client_pool_name(&self) -> Option<&str> {
1023		self.0.client_pool_name.get().map(String::as_str)
1024	}
1025
1026	pub fn ack_hibernatable_websocket_message(
1027		&self,
1028		gateway_id: &[u8],
1029		request_id: &[u8],
1030		server_message_index: u16,
1031	) -> Result<()> {
1032		let gateway_id = hibernatable_id_from_slice("gateway_id", gateway_id)?;
1033		let request_id = hibernatable_id_from_slice("request_id", request_id)?;
1034		let envoy_handle = self.sleep_envoy_handle().ok_or_else(|| {
1035			ActorRuntime::NotConfigured {
1036				component: "hibernatable websocket ack".to_owned(),
1037			}
1038			.build()
1039		})?;
1040		envoy_handle.send_hibernatable_ws_message_ack(gateway_id, request_id, server_message_index);
1041		Ok(())
1042	}
1043
1044	pub(crate) fn load_persisted_actor(&self, persisted: PersistedActor) {
1045		self.load_persisted(persisted);
1046	}
1047
1048	pub(crate) fn persisted_actor(&self) -> PersistedActor {
1049		self.persisted()
1050	}
1051
1052	/// Dispatches any scheduled actions whose deadline has already passed.
1053	///
1054	/// Foreign-runtime adapters should call this after startup callbacks complete
1055	/// so overdue scheduled work enters the normal actor event loop.
1056	pub async fn drain_overdue_scheduled_events(&self) -> Result<()> {
1057		for dispatch in self.take_due_schedule_dispatches().await? {
1058			self.dispatch_scheduled_action(dispatch).await;
1059		}
1060		Ok(())
1061	}
1062
1063	pub(crate) fn metrics(&self) -> &ActorMetrics {
1064		&self.0.metrics
1065	}
1066
1067	/// Identity fields shared by every invocation on this actor. Built once so a
1068	/// span does not re-allocate them per action.
1069	pub(crate) fn telemetry_identity(&self) -> Arc<ActorTelemetryIdentity> {
1070		self.0.telemetry_identity.clone()
1071	}
1072
1073	pub(crate) fn record_user_task_started(&self, kind: UserTaskKind) {
1074		self.0.metrics.begin_user_task(kind);
1075	}
1076
1077	pub(crate) fn record_user_task_finished(&self, kind: UserTaskKind, duration: Duration) {
1078		self.0.metrics.end_user_task(kind, duration);
1079	}
1080
1081	pub(crate) fn record_shutdown_wait(
1082		&self,
1083		reason: crate::actor::task_types::ShutdownKind,
1084		duration: Duration,
1085	) {
1086		self.0.metrics.observe_shutdown_wait(reason, duration);
1087	}
1088
1089	pub(crate) fn record_shutdown_timeout(&self, reason: crate::actor::task_types::ShutdownKind) {
1090		self.0.metrics.inc_shutdown_timeout(reason);
1091	}
1092
1093	pub(crate) fn record_direct_subsystem_shutdown_warning(
1094		&self,
1095		subsystem: &str,
1096		operation: &str,
1097	) {
1098		self.0
1099			.metrics
1100			.inc_direct_subsystem_shutdown_warning(subsystem, operation);
1101	}
1102
1103	pub(crate) fn warn_work_sent_to_stopping_instance(&self, operation: &'static str) {
1104		if let Some(suppression) = self.0.diagnostics.record("work_sent_to_stopping_instance") {
1105			tracing::warn!(
1106				actor_id = %suppression.actor_id,
1107				operation,
1108				per_actor_suppressed = suppression.per_actor_suppressed,
1109				global_suppressed = suppression.global_suppressed,
1110				"work sent to stopping actor instance"
1111			);
1112		}
1113	}
1114
1115	pub(crate) fn warn_self_call_risk(&self, operation: &'static str) {
1116		if let Some(suppression) = self.0.diagnostics.record("self_call_risk") {
1117			tracing::warn!(
1118				actor_id = %suppression.actor_id,
1119				operation,
1120				per_actor_suppressed = suppression.per_actor_suppressed,
1121				global_suppressed = suppression.global_suppressed,
1122				"actor dispatch may be parked behind the current instance"
1123			);
1124		}
1125	}
1126
1127	#[cfg(test)]
1128	pub(crate) fn add_conn(&self, conn: ConnHandle) {
1129		self.insert_existing(conn);
1130		self.record_connections_updated();
1131		self.reset_sleep_timer();
1132	}
1133
1134	pub(crate) fn remove_conn(&self, conn_id: &str) -> Option<ConnHandle> {
1135		let removed = self.remove_existing(conn_id);
1136		if removed.is_some() {
1137			self.record_connections_updated();
1138			self.reset_sleep_timer();
1139		}
1140		removed
1141	}
1142
1143	pub(crate) fn configure_connection_runtime(&self, config: ActorConfig) {
1144		self.configure_sleep_state(config.clone());
1145		self.configure_connection_storage(config);
1146	}
1147
1148	pub(crate) fn configure_actor_events(&self, sender: Option<mpsc::UnboundedSender<ActorEvent>>) {
1149		*self.0.actor_events.write() = sender;
1150	}
1151
1152	pub(crate) fn try_send_actor_event(
1153		&self,
1154		event: ActorEvent,
1155		operation: &'static str,
1156	) -> Result<()> {
1157		let sender = self.0.actor_events.read().clone().ok_or_else(|| {
1158			ActorRuntime::NotConfigured {
1159				component: "actor event inbox".to_owned(),
1160			}
1161			.build()
1162		})?;
1163		tracing::debug!(
1164			actor_id = %self.actor_id(),
1165			operation,
1166			event = event.kind(),
1167			"actor event enqueued"
1168		);
1169		sender.send(event).map_err(|_| {
1170			ActorRuntime::NotConfigured {
1171				component: "actor event inbox".to_owned(),
1172			}
1173			.build()
1174		})
1175	}
1176
1177	#[doc(hidden)]
1178	pub fn configure_envoy(&self, envoy_handle: EnvoyHandle, generation: Option<u32>) {
1179		let _ = self
1180			.0
1181			.client_endpoint
1182			.set(envoy_handle.endpoint().to_owned());
1183		if let Some(token) = envoy_handle.token() {
1184			let _ = self.0.client_token.set(token.to_owned());
1185		}
1186		let _ = self
1187			.0
1188			.client_namespace
1189			.set(envoy_handle.namespace().to_owned());
1190		let _ = self
1191			.0
1192			.client_pool_name
1193			.set(envoy_handle.pool_name().to_owned());
1194		self.configure_sleep_envoy(envoy_handle.clone(), generation);
1195		self.configure_schedule_envoy(envoy_handle, generation);
1196	}
1197
1198	pub(crate) async fn connect_conn<F>(
1199		&self,
1200		params: Vec<u8>,
1201		is_hibernatable: bool,
1202		hibernation: Option<HibernatableConnectionMetadata>,
1203		request: Option<Request>,
1204		create_state: F,
1205	) -> Result<ConnHandle>
1206	where
1207		F: Future<Output = Result<Vec<u8>>> + Send,
1208	{
1209		let conn = self
1210			.connect_with_state(params, is_hibernatable, hibernation, request, create_state)
1211			.await?;
1212		self.record_connections_updated();
1213		self.reset_sleep_timer();
1214		Ok(conn)
1215	}
1216
1217	pub(crate) async fn connect_conn_with_prepare<F, P>(
1218		&self,
1219		params: Vec<u8>,
1220		is_hibernatable: bool,
1221		hibernation: Option<HibernatableConnectionMetadata>,
1222		request: Option<Request>,
1223		create_state: F,
1224		prepare_connection: P,
1225	) -> Result<ConnHandle>
1226	where
1227		F: Future<Output = Result<Vec<u8>>> + Send,
1228		P: FnOnce(&ConnHandle) -> Result<()>,
1229	{
1230		let conn = self
1231			.connect_with_state_and_prepare(
1232				params,
1233				is_hibernatable,
1234				hibernation,
1235				request,
1236				create_state,
1237				prepare_connection,
1238			)
1239			.await?;
1240		self.record_connections_updated();
1241		self.reset_sleep_timer();
1242		Ok(conn)
1243	}
1244
1245	pub async fn connect_conn_with_request<F>(
1246		&self,
1247		params: Vec<u8>,
1248		request: Option<Request>,
1249		create_state: F,
1250	) -> Result<ConnHandle>
1251	where
1252		F: Future<Output = Result<Vec<u8>>> + Send,
1253	{
1254		self.connect_conn(params, false, None, request, create_state)
1255			.await
1256	}
1257
1258	pub(crate) fn reconnect_hibernatable_conn(
1259		&self,
1260		gateway_id: &[u8],
1261		request_id: &[u8],
1262	) -> Result<ConnHandle> {
1263		self.reconnect_hibernatable(gateway_id, request_id)
1264	}
1265
1266	pub async fn disconnect_conn(&self, id: ConnId) -> Result<()> {
1267		self.disconnect_transport_only(|conn| conn.id() == id).await
1268	}
1269
1270	pub async fn disconnect_conns<F>(&self, predicate: F) -> Result<()>
1271	where
1272		F: FnMut(&ConnHandle) -> bool,
1273	{
1274		self.disconnect_transport_only(predicate).await
1275	}
1276
1277	pub(crate) fn request_hibernation_transport_save(&self, conn_id: &str) {
1278		self.queue_hibernation_update(conn_id.to_owned());
1279		self.request_save(RequestSaveOpts::default());
1280	}
1281
1282	pub(crate) fn request_hibernation_transport_removal(&self, conn_id: impl Into<String>) {
1283		self.queue_hibernation_removal_inner(conn_id.into());
1284		self.request_save(RequestSaveOpts::default());
1285	}
1286
1287	pub fn queue_hibernation_removal(&self, conn_id: impl Into<String>) {
1288		self.request_hibernation_transport_removal(conn_id);
1289	}
1290
1291	pub fn has_pending_hibernation_changes(&self) -> bool {
1292		self.has_pending_hibernation_changes_inner()
1293	}
1294
1295	pub fn take_pending_hibernation_changes(&self) -> Vec<ConnId> {
1296		self.pending_hibernation_removals()
1297	}
1298
1299	pub fn dirty_hibernatable_conns(&self) -> Vec<ConnHandle> {
1300		self.dirty_hibernatable_conns_inner()
1301	}
1302
1303	pub(crate) fn hibernated_connection_is_live(
1304		&self,
1305		gateway_id: &[u8],
1306		request_id: &[u8],
1307	) -> Result<bool> {
1308		let gateway_id = hibernatable_id_from_slice("gateway_id", gateway_id)?;
1309		let request_id = hibernatable_id_from_slice("request_id", request_id)?;
1310
1311		if let Some(override_pairs) = self
1312			.0
1313			.hibernated_connection_liveness_override
1314			.read()
1315			.as_ref()
1316		{
1317			return Ok(override_pairs.contains(&(gateway_id.to_vec(), request_id.to_vec())));
1318		}
1319
1320		let Some(envoy_handle) = self.sleep_envoy_handle() else {
1321			return Ok(false);
1322		};
1323		let is_live = envoy_handle.hibernatable_connection_is_live(
1324			self.actor_id(),
1325			self.sleep_generation(),
1326			gateway_id,
1327			request_id,
1328		);
1329		Ok(is_live)
1330	}
1331
1332	#[cfg(test)]
1333	pub(crate) fn set_hibernated_connection_liveness_override<I>(&self, pairs: I)
1334	where
1335		I: IntoIterator<Item = (Vec<u8>, Vec<u8>)>,
1336	{
1337		*self.0.hibernated_connection_liveness_override.write() = Some(pairs.into_iter().collect());
1338	}
1339
1340	pub(super) fn prepare_state_deltas(
1341		&self,
1342		deltas: Vec<StateDelta>,
1343	) -> Result<(Vec<StateDelta>, PendingHibernationChanges)> {
1344		fn finish_with_error(
1345			ctx: &ActorContext,
1346			pending: PendingHibernationChanges,
1347			error: anyhow::Error,
1348		) -> Result<(Vec<StateDelta>, PendingHibernationChanges)> {
1349			ctx.restore_pending_hibernation_changes(pending);
1350			Err(error)
1351		}
1352
1353		let mut next_deltas = Vec::new();
1354		let mut explicit_updates = std::collections::BTreeMap::new();
1355		let mut explicit_removals = std::collections::BTreeSet::new();
1356
1357		for delta in deltas {
1358			match delta {
1359				StateDelta::ConnHibernation { conn, bytes } => {
1360					explicit_updates.insert(conn, bytes);
1361				}
1362				StateDelta::ConnHibernationRemoved(conn) => {
1363					explicit_removals.insert(conn);
1364				}
1365				other => next_deltas.push(other),
1366			}
1367		}
1368
1369		let mut pending = self.take_pending_hibernation_changes_inner();
1370		let mut removal_ids = pending.removed.clone();
1371		removal_ids.extend(explicit_removals.iter().cloned());
1372
1373		let explicit_update_ids: std::collections::BTreeSet<_> =
1374			explicit_updates.keys().cloned().collect();
1375		pending.updated.extend(explicit_update_ids.iter().cloned());
1376		pending.removed.extend(explicit_removals.iter().cloned());
1377
1378		for (conn, bytes) in explicit_updates {
1379			if removal_ids.contains(&conn) {
1380				continue;
1381			}
1382			let encoded = match self.encode_hibernation_delta(&conn, bytes) {
1383				Ok(encoded) => encoded,
1384				Err(error) => {
1385					return finish_with_error(self, pending, error);
1386				}
1387			};
1388			next_deltas.push(StateDelta::ConnHibernation {
1389				conn,
1390				bytes: encoded,
1391			});
1392		}
1393
1394		for conn in &pending.updated {
1395			if removal_ids.contains(conn)
1396				|| explicit_removals.contains(conn)
1397				|| explicit_update_ids.contains(conn)
1398			{
1399				continue;
1400			}
1401			let Some(handle) = self.connection(conn) else {
1402				continue;
1403			};
1404			if !handle.is_hibernatable() || handle.hibernation().is_none() {
1405				continue;
1406			}
1407			let encoded = match self.encode_hibernation_delta(conn, handle.state()) {
1408				Ok(encoded) => encoded,
1409				Err(error) => {
1410					return finish_with_error(self, pending, error);
1411				}
1412			};
1413			next_deltas.push(StateDelta::ConnHibernation {
1414				conn: conn.clone(),
1415				bytes: encoded,
1416			});
1417		}
1418
1419		for conn in removal_ids {
1420			next_deltas.push(StateDelta::ConnHibernationRemoved(conn));
1421		}
1422
1423		Ok((next_deltas, pending))
1424	}
1425
1426	pub(crate) async fn restore_hibernatable_connections(&self) -> Result<Vec<ConnHandle>> {
1427		let restored = self.restore_persisted().await?;
1428		if !restored.is_empty() {
1429			if let Some(envoy_handle) = self.sleep_envoy_handle() {
1430				let meta_entries: Vec<_> = restored
1431					.iter()
1432					.filter_map(|conn| {
1433						let hibernation = conn.hibernation()?;
1434						Some(HibernatingWebSocketMetadata {
1435							gateway_id: hibernation.gateway_id,
1436							request_id: hibernation.request_id,
1437							envoy_message_index: hibernation.client_message_index,
1438							rivet_message_index: hibernation.server_message_index,
1439							path: hibernation.request_path,
1440							headers: hibernation.request_headers.into_iter().collect(),
1441						})
1442					})
1443					.collect();
1444				envoy_handle.restore_hibernating_requests(self.actor_id().to_owned(), meta_entries);
1445			}
1446			self.record_connections_updated();
1447			self.reset_sleep_timer();
1448		}
1449		Ok(restored)
1450	}
1451
1452	pub(crate) fn configure_inspector(&self, inspector: Option<Inspector>) {
1453		*self.0.inspector.write() = inspector;
1454	}
1455
1456	pub(crate) fn inspector(&self) -> Option<Inspector> {
1457		self.0.inspector.read().clone()
1458	}
1459
1460	pub fn inspector_snapshot(&self) -> InspectorSnapshot {
1461		self.inspector()
1462			.map(|inspector| inspector.snapshot())
1463			.unwrap_or_default()
1464	}
1465
1466	pub(crate) fn configure_inspector_runtime(
1467		&self,
1468		attach_count: Arc<AtomicU32>,
1469		overlay_tx: broadcast::Sender<Arc<Vec<u8>>>,
1470	) {
1471		*self.0.inspector_attach_count.write() = Some(attach_count);
1472		*self.0.inspector_overlay_tx.write() = Some(overlay_tx);
1473	}
1474
1475	pub(crate) fn inspector_attach(&self) -> Option<InspectorAttachGuard> {
1476		InspectorAttachGuard::new(self.clone())
1477	}
1478
1479	#[cfg(test)]
1480	pub(crate) fn inspector_attach_count(&self) -> u32 {
1481		self.inspector_attach_count_arc()
1482			.map(|attach_count| attach_count.load(Ordering::SeqCst))
1483			.unwrap_or(0)
1484	}
1485
1486	pub(crate) fn subscribe_inspector(&self) -> Option<broadcast::Receiver<Arc<Vec<u8>>>> {
1487		self.0
1488			.inspector_overlay_tx
1489			.read()
1490			.clone()
1491			.map(|overlay_tx| overlay_tx.subscribe())
1492	}
1493
1494	pub(crate) fn downgrade(&self) -> Weak<ActorContextInner> {
1495		Arc::downgrade(&self.0)
1496	}
1497
1498	pub(crate) fn from_weak(weak: &Weak<ActorContextInner>) -> Option<Self> {
1499		weak.upgrade().map(|inner| Self(inner, None))
1500	}
1501
1502	#[doc(hidden)]
1503	pub fn set_started(&self, started: bool) {
1504		self.set_lifecycle_started(started);
1505		self.reset_sleep_timer();
1506	}
1507
1508	#[doc(hidden)]
1509	pub fn started(&self) -> bool {
1510		self.lifecycle_started()
1511	}
1512
1513	pub(crate) fn destroy_requested(&self) -> bool {
1514		self.0.destroy_requested.load(Ordering::SeqCst)
1515	}
1516
1517	pub fn is_destroy_requested(&self) -> bool {
1518		self.destroy_requested()
1519	}
1520
1521	pub(crate) async fn wait_for_destroy_completion(&self) {
1522		if self.0.destroy_completed.load(Ordering::SeqCst) {
1523			return;
1524		}
1525
1526		loop {
1527			let notified = self.0.destroy_completion_notify.notified();
1528			if self.0.destroy_completed.load(Ordering::SeqCst) {
1529				return;
1530			}
1531			notified.await;
1532			if self.0.destroy_completed.load(Ordering::SeqCst) {
1533				return;
1534			}
1535		}
1536	}
1537
1538	pub async fn wait_for_destroy_completion_public(&self) {
1539		self.wait_for_destroy_completion().await;
1540	}
1541
1542	pub(crate) fn mark_destroy_completed(&self) {
1543		self.0.destroy_completed.store(true, Ordering::SeqCst);
1544		self.0.destroy_completion_notify.notify_waiters();
1545	}
1546
1547	pub(crate) async fn can_sleep(&self) -> CanSleep {
1548		self.can_arm_sleep_timer().await
1549	}
1550
1551	pub(crate) fn pending_disconnect_count(&self) -> usize {
1552		self.0.sleep.work.disconnect_callback.load()
1553	}
1554
1555	pub async fn with_disconnect_callback<F, Fut, T>(&self, run: F) -> T
1556	where
1557		F: FnOnce() -> Fut,
1558		Fut: Future<Output = T>,
1559	{
1560		self.track_work(ActorWorkKind::DisconnectCallback, run())
1561			.await
1562	}
1563
1564	pub(crate) fn configure_lifecycle_events(
1565		&self,
1566		sender: Option<mpsc::UnboundedSender<LifecycleEvent>>,
1567	) {
1568		*self.0.lifecycle_events.write() = sender;
1569	}
1570
1571	pub(crate) fn notify_inspector_serialize_requested(&self) {
1572		self.try_send_lifecycle_event(
1573			LifecycleEvent::InspectorSerializeRequested,
1574			"inspector_serialize_requested",
1575		);
1576	}
1577
1578	pub(crate) fn notify_activity_dirty(&self) -> bool {
1579		if self.0.lifecycle_events.read().is_none() {
1580			return false;
1581		}
1582		if self.0.activity.mark_dirty() {
1583			self.sleep_activity_notify().notify_one();
1584		}
1585		true
1586	}
1587
1588	pub(crate) fn acknowledge_activity_dirty(&self) -> bool {
1589		self.0.activity.take_dirty()
1590	}
1591
1592	/// Notify the ActorTask that a `can_sleep` input has changed so the sleep
1593	/// deadline gets re-evaluated. Falls back to the detached compat timer
1594	/// when the actor has no wired `ActorTask` (test-only contexts).
1595	pub(crate) fn reset_sleep_timer(&self) {
1596		if self.notify_activity_dirty() {
1597			return;
1598		}
1599
1600		#[cfg(feature = "wasm-runtime")]
1601		return;
1602
1603		#[cfg(not(feature = "wasm-runtime"))]
1604		self.reset_sleep_timer_state();
1605	}
1606
1607	fn notify_inspector_attachments_changed(&self) {
1608		self.try_send_lifecycle_event(
1609			LifecycleEvent::InspectorAttachmentsChanged,
1610			"inspector_attachments_changed",
1611		);
1612	}
1613
1614	pub(crate) fn configure_sleep(&self, config: ActorConfig) {
1615		self.configure_sleep_state(config.clone());
1616		self.configure_queue(config);
1617		self.reset_sleep_timer();
1618	}
1619
1620	pub(crate) fn sleep_config(&self) -> ActorConfig {
1621		self.sleep_state_config()
1622	}
1623
1624	pub(crate) fn sleep_requested(&self) -> bool {
1625		self.0.sleep_requested.load(Ordering::SeqCst)
1626	}
1627
1628	pub(crate) fn clear_sleep_requested(&self) {
1629		self.0.sleep_requested.store(false, Ordering::SeqCst);
1630	}
1631
1632	pub(crate) async fn internal_keep_awake_task(
1633		&self,
1634		future: BoxFuture<'static, Result<()>>,
1635	) -> Result<()> {
1636		self.internal_keep_awake(future).await
1637	}
1638
1639	pub fn websocket_callback_region(&self) -> WebSocketCallbackRegion {
1640		WebSocketCallbackRegion {
1641			region: Some(self.begin_work_region(ActorWorkKind::WebSocketCallback)),
1642		}
1643	}
1644
1645	pub(crate) async fn with_websocket_callback<F, Fut, T>(&self, run: F) -> T
1646	where
1647		F: FnOnce() -> Fut,
1648		Fut: Future<Output = T>,
1649	{
1650		let _guard = self.websocket_callback_region();
1651		run().await
1652	}
1653
1654	fn idle_work_region(&self, kind: ActorWorkKind) -> Option<RegionGuard> {
1655		if !kind.policy().blocks_idle_sleep {
1656			return None;
1657		}
1658		let region = match kind {
1659			ActorWorkKind::Action => self.internal_keep_awake_region(),
1660			ActorWorkKind::KeepAwake => self.keep_awake_region_state(),
1661			ActorWorkKind::InternalKeepAwake => self.internal_keep_awake_region(),
1662			ActorWorkKind::WaitUntil => return None,
1663			ActorWorkKind::RegisteredTask => return None,
1664			ActorWorkKind::WebSocketCallback => self.websocket_callback_region_state(),
1665			ActorWorkKind::DisconnectCallback => self.disconnect_callback_region_state(),
1666		};
1667		Some(region.with_log_fields(kind.label(), Some(self.actor_id().to_owned())))
1668	}
1669
1670	fn shutdown_work_region(&self) -> CountGuard {
1671		let counter = self.0.sleep.work.shutdown_counter.clone();
1672		counter.increment();
1673		CountGuard::from_incremented(counter)
1674	}
1675
1676	fn configure_sleep_hooks(&self) {
1677		let keep_awake_ctx = self.downgrade();
1678		self.0
1679			.sleep
1680			.work
1681			.keep_awake
1682			.register_change_callback(Arc::new(move || {
1683				if let Some(ctx) = ActorContext::from_weak(&keep_awake_ctx) {
1684					ctx.0
1685						.metrics
1686						.set_keep_awake_active(ctx.sleep_keep_awake_count());
1687				}
1688			}));
1689
1690		let internal_keep_awake_metric_ctx = self.downgrade();
1691		self.0
1692			.sleep
1693			.work
1694			.internal_keep_awake
1695			.register_change_callback(Arc::new(move || {
1696				if let Some(ctx) = ActorContext::from_weak(&internal_keep_awake_metric_ctx) {
1697					ctx.0
1698						.metrics
1699						.set_internal_keep_awake_active(ctx.sleep_internal_keep_awake_count());
1700				}
1701			}));
1702
1703		let shutdown_tasks_ctx = self.downgrade();
1704		self.0
1705			.sleep
1706			.work
1707			.shutdown_counter
1708			.register_change_callback(Arc::new(move || {
1709				if let Some(ctx) = ActorContext::from_weak(&shutdown_tasks_ctx) {
1710					ctx.0
1711						.metrics
1712						.set_shutdown_tasks_active(ctx.shutdown_task_count());
1713				}
1714			}));
1715
1716		let internal_keep_awake_ctx = self.downgrade();
1717		self.set_internal_keep_awake(Some(Arc::new(move |future| {
1718			let ctx = ActorContext::from_weak(&internal_keep_awake_ctx);
1719			Box::pin(async move {
1720				let Some(ctx) = ctx else {
1721					return Err(ActorRuntime::NotConfigured {
1722						component: "actor context".to_owned(),
1723					}
1724					.build());
1725				};
1726				ctx.internal_keep_awake_task(future).await
1727			})
1728		})));
1729
1730		let queue_ctx = self.downgrade();
1731		self.set_wait_activity_callback(Some(Arc::new(move || {
1732			if let Some(ctx) = ActorContext::from_weak(&queue_ctx) {
1733				ctx.reset_sleep_timer();
1734			}
1735		})));
1736
1737		let queue_ctx = self.downgrade();
1738		self.set_inspector_update_callback(Some(Arc::new(move |queue_size| {
1739			if let Some(ctx) = ActorContext::from_weak(&queue_ctx) {
1740				ctx.record_queue_updated(queue_size);
1741			}
1742		})));
1743	}
1744
1745	pub(crate) fn record_state_updated(&self) {
1746		if let Some(inspector) = self.inspector() {
1747			inspector.record_state_updated();
1748		}
1749	}
1750
1751	pub(crate) fn record_connections_updated(&self) {
1752		let Some(inspector) = self.inspector() else {
1753			return;
1754		};
1755		let active_connections = self.active_connection_count();
1756		inspector.record_connections_updated(active_connections);
1757	}
1758
1759	fn record_queue_updated(&self, queue_size: u32) {
1760		if let Some(inspector) = self.inspector() {
1761			inspector.record_queue_updated(queue_size);
1762		}
1763	}
1764
1765	pub(crate) fn record_schedules_updated(&self) {
1766		if let Some(inspector) = self.inspector() {
1767			inspector.record_schedules_updated();
1768		}
1769	}
1770
1771	pub(crate) async fn save_state_with_revision(
1772		&self,
1773		deltas: Vec<StateDelta>,
1774		save_request_revision: u64,
1775	) -> Result<()> {
1776		let (deltas, pending_hibernation_changes) = match self.prepare_state_deltas(deltas) {
1777			Ok(prepared) => prepared,
1778			Err(error) => return Err(error),
1779		};
1780		if let Err(error) = self.apply_state_deltas(deltas, save_request_revision).await {
1781			self.restore_pending_hibernation_changes(pending_hibernation_changes);
1782			return Err(error);
1783		}
1784		self.record_state_updated();
1785		Ok(())
1786	}
1787
1788	pub(crate) fn state_transaction_epoch(&self) -> u64 {
1789		self.0.state_transaction_epoch.load(Ordering::SeqCst)
1790	}
1791
1792	pub(crate) async fn save_state_with_revision_at_transaction_epoch(
1793		&self,
1794		deltas: Vec<StateDelta>,
1795		save_request_revision: u64,
1796		state_transaction_epoch: u64,
1797	) -> Result<bool> {
1798		let (deltas, pending_hibernation_changes) = match self.prepare_state_deltas(deltas) {
1799			Ok(prepared) => prepared,
1800			Err(error) => return Err(error),
1801		};
1802		match self
1803			.apply_state_deltas_inner(
1804				deltas,
1805				None,
1806				save_request_revision,
1807				Some(state_transaction_epoch),
1808			)
1809			.await
1810		{
1811			Ok(true) => {
1812				self.record_state_updated();
1813				Ok(true)
1814			}
1815			Ok(false) => {
1816				self.restore_pending_hibernation_changes(pending_hibernation_changes);
1817				Ok(false)
1818			}
1819			Err(error) => {
1820				self.restore_pending_hibernation_changes(pending_hibernation_changes);
1821				Err(error)
1822			}
1823		}
1824	}
1825
1826	#[cfg(test)]
1827	pub(crate) async fn save_state_and_workflow_batch_with_revision(
1828		&self,
1829		deltas: Vec<StateDelta>,
1830		workflow_writes: Vec<WorkflowKvWrite>,
1831		save_request_revision: u64,
1832	) -> Result<()> {
1833		self.save_state_and_workflow_batch_with_revision_inner(
1834			deltas,
1835			workflow_writes,
1836			save_request_revision,
1837			None,
1838		)
1839		.await
1840		.map(|_| ())
1841	}
1842
1843	pub(crate) async fn save_state_and_workflow_batch_at_transaction_epoch(
1844		&self,
1845		deltas: Vec<StateDelta>,
1846		workflow_writes: Vec<WorkflowKvWrite>,
1847		save_request_revision: u64,
1848		state_transaction_epoch: u64,
1849	) -> Result<bool> {
1850		self.save_state_and_workflow_batch_with_revision_inner(
1851			deltas,
1852			workflow_writes,
1853			save_request_revision,
1854			Some(state_transaction_epoch),
1855		)
1856		.await
1857	}
1858
1859	async fn save_state_and_workflow_batch_with_revision_inner(
1860		&self,
1861		deltas: Vec<StateDelta>,
1862		workflow_writes: Vec<WorkflowKvWrite>,
1863		save_request_revision: u64,
1864		expected_state_transaction_epoch: Option<u64>,
1865	) -> Result<bool> {
1866		let (deltas, pending_hibernation_changes) = match self.prepare_state_deltas(deltas) {
1867			Ok(prepared) => prepared,
1868			Err(error) => return Err(error),
1869		};
1870		match self
1871			.apply_state_deltas_inner(
1872				deltas,
1873				Some(workflow_writes),
1874				save_request_revision,
1875				expected_state_transaction_epoch,
1876			)
1877			.await
1878		{
1879			Ok(true) => {
1880				self.record_state_updated();
1881				Ok(true)
1882			}
1883			Ok(false) => {
1884				self.restore_pending_hibernation_changes(pending_hibernation_changes);
1885				Ok(false)
1886			}
1887			Err(error) => {
1888				self.restore_pending_hibernation_changes(pending_hibernation_changes);
1889				Err(error)
1890			}
1891		}
1892	}
1893
1894	async fn dispatch_scheduled_action(
1895		&self,
1896		dispatch: crate::actor::schedule::DueScheduleDispatch,
1897	) {
1898		let ctx = self.clone();
1899		let event_id = dispatch.event_id;
1900		let action = dispatch.action;
1901		let args = dispatch.args;
1902		let scheduled_fire = dispatch.fire;
1903		let recurring_name = scheduled_fire.name.clone();
1904		let history_id = dispatch.history_id;
1905		let internal_keep_awake_region = self.begin_work_region(ActorWorkKind::InternalKeepAwake);
1906
1907		self.track_shutdown_task(async move {
1908			let _internal_keep_awake_region = internal_keep_awake_region;
1909			ctx.record_user_task_started(UserTaskKind::ScheduledAction);
1910			let user_task_started_at = Instant::now();
1911			let action_name = action.clone();
1912			let invocation = crate::telemetry::ActorInvocation::start_scheduled(
1913				&ctx,
1914				&action_name,
1915				dispatch.trace_context,
1916			);
1917			let invocation_telemetry = invocation.telemetry();
1918			let (reply_tx, reply_rx) = oneshot::channel();
1919
1920			let mut dispatch_error = None;
1921			match ctx.try_send_actor_event(
1922				ActorEvent::Action {
1923					name: action.clone(),
1924					args,
1925					conn: None,
1926					scheduled_fire: Some(scheduled_fire),
1927					invocation_telemetry: Some(invocation_telemetry),
1928					reply: Reply::from(reply_tx),
1929				},
1930				"scheduled_action",
1931			) {
1932				Ok(()) => match reply_rx.await {
1933					Ok(Ok(_)) => {}
1934					Ok(Err(error)) => {
1935						dispatch_error = Some(error);
1936						tracing::error!(
1937							error = ?dispatch_error.as_ref().expect("just assigned"),
1938							event_id,
1939							action_name,
1940							"scheduled event execution failed"
1941						);
1942					}
1943					Err(_) => {
1944						dispatch_error = Some(ActorLifecycleError::DroppedReply.build());
1945						tracing::error!(
1946							error = ?dispatch_error.as_ref().expect("just assigned"),
1947							event_id,
1948							action_name,
1949							"scheduled event reply dropped"
1950						);
1951					}
1952				},
1953				Err(error) => {
1954					dispatch_error = Some(error);
1955					tracing::error!(
1956						error = ?dispatch_error.as_ref().expect("just assigned"),
1957						event_id,
1958						action_name,
1959						"failed to enqueue scheduled event"
1960					);
1961				}
1962			}
1963			invocation.finish(dispatch_error.as_ref());
1964
1965			ctx.finish_schedule_dispatch(&event_id, history_id, dispatch_error.as_ref())
1966				.await;
1967			if let (Some(name), Some(error)) = (recurring_name, dispatch_error.as_ref()) {
1968				let structured = rivet_error::RivetError::extract(error);
1969				if structured.group() == "actor" && structured.code() == "action_not_found" {
1970					if let Err(delete_error) = ctx.cron_delete_if_action(&name, &action_name).await
1971					{
1972						tracing::error!(
1973							?delete_error,
1974							%name,
1975							"failed to delete recurring schedule for missing action"
1976						);
1977					}
1978				}
1979			}
1980
1981			ctx.record_user_task_finished(
1982				UserTaskKind::ScheduledAction,
1983				user_task_started_at.elapsed(),
1984			);
1985		});
1986	}
1987
1988	fn inspector_attach_count_arc(&self) -> Option<Arc<AtomicU32>> {
1989		self.0.inspector_attach_count.read().clone()
1990	}
1991
1992	fn try_send_lifecycle_event(&self, event: LifecycleEvent, operation: &'static str) {
1993		let Some(sender) = self.0.lifecycle_events.read().clone() else {
1994			return;
1995		};
1996
1997		if sender.send(event).is_err() {
1998			tracing::warn!(operation, "failed to enqueue actor lifecycle event");
1999		}
2000	}
2001}
2002
2003/// Cap on the stop error message forwarded to the engine. The message is
2004/// persisted in engine workflow state and rendered in the dashboard, so
2005/// unbounded anyhow chains must be truncated at this boundary.
2006const MAX_STOP_ERROR_MESSAGE_LEN: usize = 2048;
2007
2008fn truncate_stop_error_message(mut message: String) -> String {
2009	if message.len() <= MAX_STOP_ERROR_MESSAGE_LEN {
2010		return message;
2011	}
2012	let mut end = MAX_STOP_ERROR_MESSAGE_LEN;
2013	while !message.is_char_boundary(end) {
2014		end -= 1;
2015	}
2016	message.truncate(end);
2017	message.push_str("... (truncated)");
2018	message
2019}
2020
2021struct ActorWorkGuard {
2022	ctx: ActorContext,
2023	kind: ActorWorkKind,
2024	started_at: Option<Instant>,
2025	idle_region: Option<RegionGuard>,
2026	shutdown_region: Option<CountGuard>,
2027}
2028
2029#[must_use]
2030pub struct ActorWorkRegion {
2031	guard: Option<ActorWorkGuard>,
2032}
2033
2034impl ActorWorkGuard {
2035	fn new(ctx: ActorContext, kind: ActorWorkKind) -> Self {
2036		let policy = kind.policy();
2037		let idle_region = ctx.idle_work_region(kind);
2038		let shutdown_region = if policy.drains_shutdown_grace {
2039			Some(ctx.shutdown_work_region())
2040		} else {
2041			None
2042		};
2043		let started_at = if let Some(user_task_kind) = policy.user_task_kind {
2044			ctx.record_user_task_started(user_task_kind);
2045			Some(Instant::now())
2046		} else {
2047			None
2048		};
2049		ctx.reset_sleep_timer();
2050		Self {
2051			ctx,
2052			kind,
2053			started_at,
2054			idle_region,
2055			shutdown_region,
2056		}
2057	}
2058}
2059
2060impl Drop for ActorWorkGuard {
2061	fn drop(&mut self) {
2062		if let Some(started_at) = self.started_at.take()
2063			&& let Some(user_task_kind) = self.kind.policy().user_task_kind
2064		{
2065			self.ctx
2066				.record_user_task_finished(user_task_kind, started_at.elapsed());
2067		}
2068		self.idle_region.take();
2069		self.shutdown_region.take();
2070		self.ctx.reset_sleep_timer();
2071	}
2072}
2073
2074impl Drop for ActorWorkRegion {
2075	fn drop(&mut self) {
2076		self.guard.take();
2077	}
2078}
2079
2080#[must_use]
2081#[derive(Debug)]
2082pub(crate) struct InspectorAttachGuard {
2083	ctx: ActorContext,
2084}
2085
2086impl InspectorAttachGuard {
2087	fn new(ctx: ActorContext) -> Option<Self> {
2088		let attach_count = ctx.inspector_attach_count_arc()?;
2089		let previous = attach_count.fetch_add(1, Ordering::SeqCst);
2090		let current = previous.saturating_add(1);
2091		tracing::debug!(
2092			actor_id = %ctx.actor_id(),
2093			previous_count = previous,
2094			current_count = current,
2095			"inspector attached"
2096		);
2097		if previous == 0 {
2098			ctx.notify_inspector_attachments_changed();
2099		}
2100		Some(Self { ctx })
2101	}
2102}
2103
2104impl Drop for InspectorAttachGuard {
2105	fn drop(&mut self) {
2106		let Some(attach_count) = self.ctx.inspector_attach_count_arc() else {
2107			return;
2108		};
2109		let Ok(previous) =
2110			attach_count.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |current| {
2111				current.checked_sub(1)
2112			})
2113		else {
2114			return;
2115		};
2116		let current = previous.saturating_sub(1);
2117		tracing::debug!(
2118			actor_id = %self.ctx.actor_id(),
2119			previous_count = previous,
2120			current_count = current,
2121			"inspector detached"
2122		);
2123		if previous == 1 {
2124			self.ctx.notify_inspector_attachments_changed();
2125		}
2126	}
2127}
2128
2129pub struct WebSocketCallbackRegion {
2130	region: Option<ActorWorkRegion>,
2131}
2132
2133pub struct KeepAwakeRegion {
2134	region: Option<ActorWorkRegion>,
2135}
2136
2137impl Drop for WebSocketCallbackRegion {
2138	fn drop(&mut self) {
2139		self.region.take();
2140	}
2141}
2142
2143impl Drop for KeepAwakeRegion {
2144	fn drop(&mut self) {
2145		// Take the region explicitly to mirror WebSocketCallbackRegion.
2146		self.region.take();
2147	}
2148}
2149
2150impl std::fmt::Debug for ActorContext {
2151	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2152		f.debug_struct("ActorContext")
2153			.field("actor_id", &self.0.actor_id)
2154			.field("name", &self.0.name)
2155			.field("key", &self.0.key)
2156			.field("region", &self.0.region)
2157			.finish()
2158	}
2159}
2160
2161// Test shim keeps moved tests in crate-root tests/ with private-module access.
2162#[cfg(test)]
2163#[path = "../../tests/context.rs"]
2164pub(crate) mod tests;