1use std::pin::Pin;
2use std::sync::Arc;
3use std::time::Instant;
4
5use serde::{Deserialize, Serialize};
6use tokio::sync::{mpsc, oneshot};
7use tokio_util::sync::CancellationToken;
8
9use crate::{RuntimeError, RuntimeErrorCode};
10
11use super::super::envelope::{RuntimeEffectEnvelope, RuntimeEffectOutcome};
12use super::{RuntimeEffectControllerError, RuntimeEffectLocalExecutor};
13
14#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
24#[serde(tag = "type", rename_all = "snake_case")]
25pub enum ExecutionScope {
26 Turn {
27 session_id: String,
28 turn_id: String,
29 },
30 Process {
31 process_id: String,
32 },
33 QueueDrain {
34 session_id: String,
35 drain_id: String,
36 },
37 SessionDelete {
38 session_id: String,
39 },
40 RuntimeOperation {
41 operation_id: String,
42 },
43}
44
45impl ExecutionScope {
46 pub fn turn(session_id: impl Into<String>, turn_id: impl Into<String>) -> Self {
47 Self::Turn {
48 session_id: session_id.into(),
49 turn_id: turn_id.into(),
50 }
51 }
52
53 pub fn process(process_id: impl Into<String>) -> Self {
54 Self::Process {
55 process_id: process_id.into(),
56 }
57 }
58
59 pub fn queue_drain(session_id: impl Into<String>, drain_id: impl Into<String>) -> Self {
60 Self::QueueDrain {
61 session_id: session_id.into(),
62 drain_id: drain_id.into(),
63 }
64 }
65
66 pub fn session_delete(session_id: impl Into<String>) -> Self {
67 Self::SessionDelete {
68 session_id: session_id.into(),
69 }
70 }
71
72 pub fn runtime_operation(operation_id: impl Into<String>) -> Self {
73 Self::RuntimeOperation {
74 operation_id: operation_id.into(),
75 }
76 }
77
78 pub fn id(&self) -> &str {
79 match self {
80 Self::Turn { turn_id, .. } => turn_id,
81 Self::Process { process_id } => process_id,
82 Self::QueueDrain { drain_id, .. } => drain_id,
83 Self::SessionDelete { session_id } => session_id,
84 Self::RuntimeOperation { operation_id } => operation_id,
85 }
86 }
87
88 pub fn session_id(&self) -> Option<&str> {
89 match self {
90 Self::Turn { session_id, .. }
91 | Self::QueueDrain { session_id, .. }
92 | Self::SessionDelete { session_id } => Some(session_id),
93 Self::Process { .. } | Self::RuntimeOperation { .. } => None,
94 }
95 }
96
97 pub fn turn_id(&self) -> Option<&str> {
98 match self {
99 Self::Turn { turn_id, .. } => Some(turn_id),
100 _ => None,
101 }
102 }
103
104 pub fn validates_turn_trace_id(&self) -> bool {
105 matches!(self, Self::Turn { .. })
106 }
107
108 pub(crate) fn validate(&self) -> Result<(), RuntimeError> {
109 let missing = match self {
110 Self::Turn {
111 session_id,
112 turn_id,
113 } => session_id.trim().is_empty() || turn_id.trim().is_empty(),
114 Self::Process { process_id } => process_id.trim().is_empty(),
115 Self::QueueDrain {
116 session_id,
117 drain_id,
118 } => session_id.trim().is_empty() || drain_id.trim().is_empty(),
119 Self::SessionDelete { session_id } => session_id.trim().is_empty(),
120 Self::RuntimeOperation { operation_id } => operation_id.trim().is_empty(),
121 };
122 if missing {
123 return Err(RuntimeError::new(
124 RuntimeErrorCode::MissingExecutionScopeId,
125 "execution scopes require non-empty stable ids",
126 ));
127 }
128 Ok(())
129 }
130}
131
132#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
133#[serde(tag = "type", rename_all = "snake_case")]
134pub enum AwaitEventWaitIdentity {
135 ToolCompletion {
136 tool_call_id: String,
137 },
138 ProcessSignal {
139 process_id: String,
140 signal_name: String,
141 ordinal: u64,
142 },
143 TurnCancelGate,
146 TurnTerminal,
148 Custom {
149 key: String,
150 },
151}
152
153impl AwaitEventWaitIdentity {
154 pub fn tool_completion(tool_call_id: impl Into<String>) -> Self {
155 Self::ToolCompletion {
156 tool_call_id: tool_call_id.into(),
157 }
158 }
159
160 pub fn process_signal(
161 process_id: impl Into<String>,
162 signal_name: impl Into<String>,
163 ordinal: u64,
164 ) -> Self {
165 Self::ProcessSignal {
166 process_id: process_id.into(),
167 signal_name: signal_name.into(),
168 ordinal,
169 }
170 }
171
172 pub(in crate::runtime::effect) fn validate(&self) -> Result<(), RuntimeError> {
173 let invalid = match self {
174 Self::ToolCompletion { tool_call_id } => tool_call_id.trim().is_empty(),
175 Self::ProcessSignal {
176 process_id,
177 signal_name,
178 ordinal,
179 } => process_id.trim().is_empty() || signal_name.trim().is_empty() || *ordinal == 0,
180 Self::TurnCancelGate | Self::TurnTerminal => false,
181 Self::Custom { key } => key.trim().is_empty(),
182 };
183 if invalid {
184 return Err(RuntimeError::new(
185 "invalid_await_event_wait_identity",
186 "await-event wait identity requires non-empty stable ids",
187 ));
188 }
189 Ok(())
190 }
191
192 pub fn is_turn_control(&self) -> bool {
193 matches!(self, Self::TurnCancelGate | Self::TurnTerminal)
194 }
195}
196
197#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
198pub struct AwaitEventKey {
199 pub scope: ExecutionScope,
200 pub wait: AwaitEventWaitIdentity,
201 pub key_id: String,
202 pub signature: String,
203}
204
205impl AwaitEventKey {
206 pub fn promise_key(&self) -> String {
207 format!("lash-await-event:{}", self.key_id)
208 }
209}
210
211#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
212pub struct ExternalCompletionError {
213 pub code: String,
214 pub message: String,
215 #[serde(default, skip_serializing_if = "Option::is_none")]
216 pub raw: Option<serde_json::Value>,
217}
218
219impl ExternalCompletionError {
220 pub fn new(code: impl Into<String>, message: impl Into<String>) -> Self {
221 Self {
222 code: code.into(),
223 message: message.into(),
224 raw: None,
225 }
226 }
227}
228
229#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
230#[serde(tag = "status", content = "payload", rename_all = "snake_case")]
231pub enum Resolution {
232 Ok(serde_json::Value),
233 Err(ExternalCompletionError),
234 Timeout,
235 Cancelled,
236}
237
238#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
239#[serde(tag = "status", rename_all = "snake_case")]
240pub enum ResolveOutcome {
241 Accepted,
242 AlreadyResolved { terminal: Resolution },
243 UnknownOrRevoked,
244}
245
246pub(super) enum ScopedEffectControllerInner<'run> {
247 Borrowed(&'run dyn RuntimeEffectController),
248 Shared(Arc<dyn RuntimeEffectController>),
249}
250
251impl Clone for ScopedEffectControllerInner<'_> {
252 fn clone(&self) -> Self {
253 match self {
254 Self::Borrowed(controller) => Self::Borrowed(*controller),
255 Self::Shared(controller) => Self::Shared(Arc::clone(controller)),
256 }
257 }
258}
259
260#[derive(Clone)]
262pub struct ScopedEffectController<'run> {
263 pub(super) controller: ScopedEffectControllerInner<'run>,
264 pub(super) scope: ExecutionScope,
265}
266
267impl<'run> ScopedEffectController<'run> {
268 pub fn borrowed(
269 controller: &'run dyn RuntimeEffectController,
270 scope: ExecutionScope,
271 ) -> Result<Self, RuntimeError> {
272 scope.validate()?;
273 Ok(Self {
274 controller: ScopedEffectControllerInner::Borrowed(controller),
275 scope,
276 })
277 }
278
279 pub fn shared(
280 controller: Arc<dyn RuntimeEffectController>,
281 scope: ExecutionScope,
282 ) -> Result<Self, RuntimeError> {
283 scope.validate()?;
284 Ok(Self {
285 controller: ScopedEffectControllerInner::Shared(controller),
286 scope,
287 })
288 }
289
290 pub fn controller(&self) -> &dyn RuntimeEffectController {
291 match &self.controller {
292 ScopedEffectControllerInner::Borrowed(controller) => *controller,
293 ScopedEffectControllerInner::Shared(controller) => controller.as_ref(),
294 }
295 }
296
297 pub fn execution_scope(&self) -> &ExecutionScope {
298 &self.scope
299 }
300
301 pub fn scope_id(&self) -> &str {
302 self.scope.id()
303 }
304
305 pub fn turn_id(&self) -> Option<&str> {
306 self.scope.turn_id()
307 }
308
309 pub(crate) fn to_static(&self) -> Option<ScopedEffectController<'static>> {
310 let ScopedEffectControllerInner::Shared(controller) = &self.controller else {
311 return None;
312 };
313 Some(ScopedEffectController {
314 controller: ScopedEffectControllerInner::Shared(Arc::clone(controller)),
315 scope: self.scope.clone(),
316 })
317 }
318}
319
320type EffectControllerTaskFuture<'run> = Pin<Box<dyn Future<Output = ()> + Send + 'run>>;
321
322pub(crate) enum EffectControllerTaskRequest {
323 Execute {
324 envelope: Box<RuntimeEffectEnvelope>,
325 local_executor: Box<RuntimeEffectLocalExecutor<'static>>,
326 response: oneshot::Sender<Result<RuntimeEffectOutcome, RuntimeEffectControllerError>>,
327 },
328 AwaitEventKey {
329 scope: ExecutionScope,
330 wait: AwaitEventWaitIdentity,
331 response: oneshot::Sender<Result<AwaitEventKey, RuntimeError>>,
332 },
333 ResolveAwaitEvent {
334 key: AwaitEventKey,
335 resolution: Resolution,
336 response: oneshot::Sender<Result<ResolveOutcome, RuntimeError>>,
337 },
338}
339
340impl EffectControllerTaskRequest {
341 fn into_future<'run>(
342 self,
343 controller: &'run dyn RuntimeEffectController,
344 ) -> EffectControllerTaskFuture<'run> {
345 match self {
346 Self::Execute {
347 envelope,
348 local_executor,
349 response,
350 } => Box::pin(async move {
351 let _ = response.send(controller.execute_effect(*envelope, *local_executor).await);
352 }),
353 Self::AwaitEventKey {
354 scope,
355 wait,
356 response,
357 } => Box::pin(async move {
358 let _ = response.send(controller.await_event_key(&scope, wait).await);
359 }),
360 Self::ResolveAwaitEvent {
361 key,
362 resolution,
363 response,
364 } => Box::pin(async move {
365 let _ = response.send(controller.resolve_await_event(&key, resolution).await);
366 }),
367 }
368 }
369}
370
371pub(super) struct RemoteLocalExecutionRequest {
372 pub(super) envelope: RuntimeEffectEnvelope,
373 pub(super) response:
374 oneshot::Sender<Result<RuntimeEffectOutcome, RuntimeEffectControllerError>>,
375}
376
377#[derive(Clone)]
378pub(crate) struct EffectTaskController {
379 requests: mpsc::UnboundedSender<EffectControllerTaskRequest>,
380 durability_tier: crate::DurabilityTier,
381 allows_process_lifetime_completion_keys: bool,
382 supports_concurrent_effects: bool,
383}
384
385impl EffectTaskController {
386 pub(crate) fn scoped(
387 controller: &dyn RuntimeEffectController,
388 scope: ExecutionScope,
389 ) -> Result<
390 (
391 ScopedEffectController<'static>,
392 mpsc::UnboundedReceiver<EffectControllerTaskRequest>,
393 ),
394 RuntimeError,
395 > {
396 let (requests, request_rx) = mpsc::unbounded_channel();
397 let proxy = Self {
398 requests,
399 durability_tier: controller.durability_tier(),
400 allows_process_lifetime_completion_keys: controller
401 .allows_process_lifetime_completion_keys(),
402 supports_concurrent_effects: controller.supports_concurrent_effects(),
403 };
404 Ok((
405 ScopedEffectController::shared(Arc::new(proxy), scope)?,
406 request_rx,
407 ))
408 }
409}
410
411#[async_trait::async_trait]
412impl AwaitEventResolver for EffectTaskController {
413 fn durability_tier(&self) -> crate::DurabilityTier {
414 self.durability_tier
415 }
416
417 fn allows_process_lifetime_completion_keys(&self) -> bool {
418 self.allows_process_lifetime_completion_keys
419 }
420
421 async fn await_event_key(
422 &self,
423 scope: &ExecutionScope,
424 wait: AwaitEventWaitIdentity,
425 ) -> Result<AwaitEventKey, RuntimeError> {
426 let (response_tx, response_rx) = oneshot::channel();
427 self.requests
428 .send(EffectControllerTaskRequest::AwaitEventKey {
429 scope: scope.clone(),
430 wait,
431 response: response_tx,
432 })
433 .map_err(|_| {
434 RuntimeError::new(
435 "runtime_effect_controller_task_closed",
436 "await-event key controller task is no longer running",
437 )
438 })?;
439 response_rx.await.map_err(|_| {
440 RuntimeError::new(
441 "runtime_effect_controller_task_closed",
442 "await-event key controller response was dropped",
443 )
444 })?
445 }
446
447 async fn resolve_await_event(
448 &self,
449 key: &AwaitEventKey,
450 resolution: Resolution,
451 ) -> Result<ResolveOutcome, RuntimeError> {
452 let (response_tx, response_rx) = oneshot::channel();
453 self.requests
454 .send(EffectControllerTaskRequest::ResolveAwaitEvent {
455 key: key.clone(),
456 resolution,
457 response: response_tx,
458 })
459 .map_err(|_| {
460 RuntimeError::new(
461 "runtime_effect_controller_task_closed",
462 "await-event resolution controller task is no longer running",
463 )
464 })?;
465 response_rx.await.map_err(|_| {
466 RuntimeError::new(
467 "runtime_effect_controller_task_closed",
468 "await-event resolution controller response was dropped",
469 )
470 })?
471 }
472}
473
474#[async_trait::async_trait]
475impl RuntimeEffectController for EffectTaskController {
476 fn supports_concurrent_effects(&self) -> bool {
477 self.supports_concurrent_effects
478 }
479
480 async fn execute_effect(
481 &self,
482 envelope: RuntimeEffectEnvelope,
483 local_executor: RuntimeEffectLocalExecutor<'_>,
484 ) -> Result<RuntimeEffectOutcome, RuntimeEffectControllerError> {
485 let (local_executor, mut local_execution) = local_executor.into_remote_execution();
486 let (response_tx, response_rx) = oneshot::channel();
487 self.requests
488 .send(EffectControllerTaskRequest::Execute {
489 envelope: Box::new(envelope),
490 local_executor: Box::new(local_executor),
491 response: response_tx,
492 })
493 .map_err(|_| {
494 RuntimeEffectControllerError::new(
495 "runtime_effect_controller_task_closed",
496 "effect controller task is no longer running",
497 )
498 })?;
499
500 tokio::pin!(response_rx);
501 loop {
502 tokio::select! {
503 response = &mut response_rx => {
504 return response.map_err(|_| {
505 RuntimeEffectControllerError::new(
506 "runtime_effect_controller_task_closed",
507 "effect controller response was dropped",
508 )
509 })?;
510 }
511 request = async {
512 match local_execution.as_mut() {
513 Some((_, requests)) => requests.recv().await,
514 None => std::future::pending().await,
515 }
516 } => {
517 let Some(request) = request else {
518 local_execution = None;
523 continue;
524 };
525 let Some((executor, _)) = local_execution.take() else {
526 unreachable!("local execution request requires a local executor");
527 };
528 let result = executor.execute_forwarded(request.envelope).await;
529 let _ = request.response.send(result);
530 }
531 }
532 }
533 }
534}
535
536pub(crate) async fn drive_effect_controller_task(
537 controller: &dyn RuntimeEffectController,
538 envelope: RuntimeEffectEnvelope,
539 local_executor: RuntimeEffectLocalExecutor<'static>,
540 mut requests: mpsc::UnboundedReceiver<EffectControllerTaskRequest>,
541) -> Result<RuntimeEffectOutcome, RuntimeEffectControllerError> {
542 let (root_tx, root_rx) = oneshot::channel();
543 let root = EffectControllerTaskRequest::Execute {
544 envelope: Box::new(envelope),
545 local_executor: Box::new(local_executor),
546 response: root_tx,
547 };
548 let mut stack = vec![root.into_future(controller)];
549 let mut requests_open = true;
550 tokio::pin!(root_rx);
551
552 loop {
553 let Some(active) = stack.last_mut() else {
554 return root_rx.await.map_err(|_| {
555 RuntimeEffectControllerError::new(
556 "runtime_effect_controller_task_closed",
557 "root effect controller response was dropped",
558 )
559 })?;
560 };
561 tokio::select! {
562 biased;
563 response = &mut root_rx => {
564 return response.map_err(|_| {
565 RuntimeEffectControllerError::new(
566 "runtime_effect_controller_task_closed",
567 "root effect controller response was dropped",
568 )
569 })?;
570 }
571 () = active => {
572 stack.pop();
573 }
574 request = async {
575 if requests_open {
576 requests.recv().await
577 } else {
578 std::future::pending().await
579 }
580 } => {
581 match request {
582 Some(request) => stack.push(request.into_future(controller)),
583 None => requests_open = false,
584 }
585 }
586 }
587 }
588}
589
590#[async_trait::async_trait]
596pub trait AwaitEventResolver: Send + Sync {
597 fn durability_tier(&self) -> crate::DurabilityTier {
598 crate::DurabilityTier::Inline
599 }
600
601 fn allows_process_lifetime_completion_keys(&self) -> bool {
608 self.durability_tier() == crate::DurabilityTier::Durable
609 }
610
611 async fn await_event_key(
612 &self,
613 _scope: &ExecutionScope,
614 _wait: AwaitEventWaitIdentity,
615 ) -> Result<AwaitEventKey, RuntimeError> {
616 Err(RuntimeError::new(
617 "await_event_unsupported",
618 "this effect boundary does not support await-event keys",
619 ))
620 }
621
622 async fn resolve_await_event(
623 &self,
624 _key: &AwaitEventKey,
625 _resolution: Resolution,
626 ) -> Result<ResolveOutcome, RuntimeError> {
627 Ok(ResolveOutcome::UnknownOrRevoked)
628 }
629
630 async fn peek_await_event(
638 &self,
639 _key: &AwaitEventKey,
640 ) -> Result<Option<Resolution>, RuntimeError> {
641 Err(RuntimeError::new(
642 "await_event_unsupported",
643 "this effect boundary does not support await-event reads",
644 ))
645 }
646
647 async fn await_await_event(
648 &self,
649 _key: &AwaitEventKey,
650 _cancel: CancellationToken,
651 _deadline: Option<Instant>,
652 ) -> Result<Resolution, RuntimeError> {
653 Err(RuntimeError::new(
654 "await_event_unsupported",
655 "this effect boundary does not support await-event waits",
656 ))
657 }
658
659 async fn revoke_await_events_for_session(&self, _session_id: &str) -> Result<(), RuntimeError> {
660 Err(RuntimeError::new(
661 "await_event_unsupported",
662 "this effect boundary does not support revoking await-event waits",
663 ))
664 }
665
666 async fn cancel_await_events_for_session(&self, _session_id: &str) -> Result<(), RuntimeError> {
677 Err(RuntimeError::new(
678 "await_event_cancel_unsupported",
679 "this effect boundary does not support cancelling durable waits",
680 ))
681 }
682}
683
684#[async_trait::async_trait]
686pub trait EffectHost: AwaitEventResolver {
687 fn scoped<'run>(
688 &'run self,
689 scope: ExecutionScope,
690 ) -> Result<ScopedEffectController<'run>, RuntimeError>;
691
692 fn scoped_static(
693 &self,
694 _scope: ExecutionScope,
695 ) -> Result<Option<ScopedEffectController<'static>>, RuntimeError> {
696 Ok(None)
697 }
698}
699
700#[async_trait::async_trait]
702pub trait RuntimeEffectController: AwaitEventResolver {
703 fn wants_segment_boundary(&self, _progress: &SegmentProgress) -> Option<BoundaryReason> {
710 None
711 }
712
713 fn supports_concurrent_effects(&self) -> bool {
723 true
724 }
725
726 async fn execute_effect(
727 &self,
728 envelope: RuntimeEffectEnvelope,
729 local_executor: RuntimeEffectLocalExecutor<'_>,
730 ) -> Result<RuntimeEffectOutcome, RuntimeEffectControllerError>;
731}
732
733#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
734pub struct SegmentProgress {
735 pub effects_executed: u64,
736 pub journaled_bytes_estimate: Option<u64>,
737}
738
739#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
740#[serde(rename_all = "snake_case")]
741pub enum BoundaryReason {
742 JournalBudget,
743 DurationCap,
744}
745
746#[derive(Clone)]
749pub(crate) enum RuntimeEffectControllerHandle<'run> {
750 Borrowed(ScopedEffectController<'run>),
751 #[cfg(any(test, feature = "testing"))]
752 Shared {
753 controller: Arc<dyn RuntimeEffectController>,
754 scope: ExecutionScope,
755 },
756}
757
758impl<'run> RuntimeEffectControllerHandle<'run> {
759 pub(crate) fn borrowed(scoped: ScopedEffectController<'run>) -> Self {
760 Self::Borrowed(scoped)
761 }
762
763 #[cfg(any(test, feature = "testing"))]
764 pub(crate) fn shared(controller: Arc<dyn RuntimeEffectController>) -> Self {
765 Self::Shared {
766 controller,
767 scope: ExecutionScope::runtime_operation("test-runtime-effect-controller"),
768 }
769 }
770
771 pub(crate) fn controller(&self) -> &dyn RuntimeEffectController {
772 match self {
773 Self::Borrowed(scoped) => scoped.controller(),
774 #[cfg(any(test, feature = "testing"))]
775 Self::Shared { controller, .. } => controller.as_ref(),
776 }
777 }
778
779 pub(crate) fn scoped(&self) -> ScopedEffectController<'_> {
780 match self {
781 Self::Borrowed(scoped) => scoped.clone(),
782 #[cfg(any(test, feature = "testing"))]
783 Self::Shared { controller, scope } => {
784 ScopedEffectController::shared(Arc::clone(controller), scope.clone())
785 .expect("runtime effect controller handle carries a valid scope")
786 }
787 }
788 }
789
790 pub(crate) fn clone_scoped(&self) -> RuntimeEffectControllerHandle<'run> {
791 self.clone()
792 }
793
794 pub(crate) fn to_static(&self) -> Option<RuntimeEffectControllerHandle<'static>> {
795 match self {
796 Self::Borrowed(scoped) => scoped
797 .to_static()
798 .map(RuntimeEffectControllerHandle::Borrowed),
799 #[cfg(any(test, feature = "testing"))]
800 Self::Shared { controller, scope } => Some(RuntimeEffectControllerHandle::Shared {
801 controller: Arc::clone(controller),
802 scope: scope.clone(),
803 }),
804 }
805 }
806}