1use std::collections::{HashMap, VecDeque};
2use std::fmt;
3use std::future::Future;
4use std::pin::Pin;
5use std::sync::atomic::{AtomicU64, Ordering};
6use std::sync::{Arc, Mutex, RwLock};
7#[cfg(not(target_arch = "wasm32"))]
8use std::thread::JoinHandle;
9use std::time::Duration;
10
11use serde::{Deserialize, Serialize};
12use serde_json::{json, Value};
13#[cfg(not(target_arch = "wasm32"))]
14use tokio::sync::mpsc;
15use tokio::sync::{watch, Notify};
16use web_time::Instant;
17
18use crate::{
19 EffectRequest, EffectResult, LocalProviderBinding, ProjectSettings, RuntimeManifest,
20 RuntimeRelease, RuntimeSnapshot, RuntimeTraceRecord, MAX_ENDPOINT_TIMEOUT_MS,
21};
22
23const SNAPSHOT_VERSION: u32 = 1;
24const DEFAULT_TIMEOUT_MS: u64 = 30_000;
25const DEFAULT_EFFECT_LIMIT: usize = 64;
26#[cfg(not(target_arch = "wasm32"))]
27const MAX_IN_FLIGHT_INVOCATIONS: usize = 64;
28const MAX_RETAINED_INVOCATIONS: usize = 256;
29const MAX_RETAINED_INVOCATION_EVENTS: usize = 256;
30const MAX_COALESCED_EVENT_BYTES: usize = 64 * 1024;
31#[cfg(not(target_arch = "wasm32"))]
32const WORKER_QUEUE_CAPACITY: usize = 64;
33const MAX_RUNTIME_MONITOR_IO_BYTES: usize = 128 * 1024;
34
35#[cfg(not(target_arch = "wasm32"))]
37pub type ProviderFuture<'a> =
38 Pin<Box<dyn Future<Output = Result<ProviderResponse, RuntimeError>> + Send + 'a>>;
39
40#[cfg(target_arch = "wasm32")]
41pub type ProviderFuture<'a> =
42 Pin<Box<dyn Future<Output = Result<ProviderResponse, RuntimeError>> + 'a>>;
43
44#[derive(Clone, PartialEq, Serialize, Deserialize)]
46#[serde(tag = "format", content = "value", rename_all = "camelCase")]
47pub enum InvocationData {
48 Json(Value),
49 Binary(Vec<u8>),
50}
51
52impl fmt::Debug for InvocationData {
53 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
54 match self {
55 Self::Json(_) => formatter.write_str("InvocationData::Json([REDACTED])"),
56 Self::Binary(bytes) => formatter
57 .debug_tuple("InvocationData::Binary")
58 .field(&format_args!("{} bytes", bytes.len()))
59 .finish(),
60 }
61 }
62}
63
64impl Default for InvocationData {
65 fn default() -> Self {
66 Self::Json(Value::Null)
67 }
68}
69
70#[derive(Clone, PartialEq, Serialize, Deserialize)]
72#[serde(rename_all = "camelCase")]
73pub struct InvocationInput {
74 pub endpoint: String,
75 #[serde(default = "default_session_id")]
76 pub session_id: String,
77 #[serde(default)]
78 pub data: InvocationData,
79 #[serde(default)]
80 pub metadata: Value,
81}
82
83impl InvocationInput {
84 pub fn json(endpoint: impl Into<String>, data: Value) -> Self {
85 Self {
86 endpoint: endpoint.into(),
87 session_id: default_session_id(),
88 data: InvocationData::Json(data),
89 metadata: Value::Object(Default::default()),
90 }
91 }
92
93 pub fn with_session(mut self, session_id: impl Into<String>) -> Self {
94 self.session_id = session_id.into();
95 self
96 }
97}
98
99impl fmt::Debug for InvocationInput {
100 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
101 formatter
102 .debug_struct("InvocationInput")
103 .field("endpoint", &self.endpoint)
104 .field("session_id", &self.session_id)
105 .field("data", &"[REDACTED]")
106 .field("metadata", &"[REDACTED]")
107 .finish()
108 }
109}
110
111#[derive(Clone, PartialEq, Serialize, Deserialize)]
113#[serde(rename_all = "camelCase")]
114pub struct AgentDefinition {
115 pub id: String,
116 pub name: String,
117 pub provider: String,
118 pub capabilities: Vec<String>,
119 #[serde(default)]
120 pub metadata: Value,
121}
122
123impl fmt::Debug for AgentDefinition {
124 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
125 formatter
126 .debug_struct("AgentDefinition")
127 .field("id", &self.id)
128 .field("name", &self.name)
129 .field("provider", &self.provider)
130 .field("capabilities", &self.capabilities)
131 .field("metadata", &"[REDACTED]")
132 .finish()
133 }
134}
135
136#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
138#[serde(rename_all = "camelCase")]
139pub struct EndpointDefinition {
140 pub name: String,
141 pub agent: String,
142 pub capability: String,
143 #[serde(default = "default_timeout_ms")]
145 pub timeout_ms: u64,
146}
147
148#[derive(Clone)]
150pub struct ProviderRequest {
151 pub project_id: String,
152 pub endpoint: String,
153 pub session_id: String,
154 pub agent: AgentDefinition,
155 pub capability: String,
156 pub data: InvocationData,
157 pub metadata: Value,
158 pub snapshot: RuntimeSnapshot,
159}
160
161impl fmt::Debug for ProviderRequest {
162 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
163 formatter
164 .debug_struct("ProviderRequest")
165 .field("project_id", &self.project_id)
166 .field("endpoint", &self.endpoint)
167 .field("session_id", &self.session_id)
168 .field("agent", &self.agent.id)
169 .field("capability", &self.capability)
170 .field("data", &"[REDACTED]")
171 .field("metadata", &"[REDACTED]")
172 .field("snapshot_revision", &self.snapshot.revision)
173 .finish()
174 }
175}
176
177#[derive(Clone, PartialEq, Serialize, Deserialize)]
179#[serde(rename_all = "camelCase")]
180pub struct ProviderResponse {
181 #[serde(default)]
182 pub data: InvocationData,
183 #[serde(default)]
184 pub metadata: Value,
185 #[serde(default, skip_serializing_if = "Option::is_none")]
186 pub state: Option<Value>,
187}
188
189impl ProviderResponse {
190 pub fn json(data: Value) -> Self {
191 Self {
192 data: InvocationData::Json(data),
193 metadata: Value::Object(Default::default()),
194 state: None,
195 }
196 }
197}
198
199impl fmt::Debug for ProviderResponse {
200 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
201 formatter
202 .debug_struct("ProviderResponse")
203 .field("data", &"[REDACTED]")
204 .field("metadata", &"[REDACTED]")
205 .field("state", &self.state.as_ref().map(|_| "[REDACTED]"))
206 .finish()
207 }
208}
209
210#[derive(Clone, Default)]
212pub struct CancellationToken {
213 inner: Arc<CancellationState>,
214}
215
216#[derive(Default)]
217struct CancellationState {
218 cancelled: std::sync::atomic::AtomicBool,
219 notify: Notify,
220}
221
222impl CancellationToken {
223 pub fn cancel(&self) {
224 if !self.inner.cancelled.swap(true, Ordering::AcqRel) {
225 self.inner.notify.notify_waiters();
226 }
227 }
228
229 pub fn is_cancelled(&self) -> bool {
230 self.inner.cancelled.load(Ordering::Acquire)
231 }
232
233 pub async fn cancelled(&self) {
234 if self.is_cancelled() {
235 return;
236 }
237 self.inner.notify.notified().await;
238 }
239}
240
241impl fmt::Debug for CancellationToken {
242 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
243 formatter
244 .debug_struct("CancellationToken")
245 .field("cancelled", &self.is_cancelled())
246 .finish()
247 }
248}
249
250#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
252#[serde(rename_all = "camelCase")]
253pub enum InvocationEventKind {
254 Started,
255 OutputDelta,
256 Completed,
257 Failed,
258 Cancelled,
259}
260
261#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Serialize, Deserialize)]
263#[serde(rename_all = "camelCase")]
264pub enum ProviderStage {
265 Queue,
266 Load,
267 Tokenize,
268 Prefill,
269 FirstToken,
270 Decode,
271 Validate,
272}
273
274#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
276#[serde(tag = "type", rename_all = "camelCase")]
277pub enum ProviderEvent {
278 Activity,
281 OutputDelta {
282 data: InvocationData,
283 },
284 StageStarted {
285 stage: ProviderStage,
286 #[serde(default, skip_serializing_if = "is_null")]
287 metadata: Value,
288 },
289 StageCompleted {
290 stage: ProviderStage,
291 elapsed_ms: u64,
292 #[serde(default, skip_serializing_if = "is_null")]
293 metadata: Value,
294 },
295 StageFailed {
296 stage: ProviderStage,
297 elapsed_ms: u64,
298 error: String,
299 #[serde(default, skip_serializing_if = "is_null")]
300 metadata: Value,
301 },
302}
303
304#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
306#[serde(rename_all = "camelCase")]
307pub enum RuntimeMonitorStatus {
308 Completed,
309 Cancelled,
310 Error,
311}
312
313#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
315#[serde(rename_all = "camelCase")]
316pub enum RuntimeMonitorStageStatus {
317 Started,
318 Completed,
319 Failed,
320}
321
322#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
328#[serde(tag = "type", rename_all = "camelCase")]
329pub enum RuntimeMonitorEvent {
330 InvocationStarted {
331 trace_id: String,
332 invocation_id: String,
333 project_id: String,
334 endpoint: String,
335 agent_id: String,
336 provider_id: String,
337 capability: String,
338 started_at_ms: u64,
339 },
340 ProviderStage {
341 trace_id: String,
342 invocation_id: String,
343 stage: ProviderStage,
344 status: RuntimeMonitorStageStatus,
345 #[serde(default, skip_serializing_if = "Option::is_none")]
346 elapsed_ms: Option<u64>,
347 request_elapsed_ms: u64,
348 #[serde(default, skip_serializing_if = "Option::is_none")]
349 input_tokens: Option<u64>,
350 #[serde(default, skip_serializing_if = "Option::is_none")]
351 output_tokens: Option<u64>,
352 #[serde(default, skip_serializing_if = "Option::is_none")]
353 resident: Option<bool>,
354 #[serde(default, skip_serializing_if = "Option::is_none")]
355 error: Option<String>,
356 },
357 InvocationFinished {
358 trace_id: String,
359 invocation_id: String,
360 status: RuntimeMonitorStatus,
361 duration_ms: u64,
362 ended_at_ms: u64,
363 #[serde(default, skip_serializing_if = "Option::is_none")]
364 error: Option<String>,
365 },
366}
367
368pub type RuntimeMonitorObserver = Arc<dyn Fn(RuntimeMonitorEvent) + Send + Sync>;
371
372#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
374#[serde(rename_all = "camelCase")]
375pub struct RuntimeMonitorIoSummary {
376 pub value: Value,
377 pub truncated: bool,
378}
379
380#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
382#[serde(tag = "type", rename_all = "camelCase")]
383pub enum RuntimeMonitorIoEvent {
384 InvocationInput {
385 trace_id: String,
386 invocation_id: String,
387 summary: RuntimeMonitorIoSummary,
388 },
389 InvocationOutput {
390 trace_id: String,
391 invocation_id: String,
392 summary: RuntimeMonitorIoSummary,
393 },
394}
395
396pub type RuntimeMonitorIoObserver = Arc<dyn Fn(RuntimeMonitorIoEvent) + Send + Sync>;
398
399#[derive(Clone, PartialEq, Serialize, Deserialize)]
401#[serde(rename_all = "camelCase")]
402pub struct InvocationEvent {
403 pub sequence: u64,
404 pub kind: InvocationEventKind,
405 #[serde(default, skip_serializing_if = "Option::is_none")]
406 pub data: Option<InvocationData>,
407 #[serde(default, skip_serializing_if = "Option::is_none")]
408 pub error: Option<String>,
409}
410
411impl fmt::Debug for InvocationEvent {
412 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
413 formatter
414 .debug_struct("InvocationEvent")
415 .field("sequence", &self.sequence)
416 .field("kind", &self.kind)
417 .field("data", &self.data.as_ref().map(|_| "[REDACTED]"))
418 .field("error", &self.error.as_ref().map(|_| "[REDACTED]"))
419 .finish()
420 }
421}
422
423#[derive(Clone)]
429pub struct ProviderEventSink {
430 emit: Arc<dyn ProviderEventCallback>,
431}
432
433#[cfg(not(target_arch = "wasm32"))]
434trait ProviderEventCallback: Fn(ProviderEvent) + Send + Sync {}
435
436#[cfg(not(target_arch = "wasm32"))]
437impl<T> ProviderEventCallback for T where T: Fn(ProviderEvent) + Send + Sync {}
438
439#[cfg(target_arch = "wasm32")]
440trait ProviderEventCallback: Fn(ProviderEvent) {}
441
442#[cfg(target_arch = "wasm32")]
443impl<T> ProviderEventCallback for T where T: Fn(ProviderEvent) {}
444
445impl ProviderEventSink {
446 fn new(emit: impl ProviderEventCallback + 'static) -> Self {
447 Self {
448 emit: Arc::new(emit),
449 }
450 }
451
452 #[cfg(not(target_arch = "wasm32"))]
454 pub fn from_fn(emit: impl Fn(ProviderEvent) + Send + Sync + 'static) -> Self {
455 Self::new(emit)
456 }
457
458 #[cfg(target_arch = "wasm32")]
460 pub fn from_fn(emit: impl Fn(ProviderEvent) + 'static) -> Self {
461 Self::new(emit)
462 }
463
464 pub fn discard() -> Self {
465 Self::new(|_event| {})
466 }
467
468 pub fn output_delta(&self, data: InvocationData) {
469 (self.emit)(ProviderEvent::OutputDelta { data });
470 }
471
472 pub fn activity(&self) {
473 (self.emit)(ProviderEvent::Activity);
474 }
475
476 pub fn stage_started(&self, stage: ProviderStage, metadata: Value) {
477 (self.emit)(ProviderEvent::StageStarted { stage, metadata });
478 }
479
480 pub fn stage_completed(&self, stage: ProviderStage, elapsed_ms: u64, metadata: Value) {
481 (self.emit)(ProviderEvent::StageCompleted {
482 stage,
483 elapsed_ms,
484 metadata,
485 });
486 }
487
488 pub fn stage_failed(
489 &self,
490 stage: ProviderStage,
491 elapsed_ms: u64,
492 error: impl Into<String>,
493 metadata: Value,
494 ) {
495 (self.emit)(ProviderEvent::StageFailed {
496 stage,
497 elapsed_ms,
498 error: error.into(),
499 metadata,
500 });
501 }
502}
503
504impl fmt::Debug for ProviderEventSink {
505 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
506 formatter.write_str("ProviderEventSink")
507 }
508}
509
510#[cfg(not(target_arch = "wasm32"))]
516pub trait AgentProviderBounds: Send + Sync {}
517
518#[cfg(not(target_arch = "wasm32"))]
519impl<T: Send + Sync> AgentProviderBounds for T {}
520
521#[cfg(target_arch = "wasm32")]
522pub trait AgentProviderBounds {}
523
524#[cfg(target_arch = "wasm32")]
525impl<T> AgentProviderBounds for T {}
526
527pub trait AgentProvider: AgentProviderBounds + 'static {
528 fn supports(&self, capability: &str) -> bool;
529
530 fn invoke<'a>(
531 &'a self,
532 request: ProviderRequest,
533 cancellation: CancellationToken,
534 ) -> ProviderFuture<'a>;
535
536 fn invoke_with_events<'a>(
537 &'a self,
538 request: ProviderRequest,
539 cancellation: CancellationToken,
540 _events: ProviderEventSink,
541 ) -> ProviderFuture<'a> {
542 self.invoke(request, cancellation)
543 }
544}
545
546pub trait RuntimeStore: Send + Sync + 'static {
551 fn load(
552 &self,
553 project_id: &str,
554 session_id: &str,
555 ) -> Result<Option<RuntimeSnapshot>, RuntimeError>;
556
557 fn save(
558 &self,
559 project_id: &str,
560 session_id: &str,
561 snapshot: &RuntimeSnapshot,
562 ) -> Result<(), RuntimeError>;
563
564 fn save_release(&self, _release: &RuntimeRelease) -> Result<(), RuntimeError> {
565 Err(RuntimeError::store(
566 "this runtime store does not support releases".to_string(),
567 ))
568 }
569
570 fn load_release(
571 &self,
572 _project_id: &str,
573 _version: u64,
574 ) -> Result<Option<RuntimeRelease>, RuntimeError> {
575 Ok(None)
576 }
577
578 fn list_releases(&self, _project_id: &str) -> Result<Vec<RuntimeRelease>, RuntimeError> {
579 Ok(Vec::new())
580 }
581
582 fn active_release(&self, _project_id: &str) -> Result<Option<u64>, RuntimeError> {
583 Ok(None)
584 }
585
586 fn set_active_release(&self, _project_id: &str, _version: u64) -> Result<(), RuntimeError> {
587 Err(RuntimeError::store(
588 "this runtime store does not support releases".to_string(),
589 ))
590 }
591
592 fn save_local_provider_binding(
593 &self,
594 _project_id: &str,
595 _binding: &LocalProviderBinding,
596 ) -> Result<(), RuntimeError> {
597 Err(RuntimeError::store(
598 "this runtime store does not support provider bindings".to_string(),
599 ))
600 }
601
602 fn local_provider_bindings(
603 &self,
604 _project_id: &str,
605 ) -> Result<Vec<LocalProviderBinding>, RuntimeError> {
606 Ok(Vec::new())
607 }
608
609 fn enqueue_trace(&self, _trace: &RuntimeTraceRecord) -> Result<(), RuntimeError> {
610 Ok(())
611 }
612
613 fn pending_traces(&self, _limit: usize) -> Result<Vec<RuntimeTraceRecord>, RuntimeError> {
614 Ok(Vec::new())
615 }
616
617 fn acknowledge_traces(&self, _trace_ids: &[String]) -> Result<(), RuntimeError> {
618 Ok(())
619 }
620}
621
622#[derive(Default)]
624pub struct MemoryRuntimeStore {
625 snapshots: RwLock<HashMap<(String, String), RuntimeSnapshot>>,
626 releases: RwLock<HashMap<(String, u64), RuntimeRelease>>,
627 active_releases: RwLock<HashMap<String, u64>>,
628 provider_bindings: RwLock<HashMap<(String, String), LocalProviderBinding>>,
629 trace_outbox: RwLock<VecDeque<RuntimeTraceRecord>>,
630}
631
632impl RuntimeStore for MemoryRuntimeStore {
633 fn load(
634 &self,
635 project_id: &str,
636 session_id: &str,
637 ) -> Result<Option<RuntimeSnapshot>, RuntimeError> {
638 let snapshots = self.snapshots.read().map_err(|_| RuntimeError::Internal)?;
639 Ok(snapshots
640 .get(&(project_id.to_string(), session_id.to_string()))
641 .cloned())
642 }
643
644 fn save(
645 &self,
646 project_id: &str,
647 session_id: &str,
648 snapshot: &RuntimeSnapshot,
649 ) -> Result<(), RuntimeError> {
650 let mut snapshots = self.snapshots.write().map_err(|_| RuntimeError::Internal)?;
651 snapshots.insert(
652 (project_id.to_string(), session_id.to_string()),
653 snapshot.clone(),
654 );
655 Ok(())
656 }
657
658 fn save_release(&self, release: &RuntimeRelease) -> Result<(), RuntimeError> {
659 release.validate()?;
660 let key = (release.manifest.project_id.clone(), release.version);
661 let mut releases = self.releases.write().map_err(|_| RuntimeError::Internal)?;
662 if let Some(existing) = releases.get(&key) {
663 if existing != release {
664 return Err(RuntimeError::store(
665 "runtime release versions are immutable".to_string(),
666 ));
667 }
668 return Ok(());
669 }
670 releases.insert(key, release.clone());
671 Ok(())
672 }
673
674 fn load_release(
675 &self,
676 project_id: &str,
677 version: u64,
678 ) -> Result<Option<RuntimeRelease>, RuntimeError> {
679 Ok(self
680 .releases
681 .read()
682 .map_err(|_| RuntimeError::Internal)?
683 .get(&(project_id.to_string(), version))
684 .cloned())
685 }
686
687 fn list_releases(&self, project_id: &str) -> Result<Vec<RuntimeRelease>, RuntimeError> {
688 let mut releases = self
689 .releases
690 .read()
691 .map_err(|_| RuntimeError::Internal)?
692 .iter()
693 .filter(|((stored_project_id, _), _)| stored_project_id == project_id)
694 .map(|(_, release)| release.clone())
695 .collect::<Vec<_>>();
696 releases.sort_by_key(|release| std::cmp::Reverse(release.version));
697 Ok(releases)
698 }
699
700 fn active_release(&self, project_id: &str) -> Result<Option<u64>, RuntimeError> {
701 Ok(self
702 .active_releases
703 .read()
704 .map_err(|_| RuntimeError::Internal)?
705 .get(project_id)
706 .copied())
707 }
708
709 fn set_active_release(&self, project_id: &str, version: u64) -> Result<(), RuntimeError> {
710 if !self
711 .releases
712 .read()
713 .map_err(|_| RuntimeError::Internal)?
714 .contains_key(&(project_id.to_string(), version))
715 {
716 return Err(RuntimeError::store("runtime release was not found"));
717 }
718 self.active_releases
719 .write()
720 .map_err(|_| RuntimeError::Internal)?
721 .insert(project_id.to_string(), version);
722 Ok(())
723 }
724
725 fn save_local_provider_binding(
726 &self,
727 project_id: &str,
728 binding: &LocalProviderBinding,
729 ) -> Result<(), RuntimeError> {
730 self.provider_bindings
731 .write()
732 .map_err(|_| RuntimeError::Internal)?
733 .insert(
734 (project_id.to_string(), binding.provider_id.clone()),
735 binding.clone(),
736 );
737 Ok(())
738 }
739
740 fn local_provider_bindings(
741 &self,
742 project_id: &str,
743 ) -> Result<Vec<LocalProviderBinding>, RuntimeError> {
744 let mut bindings = self
745 .provider_bindings
746 .read()
747 .map_err(|_| RuntimeError::Internal)?
748 .iter()
749 .filter(|((stored_project_id, _), _)| stored_project_id == project_id)
750 .map(|(_, binding)| binding.clone())
751 .collect::<Vec<_>>();
752 bindings.sort_by(|left, right| left.provider_id.cmp(&right.provider_id));
753 Ok(bindings)
754 }
755
756 fn enqueue_trace(&self, trace: &RuntimeTraceRecord) -> Result<(), RuntimeError> {
757 const MAX_MEMORY_TRACES: usize = 1_000;
758 let mut traces = self
759 .trace_outbox
760 .write()
761 .map_err(|_| RuntimeError::Internal)?;
762 if traces.iter().any(|stored| stored.id == trace.id) {
763 return Ok(());
764 }
765 traces.push_back(trace.clone());
766 while traces.len() > MAX_MEMORY_TRACES {
767 traces.pop_front();
768 }
769 Ok(())
770 }
771
772 fn pending_traces(&self, limit: usize) -> Result<Vec<RuntimeTraceRecord>, RuntimeError> {
773 Ok(self
774 .trace_outbox
775 .read()
776 .map_err(|_| RuntimeError::Internal)?
777 .iter()
778 .take(limit)
779 .cloned()
780 .collect())
781 }
782
783 fn acknowledge_traces(&self, trace_ids: &[String]) -> Result<(), RuntimeError> {
784 self.trace_outbox
785 .write()
786 .map_err(|_| RuntimeError::Internal)?
787 .retain(|trace| !trace_ids.contains(&trace.id));
788 Ok(())
789 }
790}
791
792#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
794#[serde(rename_all = "camelCase")]
795pub struct InvocationTraceEvent {
796 pub name: String,
797 pub status: String,
798 pub duration_ms: u64,
799 #[serde(default)]
800 pub attributes: Value,
801}
802
803#[derive(Clone, PartialEq, Serialize, Deserialize)]
805#[serde(rename_all = "camelCase")]
806pub struct InvocationOutput {
807 pub invocation_id: String,
808 pub project_id: String,
809 pub endpoint: String,
810 pub session_id: String,
811 pub agent: String,
812 pub provider: String,
813 pub capability: String,
814 pub data: InvocationData,
815 #[serde(default)]
816 pub metadata: Value,
817 pub snapshot: RuntimeSnapshot,
818 pub trace: Vec<InvocationTraceEvent>,
819}
820
821impl fmt::Debug for InvocationOutput {
822 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
823 formatter
824 .debug_struct("InvocationOutput")
825 .field("invocation_id", &self.invocation_id)
826 .field("project_id", &self.project_id)
827 .field("endpoint", &self.endpoint)
828 .field("session_id", &self.session_id)
829 .field("agent", &self.agent)
830 .field("provider", &self.provider)
831 .field("capability", &self.capability)
832 .field("data", &"[REDACTED]")
833 .field("metadata", &"[REDACTED]")
834 .field("snapshot_revision", &self.snapshot.revision)
835 .field("trace_count", &self.trace.len())
836 .finish()
837 }
838}
839
840#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
842pub struct InvocationHandle(pub String);
843
844#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
846#[serde(rename_all = "camelCase")]
847pub enum InvocationStatus {
848 Pending,
849 Running,
850 Completed,
851 Failed,
852 Cancelled,
853}
854
855#[derive(Clone, PartialEq, Serialize, Deserialize)]
857#[serde(rename_all = "camelCase")]
858pub struct InvocationPoll {
859 pub handle: InvocationHandle,
860 pub status: InvocationStatus,
861 #[serde(default, skip_serializing_if = "Option::is_none")]
862 pub output: Option<InvocationOutput>,
863 #[serde(default, skip_serializing_if = "Option::is_none")]
864 pub error: Option<String>,
865}
866
867impl fmt::Debug for InvocationPoll {
868 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
869 formatter
870 .debug_struct("InvocationPoll")
871 .field("handle", &self.handle)
872 .field("status", &self.status)
873 .field("output", &self.output.as_ref().map(|_| "[REDACTED]"))
874 .field("error", &self.error.as_ref().map(|_| "[REDACTED]"))
875 .finish()
876 }
877}
878
879#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
881#[serde(rename_all = "camelCase")]
882pub struct EffectExecution {
883 pub results: Vec<EffectResult>,
884 pub unhandled: Vec<EffectRequest>,
885}
886
887pub enum RuntimeError {
889 InvalidDefinition(String),
890 EndpointNotFound(String),
891 AgentNotFound(String),
892 ProviderNotFound(String),
893 CapabilityUnavailable {
894 provider: String,
895 capability: String,
896 },
897 Timeout(u64),
898 Cancelled,
899 Unavailable(String),
900 Backpressure(String),
901 Provider {
902 provider: String,
903 message: String,
904 },
905 Store(String),
906 Snapshot(String),
907 EffectLimitExceeded(usize),
908 InvocationNotFound(String),
909 Internal,
910}
911
912impl RuntimeError {
913 pub fn provider(provider: impl Into<String>, message: impl Into<String>) -> Self {
914 Self::Provider {
915 provider: provider.into(),
916 message: message.into(),
917 }
918 }
919
920 pub fn store(message: impl Into<String>) -> Self {
921 Self::Store(message.into())
922 }
923
924 pub fn public_message(&self) -> String {
925 match self {
926 Self::Provider { provider, .. } => {
927 format!("provider {provider} request failed")
928 }
929 Self::Store(_) => "runtime state could not be persisted".to_string(),
930 Self::Snapshot(_) => "runtime snapshot is invalid".to_string(),
931 Self::Unavailable(_) => "provider is not available".to_string(),
932 Self::Backpressure(_) => "runtime is busy".to_string(),
933 _ => self.to_string(),
934 }
935 }
936}
937
938impl fmt::Debug for RuntimeError {
939 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
940 match self {
941 Self::InvalidDefinition(_) => formatter.write_str("InvalidDefinition([REDACTED])"),
942 Self::EndpointNotFound(endpoint) => formatter
943 .debug_tuple("EndpointNotFound")
944 .field(endpoint)
945 .finish(),
946 Self::AgentNotFound(agent) => {
947 formatter.debug_tuple("AgentNotFound").field(agent).finish()
948 }
949 Self::ProviderNotFound(provider) => formatter
950 .debug_tuple("ProviderNotFound")
951 .field(provider)
952 .finish(),
953 Self::CapabilityUnavailable {
954 provider,
955 capability,
956 } => formatter
957 .debug_struct("CapabilityUnavailable")
958 .field("provider", provider)
959 .field("capability", capability)
960 .finish(),
961 Self::Timeout(timeout) => formatter.debug_tuple("Timeout").field(timeout).finish(),
962 Self::Cancelled => formatter.write_str("Cancelled"),
963 Self::Unavailable(_) => formatter.write_str("Unavailable([REDACTED])"),
964 Self::Backpressure(_) => formatter.write_str("Backpressure([REDACTED])"),
965 Self::Provider { provider, .. } => formatter
966 .debug_struct("Provider")
967 .field("provider", provider)
968 .field("message", &"[REDACTED]")
969 .finish(),
970 Self::Store(_) => formatter.write_str("Store([REDACTED])"),
971 Self::Snapshot(_) => formatter.write_str("Snapshot([REDACTED])"),
972 Self::EffectLimitExceeded(limit) => formatter
973 .debug_tuple("EffectLimitExceeded")
974 .field(limit)
975 .finish(),
976 Self::InvocationNotFound(handle) => formatter
977 .debug_tuple("InvocationNotFound")
978 .field(handle)
979 .finish(),
980 Self::Internal => formatter.write_str("Internal"),
981 }
982 }
983}
984
985impl fmt::Display for RuntimeError {
986 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
987 match self {
988 Self::InvalidDefinition(message) => {
989 write!(formatter, "invalid runtime definition: {message}")
990 }
991 Self::EndpointNotFound(endpoint) => {
992 write!(formatter, "endpoint {endpoint} is not registered")
993 }
994 Self::AgentNotFound(agent) => write!(formatter, "agent {agent} is not registered"),
995 Self::ProviderNotFound(provider) => {
996 write!(formatter, "provider {provider} is not registered")
997 }
998 Self::CapabilityUnavailable {
999 provider,
1000 capability,
1001 } => write!(
1002 formatter,
1003 "provider {provider} does not support capability {capability}"
1004 ),
1005 Self::Timeout(timeout_ms) => {
1006 write!(formatter, "agent invocation was idle for {timeout_ms} ms")
1007 }
1008 Self::Cancelled => formatter.write_str("agent invocation was cancelled"),
1009 Self::Unavailable(message) => {
1010 write!(formatter, "provider is not available: {message}")
1011 }
1012 Self::Backpressure(message) => write!(formatter, "runtime is busy: {message}"),
1013 Self::Provider { provider, message } => {
1014 write!(formatter, "provider {provider} failed: {message}")
1015 }
1016 Self::Store(message) => write!(formatter, "runtime store failed: {message}"),
1017 Self::Snapshot(message) => write!(formatter, "runtime snapshot failed: {message}"),
1018 Self::EffectLimitExceeded(limit) => {
1019 write!(formatter, "runtime effect limit {limit} was exceeded")
1020 }
1021 Self::InvocationNotFound(handle) => {
1022 write!(formatter, "invocation {handle} was not found")
1023 }
1024 Self::Internal => formatter.write_str("runtime internal error"),
1025 }
1026 }
1027}
1028
1029impl std::error::Error for RuntimeError {}
1030
1031#[derive(Default)]
1032struct RuntimeRegistry {
1033 providers: HashMap<String, Arc<dyn AgentProvider>>,
1034 agents: HashMap<String, AgentDefinition>,
1035 endpoints: HashMap<String, EndpointDefinition>,
1036}
1037
1038struct RuntimeCore {
1039 project_id: String,
1040 registry: RwLock<RuntimeRegistry>,
1041 manifest: RwLock<Option<RuntimeManifest>>,
1042 store: Arc<dyn RuntimeStore>,
1043 sessions: RwLock<HashMap<String, RuntimeSnapshot>>,
1044 session_locks: Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>,
1045 invocations: Mutex<InvocationRegistry>,
1046 next_invocation: AtomicU64,
1047 monitor_observer: RwLock<Option<RuntimeMonitorObserver>>,
1048 monitor_io_observer: RwLock<Option<RuntimeMonitorIoObserver>>,
1049}
1050
1051struct InvocationEntry {
1052 poll: InvocationPoll,
1053 cancellation: CancellationToken,
1054 events: VecDeque<InvocationEvent>,
1055 next_event_sequence: u64,
1056}
1057
1058#[derive(Default)]
1059struct InvocationRegistry {
1060 entries: HashMap<String, InvocationEntry>,
1061 terminal_order: VecDeque<String>,
1062 active_count: usize,
1063}
1064
1065impl InvocationRegistry {
1066 #[cfg(not(target_arch = "wasm32"))]
1067 fn insert(
1068 &mut self,
1069 handle: InvocationHandle,
1070 cancellation: CancellationToken,
1071 ) -> Result<(), RuntimeError> {
1072 if self.active_count >= MAX_IN_FLIGHT_INVOCATIONS {
1073 return Err(RuntimeError::Backpressure(
1074 "too many invocations are already running".to_string(),
1075 ));
1076 }
1077 self.entries.insert(
1078 handle.0.clone(),
1079 InvocationEntry {
1080 poll: InvocationPoll {
1081 handle,
1082 status: InvocationStatus::Pending,
1083 output: None,
1084 error: None,
1085 },
1086 cancellation,
1087 events: VecDeque::new(),
1088 next_event_sequence: 1,
1089 },
1090 );
1091 self.active_count += 1;
1092 Ok(())
1093 }
1094
1095 fn update(
1096 &mut self,
1097 handle: &InvocationHandle,
1098 status: InvocationStatus,
1099 output: Option<InvocationOutput>,
1100 error: Option<String>,
1101 ) {
1102 let Some(entry) = self.entries.get_mut(&handle.0) else {
1103 return;
1104 };
1105 if is_terminal_status(entry.poll.status) {
1106 return;
1107 }
1108 entry.poll.status = status;
1109 entry.poll.output = output;
1110 entry.poll.error = error;
1111 let event = match status {
1112 InvocationStatus::Pending => None,
1113 InvocationStatus::Running => Some((InvocationEventKind::Started, None, None)),
1114 InvocationStatus::Completed => Some((
1115 InvocationEventKind::Completed,
1116 entry.poll.output.as_ref().map(|value| value.data.clone()),
1117 None,
1118 )),
1119 InvocationStatus::Failed => {
1120 Some((InvocationEventKind::Failed, None, entry.poll.error.clone()))
1121 }
1122 InvocationStatus::Cancelled => Some((InvocationEventKind::Cancelled, None, None)),
1123 };
1124 if let Some((kind, data, error)) = event {
1125 entry.push_event(kind, data, error);
1126 }
1127 if is_terminal_status(status) {
1128 self.active_count = self.active_count.saturating_sub(1);
1129 self.terminal_order.push_back(handle.0.clone());
1130 self.evict_old_terminal_entries();
1131 }
1132 }
1133
1134 fn remove(&mut self, handle: &InvocationHandle) {
1135 if let Some(entry) = self.entries.remove(&handle.0) {
1136 if !is_terminal_status(entry.poll.status) {
1137 self.active_count = self.active_count.saturating_sub(1);
1138 }
1139 }
1140 self.terminal_order.retain(|stored| stored != &handle.0);
1141 }
1142
1143 fn take(&mut self, handle: &InvocationHandle) -> Result<InvocationPoll, RuntimeError> {
1144 let poll = self
1145 .entries
1146 .get(&handle.0)
1147 .map(|entry| entry.poll.clone())
1148 .ok_or_else(|| RuntimeError::InvocationNotFound(handle.0.clone()))?;
1149 if is_terminal_status(poll.status) {
1150 self.remove(handle);
1151 }
1152 Ok(poll)
1153 }
1154
1155 fn push_provider_event(&mut self, handle: &InvocationHandle, event: ProviderEvent) {
1156 let Some(entry) = self.entries.get_mut(&handle.0) else {
1157 return;
1158 };
1159 if entry.poll.status != InvocationStatus::Running {
1160 return;
1161 }
1162 match event {
1163 ProviderEvent::Activity => {}
1164 ProviderEvent::OutputDelta { data } => {
1165 entry.push_event(InvocationEventKind::OutputDelta, Some(data), None);
1166 }
1167 ProviderEvent::StageStarted { .. }
1168 | ProviderEvent::StageCompleted { .. }
1169 | ProviderEvent::StageFailed { .. } => {}
1170 }
1171 }
1172
1173 fn drain_events(
1174 &mut self,
1175 handle: &InvocationHandle,
1176 ) -> Result<Vec<InvocationEvent>, RuntimeError> {
1177 let entry = self
1178 .entries
1179 .get_mut(&handle.0)
1180 .ok_or_else(|| RuntimeError::InvocationNotFound(handle.0.clone()))?;
1181 Ok(entry.events.drain(..).collect())
1182 }
1183
1184 fn evict_old_terminal_entries(&mut self) {
1185 while self.terminal_order.len() > MAX_RETAINED_INVOCATIONS {
1186 if let Some(handle) = self.terminal_order.pop_front() {
1187 self.entries.remove(&handle);
1188 }
1189 }
1190 }
1191}
1192
1193impl InvocationEntry {
1194 fn push_event(
1195 &mut self,
1196 kind: InvocationEventKind,
1197 data: Option<InvocationData>,
1198 error: Option<String>,
1199 ) {
1200 if kind == InvocationEventKind::OutputDelta {
1201 if let (
1202 Some(InvocationEvent {
1203 kind: InvocationEventKind::OutputDelta,
1204 data: Some(previous),
1205 ..
1206 }),
1207 Some(next),
1208 ) = (self.events.back_mut(), data.as_ref())
1209 {
1210 if merge_invocation_data(previous, next) {
1211 return;
1212 }
1213 }
1214 }
1215 self.events.push_back(InvocationEvent {
1216 sequence: self.next_event_sequence,
1217 kind,
1218 data,
1219 error,
1220 });
1221 self.next_event_sequence = self.next_event_sequence.saturating_add(1);
1222 while self.events.len() > MAX_RETAINED_INVOCATION_EVENTS {
1223 self.events.pop_front();
1224 }
1225 }
1226}
1227
1228impl RuntimeCore {
1229 async fn invoke(
1230 self: &Arc<Self>,
1231 invocation_id: String,
1232 input: InvocationInput,
1233 cancellation: CancellationToken,
1234 forwarded_events: ProviderEventSink,
1235 ) -> Result<InvocationOutput, RuntimeError> {
1236 let endpoint = input.endpoint.clone();
1237 let created_at_ms = crate::unix_time_ms();
1238 let trace_id = format!("trace-{created_at_ms}-{invocation_id}");
1239 let started = Instant::now();
1240 let result = self
1241 .invoke_provider(
1242 trace_id.clone(),
1243 created_at_ms,
1244 invocation_id.clone(),
1245 input,
1246 cancellation,
1247 forwarded_events,
1248 )
1249 .await;
1250 let elapsed_ms = duration_ms(started.elapsed());
1251 let trace = match &result {
1252 Ok(output) => RuntimeTraceRecord {
1253 id: trace_id.clone(),
1254 project_id: self.project_id.clone(),
1255 invocation_id: invocation_id.clone(),
1256 endpoint: endpoint.clone(),
1257 agent: Some(output.agent.clone()),
1258 provider: Some(output.provider.clone()),
1259 capability: Some(output.capability.clone()),
1260 status: "completed".to_string(),
1261 duration_ms: elapsed_ms,
1262 created_at_ms,
1263 },
1264 Err(error) => RuntimeTraceRecord {
1265 id: trace_id.clone(),
1266 project_id: self.project_id.clone(),
1267 invocation_id: invocation_id.clone(),
1268 endpoint,
1269 agent: None,
1270 provider: None,
1271 capability: None,
1272 status: match error {
1273 RuntimeError::Cancelled => "cancelled",
1274 _ => "error",
1275 }
1276 .to_string(),
1277 duration_ms: elapsed_ms,
1278 created_at_ms,
1279 },
1280 };
1281 let _ = self.store.enqueue_trace(&trace);
1282 if let Ok(output) = &result {
1283 self.emit_monitor_io_event(RuntimeMonitorIoEvent::InvocationOutput {
1284 trace_id: trace_id.clone(),
1285 invocation_id: invocation_id.clone(),
1286 summary: runtime_monitor_io_summary(&output.data),
1287 });
1288 }
1289 self.emit_monitor_event(RuntimeMonitorEvent::InvocationFinished {
1290 trace_id,
1291 invocation_id,
1292 status: match &result {
1293 Ok(_) => RuntimeMonitorStatus::Completed,
1294 Err(RuntimeError::Cancelled) => RuntimeMonitorStatus::Cancelled,
1295 Err(_) => RuntimeMonitorStatus::Error,
1296 },
1297 duration_ms: elapsed_ms,
1298 ended_at_ms: created_at_ms.saturating_add(elapsed_ms),
1302 error: result.as_ref().err().map(RuntimeError::public_message),
1303 });
1304 result
1305 }
1306
1307 async fn invoke_provider(
1308 self: &Arc<Self>,
1309 trace_id: String,
1310 started_at_ms: u64,
1311 invocation_id: String,
1312 input: InvocationInput,
1313 cancellation: CancellationToken,
1314 forwarded_events: ProviderEventSink,
1315 ) -> Result<InvocationOutput, RuntimeError> {
1316 validate_identifier("endpoint", &input.endpoint)?;
1317 validate_identifier("session", &input.session_id)?;
1318 let session_lock = {
1319 let mut locks = self
1320 .session_locks
1321 .lock()
1322 .map_err(|_| RuntimeError::Internal)?;
1323 Arc::clone(
1324 locks
1325 .entry(input.session_id.clone())
1326 .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(()))),
1327 )
1328 };
1329 let _session_guard = session_lock.lock().await;
1330 let (endpoint, agent, provider) = {
1331 let registry = self.registry.read().map_err(|_| RuntimeError::Internal)?;
1332 let endpoint = registry
1333 .endpoints
1334 .get(&input.endpoint)
1335 .cloned()
1336 .ok_or_else(|| RuntimeError::EndpointNotFound(input.endpoint.clone()))?;
1337 let agent = registry
1338 .agents
1339 .get(&endpoint.agent)
1340 .cloned()
1341 .ok_or_else(|| RuntimeError::AgentNotFound(endpoint.agent.clone()))?;
1342 let provider = registry
1343 .providers
1344 .get(&agent.provider)
1345 .cloned()
1346 .ok_or_else(|| RuntimeError::ProviderNotFound(agent.provider.clone()))?;
1347 (endpoint, agent, provider)
1348 };
1349 if !agent
1350 .capabilities
1351 .iter()
1352 .any(|capability| capability == &endpoint.capability)
1353 || !provider.supports(&endpoint.capability)
1354 {
1355 return Err(RuntimeError::CapabilityUnavailable {
1356 provider: agent.provider.clone(),
1357 capability: endpoint.capability,
1358 });
1359 }
1360 if cancellation.is_cancelled() {
1361 return Err(RuntimeError::Cancelled);
1362 }
1363
1364 self.emit_monitor_event(RuntimeMonitorEvent::InvocationStarted {
1365 trace_id: trace_id.clone(),
1366 invocation_id: invocation_id.clone(),
1367 project_id: self.project_id.clone(),
1368 endpoint: endpoint.name.clone(),
1369 agent_id: agent.id.clone(),
1370 provider_id: agent.provider.clone(),
1371 capability: endpoint.capability.clone(),
1372 started_at_ms,
1373 });
1374 self.emit_monitor_io_event(RuntimeMonitorIoEvent::InvocationInput {
1375 trace_id: trace_id.clone(),
1376 invocation_id: invocation_id.clone(),
1377 summary: runtime_monitor_io_summary(&input.data),
1378 });
1379
1380 let snapshot = self.load_snapshot(&input.session_id)?;
1381 let request = ProviderRequest {
1382 project_id: self.project_id.clone(),
1383 endpoint: endpoint.name.clone(),
1384 session_id: input.session_id.clone(),
1385 agent: agent.clone(),
1386 capability: endpoint.capability.clone(),
1387 data: input.data,
1388 metadata: input.metadata,
1389 snapshot: snapshot.clone(),
1390 };
1391 let started = Instant::now();
1392 let (activity_sender, mut activity_receiver) = watch::channel(0_u64);
1393 let provider_trace = Arc::new(Mutex::new(Vec::new()));
1394 let events = self.provider_event_sink(
1395 &InvocationHandle(invocation_id.clone()),
1396 trace_id,
1397 started,
1398 activity_sender,
1399 forwarded_events,
1400 Arc::clone(&provider_trace),
1401 );
1402 let provider_call = provider.invoke_with_events(request, cancellation.clone(), events);
1403 tokio::pin!(provider_call);
1404 let idle_timeout = Duration::from_millis(endpoint.timeout_ms);
1405 let mut activity_open = true;
1406 let response = loop {
1407 let idle_deadline = runtime_sleep(idle_timeout);
1408 tokio::pin!(idle_deadline);
1409 tokio::select! {
1410 biased;
1411 _ = cancellation.cancelled() => return Err(RuntimeError::Cancelled),
1412 response = &mut provider_call => break response?,
1413 changed = activity_receiver.changed(), if activity_open => {
1414 if changed.is_err() {
1415 activity_open = false;
1416 }
1417 }
1418 _ = &mut idle_deadline => {
1419 cancellation.cancel();
1420 return Err(RuntimeError::Timeout(endpoint.timeout_ms));
1421 }
1422 }
1423 };
1424 if cancellation.is_cancelled() {
1425 return Err(RuntimeError::Cancelled);
1426 }
1427
1428 let next_snapshot = RuntimeSnapshot {
1429 revision: snapshot.revision.saturating_add(1),
1430 state: response.state.unwrap_or(snapshot.state),
1431 };
1432 self.store
1433 .save(&self.project_id, &input.session_id, &next_snapshot)?;
1434 self.sessions
1435 .write()
1436 .map_err(|_| RuntimeError::Internal)?
1437 .insert(input.session_id.clone(), next_snapshot.clone());
1438 let mut trace = provider_trace
1439 .lock()
1440 .map_err(|_| RuntimeError::Internal)?
1441 .clone();
1442 trace.push(InvocationTraceEvent {
1443 name: "provider.invoke".to_string(),
1444 status: "completed".to_string(),
1445 duration_ms: duration_ms(started.elapsed()),
1446 attributes: json!({
1447 "endpoint": input.endpoint,
1448 }),
1449 });
1450 Ok(InvocationOutput {
1451 invocation_id,
1452 project_id: self.project_id.clone(),
1453 endpoint: endpoint.name,
1454 session_id: input.session_id,
1455 agent: agent.id,
1456 provider: agent.provider,
1457 capability: endpoint.capability,
1458 data: response.data,
1459 metadata: response.metadata,
1460 snapshot: next_snapshot,
1461 trace,
1462 })
1463 }
1464
1465 fn load_snapshot(&self, session_id: &str) -> Result<RuntimeSnapshot, RuntimeError> {
1466 if let Some(snapshot) = self
1467 .sessions
1468 .read()
1469 .map_err(|_| RuntimeError::Internal)?
1470 .get(session_id)
1471 .cloned()
1472 {
1473 return Ok(snapshot);
1474 }
1475 let snapshot = self
1476 .store
1477 .load(&self.project_id, session_id)?
1478 .unwrap_or_default();
1479 self.sessions
1480 .write()
1481 .map_err(|_| RuntimeError::Internal)?
1482 .insert(session_id.to_string(), snapshot.clone());
1483 Ok(snapshot)
1484 }
1485
1486 fn next_invocation_id(&self) -> String {
1487 let id = self.next_invocation.fetch_add(1, Ordering::Relaxed);
1488 format!("invocation-{id}")
1489 }
1490
1491 fn update_poll(
1492 &self,
1493 handle: &InvocationHandle,
1494 status: InvocationStatus,
1495 output: Option<InvocationOutput>,
1496 error: Option<String>,
1497 ) {
1498 if let Ok(mut invocations) = self.invocations.lock() {
1499 invocations.update(handle, status, output, error);
1500 }
1501 }
1502
1503 fn provider_event_sink(
1504 self: &Arc<Self>,
1505 handle: &InvocationHandle,
1506 trace_id: String,
1507 request_started: Instant,
1508 activity: watch::Sender<u64>,
1509 forwarded_events: ProviderEventSink,
1510 provider_trace: Arc<Mutex<Vec<InvocationTraceEvent>>>,
1511 ) -> ProviderEventSink {
1512 let core = Arc::clone(self);
1513 let handle = handle.clone();
1514 ProviderEventSink::new(move |event| {
1515 activity.send_modify(|sequence| *sequence = sequence.saturating_add(1));
1516 (forwarded_events.emit)(event.clone());
1517 if let Ok(mut invocations) = core.invocations.lock() {
1518 invocations.push_provider_event(&handle, event.clone());
1519 }
1520 let trace_event = match &event {
1521 ProviderEvent::StageCompleted {
1522 stage,
1523 elapsed_ms,
1524 metadata,
1525 } => Some(InvocationTraceEvent {
1526 name: provider_stage_name(*stage).to_string(),
1527 status: "completed".to_string(),
1528 duration_ms: *elapsed_ms,
1529 attributes: metadata.clone(),
1530 }),
1531 ProviderEvent::StageFailed {
1532 stage,
1533 elapsed_ms,
1534 metadata,
1535 ..
1536 } => Some(InvocationTraceEvent {
1537 name: provider_stage_name(*stage).to_string(),
1538 status: "failed".to_string(),
1539 duration_ms: *elapsed_ms,
1540 attributes: metadata.clone(),
1541 }),
1542 ProviderEvent::Activity
1543 | ProviderEvent::OutputDelta { .. }
1544 | ProviderEvent::StageStarted { .. } => None,
1545 };
1546 if let Some(trace_event) = trace_event {
1547 if let Ok(mut trace) = provider_trace.lock() {
1548 trace.push(trace_event);
1549 }
1550 }
1551 let (stage, status, elapsed_ms, metadata, error) = match event {
1552 ProviderEvent::Activity => return,
1553 ProviderEvent::OutputDelta { .. } => return,
1554 ProviderEvent::StageStarted { stage, metadata } => (
1555 stage,
1556 RuntimeMonitorStageStatus::Started,
1557 None,
1558 metadata,
1559 None,
1560 ),
1561 ProviderEvent::StageCompleted {
1562 stage,
1563 elapsed_ms,
1564 metadata,
1565 } => (
1566 stage,
1567 RuntimeMonitorStageStatus::Completed,
1568 Some(elapsed_ms),
1569 metadata,
1570 None,
1571 ),
1572 ProviderEvent::StageFailed {
1573 stage,
1574 elapsed_ms,
1575 error,
1576 metadata,
1577 } => (
1578 stage,
1579 RuntimeMonitorStageStatus::Failed,
1580 Some(elapsed_ms),
1581 metadata,
1582 Some(error),
1583 ),
1584 };
1585 core.emit_monitor_event(RuntimeMonitorEvent::ProviderStage {
1586 trace_id: trace_id.clone(),
1587 invocation_id: handle.0.clone(),
1588 stage,
1589 status,
1590 elapsed_ms,
1591 request_elapsed_ms: duration_ms(request_started.elapsed()),
1592 input_tokens: monitor_u64(&metadata, "inputTokens"),
1593 output_tokens: monitor_u64(&metadata, "outputTokens"),
1594 resident: metadata.get("resident").and_then(Value::as_bool),
1595 error,
1596 });
1597 })
1598 }
1599
1600 fn emit_monitor_event(&self, event: RuntimeMonitorEvent) {
1601 let observer = self
1602 .monitor_observer
1603 .read()
1604 .ok()
1605 .and_then(|observer| observer.clone());
1606 if let Some(observer) = observer {
1607 observer(event);
1608 }
1609 }
1610
1611 fn emit_monitor_io_event(&self, event: RuntimeMonitorIoEvent) {
1612 let observer = self
1613 .monitor_io_observer
1614 .read()
1615 .ok()
1616 .and_then(|observer| observer.clone());
1617 if let Some(observer) = observer {
1618 observer(event);
1619 }
1620 }
1621}
1622
1623fn provider_stage_name(stage: ProviderStage) -> &'static str {
1624 match stage {
1625 ProviderStage::Queue => "queue",
1626 ProviderStage::Load => "load",
1627 ProviderStage::Tokenize => "tokenize",
1628 ProviderStage::Prefill => "prefill",
1629 ProviderStage::FirstToken => "first_token",
1630 ProviderStage::Decode => "decode",
1631 ProviderStage::Validate => "validate",
1632 }
1633}
1634
1635#[cfg(not(target_arch = "wasm32"))]
1636async fn runtime_sleep(duration: Duration) {
1637 tokio::time::sleep(duration).await;
1638}
1639
1640#[cfg(target_arch = "wasm32")]
1641async fn runtime_sleep(duration: Duration) {
1642 use wasm_bindgen::JsCast;
1643
1644 let milliseconds = duration.as_millis().min(i32::MAX as u128) as i32;
1645 let promise = js_sys::Promise::new(&mut |resolve, _reject| {
1646 let global = js_sys::global();
1647 let set_timeout = js_sys::Reflect::get(&global, &"setTimeout".into())
1648 .ok()
1649 .and_then(|value| value.dyn_into::<js_sys::Function>().ok());
1650 if let Some(set_timeout) = set_timeout {
1651 if let Ok(handle) = set_timeout.call2(&global, &resolve, &milliseconds.into()) {
1652 let unref = js_sys::Reflect::get(&handle, &"unref".into())
1653 .ok()
1654 .and_then(|value| value.dyn_into::<js_sys::Function>().ok());
1655 if let Some(unref) = unref {
1656 let _ = unref.call0(&handle);
1657 }
1658 }
1659 } else {
1660 let _ = resolve.call0(&wasm_bindgen::JsValue::UNDEFINED);
1661 }
1662 });
1663 let _ = wasm_bindgen_futures::JsFuture::from(promise).await;
1664}
1665
1666fn runtime_monitor_io_summary(data: &InvocationData) -> RuntimeMonitorIoSummary {
1667 match data {
1668 InvocationData::Binary(bytes) => RuntimeMonitorIoSummary {
1669 value: json!({
1670 "_vifuBinary": true,
1671 "bytes": bytes.len(),
1672 }),
1673 truncated: true,
1674 },
1675 InvocationData::Json(value)
1676 if serde_json::to_vec(value)
1677 .is_ok_and(|encoded| encoded.len() <= MAX_RUNTIME_MONITOR_IO_BYTES) =>
1678 {
1679 RuntimeMonitorIoSummary {
1680 value: value.clone(),
1681 truncated: false,
1682 }
1683 }
1684 InvocationData::Json(value) => RuntimeMonitorIoSummary {
1685 value: json!({
1686 "summary": monitor_value_shape(value),
1687 "truncated": true,
1688 }),
1689 truncated: true,
1690 },
1691 }
1692}
1693
1694fn monitor_value_shape(value: &Value) -> &'static str {
1695 match value {
1696 Value::Null => "null",
1697 Value::Bool(_) => "boolean",
1698 Value::Number(_) => "number",
1699 Value::String(_) => "string",
1700 Value::Array(_) => "array",
1701 Value::Object(_) => "object",
1702 }
1703}
1704
1705fn monitor_u64(metadata: &Value, key: &str) -> Option<u64> {
1706 metadata.get(key).and_then(Value::as_u64)
1707}
1708
1709fn is_null(value: &Value) -> bool {
1710 value.is_null()
1711}
1712
1713fn merge_invocation_data(previous: &mut InvocationData, next: &InvocationData) -> bool {
1714 match (previous, next) {
1715 (
1716 InvocationData::Json(Value::String(previous)),
1717 InvocationData::Json(Value::String(next)),
1718 ) if previous.len().saturating_add(next.len()) <= MAX_COALESCED_EVENT_BYTES => {
1719 previous.push_str(next);
1720 true
1721 }
1722 (InvocationData::Binary(previous), InvocationData::Binary(next))
1723 if previous.len().saturating_add(next.len()) <= MAX_COALESCED_EVENT_BYTES =>
1724 {
1725 previous.extend_from_slice(next);
1726 true
1727 }
1728 _ => false,
1729 }
1730}
1731
1732#[cfg(not(target_arch = "wasm32"))]
1733enum WorkerCommand {
1734 Start {
1735 handle: InvocationHandle,
1736 input: InvocationInput,
1737 cancellation: CancellationToken,
1738 },
1739}
1740
1741#[cfg(not(target_arch = "wasm32"))]
1742struct RuntimeWorker {
1743 sender: Mutex<Option<mpsc::Sender<WorkerCommand>>>,
1744 thread: Mutex<Option<JoinHandle<()>>>,
1745}
1746
1747#[cfg(not(target_arch = "wasm32"))]
1748impl RuntimeWorker {
1749 fn spawn(core: Arc<RuntimeCore>) -> Result<Self, RuntimeError> {
1750 let (sender, mut receiver) = mpsc::channel(WORKER_QUEUE_CAPACITY);
1751 let thread = std::thread::Builder::new()
1752 .name(format!("vifu-runtime-{}", core.project_id))
1753 .spawn(move || {
1754 let Ok(runtime) = tokio::runtime::Builder::new_current_thread()
1755 .enable_time()
1756 .build()
1757 else {
1758 return;
1759 };
1760 runtime.block_on(async move {
1761 while let Some(command) = receiver.recv().await {
1762 match command {
1763 WorkerCommand::Start {
1764 handle,
1765 input,
1766 cancellation,
1767 } => {
1768 let invocation_core = Arc::clone(&core);
1769 tokio::spawn(async move {
1770 invocation_core.update_poll(
1771 &handle,
1772 InvocationStatus::Running,
1773 None,
1774 None,
1775 );
1776 let result = invocation_core
1777 .invoke(
1778 handle.0.clone(),
1779 input,
1780 cancellation,
1781 ProviderEventSink::discard(),
1782 )
1783 .await;
1784 match result {
1785 Ok(output) => invocation_core.update_poll(
1786 &handle,
1787 InvocationStatus::Completed,
1788 Some(output),
1789 None,
1790 ),
1791 Err(RuntimeError::Cancelled) => invocation_core
1792 .update_poll(
1793 &handle,
1794 InvocationStatus::Cancelled,
1795 None,
1796 None,
1797 ),
1798 Err(error) => invocation_core.update_poll(
1799 &handle,
1800 InvocationStatus::Failed,
1801 None,
1802 Some(error.public_message()),
1803 ),
1804 }
1805 });
1806 }
1807 }
1808 }
1809 });
1810 })
1811 .map_err(|_error| RuntimeError::Internal)?;
1812 Ok(Self {
1813 sender: Mutex::new(Some(sender)),
1814 thread: Mutex::new(Some(thread)),
1815 })
1816 }
1817
1818 fn send(&self, command: WorkerCommand) -> Result<(), RuntimeError> {
1819 self.sender
1820 .lock()
1821 .map_err(|_| RuntimeError::Internal)?
1822 .as_ref()
1823 .ok_or(RuntimeError::Internal)?
1824 .try_send(command)
1825 .map_err(|error| match error {
1826 mpsc::error::TrySendError::Full(_) => {
1827 RuntimeError::Backpressure("invocation queue is full".to_string())
1828 }
1829 mpsc::error::TrySendError::Closed(_) => RuntimeError::Internal,
1830 })
1831 }
1832}
1833
1834#[cfg(not(target_arch = "wasm32"))]
1835impl Drop for RuntimeWorker {
1836 fn drop(&mut self) {
1837 if let Ok(sender) = self.sender.get_mut() {
1838 sender.take();
1839 }
1840 if let Ok(thread) = self.thread.get_mut() {
1841 if let Some(thread) = thread.take() {
1842 let _ = thread.join();
1843 }
1844 }
1845 }
1846}
1847
1848#[derive(Clone)]
1854pub struct VifuRuntime {
1855 core: Arc<RuntimeCore>,
1856 #[cfg(not(target_arch = "wasm32"))]
1857 worker: Arc<Mutex<Option<RuntimeWorker>>>,
1858}
1859
1860impl fmt::Debug for VifuRuntime {
1861 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
1862 let counts = self.core.registry.read().ok().map(|registry| {
1863 (
1864 registry.providers.len(),
1865 registry.agents.len(),
1866 registry.endpoints.len(),
1867 )
1868 });
1869 formatter
1870 .debug_struct("VifuRuntime")
1871 .field("project_id", &self.core.project_id)
1872 .field("resource_counts", &counts)
1873 .finish()
1874 }
1875}
1876
1877impl VifuRuntime {
1878 pub fn new(project_id: impl Into<String>) -> Result<Self, RuntimeError> {
1879 Self::with_store(project_id, Arc::new(MemoryRuntimeStore::default()))
1880 }
1881
1882 pub fn with_store(
1883 project_id: impl Into<String>,
1884 store: Arc<dyn RuntimeStore>,
1885 ) -> Result<Self, RuntimeError> {
1886 let project_id = project_id.into();
1887 validate_identifier("project", &project_id)?;
1888 let core = Arc::new(RuntimeCore {
1889 project_id,
1890 registry: RwLock::new(RuntimeRegistry::default()),
1891 manifest: RwLock::new(None),
1892 store,
1893 sessions: RwLock::new(HashMap::new()),
1894 session_locks: Mutex::new(HashMap::new()),
1895 invocations: Mutex::new(InvocationRegistry::default()),
1896 next_invocation: AtomicU64::new(1),
1897 monitor_observer: RwLock::new(None),
1898 monitor_io_observer: RwLock::new(None),
1899 });
1900 #[cfg(not(target_arch = "wasm32"))]
1901 let worker = Arc::new(Mutex::new(None));
1902 Ok(Self {
1903 core,
1904 #[cfg(not(target_arch = "wasm32"))]
1905 worker,
1906 })
1907 }
1908
1909 pub fn project_id(&self) -> &str {
1910 &self.core.project_id
1911 }
1912
1913 pub fn set_monitor_observer(
1915 &self,
1916 observer: Option<RuntimeMonitorObserver>,
1917 ) -> Result<(), RuntimeError> {
1918 *self
1919 .core
1920 .monitor_observer
1921 .write()
1922 .map_err(|_| RuntimeError::Internal)? = observer;
1923 Ok(())
1924 }
1925
1926 pub fn set_monitor_io_observer(
1928 &self,
1929 observer: Option<RuntimeMonitorIoObserver>,
1930 ) -> Result<(), RuntimeError> {
1931 *self
1932 .core
1933 .monitor_io_observer
1934 .write()
1935 .map_err(|_| RuntimeError::Internal)? = observer;
1936 Ok(())
1937 }
1938
1939 pub fn register_provider(
1940 &self,
1941 name: impl Into<String>,
1942 provider: Arc<dyn AgentProvider>,
1943 ) -> Result<(), RuntimeError> {
1944 let name = name.into();
1945 validate_identifier("provider", &name)?;
1946 self.core
1947 .registry
1948 .write()
1949 .map_err(|_| RuntimeError::Internal)?
1950 .providers
1951 .insert(name, provider);
1952 Ok(())
1953 }
1954
1955 pub fn unregister_provider(&self, name: &str) -> Result<bool, RuntimeError> {
1961 validate_identifier("provider", name)?;
1962 Ok(self
1963 .core
1964 .registry
1965 .write()
1966 .map_err(|_| RuntimeError::Internal)?
1967 .providers
1968 .remove(name)
1969 .is_some())
1970 }
1971
1972 pub fn unregister_agent(&self, id: &str) -> Result<bool, RuntimeError> {
1973 validate_identifier("agent", id)?;
1974 Ok(self
1975 .core
1976 .registry
1977 .write()
1978 .map_err(|_| RuntimeError::Internal)?
1979 .agents
1980 .remove(id)
1981 .is_some())
1982 }
1983
1984 pub fn unregister_endpoint(&self, name: &str) -> Result<bool, RuntimeError> {
1985 validate_identifier("endpoint", name)?;
1986 Ok(self
1987 .core
1988 .registry
1989 .write()
1990 .map_err(|_| RuntimeError::Internal)?
1991 .endpoints
1992 .remove(name)
1993 .is_some())
1994 }
1995
1996 pub fn register_agent(&self, mut agent: AgentDefinition) -> Result<(), RuntimeError> {
1997 validate_identifier("agent", &agent.id)?;
1998 validate_identifier("provider", &agent.provider)?;
1999 if agent.name.trim().is_empty() || agent.capabilities.is_empty() {
2000 return Err(RuntimeError::InvalidDefinition(
2001 "agent name and at least one capability are required".to_string(),
2002 ));
2003 }
2004 for capability in &mut agent.capabilities {
2005 *capability = capability.trim().to_ascii_lowercase();
2006 validate_identifier("capability", capability)?;
2007 }
2008 agent.capabilities.sort();
2009 agent.capabilities.dedup();
2010 let mut registry = self
2011 .core
2012 .registry
2013 .write()
2014 .map_err(|_| RuntimeError::Internal)?;
2015 if !registry.providers.contains_key(&agent.provider) {
2016 return Err(RuntimeError::ProviderNotFound(agent.provider));
2017 }
2018 registry.agents.insert(agent.id.clone(), agent);
2019 Ok(())
2020 }
2021
2022 pub fn register_endpoint(&self, mut endpoint: EndpointDefinition) -> Result<(), RuntimeError> {
2023 validate_identifier("endpoint", &endpoint.name)?;
2024 validate_identifier("agent", &endpoint.agent)?;
2025 endpoint.capability = endpoint.capability.trim().to_ascii_lowercase();
2026 validate_identifier("capability", &endpoint.capability)?;
2027 if !(1..=MAX_ENDPOINT_TIMEOUT_MS).contains(&endpoint.timeout_ms) {
2028 return Err(RuntimeError::InvalidDefinition(format!(
2029 "endpoint timeout must be between 1 and {MAX_ENDPOINT_TIMEOUT_MS} ms"
2030 )));
2031 }
2032 let mut registry = self
2033 .core
2034 .registry
2035 .write()
2036 .map_err(|_| RuntimeError::Internal)?;
2037 let agent = registry
2038 .agents
2039 .get(&endpoint.agent)
2040 .ok_or_else(|| RuntimeError::AgentNotFound(endpoint.agent.clone()))?;
2041 if !agent
2042 .capabilities
2043 .iter()
2044 .any(|capability| capability == &endpoint.capability)
2045 {
2046 return Err(RuntimeError::CapabilityUnavailable {
2047 provider: agent.provider.clone(),
2048 capability: endpoint.capability,
2049 });
2050 }
2051 registry.endpoints.insert(endpoint.name.clone(), endpoint);
2052 Ok(())
2053 }
2054
2055 pub fn agent_definitions(&self) -> Result<Vec<AgentDefinition>, RuntimeError> {
2056 let mut agents = self
2057 .core
2058 .registry
2059 .read()
2060 .map_err(|_| RuntimeError::Internal)?
2061 .agents
2062 .values()
2063 .cloned()
2064 .collect::<Vec<_>>();
2065 agents.sort_by(|left, right| left.id.cmp(&right.id));
2066 Ok(agents)
2067 }
2068
2069 pub fn endpoint_definitions(&self) -> Result<Vec<EndpointDefinition>, RuntimeError> {
2070 let mut endpoints = self
2071 .core
2072 .registry
2073 .read()
2074 .map_err(|_| RuntimeError::Internal)?
2075 .endpoints
2076 .values()
2077 .cloned()
2078 .collect::<Vec<_>>();
2079 endpoints.sort_by(|left, right| left.name.cmp(&right.name));
2080 Ok(endpoints)
2081 }
2082
2083 pub fn apply_manifest(&self, manifest: RuntimeManifest) -> Result<(), RuntimeError> {
2086 manifest.validate()?;
2087 if manifest.project_id != self.core.project_id {
2088 return Err(RuntimeError::InvalidDefinition(
2089 "project settings belong to another project".to_string(),
2090 ));
2091 }
2092 let mut registry = self
2093 .core
2094 .registry
2095 .write()
2096 .map_err(|_| RuntimeError::Internal)?;
2097 for requirement in &manifest.providers {
2098 let provider = registry
2099 .providers
2100 .get(&requirement.id)
2101 .ok_or_else(|| RuntimeError::ProviderNotFound(requirement.id.clone()))?;
2102 for capability in &requirement.capabilities {
2103 if !provider.supports(capability) {
2104 return Err(RuntimeError::CapabilityUnavailable {
2105 provider: requirement.id.clone(),
2106 capability: capability.clone(),
2107 });
2108 }
2109 }
2110 }
2111 registry.agents = manifest
2112 .agents
2113 .iter()
2114 .cloned()
2115 .map(|agent| (agent.id.clone(), agent))
2116 .collect();
2117 registry.endpoints = manifest
2118 .endpoints
2119 .iter()
2120 .cloned()
2121 .map(|endpoint| (endpoint.name.clone(), endpoint))
2122 .collect();
2123 *self
2124 .core
2125 .manifest
2126 .write()
2127 .map_err(|_| RuntimeError::Internal)? = Some(manifest);
2128 Ok(())
2129 }
2130
2131 pub fn apply_project_settings(&self, settings: ProjectSettings) -> Result<(), RuntimeError> {
2132 self.apply_manifest(settings)
2133 }
2134
2135 pub fn current_manifest(&self) -> Result<Option<RuntimeManifest>, RuntimeError> {
2136 Ok(self
2137 .core
2138 .manifest
2139 .read()
2140 .map_err(|_| RuntimeError::Internal)?
2141 .clone())
2142 }
2143
2144 pub fn current_project_settings(&self) -> Result<Option<ProjectSettings>, RuntimeError> {
2145 self.current_manifest()
2146 }
2147
2148 pub fn install_release(&self, release: &RuntimeRelease) -> Result<(), RuntimeError> {
2149 release.validate()?;
2150 if release.manifest.project_id != self.core.project_id {
2151 return Err(RuntimeError::InvalidDefinition(
2152 "runtime release belongs to another project".to_string(),
2153 ));
2154 }
2155 self.core.store.save_release(release)
2156 }
2157
2158 pub fn releases(&self) -> Result<Vec<RuntimeRelease>, RuntimeError> {
2159 self.core.store.list_releases(&self.core.project_id)
2160 }
2161
2162 pub fn active_release_version(&self) -> Result<Option<u64>, RuntimeError> {
2163 self.core.store.active_release(&self.core.project_id)
2164 }
2165
2166 pub fn activate_release(&self, version: u64) -> Result<RuntimeRelease, RuntimeError> {
2167 let release = self
2168 .core
2169 .store
2170 .load_release(&self.core.project_id, version)?
2171 .ok_or_else(|| RuntimeError::store("runtime release was not found"))?;
2172 self.apply_manifest(release.manifest.clone())?;
2173 self.core
2174 .store
2175 .set_active_release(&self.core.project_id, version)?;
2176 Ok(release)
2177 }
2178
2179 pub fn restore_active_release(&self) -> Result<Option<RuntimeRelease>, RuntimeError> {
2180 self.active_release_version()?
2181 .map(|version| self.activate_release(version))
2182 .transpose()
2183 }
2184
2185 pub fn bootstrap_release(
2186 &self,
2187 manifest: RuntimeManifest,
2188 ) -> Result<RuntimeRelease, RuntimeError> {
2189 if let Some(active) = self.restore_active_release()? {
2190 return Ok(active);
2191 }
2192 let release = RuntimeRelease::new(1, manifest)?;
2193 self.install_release(&release)?;
2194 self.activate_release(release.version)
2195 }
2196
2197 pub fn bootstrap_project_settings(
2198 &self,
2199 settings: ProjectSettings,
2200 ) -> Result<RuntimeRelease, RuntimeError> {
2201 self.bootstrap_release(settings)
2202 }
2203
2204 pub fn save_local_provider_binding(
2205 &self,
2206 binding: &LocalProviderBinding,
2207 ) -> Result<(), RuntimeError> {
2208 validate_identifier("provider", &binding.provider_id)?;
2209 self.core
2210 .store
2211 .save_local_provider_binding(&self.core.project_id, binding)
2212 }
2213
2214 pub fn local_provider_bindings(&self) -> Result<Vec<LocalProviderBinding>, RuntimeError> {
2215 self.core
2216 .store
2217 .local_provider_bindings(&self.core.project_id)
2218 }
2219
2220 pub fn pending_traces(&self, limit: usize) -> Result<Vec<RuntimeTraceRecord>, RuntimeError> {
2221 self.core.store.pending_traces(limit.min(1_000))
2222 }
2223
2224 pub fn acknowledge_traces(&self, trace_ids: &[String]) -> Result<(), RuntimeError> {
2225 self.core.store.acknowledge_traces(trace_ids)
2226 }
2227
2228 pub fn session(&self, session_id: impl Into<String>) -> Result<RuntimeSession, RuntimeError> {
2229 let session_id = session_id.into();
2230 validate_identifier("session", &session_id)?;
2231 Ok(RuntimeSession {
2232 runtime: self.clone(),
2233 session_id,
2234 })
2235 }
2236
2237 pub async fn invoke(&self, input: InvocationInput) -> Result<InvocationOutput, RuntimeError> {
2238 self.invoke_with_cancellation(input, CancellationToken::default())
2239 .await
2240 }
2241
2242 pub async fn invoke_with_cancellation(
2245 &self,
2246 input: InvocationInput,
2247 cancellation: CancellationToken,
2248 ) -> Result<InvocationOutput, RuntimeError> {
2249 self.invoke_with_events_and_cancellation(input, cancellation, ProviderEventSink::discard())
2250 .await
2251 }
2252
2253 pub async fn invoke_with_events_and_cancellation(
2256 &self,
2257 input: InvocationInput,
2258 cancellation: CancellationToken,
2259 events: ProviderEventSink,
2260 ) -> Result<InvocationOutput, RuntimeError> {
2261 let invocation_id = self.core.next_invocation_id();
2262 self.core
2263 .invoke(invocation_id, input, cancellation, events)
2264 .await
2265 }
2266
2267 pub fn start_invoke(&self, input: InvocationInput) -> Result<InvocationHandle, RuntimeError> {
2268 #[cfg(target_arch = "wasm32")]
2269 {
2270 let _ = input;
2271 return Err(RuntimeError::InvalidDefinition(
2272 "background invocation polling is unavailable in WASM; use invoke".to_string(),
2273 ));
2274 }
2275 #[cfg(not(target_arch = "wasm32"))]
2276 {
2277 validate_identifier("endpoint", &input.endpoint)?;
2278 validate_identifier("session", &input.session_id)?;
2279 let handle = InvocationHandle(self.core.next_invocation_id());
2280 let cancellation = CancellationToken::default();
2281 let mut worker = self.worker.lock().map_err(|_| RuntimeError::Internal)?;
2282 if worker.is_none() {
2283 *worker = Some(RuntimeWorker::spawn(Arc::clone(&self.core))?);
2284 }
2285 self.core
2286 .invocations
2287 .lock()
2288 .map_err(|_| RuntimeError::Internal)?
2289 .insert(handle.clone(), cancellation.clone())?;
2290 let send_result =
2291 worker
2292 .as_ref()
2293 .ok_or(RuntimeError::Internal)?
2294 .send(WorkerCommand::Start {
2295 handle: handle.clone(),
2296 input,
2297 cancellation,
2298 });
2299 if let Err(error) = send_result {
2300 self.core
2301 .invocations
2302 .lock()
2303 .map_err(|_| RuntimeError::Internal)?
2304 .remove(&handle);
2305 return Err(error);
2306 }
2307 Ok(handle)
2308 }
2309 }
2310
2311 pub fn poll_invocation(
2312 &self,
2313 handle: &InvocationHandle,
2314 ) -> Result<InvocationPoll, RuntimeError> {
2315 self.core
2316 .invocations
2317 .lock()
2318 .map_err(|_| RuntimeError::Internal)?
2319 .entries
2320 .get(&handle.0)
2321 .map(|entry| entry.poll.clone())
2322 .ok_or_else(|| RuntimeError::InvocationNotFound(handle.0.clone()))
2323 }
2324
2325 pub fn drain_invocation_events(
2327 &self,
2328 handle: &InvocationHandle,
2329 ) -> Result<Vec<InvocationEvent>, RuntimeError> {
2330 self.core
2331 .invocations
2332 .lock()
2333 .map_err(|_| RuntimeError::Internal)?
2334 .drain_events(handle)
2335 }
2336
2337 pub fn take_invocation(
2342 &self,
2343 handle: &InvocationHandle,
2344 ) -> Result<InvocationPoll, RuntimeError> {
2345 self.core
2346 .invocations
2347 .lock()
2348 .map_err(|_| RuntimeError::Internal)?
2349 .take(handle)
2350 }
2351
2352 pub fn cancel_invocation(&self, handle: &InvocationHandle) -> Result<(), RuntimeError> {
2353 let cancellation = self
2354 .core
2355 .invocations
2356 .lock()
2357 .map_err(|_| RuntimeError::Internal)?
2358 .entries
2359 .get(&handle.0)
2360 .map(|entry| entry.cancellation.clone())
2361 .ok_or_else(|| RuntimeError::InvocationNotFound(handle.0.clone()))?;
2362 cancellation.cancel();
2363 self.core
2364 .update_poll(handle, InvocationStatus::Cancelled, None, None);
2365 Ok(())
2366 }
2367
2368 pub async fn execute_effects(
2369 &self,
2370 effects: Vec<EffectRequest>,
2371 ) -> Result<EffectExecution, RuntimeError> {
2372 self.execute_effects_with_limit(effects, DEFAULT_EFFECT_LIMIT)
2373 .await
2374 }
2375
2376 pub async fn execute_effects_with_limit(
2377 &self,
2378 effects: Vec<EffectRequest>,
2379 limit: usize,
2380 ) -> Result<EffectExecution, RuntimeError> {
2381 if effects.len() > limit {
2382 return Err(RuntimeError::EffectLimitExceeded(limit));
2383 }
2384 let mut results = Vec::new();
2385 let mut unhandled = Vec::new();
2386 for effect in effects {
2387 if effect.kind != "agent.invoke" {
2388 unhandled.push(effect);
2389 continue;
2390 }
2391 let input = serde_json::from_value::<InvocationInput>(effect.payload.clone())
2392 .map_err(|error| RuntimeError::InvalidDefinition(error.to_string()))?;
2393 let result = self.invoke(input).await;
2394 match result {
2395 Ok(output) => results.push(EffectResult {
2396 effect_id: effect.id,
2397 succeeded: true,
2398 output: serde_json::to_value(output)
2399 .map_err(|_error| RuntimeError::Internal)?,
2400 }),
2401 Err(error) => results.push(EffectResult {
2402 effect_id: effect.id,
2403 succeeded: false,
2404 output: json!({ "error": error.public_message() }),
2405 }),
2406 }
2407 }
2408 Ok(EffectExecution { results, unhandled })
2409 }
2410
2411 pub fn export_snapshot(&self) -> Result<Vec<u8>, RuntimeError> {
2412 let snapshot = PortableProjectSnapshot {
2413 version: SNAPSHOT_VERSION,
2414 project_id: self.core.project_id.clone(),
2415 sessions: self
2416 .core
2417 .sessions
2418 .read()
2419 .map_err(|_| RuntimeError::Internal)?
2420 .clone(),
2421 };
2422 serde_json::to_vec(&snapshot).map_err(|error| RuntimeError::Snapshot(error.to_string()))
2423 }
2424
2425 pub fn restore_snapshot(&self, bytes: &[u8]) -> Result<(), RuntimeError> {
2426 let snapshot = serde_json::from_slice::<PortableProjectSnapshot>(bytes)
2427 .map_err(|error| RuntimeError::Snapshot(error.to_string()))?;
2428 if snapshot.version != SNAPSHOT_VERSION || snapshot.project_id != self.core.project_id {
2429 return Err(RuntimeError::Snapshot(
2430 "snapshot version or project does not match".to_string(),
2431 ));
2432 }
2433 for (session_id, state) in &snapshot.sessions {
2434 validate_identifier("session", session_id)?;
2435 self.core
2436 .store
2437 .save(&self.core.project_id, session_id, state)?;
2438 }
2439 *self
2440 .core
2441 .sessions
2442 .write()
2443 .map_err(|_| RuntimeError::Internal)? = snapshot.sessions;
2444 Ok(())
2445 }
2446}
2447
2448#[derive(Clone, Debug)]
2450pub struct RuntimeSession {
2451 runtime: VifuRuntime,
2452 session_id: String,
2453}
2454
2455impl RuntimeSession {
2456 pub fn id(&self) -> &str {
2457 &self.session_id
2458 }
2459
2460 pub async fn invoke(
2461 &self,
2462 mut input: InvocationInput,
2463 ) -> Result<InvocationOutput, RuntimeError> {
2464 input.session_id.clone_from(&self.session_id);
2465 self.runtime.invoke(input).await
2466 }
2467
2468 pub fn start_invoke(
2469 &self,
2470 mut input: InvocationInput,
2471 ) -> Result<InvocationHandle, RuntimeError> {
2472 input.session_id.clone_from(&self.session_id);
2473 self.runtime.start_invoke(input)
2474 }
2475}
2476
2477#[derive(Serialize, Deserialize)]
2478#[serde(rename_all = "camelCase")]
2479struct PortableProjectSnapshot {
2480 version: u32,
2481 project_id: String,
2482 sessions: HashMap<String, RuntimeSnapshot>,
2483}
2484
2485fn default_session_id() -> String {
2486 "default".to_string()
2487}
2488
2489const fn default_timeout_ms() -> u64 {
2490 DEFAULT_TIMEOUT_MS
2491}
2492
2493fn validate_identifier(kind: &str, value: &str) -> Result<(), RuntimeError> {
2494 if value.is_empty()
2495 || value.len() > 128
2496 || !value
2497 .bytes()
2498 .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b':'))
2499 {
2500 return Err(RuntimeError::InvalidDefinition(format!(
2501 "{kind} must be a portable identifier"
2502 )));
2503 }
2504 Ok(())
2505}
2506
2507fn duration_ms(duration: Duration) -> u64 {
2508 u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
2509}
2510
2511const fn is_terminal_status(status: InvocationStatus) -> bool {
2512 matches!(
2513 status,
2514 InvocationStatus::Completed | InvocationStatus::Failed | InvocationStatus::Cancelled
2515 )
2516}
2517
2518#[cfg(test)]
2519mod tests {
2520 use super::*;
2521
2522 struct TestProvider {
2523 fail: bool,
2524 delay: Duration,
2525 }
2526
2527 impl TestProvider {
2528 fn immediate() -> Self {
2529 Self {
2530 fail: false,
2531 delay: Duration::ZERO,
2532 }
2533 }
2534 }
2535
2536 impl AgentProvider for TestProvider {
2537 fn supports(&self, capability: &str) -> bool {
2538 matches!(capability, "chat" | "speech" | "transcription")
2539 }
2540
2541 fn invoke<'a>(
2542 &'a self,
2543 request: ProviderRequest,
2544 cancellation: CancellationToken,
2545 ) -> ProviderFuture<'a> {
2546 Box::pin(async move {
2547 if !self.delay.is_zero() {
2548 tokio::select! {
2549 _ = tokio::time::sleep(self.delay) => {}
2550 _ = cancellation.cancelled() => {
2551 return Err(RuntimeError::Cancelled);
2552 }
2553 }
2554 }
2555 if self.fail {
2556 return Err(RuntimeError::provider(
2557 request.agent.provider,
2558 "synthetic provider failure",
2559 ));
2560 }
2561 Ok(ProviderResponse {
2562 data: match request.data {
2563 InvocationData::Json(data) => InvocationData::Json(json!({
2564 "capability": request.capability,
2565 "input": data,
2566 })),
2567 InvocationData::Binary(bytes) => InvocationData::Binary(bytes),
2568 },
2569 metadata: json!({}),
2570 state: Some(json!({
2571 "lastEndpoint": request.endpoint,
2572 "previousRevision": request.snapshot.revision,
2573 })),
2574 })
2575 })
2576 }
2577 }
2578
2579 struct StreamingTestProvider;
2580
2581 impl AgentProvider for StreamingTestProvider {
2582 fn supports(&self, capability: &str) -> bool {
2583 capability == "chat"
2584 }
2585
2586 fn invoke<'a>(
2587 &'a self,
2588 request: ProviderRequest,
2589 cancellation: CancellationToken,
2590 ) -> ProviderFuture<'a> {
2591 self.invoke_with_events(request, cancellation, ProviderEventSink::discard())
2592 }
2593
2594 fn invoke_with_events<'a>(
2595 &'a self,
2596 _request: ProviderRequest,
2597 cancellation: CancellationToken,
2598 events: ProviderEventSink,
2599 ) -> ProviderFuture<'a> {
2600 Box::pin(async move {
2601 if cancellation.is_cancelled() {
2602 return Err(RuntimeError::Cancelled);
2603 }
2604 events.stage_started(ProviderStage::Tokenize, Value::Null);
2605 events.stage_completed(ProviderStage::Tokenize, 2, json!({ "inputTokens": 4 }));
2606 events.output_delta(InvocationData::Json(Value::String("Hello".to_string())));
2607 events.output_delta(InvocationData::Json(Value::String(", world".to_string())));
2608 Ok(ProviderResponse::json(json!({ "text": "Hello, world" })))
2609 })
2610 }
2611 }
2612
2613 struct ActiveSlowProvider;
2614
2615 impl AgentProvider for ActiveSlowProvider {
2616 fn supports(&self, capability: &str) -> bool {
2617 capability == "chat"
2618 }
2619
2620 fn invoke<'a>(
2621 &'a self,
2622 request: ProviderRequest,
2623 cancellation: CancellationToken,
2624 ) -> ProviderFuture<'a> {
2625 self.invoke_with_events(request, cancellation, ProviderEventSink::discard())
2626 }
2627
2628 fn invoke_with_events<'a>(
2629 &'a self,
2630 _request: ProviderRequest,
2631 cancellation: CancellationToken,
2632 events: ProviderEventSink,
2633 ) -> ProviderFuture<'a> {
2634 Box::pin(async move {
2635 for _ in 0..4 {
2636 tokio::select! {
2637 _ = tokio::time::sleep(Duration::from_millis(8)) => events.activity(),
2638 _ = cancellation.cancelled() => return Err(RuntimeError::Cancelled),
2639 }
2640 }
2641 Ok(ProviderResponse::json(json!({ "ok": true })))
2642 })
2643 }
2644 }
2645
2646 fn configured_runtime(provider: Arc<dyn AgentProvider>) -> VifuRuntime {
2647 let runtime = VifuRuntime::new("test-project").expect("runtime should start");
2648 runtime
2649 .register_provider("test-provider", provider)
2650 .expect("provider should register");
2651 runtime
2652 .register_agent(AgentDefinition {
2653 id: "guide".to_string(),
2654 name: "Guide".to_string(),
2655 provider: "test-provider".to_string(),
2656 capabilities: vec![
2657 "chat".to_string(),
2658 "speech".to_string(),
2659 "transcription".to_string(),
2660 ],
2661 metadata: json!({ "public": true }),
2662 })
2663 .expect("agent should register");
2664 for capability in ["chat", "speech", "transcription"] {
2665 runtime
2666 .register_endpoint(EndpointDefinition {
2667 name: capability.to_string(),
2668 agent: "guide".to_string(),
2669 capability: capability.to_string(),
2670 timeout_ms: 500,
2671 })
2672 .expect("endpoint should register");
2673 }
2674 runtime
2675 }
2676
2677 #[test]
2678 fn dynamic_endpoint_accepts_slow_local_model_inference() {
2679 let runtime = configured_runtime(Arc::new(TestProvider::immediate()));
2680
2681 let result = runtime.register_endpoint(EndpointDefinition {
2682 name: "slow-chat".to_string(),
2683 agent: "guide".to_string(),
2684 capability: "chat".to_string(),
2685 timeout_ms: 300_000,
2686 });
2687
2688 assert!(
2689 result.is_ok(),
2690 "five-minute endpoint should register: {result:?}"
2691 );
2692 }
2693
2694 #[tokio::test(flavor = "current_thread")]
2695 async fn embedded_runtime_invokes_chat_speech_and_transcription_without_a_server() {
2696 let runtime = configured_runtime(Arc::new(TestProvider::immediate()));
2697
2698 for capability in ["chat", "speech", "transcription"] {
2699 let output = runtime
2700 .invoke(InvocationInput::json(
2701 capability,
2702 json!({ "message": capability }),
2703 ))
2704 .await
2705 .expect("endpoint should invoke");
2706 assert_eq!(output.capability, capability);
2707 }
2708 }
2709
2710 #[tokio::test(flavor = "current_thread")]
2711 async fn invocation_result_includes_completed_provider_stages() {
2712 let runtime = configured_runtime(Arc::new(StreamingTestProvider));
2713
2714 let output = runtime
2715 .invoke(InvocationInput::json("chat", json!({})))
2716 .await
2717 .expect("streaming provider should complete");
2718
2719 assert_eq!(output.trace[0].name, "tokenize");
2720 assert_eq!(output.trace[0].status, "completed");
2721 assert_eq!(output.trace[0].duration_ms, 2);
2722 assert_eq!(output.trace[0].attributes, json!({ "inputTokens": 4 }));
2723 assert_eq!(output.trace[1].name, "provider.invoke");
2724 }
2725
2726 #[tokio::test(flavor = "current_thread")]
2727 async fn monitor_observer_receives_provider_performance_metadata() {
2728 let runtime = configured_runtime(Arc::new(StreamingTestProvider));
2729 let monitor_events = Arc::new(Mutex::new(Vec::new()));
2730 let captured_events = Arc::clone(&monitor_events);
2731 runtime
2732 .set_monitor_observer(Some(Arc::new(move |event| {
2733 captured_events.lock().unwrap().push(event);
2734 })))
2735 .unwrap();
2736
2737 runtime
2738 .invoke(InvocationInput::json("chat", json!({ "text": "hello" })))
2739 .await
2740 .unwrap();
2741
2742 assert!(monitor_events.lock().unwrap().iter().any(|event| matches!(
2743 event,
2744 RuntimeMonitorEvent::ProviderStage {
2745 stage: ProviderStage::Tokenize,
2746 status: RuntimeMonitorStageStatus::Completed,
2747 input_tokens: Some(4),
2748 ..
2749 }
2750 )));
2751 }
2752
2753 #[tokio::test(flavor = "current_thread")]
2754 async fn monitor_io_observer_receives_chat_input_and_output() {
2755 let runtime = configured_runtime(Arc::new(TestProvider::immediate()));
2756 let monitor_events = Arc::new(Mutex::new(Vec::new()));
2757 let captured_events = Arc::clone(&monitor_events);
2758 runtime
2759 .set_monitor_io_observer(Some(Arc::new(move |event| {
2760 captured_events.lock().unwrap().push(event);
2761 })))
2762 .unwrap();
2763
2764 runtime
2765 .invoke(InvocationInput::json("chat", json!({ "text": "hello" })))
2766 .await
2767 .unwrap();
2768
2769 let events = monitor_events.lock().unwrap();
2770 assert!(matches!(
2771 events.as_slice(),
2772 [
2773 RuntimeMonitorIoEvent::InvocationInput { summary: input, .. },
2774 RuntimeMonitorIoEvent::InvocationOutput { summary: output, .. }
2775 ] if input.value == json!({ "text": "hello" })
2776 && output.value["input"] == json!({ "text": "hello" })
2777 ));
2778 }
2779
2780 #[tokio::test(flavor = "current_thread")]
2781 async fn runtime_sessions_keep_independent_durable_state() {
2782 let runtime = configured_runtime(Arc::new(TestProvider::immediate()));
2783 let first = runtime
2784 .session("player-one")
2785 .expect("first session should open");
2786 let second = runtime
2787 .session("player-two")
2788 .expect("second session should open");
2789
2790 let first_output = first
2791 .invoke(InvocationInput::json("chat", json!({ "text": "one" })))
2792 .await
2793 .expect("first session should invoke");
2794 let second_output = second
2795 .invoke(InvocationInput::json("chat", json!({ "text": "two" })))
2796 .await
2797 .expect("second session should invoke");
2798
2799 assert_eq!(first_output.snapshot.revision, 1);
2800 assert_eq!(second_output.snapshot.revision, 1);
2801 }
2802
2803 #[tokio::test(flavor = "current_thread")]
2804 async fn concurrent_calls_serialize_state_updates_for_one_session() {
2805 let runtime = configured_runtime(Arc::new(TestProvider {
2806 fail: false,
2807 delay: Duration::from_millis(5),
2808 }));
2809 let first = runtime.invoke(
2810 InvocationInput::json("chat", json!({ "text": "one" })).with_session("shared-session"),
2811 );
2812 let second = runtime.invoke(
2813 InvocationInput::json("chat", json!({ "text": "two" })).with_session("shared-session"),
2814 );
2815
2816 let (first, second) = tokio::join!(first, second);
2817 let mut revisions = [
2818 first
2819 .expect("first invocation should complete")
2820 .snapshot
2821 .revision,
2822 second
2823 .expect("second invocation should complete")
2824 .snapshot
2825 .revision,
2826 ];
2827 revisions.sort_unstable();
2828 assert_eq!(revisions, [1, 2]);
2829 }
2830
2831 #[tokio::test(flavor = "current_thread")]
2832 async fn runtime_round_trips_binary_provider_results() {
2833 let runtime = configured_runtime(Arc::new(TestProvider::immediate()));
2834 let output = runtime
2835 .invoke(InvocationInput {
2836 endpoint: "speech".to_string(),
2837 session_id: "audio-session".to_string(),
2838 data: InvocationData::Binary(vec![1, 2, 3, 4]),
2839 metadata: json!({}),
2840 })
2841 .await
2842 .expect("binary invocation should complete");
2843
2844 assert_eq!(output.data, InvocationData::Binary(vec![1, 2, 3, 4]));
2845 }
2846
2847 #[tokio::test(flavor = "current_thread")]
2848 async fn runtime_times_out_slow_providers() {
2849 let runtime = VifuRuntime::new("timeout-project").expect("runtime should start");
2850 runtime
2851 .register_provider(
2852 "slow",
2853 Arc::new(TestProvider {
2854 fail: false,
2855 delay: Duration::from_millis(100),
2856 }),
2857 )
2858 .expect("provider should register");
2859 runtime
2860 .register_agent(AgentDefinition {
2861 id: "slow-agent".to_string(),
2862 name: "Slow agent".to_string(),
2863 provider: "slow".to_string(),
2864 capabilities: vec!["chat".to_string()],
2865 metadata: json!({}),
2866 })
2867 .expect("agent should register");
2868 runtime
2869 .register_endpoint(EndpointDefinition {
2870 name: "slow-chat".to_string(),
2871 agent: "slow-agent".to_string(),
2872 capability: "chat".to_string(),
2873 timeout_ms: 10,
2874 })
2875 .expect("endpoint should register");
2876
2877 let error = runtime
2878 .invoke(InvocationInput::json("slow-chat", json!({})))
2879 .await
2880 .expect_err("slow invocation should time out");
2881 assert!(matches!(error, RuntimeError::Timeout(10)));
2882 }
2883
2884 #[tokio::test(flavor = "current_thread")]
2885 async fn provider_activity_resets_the_runtime_idle_timeout() {
2886 let runtime = VifuRuntime::new("active-project").expect("runtime should start");
2887 runtime
2888 .register_provider("active", Arc::new(ActiveSlowProvider))
2889 .expect("provider should register");
2890 runtime
2891 .register_agent(AgentDefinition {
2892 id: "active-agent".to_string(),
2893 name: "Active agent".to_string(),
2894 provider: "active".to_string(),
2895 capabilities: vec!["chat".to_string()],
2896 metadata: json!({}),
2897 })
2898 .expect("agent should register");
2899 runtime
2900 .register_endpoint(EndpointDefinition {
2901 name: "active-chat".to_string(),
2902 agent: "active-agent".to_string(),
2903 capability: "chat".to_string(),
2904 timeout_ms: 10,
2905 })
2906 .expect("endpoint should register");
2907
2908 let output = runtime
2909 .invoke(InvocationInput::json("active-chat", json!({})))
2910 .await
2911 .expect("ongoing provider activity should renew the idle timeout");
2912 assert_eq!(output.data, InvocationData::Json(json!({ "ok": true })));
2913 }
2914
2915 #[test]
2916 fn game_loop_api_starts_polls_and_cancels_invocations() {
2917 let runtime = configured_runtime(Arc::new(TestProvider {
2918 fail: false,
2919 delay: Duration::from_secs(5),
2920 }));
2921 let handle = runtime
2922 .start_invoke(InvocationInput::json("chat", json!({})))
2923 .expect("invocation should start");
2924 let running_deadline = Instant::now() + Duration::from_secs(1);
2925 loop {
2926 let poll = runtime
2927 .poll_invocation(&handle)
2928 .expect("invocation should remain pollable");
2929 if poll.status == InvocationStatus::Running {
2930 break;
2931 }
2932 assert!(
2933 Instant::now() < running_deadline,
2934 "invocation did not start"
2935 );
2936 std::thread::sleep(Duration::from_millis(5));
2937 }
2938 runtime
2939 .cancel_invocation(&handle)
2940 .expect("invocation should cancel");
2941
2942 let deadline = Instant::now() + Duration::from_secs(1);
2943 loop {
2944 let poll = runtime
2945 .poll_invocation(&handle)
2946 .expect("invocation should remain pollable");
2947 if poll.status == InvocationStatus::Cancelled {
2948 break;
2949 }
2950 assert!(
2951 Instant::now() < deadline,
2952 "cancelled provider did not observe cancellation"
2953 );
2954 std::thread::sleep(Duration::from_millis(5));
2955 }
2956 }
2957
2958 #[test]
2959 fn game_loop_poll_returns_the_same_provider_result_shape_as_async_invoke() {
2960 let runtime = configured_runtime(Arc::new(TestProvider::immediate()));
2961 let handle = runtime
2962 .start_invoke(
2963 InvocationInput::json("chat", json!({ "text": "hello" }))
2964 .with_session("poll-session"),
2965 )
2966 .expect("invocation should start");
2967 let deadline = Instant::now() + Duration::from_secs(1);
2968 let output = loop {
2969 let poll = runtime
2970 .poll_invocation(&handle)
2971 .expect("invocation should remain pollable");
2972 if let Some(output) = poll.output {
2973 break output;
2974 }
2975 assert!(
2976 !matches!(
2977 poll.status,
2978 InvocationStatus::Failed | InvocationStatus::Cancelled
2979 ),
2980 "invocation unexpectedly failed: {poll:?}"
2981 );
2982 assert!(Instant::now() < deadline, "invocation did not complete");
2983 std::thread::sleep(Duration::from_millis(5));
2984 };
2985
2986 assert_eq!(
2987 output.data,
2988 InvocationData::Json(json!({
2989 "capability": "chat",
2990 "input": { "text": "hello" },
2991 }))
2992 );
2993 }
2994
2995 #[test]
2996 fn game_loop_v1_event_stream_ignores_provider_stages() {
2997 let runtime = configured_runtime(Arc::new(StreamingTestProvider));
2998 let handle = runtime
2999 .start_invoke(InvocationInput::json("chat", json!({})))
3000 .expect("invocation should start");
3001 let deadline = Instant::now() + Duration::from_secs(1);
3002 loop {
3003 let poll = runtime
3004 .poll_invocation(&handle)
3005 .expect("invocation should remain pollable");
3006 if poll.status == InvocationStatus::Completed {
3007 break;
3008 }
3009 assert!(Instant::now() < deadline, "invocation did not complete");
3010 std::thread::sleep(Duration::from_millis(5));
3011 }
3012
3013 let events = runtime
3014 .drain_invocation_events(&handle)
3015 .expect("events should be available");
3016 assert_eq!(
3017 events.iter().map(|event| event.kind).collect::<Vec<_>>(),
3018 vec![
3019 InvocationEventKind::Started,
3020 InvocationEventKind::OutputDelta,
3021 InvocationEventKind::Completed,
3022 ]
3023 );
3024 assert_eq!(
3025 events[1].data,
3026 Some(InvocationData::Json(Value::String(
3027 "Hello, world".to_string()
3028 )))
3029 );
3030 }
3031
3032 #[test]
3033 fn invocation_registry_ignores_output_after_terminal_event() {
3034 let handle = InvocationHandle("late-output".to_string());
3035 let mut registry = InvocationRegistry::default();
3036 registry
3037 .insert(handle.clone(), CancellationToken::default())
3038 .expect("invocation should be registered");
3039 registry.update(&handle, InvocationStatus::Running, None, None);
3040 registry.update(
3041 &handle,
3042 InvocationStatus::Failed,
3043 None,
3044 Some("provider failed".to_string()),
3045 );
3046
3047 registry.push_provider_event(
3048 &handle,
3049 ProviderEvent::OutputDelta {
3050 data: InvocationData::Json(Value::String("too late".to_string())),
3051 },
3052 );
3053
3054 let events = registry
3055 .drain_events(&handle)
3056 .expect("events should remain available");
3057 assert_eq!(
3058 events.iter().map(|event| event.kind).collect::<Vec<_>>(),
3059 vec![InvocationEventKind::Started, InvocationEventKind::Failed]
3060 );
3061 }
3062
3063 #[test]
3064 fn taking_a_terminal_invocation_releases_its_result() {
3065 let runtime = configured_runtime(Arc::new(TestProvider::immediate()));
3066 let handle = runtime
3067 .start_invoke(InvocationInput::json("chat", json!({})))
3068 .expect("invocation should start");
3069 let deadline = Instant::now() + Duration::from_secs(1);
3070 loop {
3071 let poll = runtime
3072 .take_invocation(&handle)
3073 .expect("invocation should remain available until terminal");
3074 if is_terminal_status(poll.status) {
3075 break;
3076 }
3077 assert!(Instant::now() < deadline, "invocation did not complete");
3078 std::thread::sleep(Duration::from_millis(5));
3079 }
3080
3081 assert!(matches!(
3082 runtime.poll_invocation(&handle),
3083 Err(RuntimeError::InvocationNotFound(_))
3084 ));
3085 }
3086
3087 #[test]
3088 fn game_loop_api_applies_backpressure_to_excess_invocations() {
3089 let runtime = configured_runtime(Arc::new(TestProvider {
3090 fail: false,
3091 delay: Duration::from_secs(5),
3092 }));
3093 let handles = (0..MAX_IN_FLIGHT_INVOCATIONS)
3094 .map(|index| {
3095 runtime
3096 .start_invoke(
3097 InvocationInput::json("chat", json!({}))
3098 .with_session(format!("session-{index}")),
3099 )
3100 .expect("invocation within the bound should start")
3101 })
3102 .collect::<Vec<_>>();
3103
3104 let error = runtime
3105 .start_invoke(
3106 InvocationInput::json("chat", json!({})).with_session("one-session-too-many"),
3107 )
3108 .expect_err("invocations above the bound should be rejected");
3109 assert!(matches!(error, RuntimeError::Backpressure(_)));
3110
3111 for handle in handles {
3112 runtime
3113 .cancel_invocation(&handle)
3114 .expect("test invocation should cancel");
3115 }
3116 }
3117
3118 #[test]
3119 fn terminal_invocation_history_is_bounded() {
3120 let runtime = configured_runtime(Arc::new(TestProvider::immediate()));
3121 let first = runtime
3122 .start_invoke(
3123 InvocationInput::json("chat", json!({})).with_session("retained-session-0"),
3124 )
3125 .expect("first invocation should start");
3126 let mut last = first.clone();
3127 for index in 0..=MAX_RETAINED_INVOCATIONS {
3128 let handle = if index == 0 {
3129 first.clone()
3130 } else {
3131 runtime
3132 .start_invoke(
3133 InvocationInput::json("chat", json!({}))
3134 .with_session(format!("retained-session-{index}")),
3135 )
3136 .expect("invocation should start")
3137 };
3138 let deadline = Instant::now() + Duration::from_secs(1);
3139 loop {
3140 let poll = runtime
3141 .poll_invocation(&handle)
3142 .expect("latest invocation should remain available");
3143 if is_terminal_status(poll.status) {
3144 break;
3145 }
3146 assert!(Instant::now() < deadline, "invocation did not complete");
3147 std::thread::sleep(Duration::from_millis(2));
3148 }
3149 last = handle;
3150 }
3151
3152 assert!(matches!(
3153 runtime.poll_invocation(&first),
3154 Err(RuntimeError::InvocationNotFound(_))
3155 ));
3156 assert!(runtime.poll_invocation(&last).is_ok());
3157 }
3158
3159 #[tokio::test(flavor = "current_thread")]
3160 async fn runtime_executes_agent_effects_and_returns_custom_effects_to_the_host() {
3161 let runtime = configured_runtime(Arc::new(TestProvider::immediate()));
3162 let execution = runtime
3163 .execute_effects(vec![
3164 EffectRequest {
3165 id: "agent-effect".to_string(),
3166 kind: "agent.invoke".to_string(),
3167 payload: serde_json::to_value(InvocationInput::json(
3168 "chat",
3169 json!({ "text": "hello" }),
3170 ))
3171 .unwrap(),
3172 },
3173 EffectRequest {
3174 id: "host-effect".to_string(),
3175 kind: "game.play_animation".to_string(),
3176 payload: json!({ "name": "wave" }),
3177 },
3178 ])
3179 .await
3180 .expect("effects should execute");
3181
3182 assert_eq!(execution.results.len(), 1);
3183 assert_eq!(execution.unhandled[0].kind, "game.play_animation");
3184 }
3185
3186 #[tokio::test(flavor = "current_thread")]
3187 async fn runtime_rejects_effect_batches_above_the_bound() {
3188 let runtime = configured_runtime(Arc::new(TestProvider::immediate()));
3189 let effects = (0..3)
3190 .map(|index| EffectRequest {
3191 id: format!("effect-{index}"),
3192 kind: "host.effect".to_string(),
3193 payload: json!({}),
3194 })
3195 .collect();
3196
3197 let error = runtime
3198 .execute_effects_with_limit(effects, 2)
3199 .await
3200 .expect_err("oversized effect batch should fail");
3201 assert!(matches!(error, RuntimeError::EffectLimitExceeded(2)));
3202 }
3203
3204 #[tokio::test(flavor = "current_thread")]
3205 async fn snapshots_restore_session_state_without_runtime_definitions_or_secrets() {
3206 let secret = "synthetic-secret-must-not-leak";
3207 let runtime = configured_runtime(Arc::new(TestProvider::immediate()));
3208 runtime
3209 .invoke(
3210 InvocationInput::json("chat", json!({ "text": "hello" }))
3211 .with_session("saved-session"),
3212 )
3213 .await
3214 .expect("invocation should create state");
3215 let bytes = runtime.export_snapshot().expect("snapshot should export");
3216 assert!(!String::from_utf8_lossy(&bytes).contains(secret));
3217
3218 let restored = configured_runtime(Arc::new(TestProvider::immediate()));
3219 restored
3220 .restore_snapshot(&bytes)
3221 .expect("snapshot should restore");
3222 let output = restored
3223 .invoke(
3224 InvocationInput::json("chat", json!({ "text": "again" }))
3225 .with_session("saved-session"),
3226 )
3227 .await
3228 .expect("restored session should invoke");
3229
3230 assert_eq!(output.snapshot.revision, 2);
3231 }
3232
3233 #[test]
3234 fn debug_output_redacts_payloads_provider_errors_and_snapshots() {
3235 let secret = "synthetic-secret-must-not-leak";
3236 let input = InvocationInput::json("chat", json!({ "secret": secret }));
3237 let error = RuntimeError::provider("test-provider", secret);
3238 let runtime = configured_runtime(Arc::new(TestProvider::immediate()));
3239
3240 assert!(!format!("{input:?}").contains(secret));
3241 assert!(!format!("{error:?}").contains(secret));
3242 assert!(!format!("{runtime:?}").contains(secret));
3243 }
3244}