1use std::{
7 collections::BTreeMap,
8 sync::{Arc, Mutex},
9};
10
11use rill_handler_api::v2::{
12 HANDLER_API_VERSION, MAX_EVENT_BYTES, MAX_OUTPUT_BYTES, MAX_STATE_BYTES,
13};
14use rill_runtime_protocol::v3::{
15 EnvelopeV3, IdentityV3, PREVIEW_CHANNEL_V3, PreviewErrorCodeV3, RUNTIME_API_VERSION_V3,
16 ResourceProfileV1, RuntimeErrorCodeV3, RuntimeErrorV3, RuntimeErrorV3Preview, RuntimeRequestV3,
17 RuntimeResponseBodyV3, RuntimeResponseBodyV3Preview, RuntimeResponseV3,
18 RuntimeResponseV3Preview,
19};
20use serde::{Deserialize, Serialize};
21use sha2::{Digest, Sha256};
22
23const MAX_HANDLER_DETAIL_BYTES_V2: usize = 4 * 1024;
24
25#[derive(Debug, Clone, PartialEq, Eq)]
27pub struct StatefulHandlerMetadataV2 {
28 pub id: String,
29 pub version: String,
30 pub api_version: u32,
31 pub capabilities: Vec<String>,
32 pub state_schema_version: u32,
33}
34
35impl StatefulHandlerMetadataV2 {
36 fn validate(&self) -> Result<(), StatefulHandlerErrorV2> {
37 if self.id.is_empty() || self.id.len() > rill_handler_api::MAX_HANDLER_ID_LEN {
38 return Err(StatefulHandlerErrorV2::new(
39 StatefulHandlerErrorKindV2::MetadataMismatch,
40 ));
41 }
42 if self.version.is_empty()
43 || self.version.len() > rill_handler_api::MAX_HANDLER_VERSION_LEN
44 || self.api_version != HANDLER_API_VERSION
45 || self.state_schema_version == 0
46 {
47 return Err(StatefulHandlerErrorV2::new(
48 StatefulHandlerErrorKindV2::MetadataMismatch,
49 ));
50 }
51 if self.capabilities.is_empty()
52 || self.capabilities.len() > rill_handler_api::MAX_CAPABILITIES
53 {
54 return Err(StatefulHandlerErrorV2::new(
55 StatefulHandlerErrorKindV2::MetadataMismatch,
56 ));
57 }
58 let mut seen = std::collections::BTreeSet::new();
59 if self.capabilities.iter().any(|capability| {
60 capability.is_empty()
61 || capability.len() > rill_handler_api::MAX_CAPABILITY_LEN
62 || !seen.insert(capability)
63 }) {
64 return Err(StatefulHandlerErrorV2::new(
65 StatefulHandlerErrorKindV2::MetadataMismatch,
66 ));
67 }
68 Ok(())
69 }
70}
71
72#[derive(Debug, Clone, PartialEq)]
75pub struct StatefulHandlerResultV2 {
76 pub output: serde_json::Value,
77 pub next_state: Vec<u8>,
78}
79
80#[derive(Debug, Clone, Copy, PartialEq, Eq)]
83#[non_exhaustive]
84pub enum StatefulHandlerErrorKindV2 {
85 InvalidModel,
86 InvalidEvent,
87 InvalidState,
88 IncompatibleVersion,
89 DuplicateFeedback,
90 Timeout,
91 Trap,
92 OutputTooLarge,
93 InvalidOutput,
94 MetadataMismatch,
95 Internal,
96}
97
98#[derive(Debug, Clone)]
100pub struct StatefulHandlerErrorV2 {
101 kind: StatefulHandlerErrorKindV2,
102 detail: Option<String>,
103}
104
105impl StatefulHandlerErrorV2 {
106 pub const fn new(kind: StatefulHandlerErrorKindV2) -> Self {
107 Self { kind, detail: None }
108 }
109
110 pub fn with_detail(kind: StatefulHandlerErrorKindV2, detail: impl Into<String>) -> Self {
111 let mut detail = detail.into();
112 if detail.len() > MAX_HANDLER_DETAIL_BYTES_V2 {
113 let mut end = MAX_HANDLER_DETAIL_BYTES_V2;
114 while end > 0 && !detail.is_char_boundary(end) {
115 end -= 1;
116 }
117 detail.truncate(end);
118 }
119 Self {
120 kind,
121 detail: Some(detail),
122 }
123 }
124
125 pub const fn kind(&self) -> StatefulHandlerErrorKindV2 {
126 self.kind
127 }
128
129 pub fn detail(&self) -> Option<&str> {
130 self.detail.as_deref()
131 }
132}
133
134impl std::fmt::Display for StatefulHandlerErrorV2 {
135 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
136 write!(f, "stateful handler {:?}", self.kind)?;
137 if let Some(detail) = &self.detail {
138 write!(f, ": {detail}")?;
139 }
140 Ok(())
141 }
142}
143
144impl std::error::Error for StatefulHandlerErrorV2 {}
145
146pub trait StatefulHandlerV2: Send + Sync + std::fmt::Debug {
150 fn metadata(&self) -> &StatefulHandlerMetadataV2;
151
152 fn handle(
153 &self,
154 event_json: &[u8],
155 current_state: &[u8],
156 deterministic_seed: Option<u64>,
157 ) -> Result<StatefulHandlerResultV2, StatefulHandlerErrorV2>;
158}
159
160#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
163#[serde(rename_all = "camelCase", deny_unknown_fields)]
164pub struct StatefulStateSnapshotV2 {
165 pub state_schema_version: u32,
166 pub state_generation: u64,
167 pub state: Vec<u8>,
168 pub checksum_sha256: String,
169}
170
171#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
173#[serde(rename_all = "camelCase", deny_unknown_fields)]
174pub struct DecisionLedgerEntryV3 {
175 pub decision_id: String,
176 pub model_generation: u64,
177 pub state_generation: u64,
178 pub created_at_unix_ms: u64,
179 #[serde(default, skip_serializing_if = "Option::is_none")]
180 pub selected_arm: Option<u32>,
181 #[serde(default, skip_serializing_if = "Option::is_none")]
182 pub reward: Option<String>,
183 #[serde(default, skip_serializing_if = "Option::is_none")]
184 pub outcome_time_unix_ms: Option<u64>,
185}
186
187#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
190#[serde(rename_all = "snake_case")]
191pub enum RuntimeHealthStatusV1 {
192 Healthy,
193 ResourcePressure,
194 FailedClosed,
195}
196
197impl std::fmt::Display for RuntimeHealthStatusV1 {
198 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
199 f.write_str(match self {
200 Self::Healthy => "healthy",
201 Self::ResourcePressure => "resource_pressure",
202 Self::FailedClosed => "failed_closed",
203 })
204 }
205}
206
207#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
209#[serde(rename_all = "camelCase", deny_unknown_fields)]
210pub struct RuntimeResourceUsageV1 {
211 pub state_bytes: usize,
212 pub snapshot_bytes: usize,
213 pub pending_decisions: usize,
214 pub completed_decisions: usize,
215}
216
217#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
221#[serde(rename_all = "camelCase", deny_unknown_fields)]
222pub struct RuntimeDiagnosticsV1 {
223 pub runtime_version: String,
224 pub protocol_version: u32,
225 pub channel: String,
226 pub model_generation: u64,
227 pub state_generation: u64,
228 pub state_schema_version: u32,
229 pub observed_at_unix_ms: u64,
230 pub health: RuntimeHealthStatusV1,
231 pub reason_codes: Vec<String>,
232 pub resource_usage: RuntimeResourceUsageV1,
233 pub rollback_available: bool,
234 pub candidate_available: bool,
235 pub restart_count: u64,
236 pub last_error: Option<String>,
237}
238
239#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
242#[serde(rename_all = "camelCase", deny_unknown_fields)]
243pub struct StatefulRuntimeSnapshotV3 {
244 pub format_version: u32,
245 pub handler_snapshot: StatefulStateSnapshotV2,
246 pub pending_decisions: BTreeMap<String, DecisionLedgerEntryV3>,
247 pub completed_decisions: BTreeMap<String, DecisionLedgerEntryV3>,
248 pub checksum_sha256: String,
249}
250
251impl StatefulRuntimeSnapshotV3 {
252 pub const FORMAT_VERSION: u32 = 1;
253
254 fn checksum_input(
255 handler_snapshot: &StatefulStateSnapshotV2,
256 pending_decisions: &BTreeMap<String, DecisionLedgerEntryV3>,
257 completed_decisions: &BTreeMap<String, DecisionLedgerEntryV3>,
258 ) -> Vec<u8> {
259 serde_json::to_vec(&(handler_snapshot, pending_decisions, completed_decisions))
260 .expect("runtime snapshot fields are serializable")
261 }
262
263 fn new(
264 handler_snapshot: StatefulStateSnapshotV2,
265 pending_decisions: BTreeMap<String, DecisionLedgerEntryV3>,
266 completed_decisions: BTreeMap<String, DecisionLedgerEntryV3>,
267 ) -> Self {
268 let checksum_sha256 = state_checksum(&Self::checksum_input(
269 &handler_snapshot,
270 &pending_decisions,
271 &completed_decisions,
272 ));
273 Self {
274 format_version: Self::FORMAT_VERSION,
275 handler_snapshot,
276 pending_decisions,
277 completed_decisions,
278 checksum_sha256,
279 }
280 }
281
282 fn validate(&self, expected_schema_version: u32) -> Result<(), StatefulHandlerErrorV2> {
283 if self.format_version != Self::FORMAT_VERSION {
284 return Err(StatefulHandlerErrorV2::new(
285 StatefulHandlerErrorKindV2::IncompatibleVersion,
286 ));
287 }
288 self.handler_snapshot.validate(expected_schema_version)?;
289 if self.checksum_sha256
290 != state_checksum(&Self::checksum_input(
291 &self.handler_snapshot,
292 &self.pending_decisions,
293 &self.completed_decisions,
294 ))
295 {
296 return Err(StatefulHandlerErrorV2::new(
297 StatefulHandlerErrorKindV2::InvalidState,
298 ));
299 }
300 for (key, entry) in self
301 .pending_decisions
302 .iter()
303 .chain(self.completed_decisions.iter())
304 {
305 if key != &entry.decision_id || entry.decision_id.is_empty() {
306 return Err(StatefulHandlerErrorV2::new(
307 StatefulHandlerErrorKindV2::InvalidState,
308 ));
309 }
310 }
311 Ok(())
312 }
313}
314
315pub trait StatefulStateMigratorV1: Send + Sync {
318 fn migrate(
319 &self,
320 from_schema_version: u32,
321 state: &[u8],
322 ) -> Result<(u32, Vec<u8>), StatefulHandlerErrorV2>;
323}
324
325impl StatefulStateSnapshotV2 {
326 pub fn new(state_schema_version: u32, state_generation: u64, state: Vec<u8>) -> Self {
327 let checksum_sha256 = state_checksum(&state);
328 Self {
329 state_schema_version,
330 state_generation,
331 state,
332 checksum_sha256,
333 }
334 }
335
336 pub fn validate(&self, expected_schema_version: u32) -> Result<(), StatefulHandlerErrorV2> {
337 validate_state_bytes(
338 &self.state,
339 self.state_schema_version,
340 expected_schema_version,
341 )?;
342 if self.checksum_sha256 != state_checksum(&self.state) {
343 return Err(StatefulHandlerErrorV2::new(
344 StatefulHandlerErrorKindV2::InvalidState,
345 ));
346 }
347 Ok(())
348 }
349}
350
351#[derive(Debug, Clone)]
353#[non_exhaustive]
354pub struct StatefulRuntimeConfigV3 {
355 pub runtime_identity: IdentityV3,
356 pub model_generation: u64,
357 pub initial_state_generation: u64,
358 pub feature_schema_hash: String,
359 pub capabilities: Vec<String>,
360 pub initial_state: Vec<u8>,
361 pub resource_profile: ResourceProfileV1,
362}
363
364impl StatefulRuntimeConfigV3 {
365 pub fn new(
366 runtime_identity: IdentityV3,
367 model_generation: u64,
368 feature_schema_hash: String,
369 capabilities: Vec<String>,
370 initial_state: Vec<u8>,
371 ) -> Self {
372 Self {
373 runtime_identity,
374 model_generation,
375 initial_state_generation: 0,
376 feature_schema_hash,
377 capabilities,
378 initial_state,
379 resource_profile: ResourceProfileV1::default(),
380 }
381 }
382
383 pub fn with_resource_profile(mut self, resource_profile: ResourceProfileV1) -> Self {
384 self.resource_profile = resource_profile;
385 self
386 }
387}
388
389#[derive(Debug, Clone)]
390struct RuntimeStateV3 {
391 snapshot: StatefulStateSnapshotV2,
392 previous_good: Option<StatefulStateSnapshotV2>,
393 candidate: Option<StatefulStateSnapshotV2>,
394 pending_decisions: BTreeMap<String, DecisionLedgerEntryV3>,
395 completed_decisions: BTreeMap<String, DecisionLedgerEntryV3>,
396 restart_count: u64,
397 last_error: Option<String>,
398}
399
400#[derive(Debug)]
402pub struct StatefulRuntimeEngineV3 {
403 config: StatefulRuntimeConfigV3,
404 metadata: StatefulHandlerMetadataV2,
405 handler: Arc<dyn StatefulHandlerV2>,
406 state: Mutex<RuntimeStateV3>,
407}
408
409impl StatefulRuntimeEngineV3 {
410 pub fn new(
411 config: StatefulRuntimeConfigV3,
412 handler: Arc<dyn StatefulHandlerV2>,
413 ) -> Result<Self, StatefulHandlerErrorV2> {
414 let metadata = handler.metadata().clone();
415 metadata.validate()?;
416 config.runtime_identity.validate().map_err(|error| {
417 StatefulHandlerErrorV2::with_detail(
418 StatefulHandlerErrorKindV2::InvalidModel,
419 error.to_string(),
420 )
421 })?;
422 validate_feature_schema_hash(&config.feature_schema_hash)?;
423 validate_capabilities(&config.capabilities)?;
424 config.resource_profile.validate().map_err(|detail| {
425 StatefulHandlerErrorV2::with_detail(StatefulHandlerErrorKindV2::InvalidModel, detail)
426 })?;
427 if config.capabilities != metadata.capabilities {
428 return Err(StatefulHandlerErrorV2::new(
429 StatefulHandlerErrorKindV2::MetadataMismatch,
430 ));
431 }
432 validate_state_bytes(
433 &config.initial_state,
434 metadata.state_schema_version,
435 metadata.state_schema_version,
436 )?;
437 let snapshot = StatefulStateSnapshotV2::new(
438 metadata.state_schema_version,
439 config.initial_state_generation,
440 config.initial_state.clone(),
441 );
442 if snapshot.state.len() > config.resource_profile.max_model_state_bytes as usize {
443 return Err(StatefulHandlerErrorV2::new(
444 StatefulHandlerErrorKindV2::InvalidState,
445 ));
446 }
447 Ok(Self {
448 config,
449 metadata,
450 handler,
451 state: Mutex::new(RuntimeStateV3 {
452 snapshot,
453 previous_good: None,
454 candidate: None,
455 pending_decisions: BTreeMap::new(),
456 completed_decisions: BTreeMap::new(),
457 restart_count: 0,
458 last_error: None,
459 }),
460 })
461 }
462
463 pub fn restore_snapshot(
465 &self,
466 snapshot: StatefulStateSnapshotV2,
467 ) -> Result<(), StatefulHandlerErrorV2> {
468 snapshot.validate(self.metadata.state_schema_version)?;
469 if snapshot.state.len() > self.config.resource_profile.max_model_state_bytes as usize {
470 return Err(StatefulHandlerErrorV2::new(
471 StatefulHandlerErrorKindV2::InvalidState,
472 ));
473 }
474 let mut state = self
475 .state
476 .lock()
477 .map_err(|_| StatefulHandlerErrorV2::new(StatefulHandlerErrorKindV2::Internal))?;
478 state.previous_good = Some(state.snapshot.clone());
479 state.snapshot = snapshot;
480 state.restart_count = state.restart_count.saturating_add(1);
481 Ok(())
482 }
483
484 pub fn snapshot(&self) -> Result<StatefulStateSnapshotV2, StatefulHandlerErrorV2> {
485 self.state
486 .lock()
487 .map(|state| state.snapshot.clone())
488 .map_err(|_| StatefulHandlerErrorV2::new(StatefulHandlerErrorKindV2::Internal))
489 }
490
491 pub fn runtime_snapshot(&self) -> Result<StatefulRuntimeSnapshotV3, StatefulHandlerErrorV2> {
493 let state = self
494 .state
495 .lock()
496 .map_err(|_| StatefulHandlerErrorV2::new(StatefulHandlerErrorKindV2::Internal))?;
497 let snapshot = StatefulRuntimeSnapshotV3::new(
498 state.snapshot.clone(),
499 state.pending_decisions.clone(),
500 state.completed_decisions.clone(),
501 );
502 self.validate_runtime_snapshot_size(&snapshot)?;
503 Ok(snapshot)
504 }
505
506 pub fn restore_runtime_snapshot(
508 &self,
509 snapshot: StatefulRuntimeSnapshotV3,
510 ) -> Result<(), StatefulHandlerErrorV2> {
511 snapshot.validate(self.metadata.state_schema_version)?;
512 if snapshot.handler_snapshot.state.len()
513 > self.config.resource_profile.max_model_state_bytes as usize
514 || snapshot.pending_decisions.len()
515 > self.config.resource_profile.max_pending_decisions as usize
516 || snapshot.completed_decisions.len()
517 > self.config.resource_profile.max_completed_decisions as usize
518 {
519 return Err(StatefulHandlerErrorV2::new(
520 StatefulHandlerErrorKindV2::InvalidState,
521 ));
522 }
523 self.validate_runtime_snapshot_size(&snapshot)?;
524 let mut state = self
525 .state
526 .lock()
527 .map_err(|_| StatefulHandlerErrorV2::new(StatefulHandlerErrorKindV2::Internal))?;
528 state.previous_good = Some(state.snapshot.clone());
529 state.snapshot = snapshot.handler_snapshot;
530 state.pending_decisions = snapshot.pending_decisions;
531 state.completed_decisions = snapshot.completed_decisions;
532 state.restart_count = state.restart_count.saturating_add(1);
533 Ok(())
534 }
535
536 pub fn stage_candidate(
538 &self,
539 snapshot: StatefulStateSnapshotV2,
540 ) -> Result<(), StatefulHandlerErrorV2> {
541 snapshot.validate(self.metadata.state_schema_version)?;
542 if snapshot.state.len() > self.config.resource_profile.max_model_state_bytes as usize {
543 return Err(StatefulHandlerErrorV2::new(
544 StatefulHandlerErrorKindV2::InvalidState,
545 ));
546 }
547 let mut state = self
548 .state
549 .lock()
550 .map_err(|_| StatefulHandlerErrorV2::new(StatefulHandlerErrorKindV2::Internal))?;
551 state.candidate = Some(snapshot);
552 Ok(())
553 }
554
555 pub fn promote_candidate(&self) -> Result<u64, StatefulHandlerErrorV2> {
557 let mut state = self
558 .state
559 .lock()
560 .map_err(|_| StatefulHandlerErrorV2::new(StatefulHandlerErrorKindV2::Internal))?;
561 let candidate = state
562 .candidate
563 .take()
564 .ok_or_else(|| StatefulHandlerErrorV2::new(StatefulHandlerErrorKindV2::InvalidState))?;
565 state.previous_good = Some(state.snapshot.clone());
566 state.snapshot = candidate;
567 state.pending_decisions.clear();
568 state.completed_decisions.clear();
569 Ok(state.snapshot.state_generation)
570 }
571
572 pub fn rollback_previous_good(&self) -> Result<u64, StatefulHandlerErrorV2> {
574 let mut state = self
575 .state
576 .lock()
577 .map_err(|_| StatefulHandlerErrorV2::new(StatefulHandlerErrorKindV2::Internal))?;
578 let previous = state
579 .previous_good
580 .take()
581 .ok_or_else(|| StatefulHandlerErrorV2::new(StatefulHandlerErrorKindV2::InvalidState))?;
582 let failed = std::mem::replace(&mut state.snapshot, previous);
583 state.previous_good = Some(failed);
584 state.candidate = None;
585 state.pending_decisions.clear();
586 state.completed_decisions.clear();
587 Ok(state.snapshot.state_generation)
588 }
589
590 pub fn restore_snapshot_with_migration(
592 &self,
593 snapshot: StatefulStateSnapshotV2,
594 migrator: &dyn StatefulStateMigratorV1,
595 ) -> Result<(), StatefulHandlerErrorV2> {
596 if snapshot.state_schema_version == self.metadata.state_schema_version {
597 return self.restore_snapshot(snapshot);
598 }
599 let (schema_version, state) =
600 migrator.migrate(snapshot.state_schema_version, &snapshot.state)?;
601 if schema_version != self.metadata.state_schema_version {
602 return Err(StatefulHandlerErrorV2::new(
603 StatefulHandlerErrorKindV2::IncompatibleVersion,
604 ));
605 }
606 self.restore_snapshot(StatefulStateSnapshotV2::new(
607 schema_version,
608 snapshot.state_generation,
609 state,
610 ))
611 }
612
613 pub fn handle_at(&self, envelope: EnvelopeV3, now_unix_ms: u64) -> RuntimeResponseV3 {
616 if let Err(error) = envelope.validate() {
617 return self.error_response(
618 envelope.request_id,
619 RuntimeErrorCodeV3::InvalidEnvelope,
620 error.to_string(),
621 self.current_generation(),
622 );
623 }
624 if envelope.is_expired_at(now_unix_ms) {
625 return self.error_response(
626 envelope.request_id,
627 RuntimeErrorCodeV3::ExpiredRequest,
628 "request deadline has expired",
629 self.current_generation(),
630 );
631 }
632
633 match envelope.request.clone() {
634 RuntimeRequestV3::Handshake {} => self.response(
635 envelope.request_id,
636 self.current_generation(),
637 RuntimeResponseBodyV3::Handshake {
638 capabilities: self.config.capabilities.clone(),
639 feature_schema_hash: self.config.feature_schema_hash.clone(),
640 handler_api_version: HANDLER_API_VERSION,
641 },
642 ),
643 RuntimeRequestV3::Health {} => self.response(
644 envelope.request_id,
645 self.current_generation(),
646 RuntimeResponseBodyV3::Health {
647 healthy: self.is_healthy(),
648 },
649 ),
650 request => self.handle_stateful(envelope, request, now_unix_ms),
651 }
652 }
653
654 pub fn handle_json_at(&self, message: &[u8], now_unix_ms: u64) -> RuntimeResponseV3 {
658 if message.len() > self.config.resource_profile.max_ipc_frame_bytes as usize {
659 return self.error_response(
660 "invalid-request".into(),
661 RuntimeErrorCodeV3::PayloadTooLarge,
662 "request exceeds the IPC message limit",
663 self.current_generation(),
664 );
665 }
666 let envelope = match serde_json::from_slice::<EnvelopeV3>(message) {
667 Ok(envelope) => envelope,
668 Err(_) => {
669 return self.error_response(
670 "invalid-request".into(),
671 RuntimeErrorCodeV3::InvalidJson,
672 "request is not valid IPC V3 JSON",
673 self.current_generation(),
674 );
675 }
676 };
677 self.handle_at(envelope, now_unix_ms)
678 }
679
680 pub fn handle_preview_at(
683 &self,
684 envelope: EnvelopeV3,
685 now_unix_ms: u64,
686 ) -> RuntimeResponseV3Preview {
687 let is_decide = matches!(envelope.request, RuntimeRequestV3::Decide { .. });
688 let response = self.handle_at(envelope, now_unix_ms);
689 let body = match response.response {
690 RuntimeResponseBodyV3::Handshake {
691 capabilities,
692 feature_schema_hash,
693 handler_api_version,
694 } => RuntimeResponseBodyV3Preview::Handshake {
695 capabilities,
696 feature_schema_hash,
697 handler_api_version,
698 channel: PREVIEW_CHANNEL_V3.into(),
699 },
700 RuntimeResponseBodyV3::Health { healthy } => RuntimeResponseBodyV3Preview::Health {
701 healthy,
702 status: if healthy {
703 "healthy".into()
704 } else {
705 "failed_closed".into()
706 },
707 reason_codes: self.health_reason_codes().unwrap_or_default(),
708 },
709 RuntimeResponseBodyV3::Result { output } => RuntimeResponseBodyV3Preview::Result {
710 output,
711 decision_id: is_decide.then_some(response.request_id.clone()),
712 decision_generation: is_decide.then_some(response.state_generation),
713 },
714 RuntimeResponseBodyV3::Inspection { summary } => {
715 RuntimeResponseBodyV3Preview::Inspection { summary }
716 }
717 RuntimeResponseBodyV3::Snapshot {
718 state_schema_version,
719 state_checksum,
720 state,
721 } => RuntimeResponseBodyV3Preview::Snapshot {
722 state_schema_version,
723 state_checksum,
724 state,
725 },
726 RuntimeResponseBodyV3::Reset { reset } => RuntimeResponseBodyV3Preview::Reset { reset },
727 RuntimeResponseBodyV3::Error { error } => {
728 let preview_code = preview_error_code(error.code, &error.message);
729 RuntimeResponseBodyV3Preview::Error {
730 error: RuntimeErrorV3Preview {
731 code: preview_code,
732 message: error.message,
733 retryable: preview_code.is_retryable(),
734 },
735 }
736 }
737 };
738 RuntimeResponseV3Preview {
739 request_id: response.request_id,
740 api_version: response.api_version,
741 runtime_identity: response.runtime_identity,
742 model_generation: response.model_generation,
743 state_generation: response.state_generation,
744 response: body,
745 }
746 }
747
748 pub fn handle_preview_json_at(
749 &self,
750 message: &[u8],
751 now_unix_ms: u64,
752 ) -> RuntimeResponseV3Preview {
753 if message.len() > self.config.resource_profile.max_ipc_frame_bytes as usize {
754 return self.preview_error_response(
755 "invalid-request".into(),
756 PreviewErrorCodeV3::PayloadTooLarge,
757 "request exceeds the IPC message limit",
758 );
759 }
760 match serde_json::from_slice::<EnvelopeV3>(message) {
761 Ok(envelope) => self.handle_preview_at(envelope, now_unix_ms),
762 Err(_) => self.preview_error_response(
763 "invalid-request".into(),
764 PreviewErrorCodeV3::InvalidJson,
765 "request is not valid IPC V3 JSON",
766 ),
767 }
768 }
769
770 fn preview_error_response(
771 &self,
772 request_id: String,
773 code: PreviewErrorCodeV3,
774 message: &str,
775 ) -> RuntimeResponseV3Preview {
776 RuntimeResponseV3Preview {
777 request_id,
778 api_version: RUNTIME_API_VERSION_V3,
779 runtime_identity: self.config.runtime_identity.clone(),
780 model_generation: self.config.model_generation,
781 state_generation: self.current_generation(),
782 response: RuntimeResponseBodyV3Preview::Error {
783 error: RuntimeErrorV3Preview {
784 code,
785 message: message.into(),
786 retryable: code.is_retryable(),
787 },
788 },
789 }
790 }
791
792 fn handle_stateful(
793 &self,
794 envelope: EnvelopeV3,
795 request: RuntimeRequestV3,
796 now_unix_ms: u64,
797 ) -> RuntimeResponseV3 {
798 let request_id = envelope.request_id;
799 let capability = envelope.capability.unwrap_or_default();
800 if !self
801 .config
802 .capabilities
803 .iter()
804 .any(|item| item == &capability)
805 {
806 return self.error_response(
807 request_id,
808 RuntimeErrorCodeV3::UnsupportedCapability,
809 "capability is not in the effective set",
810 self.current_generation(),
811 );
812 }
813 if envelope.feature_schema_hash.as_deref() != Some(self.config.feature_schema_hash.as_str())
814 {
815 return self.error_response(
816 request_id,
817 RuntimeErrorCodeV3::StateMismatch,
818 "feature schema hash does not match",
819 self.current_generation(),
820 );
821 }
822 if envelope.model_generation != self.config.model_generation {
823 return self.error_response(
824 request_id,
825 RuntimeErrorCodeV3::IncompatibleGeneration,
826 "model generation does not match",
827 self.current_generation(),
828 );
829 }
830 if let RuntimeRequestV3::Feedback { generation, .. } = &request
831 && *generation != self.config.model_generation
832 {
833 return self.error_response(
834 request_id,
835 RuntimeErrorCodeV3::IncompatibleGeneration,
836 "feedback generation does not match",
837 self.current_generation(),
838 );
839 }
840
841 let mut state = match self.state.lock() {
842 Ok(state) => state,
843 Err(_) => {
844 return self.error_response(
845 request_id,
846 RuntimeErrorCodeV3::Internal,
847 "runtime state lock is poisoned",
848 0,
849 );
850 }
851 };
852 if envelope.state_generation != state.snapshot.state_generation {
853 return self.error_response(
854 request_id,
855 RuntimeErrorCodeV3::StateMismatch,
856 "state generation does not match",
857 state.snapshot.state_generation,
858 );
859 }
860
861 if let RuntimeRequestV3::Decide { .. } = &request {
862 if state.pending_decisions.contains_key(&request_id)
863 || state.completed_decisions.contains_key(&request_id)
864 {
865 return self.error_response(
866 request_id,
867 RuntimeErrorCodeV3::Internal,
868 "decision id was already used",
869 state.snapshot.state_generation,
870 );
871 }
872 if state.pending_decisions.len()
873 >= self.config.resource_profile.max_pending_decisions as usize
874 || state.completed_decisions.len()
875 >= self.config.resource_profile.max_completed_decisions as usize
876 {
877 return self.error_response(
878 request_id,
879 RuntimeErrorCodeV3::Internal,
880 "pending decision capacity is exhausted",
881 state.snapshot.state_generation,
882 );
883 }
884 }
885 if let RuntimeRequestV3::Feedback {
886 decision_id,
887 generation,
888 ..
889 } = &request
890 {
891 if state.completed_decisions.contains_key(decision_id) {
892 return self.error_response(
893 request_id,
894 RuntimeErrorCodeV3::DuplicateFeedback,
895 "feedback was already applied",
896 state.snapshot.state_generation,
897 );
898 }
899 let Some(entry) = state.pending_decisions.get(decision_id) else {
900 return self.error_response(
901 request_id,
902 RuntimeErrorCodeV3::Internal,
903 "decision id is not pending",
904 state.snapshot.state_generation,
905 );
906 };
907 if entry.model_generation != *generation {
908 return self.error_response(
909 request_id,
910 RuntimeErrorCodeV3::IncompatibleGeneration,
911 "feedback generation is stale",
912 state.snapshot.state_generation,
913 );
914 }
915 if state.completed_decisions.len()
916 >= self.config.resource_profile.max_completed_decisions as usize
917 {
918 return self.error_response(
919 request_id,
920 RuntimeErrorCodeV3::Internal,
921 "completed decision capacity is exhausted",
922 state.snapshot.state_generation,
923 );
924 }
925 }
926
927 if let RuntimeRequestV3::Snapshot {} = request {
928 return self.response(
929 request_id,
930 state.snapshot.state_generation,
931 RuntimeResponseBodyV3::Snapshot {
932 state_schema_version: state.snapshot.state_schema_version,
933 state_checksum: state.snapshot.checksum_sha256.clone(),
934 state: hex::encode(&state.snapshot.state),
935 },
936 );
937 }
938 if let RuntimeRequestV3::Reset {
939 expected_state_generation,
940 } = request
941 {
942 if expected_state_generation != state.snapshot.state_generation {
943 return self.error_response(
944 request_id,
945 RuntimeErrorCodeV3::StateMismatch,
946 "reset generation does not match",
947 state.snapshot.state_generation,
948 );
949 }
950 let Some(next_generation) = state.snapshot.state_generation.checked_add(1) else {
951 return self.error_response(
952 request_id,
953 RuntimeErrorCodeV3::InvalidState,
954 "state generation overflow",
955 state.snapshot.state_generation,
956 );
957 };
958 state.snapshot = StatefulStateSnapshotV2::new(
959 self.metadata.state_schema_version,
960 next_generation,
961 self.config.initial_state.clone(),
962 );
963 state.pending_decisions.clear();
964 state.completed_decisions.clear();
965 return self.response(
966 request_id,
967 next_generation,
968 RuntimeResponseBodyV3::Reset { reset: true },
969 );
970 }
971
972 let deterministic_seed = match &request {
973 RuntimeRequestV3::Decide {
974 deterministic_seed, ..
975 } => *deterministic_seed,
976 _ => None,
977 };
978 let event_json = match serde_json::to_vec(&request) {
979 Ok(bytes) if bytes.len() <= MAX_EVENT_BYTES => bytes,
980 Ok(_) => {
981 return self.error_response(
982 request_id,
983 RuntimeErrorCodeV3::PayloadTooLarge,
984 "event exceeds handler limit",
985 state.snapshot.state_generation,
986 );
987 }
988 Err(_) => {
989 return self.error_response(
990 request_id,
991 RuntimeErrorCodeV3::InvalidJson,
992 "event could not be encoded",
993 state.snapshot.state_generation,
994 );
995 }
996 };
997
998 let result =
999 match self
1000 .handler
1001 .handle(&event_json, &state.snapshot.state, deterministic_seed)
1002 {
1003 Ok(result) => result,
1004 Err(error) => {
1005 let (code, message) = map_handler_error(error.kind());
1006 state.last_error = Some(message.into());
1007 return self.error_response(
1008 request_id,
1009 code,
1010 message,
1011 state.snapshot.state_generation,
1012 );
1013 }
1014 };
1015 let output_bytes = match serde_json::to_vec(&result.output) {
1016 Ok(bytes) => bytes,
1017 Err(_) => {
1018 return self.error_response(
1019 request_id,
1020 RuntimeErrorCodeV3::HandlerInvalidOutput,
1021 "handler output was not valid JSON",
1022 state.snapshot.state_generation,
1023 );
1024 }
1025 };
1026 if output_bytes.len() > MAX_OUTPUT_BYTES {
1027 return self.error_response(
1028 request_id,
1029 RuntimeErrorCodeV3::HandlerOutputTooLarge,
1030 "handler output exceeded the size limit",
1031 state.snapshot.state_generation,
1032 );
1033 }
1034 if validate_state_bytes(
1035 &result.next_state,
1036 self.metadata.state_schema_version,
1037 self.metadata.state_schema_version,
1038 )
1039 .is_err()
1040 {
1041 return self.error_response(
1042 request_id,
1043 RuntimeErrorCodeV3::InvalidState,
1044 "handler returned invalid next state",
1045 state.snapshot.state_generation,
1046 );
1047 }
1048 if result.next_state.len() > self.config.resource_profile.max_model_state_bytes as usize {
1049 return self.error_response(
1050 request_id,
1051 RuntimeErrorCodeV3::Internal,
1052 "model state resource limit exceeded",
1053 state.snapshot.state_generation,
1054 );
1055 }
1056 let Some(next_generation) = state.snapshot.state_generation.checked_add(1) else {
1057 return self.error_response(
1058 request_id,
1059 RuntimeErrorCodeV3::InvalidState,
1060 "state generation overflow",
1061 state.snapshot.state_generation,
1062 );
1063 };
1064 let next_snapshot = StatefulStateSnapshotV2::new(
1065 self.metadata.state_schema_version,
1066 next_generation,
1067 result.next_state,
1068 );
1069 let mut next_pending = state.pending_decisions.clone();
1070 let mut next_completed = state.completed_decisions.clone();
1071 if matches!(request, RuntimeRequestV3::Decide { .. }) {
1072 let entry = DecisionLedgerEntryV3 {
1073 decision_id: request_id.clone(),
1074 model_generation: self.config.model_generation,
1075 state_generation: next_generation,
1076 created_at_unix_ms: now_unix_ms,
1077 selected_arm: None,
1078 reward: None,
1079 outcome_time_unix_ms: None,
1080 };
1081 next_pending.insert(request_id.clone(), entry);
1082 } else {
1083 if let RuntimeRequestV3::Feedback {
1084 decision_id,
1085 selected_arm,
1086 reward,
1087 outcome_time_ms,
1088 ..
1089 } = &request
1090 {
1091 let mut entry = next_pending
1092 .remove(decision_id)
1093 .ok_or_else(|| {
1094 StatefulHandlerErrorV2::new(StatefulHandlerErrorKindV2::Internal)
1095 })
1096 .unwrap();
1097 entry.selected_arm = Some(*selected_arm);
1098 entry.reward = Some(reward.to_string());
1099 entry.outcome_time_unix_ms = Some(*outcome_time_ms);
1100 next_completed.insert(entry.decision_id.clone(), entry);
1101 }
1102 }
1103 let prospective = StatefulRuntimeSnapshotV3::new(
1104 next_snapshot.clone(),
1105 next_pending.clone(),
1106 next_completed.clone(),
1107 );
1108 if self.validate_runtime_snapshot_size(&prospective).is_err() {
1109 return self.error_response(
1110 request_id,
1111 RuntimeErrorCodeV3::Internal,
1112 "snapshot resource limit exceeded",
1113 state.snapshot.state_generation,
1114 );
1115 }
1116 let previous_snapshot = state.snapshot.clone();
1117 state.snapshot = next_snapshot;
1118 state.previous_good = Some(previous_snapshot);
1119 state.pending_decisions = next_pending;
1120 state.completed_decisions = next_completed;
1121 self.response(
1122 request_id,
1123 next_generation,
1124 match request {
1125 RuntimeRequestV3::Inspect {} => RuntimeResponseBodyV3::Inspection {
1126 summary: self.inspection_summary(&state, result.output),
1127 },
1128 _ => RuntimeResponseBodyV3::Result {
1129 output: result.output,
1130 },
1131 },
1132 )
1133 }
1134
1135 fn is_healthy(&self) -> bool {
1136 self.state
1137 .lock()
1138 .map(|state| state.last_error.is_none())
1139 .unwrap_or(false)
1140 }
1141
1142 fn health_reason_codes(&self) -> Option<Vec<String>> {
1143 self.state
1144 .lock()
1145 .ok()
1146 .and_then(|state| state.last_error.as_ref().map(|error| vec![error.clone()]))
1147 }
1148
1149 fn inspection_summary(
1150 &self,
1151 state: &RuntimeStateV3,
1152 handler_summary: serde_json::Value,
1153 ) -> serde_json::Value {
1154 serde_json::json!({
1155 "runtimeVersion": self.config.runtime_identity.version,
1156 "protocolVersion": RUNTIME_API_VERSION_V3,
1157 "channel": PREVIEW_CHANNEL_V3,
1158 "modelGeneration": self.config.model_generation,
1159 "stateGeneration": state.snapshot.state_generation,
1160 "stateSchemaVersion": state.snapshot.state_schema_version,
1161 "stateChecksum": state.snapshot.checksum_sha256,
1162 "pendingDecisions": state.pending_decisions.len(),
1163 "completedDecisions": state.completed_decisions.len(),
1164 "resourceProfile": self.config.resource_profile,
1165 "resourceUtilization": {
1166 "stateBytes": state.snapshot.state.len(),
1167 "pendingDecisions": state.pending_decisions.len(),
1168 "completedDecisions": state.completed_decisions.len(),
1169 },
1170 "rollbackAvailable": state.previous_good.is_some(),
1171 "candidateAvailable": state.candidate.is_some(),
1172 "restartCount": state.restart_count,
1173 "health": self.health_status_for_state(state),
1174 "lastError": state.last_error,
1175 "handler": handler_summary,
1176 })
1177 }
1178
1179 pub fn diagnostics_at(
1181 &self,
1182 observed_at_unix_ms: u64,
1183 ) -> Result<RuntimeDiagnosticsV1, StatefulHandlerErrorV2> {
1184 let state = self
1185 .state
1186 .lock()
1187 .map_err(|_| StatefulHandlerErrorV2::new(StatefulHandlerErrorKindV2::Internal))?;
1188 let snapshot = StatefulRuntimeSnapshotV3::new(
1189 state.snapshot.clone(),
1190 state.pending_decisions.clone(),
1191 state.completed_decisions.clone(),
1192 );
1193 let snapshot_bytes = serde_json::to_vec(&snapshot)
1194 .map_err(|_| StatefulHandlerErrorV2::new(StatefulHandlerErrorKindV2::Internal))?
1195 .len();
1196 let usage = RuntimeResourceUsageV1 {
1197 state_bytes: state.snapshot.state.len(),
1198 snapshot_bytes,
1199 pending_decisions: state.pending_decisions.len(),
1200 completed_decisions: state.completed_decisions.len(),
1201 };
1202 let health = self.health_status_for_state(&state);
1203 let reason_codes = state
1204 .last_error
1205 .as_ref()
1206 .map(|error| vec![error.clone()])
1207 .unwrap_or_default();
1208 Ok(RuntimeDiagnosticsV1 {
1209 runtime_version: self.config.runtime_identity.version.clone(),
1210 protocol_version: RUNTIME_API_VERSION_V3,
1211 channel: PREVIEW_CHANNEL_V3.into(),
1212 model_generation: self.config.model_generation,
1213 state_generation: state.snapshot.state_generation,
1214 state_schema_version: state.snapshot.state_schema_version,
1215 observed_at_unix_ms,
1216 health,
1217 reason_codes,
1218 resource_usage: usage,
1219 rollback_available: state.previous_good.is_some(),
1220 candidate_available: state.candidate.is_some(),
1221 restart_count: state.restart_count,
1222 last_error: state.last_error.clone(),
1223 })
1224 }
1225
1226 fn health_status_for_state(&self, state: &RuntimeStateV3) -> RuntimeHealthStatusV1 {
1227 if state.last_error.is_some() {
1228 return RuntimeHealthStatusV1::FailedClosed;
1229 }
1230 let profile = &self.config.resource_profile;
1231 if state.snapshot.state.len() * 10 >= profile.max_model_state_bytes as usize * 9
1232 || state.pending_decisions.len() * 10 >= profile.max_pending_decisions as usize * 9
1233 || state.completed_decisions.len() * 10 >= profile.max_completed_decisions as usize * 9
1234 {
1235 RuntimeHealthStatusV1::ResourcePressure
1236 } else {
1237 RuntimeHealthStatusV1::Healthy
1238 }
1239 }
1240
1241 fn validate_runtime_snapshot_size(
1242 &self,
1243 snapshot: &StatefulRuntimeSnapshotV3,
1244 ) -> Result<(), StatefulHandlerErrorV2> {
1245 let bytes = serde_json::to_vec(snapshot)
1246 .map_err(|_| StatefulHandlerErrorV2::new(StatefulHandlerErrorKindV2::Internal))?;
1247 if bytes.len() > self.config.resource_profile.max_snapshot_bytes as usize {
1248 return Err(StatefulHandlerErrorV2::with_detail(
1249 StatefulHandlerErrorKindV2::InvalidState,
1250 "snapshot exceeds resource limit",
1251 ));
1252 }
1253 Ok(())
1254 }
1255
1256 fn current_generation(&self) -> u64 {
1257 self.state
1258 .lock()
1259 .map(|state| state.snapshot.state_generation)
1260 .unwrap_or(0)
1261 }
1262
1263 fn response(
1264 &self,
1265 request_id: String,
1266 state_generation: u64,
1267 response: RuntimeResponseBodyV3,
1268 ) -> RuntimeResponseV3 {
1269 RuntimeResponseV3 {
1270 request_id,
1271 api_version: RUNTIME_API_VERSION_V3,
1272 runtime_identity: self.config.runtime_identity.clone(),
1273 model_generation: self.config.model_generation,
1274 state_generation,
1275 response,
1276 }
1277 }
1278
1279 fn error_response(
1280 &self,
1281 request_id: String,
1282 code: RuntimeErrorCodeV3,
1283 message: impl Into<String>,
1284 state_generation: u64,
1285 ) -> RuntimeResponseV3 {
1286 self.response(
1287 if request_id.is_empty() {
1288 "invalid-request".into()
1289 } else {
1290 request_id
1291 },
1292 state_generation,
1293 RuntimeResponseBodyV3::Error {
1294 error: RuntimeErrorV3::new(code, message),
1295 },
1296 )
1297 }
1298}
1299
1300fn validate_state_bytes(
1301 state: &[u8],
1302 actual_schema_version: u32,
1303 expected_schema_version: u32,
1304) -> Result<(), StatefulHandlerErrorV2> {
1305 if actual_schema_version == 0 || actual_schema_version != expected_schema_version {
1306 return Err(StatefulHandlerErrorV2::new(
1307 StatefulHandlerErrorKindV2::IncompatibleVersion,
1308 ));
1309 }
1310 if state.len() > MAX_STATE_BYTES || serde_json::from_slice::<serde_json::Value>(state).is_err()
1311 {
1312 return Err(StatefulHandlerErrorV2::new(
1313 StatefulHandlerErrorKindV2::InvalidState,
1314 ));
1315 }
1316 Ok(())
1317}
1318
1319fn validate_feature_schema_hash(hash: &str) -> Result<(), StatefulHandlerErrorV2> {
1320 if hash.len() != 64
1321 || !hash
1322 .bytes()
1323 .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
1324 {
1325 return Err(StatefulHandlerErrorV2::new(
1326 StatefulHandlerErrorKindV2::InvalidModel,
1327 ));
1328 }
1329 Ok(())
1330}
1331
1332fn validate_capabilities(capabilities: &[String]) -> Result<(), StatefulHandlerErrorV2> {
1333 if capabilities.is_empty() || capabilities.len() > rill_handler_api::MAX_CAPABILITIES {
1334 return Err(StatefulHandlerErrorV2::new(
1335 StatefulHandlerErrorKindV2::InvalidModel,
1336 ));
1337 }
1338 let mut seen = std::collections::BTreeSet::new();
1339 if capabilities.iter().any(|capability| {
1340 capability.is_empty()
1341 || capability.len() > rill_handler_api::MAX_CAPABILITY_LEN
1342 || !seen.insert(capability)
1343 }) {
1344 return Err(StatefulHandlerErrorV2::new(
1345 StatefulHandlerErrorKindV2::InvalidModel,
1346 ));
1347 }
1348 Ok(())
1349}
1350
1351fn state_checksum(state: &[u8]) -> String {
1352 hex::encode(Sha256::digest(state))
1353}
1354
1355fn map_handler_error(kind: StatefulHandlerErrorKindV2) -> (RuntimeErrorCodeV3, &'static str) {
1356 match kind {
1357 StatefulHandlerErrorKindV2::InvalidEvent => (
1358 RuntimeErrorCodeV3::InvalidEnvelope,
1359 "handler rejected the event",
1360 ),
1361 StatefulHandlerErrorKindV2::InvalidState
1362 | StatefulHandlerErrorKindV2::IncompatibleVersion => (
1363 RuntimeErrorCodeV3::InvalidState,
1364 "handler rejected the current state",
1365 ),
1366 StatefulHandlerErrorKindV2::DuplicateFeedback => (
1367 RuntimeErrorCodeV3::DuplicateFeedback,
1368 "feedback was already applied",
1369 ),
1370 StatefulHandlerErrorKindV2::Timeout => (
1371 RuntimeErrorCodeV3::HandlerTimeout,
1372 "handler exceeded the wall-clock deadline",
1373 ),
1374 StatefulHandlerErrorKindV2::Trap => (RuntimeErrorCodeV3::HandlerTrap, "handler trapped"),
1375 StatefulHandlerErrorKindV2::OutputTooLarge => (
1376 RuntimeErrorCodeV3::HandlerOutputTooLarge,
1377 "handler output exceeded the size limit",
1378 ),
1379 StatefulHandlerErrorKindV2::InvalidOutput => (
1380 RuntimeErrorCodeV3::HandlerInvalidOutput,
1381 "handler output was not valid JSON",
1382 ),
1383 StatefulHandlerErrorKindV2::InvalidModel
1384 | StatefulHandlerErrorKindV2::MetadataMismatch
1385 | StatefulHandlerErrorKindV2::Internal => {
1386 (RuntimeErrorCodeV3::Internal, "internal runtime error")
1387 }
1388 }
1389}
1390
1391fn preview_error_code(code: RuntimeErrorCodeV3, message: &str) -> PreviewErrorCodeV3 {
1392 match message {
1393 "decision id was already used" => PreviewErrorCodeV3::DuplicateDecision,
1394 "decision id is not pending" => PreviewErrorCodeV3::UnknownDecision,
1395 "feedback generation is stale" => PreviewErrorCodeV3::StaleFeedback,
1396 "pending decision capacity is exhausted"
1397 | "completed decision capacity is exhausted"
1398 | "model state resource limit exceeded"
1399 | "snapshot resource limit exceeded" => PreviewErrorCodeV3::CapacityExceeded,
1400 _ => match code {
1401 RuntimeErrorCodeV3::InvalidJson => PreviewErrorCodeV3::InvalidJson,
1402 RuntimeErrorCodeV3::InvalidRequestId
1403 | RuntimeErrorCodeV3::InvalidClientIdentity
1404 | RuntimeErrorCodeV3::IncompatibleApiVersion => PreviewErrorCodeV3::InvalidEnvelope,
1405 RuntimeErrorCodeV3::InvalidEnvelope => PreviewErrorCodeV3::InvalidEnvelope,
1406 RuntimeErrorCodeV3::PayloadTooLarge => PreviewErrorCodeV3::PayloadTooLarge,
1407 RuntimeErrorCodeV3::UnsupportedCapability => PreviewErrorCodeV3::UnsupportedCapability,
1408 RuntimeErrorCodeV3::StateMismatch => PreviewErrorCodeV3::StateMismatch,
1409 RuntimeErrorCodeV3::ExpiredRequest => PreviewErrorCodeV3::ExpiredRequest,
1410 RuntimeErrorCodeV3::IncompatibleGeneration => {
1411 PreviewErrorCodeV3::IncompatibleGeneration
1412 }
1413 RuntimeErrorCodeV3::DuplicateFeedback => PreviewErrorCodeV3::DuplicateFeedback,
1414 RuntimeErrorCodeV3::HandlerTimeout => PreviewErrorCodeV3::HandlerTimeout,
1415 RuntimeErrorCodeV3::HandlerTrap => PreviewErrorCodeV3::HandlerTrap,
1416 RuntimeErrorCodeV3::HandlerOutputTooLarge => PreviewErrorCodeV3::HandlerOutputTooLarge,
1417 RuntimeErrorCodeV3::HandlerInvalidOutput => PreviewErrorCodeV3::HandlerInvalidOutput,
1418 RuntimeErrorCodeV3::InvalidState => PreviewErrorCodeV3::InvalidState,
1419 RuntimeErrorCodeV3::Internal => PreviewErrorCodeV3::Internal,
1420 },
1421 }
1422}
1423
1424#[cfg(test)]
1425mod tests {
1426 use super::*;
1427
1428 #[derive(Debug)]
1429 struct TestHandler {
1430 metadata: StatefulHandlerMetadataV2,
1431 mode: StatefulHandlerErrorKindV2,
1432 }
1433
1434 impl StatefulHandlerV2 for TestHandler {
1435 fn metadata(&self) -> &StatefulHandlerMetadataV2 {
1436 &self.metadata
1437 }
1438
1439 fn handle(
1440 &self,
1441 _event_json: &[u8],
1442 current_state: &[u8],
1443 _deterministic_seed: Option<u64>,
1444 ) -> Result<StatefulHandlerResultV2, StatefulHandlerErrorV2> {
1445 match self.mode {
1446 StatefulHandlerErrorKindV2::Internal => {
1447 let mut value: serde_json::Value =
1448 serde_json::from_slice(current_state).unwrap();
1449 value["count"] = serde_json::json!(value["count"].as_u64().unwrap_or(0) + 1);
1450 Ok(StatefulHandlerResultV2 {
1451 output: value.clone(),
1452 next_state: serde_json::to_vec(&value).unwrap(),
1453 })
1454 }
1455 StatefulHandlerErrorKindV2::InvalidState => Ok(StatefulHandlerResultV2 {
1456 output: serde_json::json!({"ignored": true}),
1457 next_state: b"not-json".to_vec(),
1458 }),
1459 StatefulHandlerErrorKindV2::OutputTooLarge => Ok(StatefulHandlerResultV2 {
1460 output: serde_json::json!({"data": "x".repeat(MAX_OUTPUT_BYTES + 1)}),
1461 next_state: current_state.to_vec(),
1462 }),
1463 other => Err(StatefulHandlerErrorV2::new(other)),
1464 }
1465 }
1466 }
1467
1468 fn engine(mode: StatefulHandlerErrorKindV2) -> StatefulRuntimeEngineV3 {
1469 let metadata = StatefulHandlerMetadataV2 {
1470 id: "org.example.stateful".into(),
1471 version: "2.0.0".into(),
1472 api_version: HANDLER_API_VERSION,
1473 capabilities: vec!["org.example.decide".into()],
1474 state_schema_version: 1,
1475 };
1476 let config = StatefulRuntimeConfigV3::new(
1477 IdentityV3 {
1478 name: "rill-runtime".into(),
1479 version: "1.0.0".into(),
1480 },
1481 7,
1482 "ab".repeat(32),
1483 metadata.capabilities.clone(),
1484 br#"{"count":0}"#.to_vec(),
1485 );
1486 StatefulRuntimeEngineV3::new(config, Arc::new(TestHandler { metadata, mode })).unwrap()
1487 }
1488
1489 fn decide(state_generation: u64) -> EnvelopeV3 {
1490 EnvelopeV3 {
1491 request_id: "d1".into(),
1492 api_version: RUNTIME_API_VERSION_V3,
1493 client_identity: IdentityV3 {
1494 name: "host".into(),
1495 version: "1".into(),
1496 },
1497 capability: Some("org.example.decide".into()),
1498 deadline_unix_ms: Some(100),
1499 feature_schema_hash: Some("ab".repeat(32)),
1500 model_generation: 7,
1501 state_generation,
1502 payload_limit: rill_runtime_protocol::MAX_MESSAGE_BYTES as u32,
1503 request: RuntimeRequestV3::Decide {
1504 context: serde_json::json!({"features": [1.0]}),
1505 deterministic_seed: Some(42),
1506 },
1507 }
1508 }
1509
1510 #[test]
1511 fn state_update_is_atomic_and_increments_generation() {
1512 let engine = engine(StatefulHandlerErrorKindV2::Internal);
1513 let response = engine.handle_at(decide(0), 100);
1514 assert!(matches!(
1515 response.response,
1516 RuntimeResponseBodyV3::Result { .. }
1517 ));
1518 assert_eq!(response.state_generation, 1);
1519 assert_eq!(engine.snapshot().unwrap().state, br#"{"count":1}"#);
1520 }
1521
1522 #[test]
1523 fn invalid_next_state_is_fail_closed() {
1524 let engine = engine(StatefulHandlerErrorKindV2::InvalidState);
1525 let before = engine.snapshot().unwrap();
1526 let response = engine.handle_at(decide(0), 100);
1527 assert!(matches!(
1528 response.response,
1529 RuntimeResponseBodyV3::Error { error }
1530 if error.code == RuntimeErrorCodeV3::InvalidState
1531 ));
1532 assert_eq!(engine.snapshot().unwrap(), before);
1533 }
1534
1535 #[test]
1536 fn timeout_trap_and_oversize_leave_state_unchanged() {
1537 for mode in [
1538 StatefulHandlerErrorKindV2::Timeout,
1539 StatefulHandlerErrorKindV2::Trap,
1540 StatefulHandlerErrorKindV2::OutputTooLarge,
1541 ] {
1542 let engine = engine(mode);
1543 let before = engine.snapshot().unwrap();
1544 let response = engine.handle_at(decide(0), 100);
1545 assert!(matches!(
1546 response.response,
1547 RuntimeResponseBodyV3::Error { .. }
1548 ));
1549 assert_eq!(engine.snapshot().unwrap(), before, "mode={mode:?}");
1550 }
1551 }
1552
1553 #[test]
1554 fn stale_generation_and_expired_request_are_rejected() {
1555 let engine = engine(StatefulHandlerErrorKindV2::Internal);
1556 let stale = engine.handle_at(decide(1), 100);
1557 assert!(matches!(
1558 stale.response,
1559 RuntimeResponseBodyV3::Error { error }
1560 if error.code == RuntimeErrorCodeV3::StateMismatch
1561 ));
1562 let expired = engine.handle_at(decide(0), 101);
1563 assert!(matches!(
1564 expired.response,
1565 RuntimeResponseBodyV3::Error { error }
1566 if error.code == RuntimeErrorCodeV3::ExpiredRequest
1567 ));
1568 }
1569
1570 #[test]
1571 fn corrupt_snapshot_checksum_is_rejected_without_mutation() {
1572 let engine = engine(StatefulHandlerErrorKindV2::Internal);
1573 let before = engine.snapshot().unwrap();
1574 let mut corrupt = before.clone();
1575 corrupt.checksum_sha256 = "00".repeat(32);
1576 assert!(engine.restore_snapshot(corrupt).is_err());
1577 assert_eq!(engine.snapshot().unwrap(), before);
1578 }
1579
1580 #[test]
1581 fn json_entrypoint_rejects_malformed_and_oversized_messages() {
1582 let engine = engine(StatefulHandlerErrorKindV2::Internal);
1583 let before = engine.snapshot().unwrap();
1584
1585 let malformed = engine.handle_json_at(b"{", 100);
1586 assert!(matches!(
1587 malformed.response,
1588 RuntimeResponseBodyV3::Error { error }
1589 if error.code == RuntimeErrorCodeV3::InvalidJson
1590 ));
1591
1592 let oversized = vec![b' '; rill_runtime_protocol::MAX_MESSAGE_BYTES + 1];
1593 let oversized = engine.handle_json_at(&oversized, 100);
1594 assert!(matches!(
1595 oversized.response,
1596 RuntimeResponseBodyV3::Error { error }
1597 if error.code == RuntimeErrorCodeV3::PayloadTooLarge
1598 ));
1599 assert_eq!(engine.snapshot().unwrap(), before);
1600 }
1601
1602 #[test]
1603 fn inspect_and_diagnostics_do_not_reenter_the_state_lock() {
1604 let engine = engine(StatefulHandlerErrorKindV2::Internal);
1605 let mut request = decide(0);
1606 request.request = RuntimeRequestV3::Inspect {};
1607 let response = engine.handle_preview_at(request, 100);
1608 assert!(matches!(
1609 response.response,
1610 RuntimeResponseBodyV3Preview::Inspection { .. }
1611 ));
1612 let diagnostics = engine.diagnostics_at(123).unwrap();
1613 assert_eq!(diagnostics.observed_at_unix_ms, 123);
1614 assert_eq!(diagnostics.health, RuntimeHealthStatusV1::Healthy);
1615 assert_eq!(diagnostics.state_generation, 1);
1616 assert!(diagnostics.resource_usage.snapshot_bytes > diagnostics.resource_usage.state_bytes);
1617 }
1618
1619 #[test]
1620 fn ipc_and_snapshot_limits_fail_closed_without_mutation() {
1621 let profile = ResourceProfileV1 {
1622 max_ipc_frame_bytes: 128,
1623 max_snapshot_bytes: 128,
1624 ..ResourceProfileV1::default()
1625 };
1626 let metadata = StatefulHandlerMetadataV2 {
1627 id: "org.example.stateful".into(),
1628 version: "2.0.0".into(),
1629 api_version: HANDLER_API_VERSION,
1630 capabilities: vec!["org.example.decide".into()],
1631 state_schema_version: 1,
1632 };
1633 let config = StatefulRuntimeConfigV3::new(
1634 IdentityV3 {
1635 name: "rill-runtime".into(),
1636 version: "1.0.0".into(),
1637 },
1638 7,
1639 "ab".repeat(32),
1640 metadata.capabilities.clone(),
1641 br#"{"count":0}"#.to_vec(),
1642 )
1643 .with_resource_profile(profile);
1644 let engine = StatefulRuntimeEngineV3::new(
1645 config,
1646 Arc::new(TestHandler {
1647 metadata,
1648 mode: StatefulHandlerErrorKindV2::Internal,
1649 }),
1650 )
1651 .unwrap();
1652 let before = engine.snapshot().unwrap();
1653 let oversized = vec![b' '; 129];
1654 let response = engine.handle_preview_json_at(&oversized, 100);
1655 assert!(matches!(
1656 response.response,
1657 RuntimeResponseBodyV3Preview::Error { error }
1658 if error.code == PreviewErrorCodeV3::PayloadTooLarge
1659 ));
1660 assert_eq!(engine.snapshot().unwrap(), before);
1661 assert!(engine.runtime_snapshot().is_err());
1662 }
1663}