1use std::{
7 collections::BTreeMap,
8 sync::{Arc, Mutex},
9};
10
11use rill_handler_api::v2::{
12 HANDLER_API_VERSION, MAX_EVENT_BYTES, MAX_OUTPUT_BYTES, MAX_STATE_BYTES,
13};
14use rill_runtime_protocol::v3::{
15 EnvelopeV3, IdentityV3, PREVIEW_CHANNEL_V3, PreviewErrorCodeV3, RUNTIME_API_VERSION_V3,
16 ResourceProfileV1, RuntimeErrorCodeV3, RuntimeErrorV3, RuntimeErrorV3Preview, RuntimeRequestV3,
17 RuntimeResponseBodyV3, RuntimeResponseBodyV3Preview, RuntimeResponseV3,
18 RuntimeResponseV3Preview,
19};
20use serde::{Deserialize, Serialize};
21use sha2::{Digest, Sha256};
22
23const MAX_HANDLER_DETAIL_BYTES_V2: usize = 4 * 1024;
24
25#[derive(Debug, Clone, PartialEq, Eq)]
27pub struct StatefulHandlerMetadataV2 {
28 pub id: String,
29 pub version: String,
30 pub api_version: u32,
31 pub capabilities: Vec<String>,
32 pub state_schema_version: u32,
33}
34
35impl StatefulHandlerMetadataV2 {
36 fn validate(&self) -> Result<(), StatefulHandlerErrorV2> {
37 if self.id.is_empty() || self.id.len() > rill_handler_api::MAX_HANDLER_ID_LEN {
38 return Err(StatefulHandlerErrorV2::new(
39 StatefulHandlerErrorKindV2::MetadataMismatch,
40 ));
41 }
42 if self.version.is_empty()
43 || self.version.len() > rill_handler_api::MAX_HANDLER_VERSION_LEN
44 || self.api_version != HANDLER_API_VERSION
45 || self.state_schema_version == 0
46 {
47 return Err(StatefulHandlerErrorV2::new(
48 StatefulHandlerErrorKindV2::MetadataMismatch,
49 ));
50 }
51 if self.capabilities.is_empty()
52 || self.capabilities.len() > rill_handler_api::MAX_CAPABILITIES
53 {
54 return Err(StatefulHandlerErrorV2::new(
55 StatefulHandlerErrorKindV2::MetadataMismatch,
56 ));
57 }
58 let mut seen = std::collections::BTreeSet::new();
59 if self.capabilities.iter().any(|capability| {
60 capability.is_empty()
61 || capability.len() > rill_handler_api::MAX_CAPABILITY_LEN
62 || !seen.insert(capability)
63 }) {
64 return Err(StatefulHandlerErrorV2::new(
65 StatefulHandlerErrorKindV2::MetadataMismatch,
66 ));
67 }
68 Ok(())
69 }
70}
71
72#[derive(Debug, Clone, PartialEq)]
75pub struct StatefulHandlerResultV2 {
76 pub output: serde_json::Value,
77 pub next_state: Vec<u8>,
78}
79
80#[derive(Debug, Clone, Copy, PartialEq, Eq)]
83#[non_exhaustive]
84pub enum StatefulHandlerErrorKindV2 {
85 InvalidModel,
86 InvalidEvent,
87 InvalidState,
88 IncompatibleVersion,
89 DuplicateFeedback,
90 Timeout,
91 Trap,
92 OutputTooLarge,
93 InvalidOutput,
94 MetadataMismatch,
95 Internal,
96}
97
98#[derive(Debug, Clone)]
100pub struct StatefulHandlerErrorV2 {
101 kind: StatefulHandlerErrorKindV2,
102 detail: Option<String>,
103}
104
105impl StatefulHandlerErrorV2 {
106 pub const fn new(kind: StatefulHandlerErrorKindV2) -> Self {
107 Self { kind, detail: None }
108 }
109
110 pub fn with_detail(kind: StatefulHandlerErrorKindV2, detail: impl Into<String>) -> Self {
111 let mut detail = detail.into();
112 if detail.len() > MAX_HANDLER_DETAIL_BYTES_V2 {
113 let mut end = MAX_HANDLER_DETAIL_BYTES_V2;
114 while end > 0 && !detail.is_char_boundary(end) {
115 end -= 1;
116 }
117 detail.truncate(end);
118 }
119 Self {
120 kind,
121 detail: Some(detail),
122 }
123 }
124
125 pub const fn kind(&self) -> StatefulHandlerErrorKindV2 {
126 self.kind
127 }
128
129 pub fn detail(&self) -> Option<&str> {
130 self.detail.as_deref()
131 }
132}
133
134impl std::fmt::Display for StatefulHandlerErrorV2 {
135 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
136 write!(f, "stateful handler {:?}", self.kind)?;
137 if let Some(detail) = &self.detail {
138 write!(f, ": {detail}")?;
139 }
140 Ok(())
141 }
142}
143
144impl std::error::Error for StatefulHandlerErrorV2 {}
145
146pub trait StatefulHandlerV2: Send + Sync + std::fmt::Debug {
150 fn metadata(&self) -> &StatefulHandlerMetadataV2;
151
152 fn handle(
153 &self,
154 event_json: &[u8],
155 current_state: &[u8],
156 deterministic_seed: Option<u64>,
157 ) -> Result<StatefulHandlerResultV2, StatefulHandlerErrorV2>;
158}
159
160#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
163#[serde(rename_all = "camelCase", deny_unknown_fields)]
164pub struct StatefulStateSnapshotV2 {
165 pub state_schema_version: u32,
166 pub state_generation: u64,
167 pub state: Vec<u8>,
168 pub checksum_sha256: String,
169}
170
171#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
173#[serde(rename_all = "camelCase", deny_unknown_fields)]
174pub struct DecisionLedgerEntryV3 {
175 pub decision_id: String,
176 pub model_generation: u64,
177 pub state_generation: u64,
178 pub created_at_unix_ms: u64,
179 #[serde(default, skip_serializing_if = "Option::is_none")]
180 pub selected_arm: Option<u32>,
181 #[serde(default, skip_serializing_if = "Option::is_none")]
182 pub reward: Option<String>,
183 #[serde(default, skip_serializing_if = "Option::is_none")]
184 pub outcome_time_unix_ms: Option<u64>,
185}
186
187#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
190#[serde(rename_all = "snake_case")]
191pub enum RuntimeHealthStatusV1 {
192 Healthy,
193 ResourcePressure,
194 FailedClosed,
195}
196
197impl std::fmt::Display for RuntimeHealthStatusV1 {
198 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
199 f.write_str(match self {
200 Self::Healthy => "healthy",
201 Self::ResourcePressure => "resource_pressure",
202 Self::FailedClosed => "failed_closed",
203 })
204 }
205}
206
207#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
209#[serde(rename_all = "camelCase", deny_unknown_fields)]
210pub struct RuntimeResourceUsageV1 {
211 pub state_bytes: usize,
212 pub snapshot_bytes: usize,
213 pub pending_decisions: usize,
214 pub completed_decisions: usize,
215}
216
217#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
221#[serde(rename_all = "camelCase", deny_unknown_fields)]
222pub struct RuntimeDiagnosticsV1 {
223 pub runtime_version: String,
224 pub protocol_version: u32,
225 pub channel: String,
226 pub model_generation: u64,
227 pub state_generation: u64,
228 pub state_schema_version: u32,
229 pub observed_at_unix_ms: u64,
230 pub health: RuntimeHealthStatusV1,
231 pub reason_codes: Vec<String>,
232 pub resource_usage: RuntimeResourceUsageV1,
233 pub rollback_available: bool,
234 pub candidate_available: bool,
235 pub restart_count: u64,
236 pub last_error: Option<String>,
237}
238
239#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
242#[serde(rename_all = "camelCase", deny_unknown_fields)]
243pub struct StatefulRuntimeSnapshotV3 {
244 pub format_version: u32,
245 pub handler_snapshot: StatefulStateSnapshotV2,
246 pub pending_decisions: BTreeMap<String, DecisionLedgerEntryV3>,
247 pub completed_decisions: BTreeMap<String, DecisionLedgerEntryV3>,
248 pub checksum_sha256: String,
249}
250
251impl StatefulRuntimeSnapshotV3 {
252 pub const FORMAT_VERSION: u32 = 1;
253
254 fn checksum_input(
255 handler_snapshot: &StatefulStateSnapshotV2,
256 pending_decisions: &BTreeMap<String, DecisionLedgerEntryV3>,
257 completed_decisions: &BTreeMap<String, DecisionLedgerEntryV3>,
258 ) -> Vec<u8> {
259 serde_json::to_vec(&(handler_snapshot, pending_decisions, completed_decisions))
260 .expect("runtime snapshot fields are serializable")
261 }
262
263 fn new(
264 handler_snapshot: StatefulStateSnapshotV2,
265 pending_decisions: BTreeMap<String, DecisionLedgerEntryV3>,
266 completed_decisions: BTreeMap<String, DecisionLedgerEntryV3>,
267 ) -> Self {
268 let checksum_sha256 = state_checksum(&Self::checksum_input(
269 &handler_snapshot,
270 &pending_decisions,
271 &completed_decisions,
272 ));
273 Self {
274 format_version: Self::FORMAT_VERSION,
275 handler_snapshot,
276 pending_decisions,
277 completed_decisions,
278 checksum_sha256,
279 }
280 }
281
282 fn validate(&self, expected_schema_version: u32) -> Result<(), StatefulHandlerErrorV2> {
283 if self.format_version != Self::FORMAT_VERSION {
284 return Err(StatefulHandlerErrorV2::new(
285 StatefulHandlerErrorKindV2::IncompatibleVersion,
286 ));
287 }
288 self.handler_snapshot.validate(expected_schema_version)?;
289 if self.checksum_sha256
290 != state_checksum(&Self::checksum_input(
291 &self.handler_snapshot,
292 &self.pending_decisions,
293 &self.completed_decisions,
294 ))
295 {
296 return Err(StatefulHandlerErrorV2::new(
297 StatefulHandlerErrorKindV2::InvalidState,
298 ));
299 }
300 for (key, entry) in self
301 .pending_decisions
302 .iter()
303 .chain(self.completed_decisions.iter())
304 {
305 if key != &entry.decision_id || entry.decision_id.is_empty() {
306 return Err(StatefulHandlerErrorV2::new(
307 StatefulHandlerErrorKindV2::InvalidState,
308 ));
309 }
310 }
311 Ok(())
312 }
313}
314
315pub trait StatefulStateMigratorV1: Send + Sync {
318 fn migrate(
319 &self,
320 from_schema_version: u32,
321 state: &[u8],
322 ) -> Result<(u32, Vec<u8>), StatefulHandlerErrorV2>;
323}
324
325impl StatefulStateSnapshotV2 {
326 pub fn new(state_schema_version: u32, state_generation: u64, state: Vec<u8>) -> Self {
327 let checksum_sha256 = state_checksum(&state);
328 Self {
329 state_schema_version,
330 state_generation,
331 state,
332 checksum_sha256,
333 }
334 }
335
336 pub fn validate(&self, expected_schema_version: u32) -> Result<(), StatefulHandlerErrorV2> {
337 validate_state_bytes(
338 &self.state,
339 self.state_schema_version,
340 expected_schema_version,
341 )?;
342 if self.checksum_sha256 != state_checksum(&self.state) {
343 return Err(StatefulHandlerErrorV2::new(
344 StatefulHandlerErrorKindV2::InvalidState,
345 ));
346 }
347 Ok(())
348 }
349}
350
351#[derive(Debug, Clone)]
353#[non_exhaustive]
354pub struct StatefulRuntimeConfigV3 {
355 pub runtime_identity: IdentityV3,
356 pub model_generation: u64,
357 pub initial_state_generation: u64,
358 pub feature_schema_hash: String,
359 pub capabilities: Vec<String>,
360 pub initial_state: Vec<u8>,
361 pub resource_profile: ResourceProfileV1,
362}
363
364impl StatefulRuntimeConfigV3 {
365 pub fn new(
366 runtime_identity: IdentityV3,
367 model_generation: u64,
368 feature_schema_hash: String,
369 capabilities: Vec<String>,
370 initial_state: Vec<u8>,
371 ) -> Self {
372 Self {
373 runtime_identity,
374 model_generation,
375 initial_state_generation: 0,
376 feature_schema_hash,
377 capabilities,
378 initial_state,
379 resource_profile: ResourceProfileV1::default(),
380 }
381 }
382
383 pub fn with_resource_profile(mut self, resource_profile: ResourceProfileV1) -> Self {
384 self.resource_profile = resource_profile;
385 self
386 }
387}
388
389#[derive(Debug, Clone)]
390struct RuntimeStateV3 {
391 snapshot: StatefulStateSnapshotV2,
392 previous_good: Option<StatefulStateSnapshotV2>,
393 candidate: Option<StatefulStateSnapshotV2>,
394 pending_decisions: BTreeMap<String, DecisionLedgerEntryV3>,
395 completed_decisions: BTreeMap<String, DecisionLedgerEntryV3>,
396 restart_count: u64,
397 last_error: Option<String>,
398}
399
400#[derive(Debug)]
402pub struct StatefulRuntimeEngineV3 {
403 config: StatefulRuntimeConfigV3,
404 metadata: StatefulHandlerMetadataV2,
405 handler: Arc<dyn StatefulHandlerV2>,
406 state: Mutex<RuntimeStateV3>,
407}
408
409impl StatefulRuntimeEngineV3 {
410 pub fn new(
411 config: StatefulRuntimeConfigV3,
412 handler: Arc<dyn StatefulHandlerV2>,
413 ) -> Result<Self, StatefulHandlerErrorV2> {
414 let metadata = handler.metadata().clone();
415 metadata.validate()?;
416 config.runtime_identity.validate().map_err(|error| {
417 StatefulHandlerErrorV2::with_detail(
418 StatefulHandlerErrorKindV2::InvalidModel,
419 error.to_string(),
420 )
421 })?;
422 validate_feature_schema_hash(&config.feature_schema_hash)?;
423 validate_capabilities(&config.capabilities)?;
424 config.resource_profile.validate().map_err(|detail| {
425 StatefulHandlerErrorV2::with_detail(StatefulHandlerErrorKindV2::InvalidModel, detail)
426 })?;
427 if config.capabilities != metadata.capabilities {
428 return Err(StatefulHandlerErrorV2::new(
429 StatefulHandlerErrorKindV2::MetadataMismatch,
430 ));
431 }
432 validate_state_bytes(
433 &config.initial_state,
434 metadata.state_schema_version,
435 metadata.state_schema_version,
436 )?;
437 let snapshot = StatefulStateSnapshotV2::new(
438 metadata.state_schema_version,
439 config.initial_state_generation,
440 config.initial_state.clone(),
441 );
442 if snapshot.state.len() > config.resource_profile.max_model_state_bytes as usize {
443 return Err(StatefulHandlerErrorV2::new(
444 StatefulHandlerErrorKindV2::InvalidState,
445 ));
446 }
447 Ok(Self {
448 config,
449 metadata,
450 handler,
451 state: Mutex::new(RuntimeStateV3 {
452 snapshot,
453 previous_good: None,
454 candidate: None,
455 pending_decisions: BTreeMap::new(),
456 completed_decisions: BTreeMap::new(),
457 restart_count: 0,
458 last_error: None,
459 }),
460 })
461 }
462
463 pub fn restore_snapshot(
465 &self,
466 snapshot: StatefulStateSnapshotV2,
467 ) -> Result<(), StatefulHandlerErrorV2> {
468 snapshot.validate(self.metadata.state_schema_version)?;
469 if snapshot.state.len() > self.config.resource_profile.max_model_state_bytes as usize {
470 return Err(StatefulHandlerErrorV2::new(
471 StatefulHandlerErrorKindV2::InvalidState,
472 ));
473 }
474 let mut state = self
475 .state
476 .lock()
477 .map_err(|_| StatefulHandlerErrorV2::new(StatefulHandlerErrorKindV2::Internal))?;
478 state.previous_good = Some(state.snapshot.clone());
479 state.snapshot = snapshot;
480 state.restart_count = state.restart_count.saturating_add(1);
481 Ok(())
482 }
483
484 pub fn snapshot(&self) -> Result<StatefulStateSnapshotV2, StatefulHandlerErrorV2> {
485 self.state
486 .lock()
487 .map(|state| state.snapshot.clone())
488 .map_err(|_| StatefulHandlerErrorV2::new(StatefulHandlerErrorKindV2::Internal))
489 }
490
491 pub fn runtime_snapshot(&self) -> Result<StatefulRuntimeSnapshotV3, StatefulHandlerErrorV2> {
493 let state = self
494 .state
495 .lock()
496 .map_err(|_| StatefulHandlerErrorV2::new(StatefulHandlerErrorKindV2::Internal))?;
497 let snapshot = StatefulRuntimeSnapshotV3::new(
498 state.snapshot.clone(),
499 state.pending_decisions.clone(),
500 state.completed_decisions.clone(),
501 );
502 self.validate_runtime_snapshot_size(&snapshot)?;
503 Ok(snapshot)
504 }
505
506 pub fn restore_runtime_snapshot(
508 &self,
509 snapshot: StatefulRuntimeSnapshotV3,
510 ) -> Result<(), StatefulHandlerErrorV2> {
511 snapshot.validate(self.metadata.state_schema_version)?;
512 if snapshot.handler_snapshot.state.len()
513 > self.config.resource_profile.max_model_state_bytes as usize
514 || snapshot.pending_decisions.len()
515 > self.config.resource_profile.max_pending_decisions as usize
516 || snapshot.completed_decisions.len()
517 > self.config.resource_profile.max_completed_decisions as usize
518 {
519 return Err(StatefulHandlerErrorV2::new(
520 StatefulHandlerErrorKindV2::InvalidState,
521 ));
522 }
523 self.validate_runtime_snapshot_size(&snapshot)?;
524 let mut state = self
525 .state
526 .lock()
527 .map_err(|_| StatefulHandlerErrorV2::new(StatefulHandlerErrorKindV2::Internal))?;
528 state.previous_good = Some(state.snapshot.clone());
529 state.snapshot = snapshot.handler_snapshot;
530 state.pending_decisions = snapshot.pending_decisions;
531 state.completed_decisions = snapshot.completed_decisions;
532 state.restart_count = state.restart_count.saturating_add(1);
533 Ok(())
534 }
535
536 pub fn stage_candidate(
538 &self,
539 snapshot: StatefulStateSnapshotV2,
540 ) -> Result<(), StatefulHandlerErrorV2> {
541 snapshot.validate(self.metadata.state_schema_version)?;
542 if snapshot.state.len() > self.config.resource_profile.max_model_state_bytes as usize {
543 return Err(StatefulHandlerErrorV2::new(
544 StatefulHandlerErrorKindV2::InvalidState,
545 ));
546 }
547 let mut state = self
548 .state
549 .lock()
550 .map_err(|_| StatefulHandlerErrorV2::new(StatefulHandlerErrorKindV2::Internal))?;
551 state.candidate = Some(snapshot);
552 Ok(())
553 }
554
555 pub fn promote_candidate(&self) -> Result<u64, StatefulHandlerErrorV2> {
557 let mut state = self
558 .state
559 .lock()
560 .map_err(|_| StatefulHandlerErrorV2::new(StatefulHandlerErrorKindV2::Internal))?;
561 let candidate = state
562 .candidate
563 .take()
564 .ok_or_else(|| StatefulHandlerErrorV2::new(StatefulHandlerErrorKindV2::InvalidState))?;
565 state.previous_good = Some(state.snapshot.clone());
566 state.snapshot = candidate;
567 state.pending_decisions.clear();
568 state.completed_decisions.clear();
569 Ok(state.snapshot.state_generation)
570 }
571
572 pub fn rollback_previous_good(&self) -> Result<u64, StatefulHandlerErrorV2> {
574 let mut state = self
575 .state
576 .lock()
577 .map_err(|_| StatefulHandlerErrorV2::new(StatefulHandlerErrorKindV2::Internal))?;
578 let previous = state
579 .previous_good
580 .take()
581 .ok_or_else(|| StatefulHandlerErrorV2::new(StatefulHandlerErrorKindV2::InvalidState))?;
582 let failed = std::mem::replace(&mut state.snapshot, previous);
583 state.previous_good = Some(failed);
584 state.candidate = None;
585 state.pending_decisions.clear();
586 state.completed_decisions.clear();
587 Ok(state.snapshot.state_generation)
588 }
589
590 pub fn restore_snapshot_with_migration(
592 &self,
593 snapshot: StatefulStateSnapshotV2,
594 migrator: &dyn StatefulStateMigratorV1,
595 ) -> Result<(), StatefulHandlerErrorV2> {
596 if snapshot.state_schema_version == self.metadata.state_schema_version {
597 return self.restore_snapshot(snapshot);
598 }
599 let (schema_version, state) =
600 migrator.migrate(snapshot.state_schema_version, &snapshot.state)?;
601 if schema_version != self.metadata.state_schema_version {
602 return Err(StatefulHandlerErrorV2::new(
603 StatefulHandlerErrorKindV2::IncompatibleVersion,
604 ));
605 }
606 self.restore_snapshot(StatefulStateSnapshotV2::new(
607 schema_version,
608 snapshot.state_generation,
609 state,
610 ))
611 }
612
613 pub fn handle_at(&self, envelope: EnvelopeV3, now_unix_ms: u64) -> RuntimeResponseV3 {
616 if let Err(error) = envelope.validate() {
617 return self.error_response(
618 envelope.request_id,
619 RuntimeErrorCodeV3::InvalidEnvelope,
620 error.to_string(),
621 self.current_generation(),
622 );
623 }
624 if envelope.is_expired_at(now_unix_ms) {
625 return self.error_response(
626 envelope.request_id,
627 RuntimeErrorCodeV3::ExpiredRequest,
628 "request deadline has expired",
629 self.current_generation(),
630 );
631 }
632
633 match envelope.request.clone() {
634 RuntimeRequestV3::Handshake {} => self.response(
635 envelope.request_id,
636 self.current_generation(),
637 RuntimeResponseBodyV3::Handshake {
638 capabilities: self.config.capabilities.clone(),
639 feature_schema_hash: self.config.feature_schema_hash.clone(),
640 handler_api_version: HANDLER_API_VERSION,
641 },
642 ),
643 RuntimeRequestV3::Health {} => self.response(
644 envelope.request_id,
645 self.current_generation(),
646 RuntimeResponseBodyV3::Health {
647 healthy: self.is_healthy(),
648 },
649 ),
650 request => self.handle_stateful(envelope, request, now_unix_ms),
651 }
652 }
653
654 pub fn handle_json_at(&self, message: &[u8], now_unix_ms: u64) -> RuntimeResponseV3 {
658 if message.len() > self.config.resource_profile.max_ipc_frame_bytes as usize {
659 return self.error_response(
660 "invalid-request".into(),
661 RuntimeErrorCodeV3::PayloadTooLarge,
662 "request exceeds the IPC message limit",
663 self.current_generation(),
664 );
665 }
666 let envelope = match serde_json::from_slice::<EnvelopeV3>(message) {
667 Ok(envelope) => envelope,
668 Err(_) => {
669 return self.error_response(
670 "invalid-request".into(),
671 RuntimeErrorCodeV3::InvalidJson,
672 "request is not valid IPC V3 JSON",
673 self.current_generation(),
674 );
675 }
676 };
677 self.handle_at(envelope, now_unix_ms)
678 }
679
680 pub fn handle_preview_at(
683 &self,
684 envelope: EnvelopeV3,
685 now_unix_ms: u64,
686 ) -> RuntimeResponseV3Preview {
687 let is_decide = matches!(envelope.request, RuntimeRequestV3::Decide { .. });
688 let response = self.handle_at(envelope, now_unix_ms);
689 let body = match response.response {
690 RuntimeResponseBodyV3::Handshake {
691 capabilities,
692 feature_schema_hash,
693 handler_api_version,
694 } => RuntimeResponseBodyV3Preview::Handshake {
695 capabilities,
696 feature_schema_hash,
697 handler_api_version,
698 channel: PREVIEW_CHANNEL_V3.into(),
699 },
700 RuntimeResponseBodyV3::Health { healthy } => RuntimeResponseBodyV3Preview::Health {
701 healthy,
702 status: if healthy {
703 "healthy".into()
704 } else {
705 "failed_closed".into()
706 },
707 reason_codes: self.health_reason_codes().unwrap_or_default(),
708 },
709 RuntimeResponseBodyV3::Result { output } => RuntimeResponseBodyV3Preview::Result {
710 output,
711 decision_id: is_decide.then_some(response.request_id.clone()),
712 decision_generation: is_decide.then_some(response.state_generation),
713 },
714 RuntimeResponseBodyV3::Inspection { summary } => {
715 RuntimeResponseBodyV3Preview::Inspection { summary }
716 }
717 RuntimeResponseBodyV3::Snapshot {
718 state_schema_version,
719 state_checksum,
720 state,
721 } => RuntimeResponseBodyV3Preview::Snapshot {
722 state_schema_version,
723 state_checksum,
724 state,
725 },
726 RuntimeResponseBodyV3::Reset { reset } => RuntimeResponseBodyV3Preview::Reset { reset },
727 RuntimeResponseBodyV3::Error { error } => RuntimeResponseBodyV3Preview::Error {
728 error: RuntimeErrorV3Preview {
729 code: preview_error_code(error.code, &error.message),
730 message: error.message,
731 retryable: preview_error_code(error.code, "").is_retryable(),
732 },
733 },
734 };
735 RuntimeResponseV3Preview {
736 request_id: response.request_id,
737 api_version: response.api_version,
738 runtime_identity: response.runtime_identity,
739 model_generation: response.model_generation,
740 state_generation: response.state_generation,
741 response: body,
742 }
743 }
744
745 pub fn handle_preview_json_at(
746 &self,
747 message: &[u8],
748 now_unix_ms: u64,
749 ) -> RuntimeResponseV3Preview {
750 if message.len() > self.config.resource_profile.max_ipc_frame_bytes as usize {
751 return self.preview_error_response(
752 "invalid-request".into(),
753 PreviewErrorCodeV3::PayloadTooLarge,
754 "request exceeds the IPC message limit",
755 );
756 }
757 match serde_json::from_slice::<EnvelopeV3>(message) {
758 Ok(envelope) => self.handle_preview_at(envelope, now_unix_ms),
759 Err(_) => self.preview_error_response(
760 "invalid-request".into(),
761 PreviewErrorCodeV3::InvalidJson,
762 "request is not valid IPC V3 JSON",
763 ),
764 }
765 }
766
767 fn preview_error_response(
768 &self,
769 request_id: String,
770 code: PreviewErrorCodeV3,
771 message: &str,
772 ) -> RuntimeResponseV3Preview {
773 RuntimeResponseV3Preview {
774 request_id,
775 api_version: RUNTIME_API_VERSION_V3,
776 runtime_identity: self.config.runtime_identity.clone(),
777 model_generation: self.config.model_generation,
778 state_generation: self.current_generation(),
779 response: RuntimeResponseBodyV3Preview::Error {
780 error: RuntimeErrorV3Preview {
781 code,
782 message: message.into(),
783 retryable: code.is_retryable(),
784 },
785 },
786 }
787 }
788
789 fn handle_stateful(
790 &self,
791 envelope: EnvelopeV3,
792 request: RuntimeRequestV3,
793 now_unix_ms: u64,
794 ) -> RuntimeResponseV3 {
795 let request_id = envelope.request_id;
796 let capability = envelope.capability.unwrap_or_default();
797 if !self
798 .config
799 .capabilities
800 .iter()
801 .any(|item| item == &capability)
802 {
803 return self.error_response(
804 request_id,
805 RuntimeErrorCodeV3::UnsupportedCapability,
806 "capability is not in the effective set",
807 self.current_generation(),
808 );
809 }
810 if envelope.feature_schema_hash.as_deref() != Some(self.config.feature_schema_hash.as_str())
811 {
812 return self.error_response(
813 request_id,
814 RuntimeErrorCodeV3::StateMismatch,
815 "feature schema hash does not match",
816 self.current_generation(),
817 );
818 }
819 if envelope.model_generation != self.config.model_generation {
820 return self.error_response(
821 request_id,
822 RuntimeErrorCodeV3::IncompatibleGeneration,
823 "model generation does not match",
824 self.current_generation(),
825 );
826 }
827 if let RuntimeRequestV3::Feedback { generation, .. } = &request
828 && *generation != self.config.model_generation
829 {
830 return self.error_response(
831 request_id,
832 RuntimeErrorCodeV3::IncompatibleGeneration,
833 "feedback generation does not match",
834 self.current_generation(),
835 );
836 }
837
838 let mut state = match self.state.lock() {
839 Ok(state) => state,
840 Err(_) => {
841 return self.error_response(
842 request_id,
843 RuntimeErrorCodeV3::Internal,
844 "runtime state lock is poisoned",
845 0,
846 );
847 }
848 };
849 if envelope.state_generation != state.snapshot.state_generation {
850 return self.error_response(
851 request_id,
852 RuntimeErrorCodeV3::StateMismatch,
853 "state generation does not match",
854 state.snapshot.state_generation,
855 );
856 }
857
858 if let RuntimeRequestV3::Decide { .. } = &request {
859 if state.pending_decisions.contains_key(&request_id)
860 || state.completed_decisions.contains_key(&request_id)
861 {
862 return self.error_response(
863 request_id,
864 RuntimeErrorCodeV3::Internal,
865 "decision id was already used",
866 state.snapshot.state_generation,
867 );
868 }
869 if state.pending_decisions.len()
870 >= self.config.resource_profile.max_pending_decisions as usize
871 {
872 return self.error_response(
873 request_id,
874 RuntimeErrorCodeV3::Internal,
875 "pending decision capacity is exhausted",
876 state.snapshot.state_generation,
877 );
878 }
879 }
880 if let RuntimeRequestV3::Feedback {
881 decision_id,
882 generation,
883 ..
884 } = &request
885 {
886 if state.completed_decisions.contains_key(decision_id) {
887 return self.error_response(
888 request_id,
889 RuntimeErrorCodeV3::DuplicateFeedback,
890 "feedback was already applied",
891 state.snapshot.state_generation,
892 );
893 }
894 let Some(entry) = state.pending_decisions.get(decision_id) else {
895 return self.error_response(
896 request_id,
897 RuntimeErrorCodeV3::Internal,
898 "decision id is not pending",
899 state.snapshot.state_generation,
900 );
901 };
902 if entry.model_generation != *generation {
903 return self.error_response(
904 request_id,
905 RuntimeErrorCodeV3::IncompatibleGeneration,
906 "feedback generation is stale",
907 state.snapshot.state_generation,
908 );
909 }
910 if state.completed_decisions.len()
911 >= self.config.resource_profile.max_completed_decisions as usize
912 {
913 return self.error_response(
914 request_id,
915 RuntimeErrorCodeV3::Internal,
916 "completed decision capacity is exhausted",
917 state.snapshot.state_generation,
918 );
919 }
920 }
921
922 if let RuntimeRequestV3::Snapshot {} = request {
923 return self.response(
924 request_id,
925 state.snapshot.state_generation,
926 RuntimeResponseBodyV3::Snapshot {
927 state_schema_version: state.snapshot.state_schema_version,
928 state_checksum: state.snapshot.checksum_sha256.clone(),
929 state: hex::encode(&state.snapshot.state),
930 },
931 );
932 }
933 if let RuntimeRequestV3::Reset {
934 expected_state_generation,
935 } = request
936 {
937 if expected_state_generation != state.snapshot.state_generation {
938 return self.error_response(
939 request_id,
940 RuntimeErrorCodeV3::StateMismatch,
941 "reset generation does not match",
942 state.snapshot.state_generation,
943 );
944 }
945 let Some(next_generation) = state.snapshot.state_generation.checked_add(1) else {
946 return self.error_response(
947 request_id,
948 RuntimeErrorCodeV3::InvalidState,
949 "state generation overflow",
950 state.snapshot.state_generation,
951 );
952 };
953 state.snapshot = StatefulStateSnapshotV2::new(
954 self.metadata.state_schema_version,
955 next_generation,
956 self.config.initial_state.clone(),
957 );
958 state.pending_decisions.clear();
959 state.completed_decisions.clear();
960 return self.response(
961 request_id,
962 next_generation,
963 RuntimeResponseBodyV3::Reset { reset: true },
964 );
965 }
966
967 let deterministic_seed = match &request {
968 RuntimeRequestV3::Decide {
969 deterministic_seed, ..
970 } => *deterministic_seed,
971 _ => None,
972 };
973 let event_json = match serde_json::to_vec(&request) {
974 Ok(bytes) if bytes.len() <= MAX_EVENT_BYTES => bytes,
975 Ok(_) => {
976 return self.error_response(
977 request_id,
978 RuntimeErrorCodeV3::PayloadTooLarge,
979 "event exceeds handler limit",
980 state.snapshot.state_generation,
981 );
982 }
983 Err(_) => {
984 return self.error_response(
985 request_id,
986 RuntimeErrorCodeV3::InvalidJson,
987 "event could not be encoded",
988 state.snapshot.state_generation,
989 );
990 }
991 };
992
993 let result =
994 match self
995 .handler
996 .handle(&event_json, &state.snapshot.state, deterministic_seed)
997 {
998 Ok(result) => result,
999 Err(error) => {
1000 let (code, message) = map_handler_error(error.kind());
1001 state.last_error = Some(message.into());
1002 return self.error_response(
1003 request_id,
1004 code,
1005 message,
1006 state.snapshot.state_generation,
1007 );
1008 }
1009 };
1010 let output_bytes = match serde_json::to_vec(&result.output) {
1011 Ok(bytes) => bytes,
1012 Err(_) => {
1013 return self.error_response(
1014 request_id,
1015 RuntimeErrorCodeV3::HandlerInvalidOutput,
1016 "handler output was not valid JSON",
1017 state.snapshot.state_generation,
1018 );
1019 }
1020 };
1021 if output_bytes.len() > MAX_OUTPUT_BYTES {
1022 return self.error_response(
1023 request_id,
1024 RuntimeErrorCodeV3::HandlerOutputTooLarge,
1025 "handler output exceeded the size limit",
1026 state.snapshot.state_generation,
1027 );
1028 }
1029 if validate_state_bytes(
1030 &result.next_state,
1031 self.metadata.state_schema_version,
1032 self.metadata.state_schema_version,
1033 )
1034 .is_err()
1035 {
1036 return self.error_response(
1037 request_id,
1038 RuntimeErrorCodeV3::InvalidState,
1039 "handler returned invalid next state",
1040 state.snapshot.state_generation,
1041 );
1042 }
1043 if result.next_state.len() > self.config.resource_profile.max_model_state_bytes as usize {
1044 state.last_error = Some("model state resource limit exceeded".into());
1045 return self.error_response(
1046 request_id,
1047 RuntimeErrorCodeV3::Internal,
1048 "model state resource limit exceeded",
1049 state.snapshot.state_generation,
1050 );
1051 }
1052 let Some(next_generation) = state.snapshot.state_generation.checked_add(1) else {
1053 return self.error_response(
1054 request_id,
1055 RuntimeErrorCodeV3::InvalidState,
1056 "state generation overflow",
1057 state.snapshot.state_generation,
1058 );
1059 };
1060 let previous_snapshot = state.snapshot.clone();
1061 state.snapshot = StatefulStateSnapshotV2::new(
1062 self.metadata.state_schema_version,
1063 next_generation,
1064 result.next_state,
1065 );
1066 state.previous_good = Some(previous_snapshot);
1067 if matches!(request, RuntimeRequestV3::Decide { .. }) {
1068 let entry = DecisionLedgerEntryV3 {
1069 decision_id: request_id.clone(),
1070 model_generation: self.config.model_generation,
1071 state_generation: next_generation,
1072 created_at_unix_ms: now_unix_ms,
1073 selected_arm: None,
1074 reward: None,
1075 outcome_time_unix_ms: None,
1076 };
1077 state.pending_decisions.insert(request_id.clone(), entry);
1078 } else {
1079 if let RuntimeRequestV3::Feedback {
1080 decision_id,
1081 selected_arm,
1082 reward,
1083 outcome_time_ms,
1084 ..
1085 } = &request
1086 {
1087 let mut entry = state
1088 .pending_decisions
1089 .remove(decision_id)
1090 .ok_or_else(|| {
1091 StatefulHandlerErrorV2::new(StatefulHandlerErrorKindV2::Internal)
1092 })
1093 .unwrap();
1094 entry.selected_arm = Some(*selected_arm);
1095 entry.reward = Some(reward.to_string());
1096 entry.outcome_time_unix_ms = Some(*outcome_time_ms);
1097 state
1098 .completed_decisions
1099 .insert(entry.decision_id.clone(), entry);
1100 }
1101 }
1102 self.response(
1103 request_id,
1104 next_generation,
1105 match request {
1106 RuntimeRequestV3::Inspect {} => RuntimeResponseBodyV3::Inspection {
1107 summary: self.inspection_summary(&state, result.output),
1108 },
1109 _ => RuntimeResponseBodyV3::Result {
1110 output: result.output,
1111 },
1112 },
1113 )
1114 }
1115
1116 fn is_healthy(&self) -> bool {
1117 self.state
1118 .lock()
1119 .map(|state| state.last_error.is_none())
1120 .unwrap_or(false)
1121 }
1122
1123 fn health_reason_codes(&self) -> Option<Vec<String>> {
1124 self.state
1125 .lock()
1126 .ok()
1127 .and_then(|state| state.last_error.as_ref().map(|error| vec![error.clone()]))
1128 }
1129
1130 fn inspection_summary(
1131 &self,
1132 state: &RuntimeStateV3,
1133 handler_summary: serde_json::Value,
1134 ) -> serde_json::Value {
1135 serde_json::json!({
1136 "runtimeVersion": self.config.runtime_identity.version,
1137 "protocolVersion": RUNTIME_API_VERSION_V3,
1138 "channel": PREVIEW_CHANNEL_V3,
1139 "modelGeneration": self.config.model_generation,
1140 "stateGeneration": state.snapshot.state_generation,
1141 "stateSchemaVersion": state.snapshot.state_schema_version,
1142 "stateChecksum": state.snapshot.checksum_sha256,
1143 "pendingDecisions": state.pending_decisions.len(),
1144 "completedDecisions": state.completed_decisions.len(),
1145 "resourceProfile": self.config.resource_profile,
1146 "resourceUtilization": {
1147 "stateBytes": state.snapshot.state.len(),
1148 "pendingDecisions": state.pending_decisions.len(),
1149 "completedDecisions": state.completed_decisions.len(),
1150 },
1151 "rollbackAvailable": state.previous_good.is_some(),
1152 "candidateAvailable": state.candidate.is_some(),
1153 "restartCount": state.restart_count,
1154 "health": self.health_status_for_state(state),
1155 "lastError": state.last_error,
1156 "handler": handler_summary,
1157 })
1158 }
1159
1160 pub fn diagnostics_at(
1162 &self,
1163 observed_at_unix_ms: u64,
1164 ) -> Result<RuntimeDiagnosticsV1, StatefulHandlerErrorV2> {
1165 let state = self
1166 .state
1167 .lock()
1168 .map_err(|_| StatefulHandlerErrorV2::new(StatefulHandlerErrorKindV2::Internal))?;
1169 let snapshot = StatefulRuntimeSnapshotV3::new(
1170 state.snapshot.clone(),
1171 state.pending_decisions.clone(),
1172 state.completed_decisions.clone(),
1173 );
1174 let snapshot_bytes = serde_json::to_vec(&snapshot)
1175 .map_err(|_| StatefulHandlerErrorV2::new(StatefulHandlerErrorKindV2::Internal))?
1176 .len();
1177 let usage = RuntimeResourceUsageV1 {
1178 state_bytes: state.snapshot.state.len(),
1179 snapshot_bytes,
1180 pending_decisions: state.pending_decisions.len(),
1181 completed_decisions: state.completed_decisions.len(),
1182 };
1183 let health = self.health_status_for_state(&state);
1184 let reason_codes = state
1185 .last_error
1186 .as_ref()
1187 .map(|error| vec![error.clone()])
1188 .unwrap_or_default();
1189 Ok(RuntimeDiagnosticsV1 {
1190 runtime_version: self.config.runtime_identity.version.clone(),
1191 protocol_version: RUNTIME_API_VERSION_V3,
1192 channel: PREVIEW_CHANNEL_V3.into(),
1193 model_generation: self.config.model_generation,
1194 state_generation: state.snapshot.state_generation,
1195 state_schema_version: state.snapshot.state_schema_version,
1196 observed_at_unix_ms,
1197 health,
1198 reason_codes,
1199 resource_usage: usage,
1200 rollback_available: state.previous_good.is_some(),
1201 candidate_available: state.candidate.is_some(),
1202 restart_count: state.restart_count,
1203 last_error: state.last_error.clone(),
1204 })
1205 }
1206
1207 fn health_status_for_state(&self, state: &RuntimeStateV3) -> RuntimeHealthStatusV1 {
1208 if state.last_error.is_some() {
1209 return RuntimeHealthStatusV1::FailedClosed;
1210 }
1211 let profile = &self.config.resource_profile;
1212 if state.snapshot.state.len() * 10 >= profile.max_model_state_bytes as usize * 9
1213 || state.pending_decisions.len() * 10 >= profile.max_pending_decisions as usize * 9
1214 || state.completed_decisions.len() * 10 >= profile.max_completed_decisions as usize * 9
1215 {
1216 RuntimeHealthStatusV1::ResourcePressure
1217 } else {
1218 RuntimeHealthStatusV1::Healthy
1219 }
1220 }
1221
1222 fn validate_runtime_snapshot_size(
1223 &self,
1224 snapshot: &StatefulRuntimeSnapshotV3,
1225 ) -> Result<(), StatefulHandlerErrorV2> {
1226 let bytes = serde_json::to_vec(snapshot)
1227 .map_err(|_| StatefulHandlerErrorV2::new(StatefulHandlerErrorKindV2::Internal))?;
1228 if bytes.len() > self.config.resource_profile.max_snapshot_bytes as usize {
1229 return Err(StatefulHandlerErrorV2::with_detail(
1230 StatefulHandlerErrorKindV2::InvalidState,
1231 "snapshot exceeds resource limit",
1232 ));
1233 }
1234 Ok(())
1235 }
1236
1237 fn current_generation(&self) -> u64 {
1238 self.state
1239 .lock()
1240 .map(|state| state.snapshot.state_generation)
1241 .unwrap_or(0)
1242 }
1243
1244 fn response(
1245 &self,
1246 request_id: String,
1247 state_generation: u64,
1248 response: RuntimeResponseBodyV3,
1249 ) -> RuntimeResponseV3 {
1250 RuntimeResponseV3 {
1251 request_id,
1252 api_version: RUNTIME_API_VERSION_V3,
1253 runtime_identity: self.config.runtime_identity.clone(),
1254 model_generation: self.config.model_generation,
1255 state_generation,
1256 response,
1257 }
1258 }
1259
1260 fn error_response(
1261 &self,
1262 request_id: String,
1263 code: RuntimeErrorCodeV3,
1264 message: impl Into<String>,
1265 state_generation: u64,
1266 ) -> RuntimeResponseV3 {
1267 self.response(
1268 if request_id.is_empty() {
1269 "invalid-request".into()
1270 } else {
1271 request_id
1272 },
1273 state_generation,
1274 RuntimeResponseBodyV3::Error {
1275 error: RuntimeErrorV3::new(code, message),
1276 },
1277 )
1278 }
1279}
1280
1281fn validate_state_bytes(
1282 state: &[u8],
1283 actual_schema_version: u32,
1284 expected_schema_version: u32,
1285) -> Result<(), StatefulHandlerErrorV2> {
1286 if actual_schema_version == 0 || actual_schema_version != expected_schema_version {
1287 return Err(StatefulHandlerErrorV2::new(
1288 StatefulHandlerErrorKindV2::IncompatibleVersion,
1289 ));
1290 }
1291 if state.len() > MAX_STATE_BYTES || serde_json::from_slice::<serde_json::Value>(state).is_err()
1292 {
1293 return Err(StatefulHandlerErrorV2::new(
1294 StatefulHandlerErrorKindV2::InvalidState,
1295 ));
1296 }
1297 Ok(())
1298}
1299
1300fn validate_feature_schema_hash(hash: &str) -> Result<(), StatefulHandlerErrorV2> {
1301 if hash.len() != 64
1302 || !hash
1303 .bytes()
1304 .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
1305 {
1306 return Err(StatefulHandlerErrorV2::new(
1307 StatefulHandlerErrorKindV2::InvalidModel,
1308 ));
1309 }
1310 Ok(())
1311}
1312
1313fn validate_capabilities(capabilities: &[String]) -> Result<(), StatefulHandlerErrorV2> {
1314 if capabilities.is_empty() || capabilities.len() > rill_handler_api::MAX_CAPABILITIES {
1315 return Err(StatefulHandlerErrorV2::new(
1316 StatefulHandlerErrorKindV2::InvalidModel,
1317 ));
1318 }
1319 let mut seen = std::collections::BTreeSet::new();
1320 if capabilities.iter().any(|capability| {
1321 capability.is_empty()
1322 || capability.len() > rill_handler_api::MAX_CAPABILITY_LEN
1323 || !seen.insert(capability)
1324 }) {
1325 return Err(StatefulHandlerErrorV2::new(
1326 StatefulHandlerErrorKindV2::InvalidModel,
1327 ));
1328 }
1329 Ok(())
1330}
1331
1332fn state_checksum(state: &[u8]) -> String {
1333 hex::encode(Sha256::digest(state))
1334}
1335
1336fn map_handler_error(kind: StatefulHandlerErrorKindV2) -> (RuntimeErrorCodeV3, &'static str) {
1337 match kind {
1338 StatefulHandlerErrorKindV2::InvalidEvent => (
1339 RuntimeErrorCodeV3::InvalidEnvelope,
1340 "handler rejected the event",
1341 ),
1342 StatefulHandlerErrorKindV2::InvalidState
1343 | StatefulHandlerErrorKindV2::IncompatibleVersion => (
1344 RuntimeErrorCodeV3::InvalidState,
1345 "handler rejected the current state",
1346 ),
1347 StatefulHandlerErrorKindV2::DuplicateFeedback => (
1348 RuntimeErrorCodeV3::DuplicateFeedback,
1349 "feedback was already applied",
1350 ),
1351 StatefulHandlerErrorKindV2::Timeout => (
1352 RuntimeErrorCodeV3::HandlerTimeout,
1353 "handler exceeded the wall-clock deadline",
1354 ),
1355 StatefulHandlerErrorKindV2::Trap => (RuntimeErrorCodeV3::HandlerTrap, "handler trapped"),
1356 StatefulHandlerErrorKindV2::OutputTooLarge => (
1357 RuntimeErrorCodeV3::HandlerOutputTooLarge,
1358 "handler output exceeded the size limit",
1359 ),
1360 StatefulHandlerErrorKindV2::InvalidOutput => (
1361 RuntimeErrorCodeV3::HandlerInvalidOutput,
1362 "handler output was not valid JSON",
1363 ),
1364 StatefulHandlerErrorKindV2::InvalidModel
1365 | StatefulHandlerErrorKindV2::MetadataMismatch
1366 | StatefulHandlerErrorKindV2::Internal => {
1367 (RuntimeErrorCodeV3::Internal, "internal runtime error")
1368 }
1369 }
1370}
1371
1372fn preview_error_code(code: RuntimeErrorCodeV3, message: &str) -> PreviewErrorCodeV3 {
1373 match message {
1374 "decision id was already used" => PreviewErrorCodeV3::DuplicateDecision,
1375 "decision id is not pending" => PreviewErrorCodeV3::UnknownDecision,
1376 "feedback generation is stale" => PreviewErrorCodeV3::StaleFeedback,
1377 "pending decision capacity is exhausted" => PreviewErrorCodeV3::CapacityExceeded,
1378 _ => match code {
1379 RuntimeErrorCodeV3::InvalidJson => PreviewErrorCodeV3::InvalidJson,
1380 RuntimeErrorCodeV3::InvalidRequestId
1381 | RuntimeErrorCodeV3::InvalidClientIdentity
1382 | RuntimeErrorCodeV3::IncompatibleApiVersion => PreviewErrorCodeV3::InvalidEnvelope,
1383 RuntimeErrorCodeV3::InvalidEnvelope => PreviewErrorCodeV3::InvalidEnvelope,
1384 RuntimeErrorCodeV3::PayloadTooLarge => PreviewErrorCodeV3::PayloadTooLarge,
1385 RuntimeErrorCodeV3::UnsupportedCapability => PreviewErrorCodeV3::UnsupportedCapability,
1386 RuntimeErrorCodeV3::StateMismatch => PreviewErrorCodeV3::StateMismatch,
1387 RuntimeErrorCodeV3::ExpiredRequest => PreviewErrorCodeV3::ExpiredRequest,
1388 RuntimeErrorCodeV3::IncompatibleGeneration => {
1389 PreviewErrorCodeV3::IncompatibleGeneration
1390 }
1391 RuntimeErrorCodeV3::DuplicateFeedback => PreviewErrorCodeV3::DuplicateFeedback,
1392 RuntimeErrorCodeV3::HandlerTimeout => PreviewErrorCodeV3::HandlerTimeout,
1393 RuntimeErrorCodeV3::HandlerTrap => PreviewErrorCodeV3::HandlerTrap,
1394 RuntimeErrorCodeV3::HandlerOutputTooLarge => PreviewErrorCodeV3::HandlerOutputTooLarge,
1395 RuntimeErrorCodeV3::HandlerInvalidOutput => PreviewErrorCodeV3::HandlerInvalidOutput,
1396 RuntimeErrorCodeV3::InvalidState => PreviewErrorCodeV3::InvalidState,
1397 RuntimeErrorCodeV3::Internal => PreviewErrorCodeV3::Internal,
1398 },
1399 }
1400}
1401
1402#[cfg(test)]
1403mod tests {
1404 use super::*;
1405
1406 #[derive(Debug)]
1407 struct TestHandler {
1408 metadata: StatefulHandlerMetadataV2,
1409 mode: StatefulHandlerErrorKindV2,
1410 }
1411
1412 impl StatefulHandlerV2 for TestHandler {
1413 fn metadata(&self) -> &StatefulHandlerMetadataV2 {
1414 &self.metadata
1415 }
1416
1417 fn handle(
1418 &self,
1419 _event_json: &[u8],
1420 current_state: &[u8],
1421 _deterministic_seed: Option<u64>,
1422 ) -> Result<StatefulHandlerResultV2, StatefulHandlerErrorV2> {
1423 match self.mode {
1424 StatefulHandlerErrorKindV2::Internal => {
1425 let mut value: serde_json::Value =
1426 serde_json::from_slice(current_state).unwrap();
1427 value["count"] = serde_json::json!(value["count"].as_u64().unwrap_or(0) + 1);
1428 Ok(StatefulHandlerResultV2 {
1429 output: value.clone(),
1430 next_state: serde_json::to_vec(&value).unwrap(),
1431 })
1432 }
1433 StatefulHandlerErrorKindV2::InvalidState => Ok(StatefulHandlerResultV2 {
1434 output: serde_json::json!({"ignored": true}),
1435 next_state: b"not-json".to_vec(),
1436 }),
1437 StatefulHandlerErrorKindV2::OutputTooLarge => Ok(StatefulHandlerResultV2 {
1438 output: serde_json::json!({"data": "x".repeat(MAX_OUTPUT_BYTES + 1)}),
1439 next_state: current_state.to_vec(),
1440 }),
1441 other => Err(StatefulHandlerErrorV2::new(other)),
1442 }
1443 }
1444 }
1445
1446 fn engine(mode: StatefulHandlerErrorKindV2) -> StatefulRuntimeEngineV3 {
1447 let metadata = StatefulHandlerMetadataV2 {
1448 id: "org.example.stateful".into(),
1449 version: "2.0.0".into(),
1450 api_version: HANDLER_API_VERSION,
1451 capabilities: vec!["org.example.decide".into()],
1452 state_schema_version: 1,
1453 };
1454 let config = StatefulRuntimeConfigV3::new(
1455 IdentityV3 {
1456 name: "rill-runtime".into(),
1457 version: "1.0.0".into(),
1458 },
1459 7,
1460 "ab".repeat(32),
1461 metadata.capabilities.clone(),
1462 br#"{"count":0}"#.to_vec(),
1463 );
1464 StatefulRuntimeEngineV3::new(config, Arc::new(TestHandler { metadata, mode })).unwrap()
1465 }
1466
1467 fn decide(state_generation: u64) -> EnvelopeV3 {
1468 EnvelopeV3 {
1469 request_id: "d1".into(),
1470 api_version: RUNTIME_API_VERSION_V3,
1471 client_identity: IdentityV3 {
1472 name: "host".into(),
1473 version: "1".into(),
1474 },
1475 capability: Some("org.example.decide".into()),
1476 deadline_unix_ms: Some(100),
1477 feature_schema_hash: Some("ab".repeat(32)),
1478 model_generation: 7,
1479 state_generation,
1480 payload_limit: rill_runtime_protocol::MAX_MESSAGE_BYTES as u32,
1481 request: RuntimeRequestV3::Decide {
1482 context: serde_json::json!({"features": [1.0]}),
1483 deterministic_seed: Some(42),
1484 },
1485 }
1486 }
1487
1488 #[test]
1489 fn state_update_is_atomic_and_increments_generation() {
1490 let engine = engine(StatefulHandlerErrorKindV2::Internal);
1491 let response = engine.handle_at(decide(0), 100);
1492 assert!(matches!(
1493 response.response,
1494 RuntimeResponseBodyV3::Result { .. }
1495 ));
1496 assert_eq!(response.state_generation, 1);
1497 assert_eq!(engine.snapshot().unwrap().state, br#"{"count":1}"#);
1498 }
1499
1500 #[test]
1501 fn invalid_next_state_is_fail_closed() {
1502 let engine = engine(StatefulHandlerErrorKindV2::InvalidState);
1503 let before = engine.snapshot().unwrap();
1504 let response = engine.handle_at(decide(0), 100);
1505 assert!(matches!(
1506 response.response,
1507 RuntimeResponseBodyV3::Error { error }
1508 if error.code == RuntimeErrorCodeV3::InvalidState
1509 ));
1510 assert_eq!(engine.snapshot().unwrap(), before);
1511 }
1512
1513 #[test]
1514 fn timeout_trap_and_oversize_leave_state_unchanged() {
1515 for mode in [
1516 StatefulHandlerErrorKindV2::Timeout,
1517 StatefulHandlerErrorKindV2::Trap,
1518 StatefulHandlerErrorKindV2::OutputTooLarge,
1519 ] {
1520 let engine = engine(mode);
1521 let before = engine.snapshot().unwrap();
1522 let response = engine.handle_at(decide(0), 100);
1523 assert!(matches!(
1524 response.response,
1525 RuntimeResponseBodyV3::Error { .. }
1526 ));
1527 assert_eq!(engine.snapshot().unwrap(), before, "mode={mode:?}");
1528 }
1529 }
1530
1531 #[test]
1532 fn stale_generation_and_expired_request_are_rejected() {
1533 let engine = engine(StatefulHandlerErrorKindV2::Internal);
1534 let stale = engine.handle_at(decide(1), 100);
1535 assert!(matches!(
1536 stale.response,
1537 RuntimeResponseBodyV3::Error { error }
1538 if error.code == RuntimeErrorCodeV3::StateMismatch
1539 ));
1540 let expired = engine.handle_at(decide(0), 101);
1541 assert!(matches!(
1542 expired.response,
1543 RuntimeResponseBodyV3::Error { error }
1544 if error.code == RuntimeErrorCodeV3::ExpiredRequest
1545 ));
1546 }
1547
1548 #[test]
1549 fn corrupt_snapshot_checksum_is_rejected_without_mutation() {
1550 let engine = engine(StatefulHandlerErrorKindV2::Internal);
1551 let before = engine.snapshot().unwrap();
1552 let mut corrupt = before.clone();
1553 corrupt.checksum_sha256 = "00".repeat(32);
1554 assert!(engine.restore_snapshot(corrupt).is_err());
1555 assert_eq!(engine.snapshot().unwrap(), before);
1556 }
1557
1558 #[test]
1559 fn json_entrypoint_rejects_malformed_and_oversized_messages() {
1560 let engine = engine(StatefulHandlerErrorKindV2::Internal);
1561 let before = engine.snapshot().unwrap();
1562
1563 let malformed = engine.handle_json_at(b"{", 100);
1564 assert!(matches!(
1565 malformed.response,
1566 RuntimeResponseBodyV3::Error { error }
1567 if error.code == RuntimeErrorCodeV3::InvalidJson
1568 ));
1569
1570 let oversized = vec![b' '; rill_runtime_protocol::MAX_MESSAGE_BYTES + 1];
1571 let oversized = engine.handle_json_at(&oversized, 100);
1572 assert!(matches!(
1573 oversized.response,
1574 RuntimeResponseBodyV3::Error { error }
1575 if error.code == RuntimeErrorCodeV3::PayloadTooLarge
1576 ));
1577 assert_eq!(engine.snapshot().unwrap(), before);
1578 }
1579
1580 #[test]
1581 fn inspect_and_diagnostics_do_not_reenter_the_state_lock() {
1582 let engine = engine(StatefulHandlerErrorKindV2::Internal);
1583 let mut request = decide(0);
1584 request.request = RuntimeRequestV3::Inspect {};
1585 let response = engine.handle_preview_at(request, 100);
1586 assert!(matches!(
1587 response.response,
1588 RuntimeResponseBodyV3Preview::Inspection { .. }
1589 ));
1590 let diagnostics = engine.diagnostics_at(123).unwrap();
1591 assert_eq!(diagnostics.observed_at_unix_ms, 123);
1592 assert_eq!(diagnostics.health, RuntimeHealthStatusV1::Healthy);
1593 assert_eq!(diagnostics.state_generation, 1);
1594 assert!(diagnostics.resource_usage.snapshot_bytes > diagnostics.resource_usage.state_bytes);
1595 }
1596
1597 #[test]
1598 fn ipc_and_snapshot_limits_fail_closed_without_mutation() {
1599 let profile = ResourceProfileV1 {
1600 max_ipc_frame_bytes: 128,
1601 max_snapshot_bytes: 128,
1602 ..ResourceProfileV1::default()
1603 };
1604 let metadata = StatefulHandlerMetadataV2 {
1605 id: "org.example.stateful".into(),
1606 version: "2.0.0".into(),
1607 api_version: HANDLER_API_VERSION,
1608 capabilities: vec!["org.example.decide".into()],
1609 state_schema_version: 1,
1610 };
1611 let config = StatefulRuntimeConfigV3::new(
1612 IdentityV3 {
1613 name: "rill-runtime".into(),
1614 version: "1.0.0".into(),
1615 },
1616 7,
1617 "ab".repeat(32),
1618 metadata.capabilities.clone(),
1619 br#"{"count":0}"#.to_vec(),
1620 )
1621 .with_resource_profile(profile);
1622 let engine = StatefulRuntimeEngineV3::new(
1623 config,
1624 Arc::new(TestHandler {
1625 metadata,
1626 mode: StatefulHandlerErrorKindV2::Internal,
1627 }),
1628 )
1629 .unwrap();
1630 let before = engine.snapshot().unwrap();
1631 let oversized = vec![b' '; 129];
1632 let response = engine.handle_preview_json_at(&oversized, 100);
1633 assert!(matches!(
1634 response.response,
1635 RuntimeResponseBodyV3Preview::Error { error }
1636 if error.code == PreviewErrorCodeV3::PayloadTooLarge
1637 ));
1638 assert_eq!(engine.snapshot().unwrap(), before);
1639 assert!(engine.runtime_snapshot().is_err());
1640 }
1641}