use rapid_structure_rs::{EngineConfig, ProviderPreference, RapidOcrError};
use serde::Serialize;
use super::limits::{RawServeLimits, ServeConfigError, ServeLimits};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum ServiceState {
Starting,
Ready,
}
impl ServiceState {
pub fn listening_succeeded(self) -> Result<Self, TransitionError> {
match self {
Self::Starting => Ok(Self::Ready),
Self::Ready => Err(TransitionError {
from: "ready",
to: "ready",
allowed_from: "starting".to_string(),
}),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(tag = "state", rename_all = "snake_case")]
pub enum EngineState {
BlockedModelsMissing { missing: Vec<String> },
Loading,
Ready {
requested: String,
selected_ep: String,
fallback_to_cpu: bool,
},
Failed { reason: String },
#[allow(dead_code)]
Rebuilding,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum StateKind {
BlockedModelsMissing,
Loading,
Ready,
Failed,
Rebuilding,
}
impl StateKind {
fn name(self) -> &'static str {
match self {
Self::BlockedModelsMissing => "blocked_models_missing",
Self::Loading => "loading",
Self::Ready => "ready",
Self::Failed => "failed",
Self::Rebuilding => "rebuilding",
}
}
}
impl EngineState {
fn kind(&self) -> StateKind {
match self {
Self::BlockedModelsMissing { .. } => StateKind::BlockedModelsMissing,
Self::Loading => StateKind::Loading,
Self::Ready { .. } => StateKind::Ready,
Self::Failed { .. } => StateKind::Failed,
Self::Rebuilding => StateKind::Rebuilding,
}
}
pub fn name(&self) -> &'static str {
self.kind().name()
}
pub fn provider_status(&self, requested: &str) -> ProviderStatus {
match self {
Self::Ready {
requested: state_requested,
selected_ep,
fallback_to_cpu,
} => ProviderStatus {
requested: state_requested.clone(),
selected_ep: Some(selected_ep.clone()),
fallback_to_cpu: Some(*fallback_to_cpu),
},
_ => ProviderStatus {
requested: requested.to_string(),
selected_ep: None,
fallback_to_cpu: None,
},
}
}
pub fn ocr_admission(&self) -> OcrAdmission {
match self {
Self::Ready { .. } => OcrAdmission::Run,
Self::BlockedModelsMissing { missing } => OcrAdmission::ModelsMissing {
missing: missing.clone(),
},
Self::Loading => OcrAdmission::Queue {
waiting_for: "loading",
},
Self::Rebuilding => OcrAdmission::Queue {
waiting_for: "rebuilding",
},
Self::Failed { reason } => OcrAdmission::Unavailable {
reason: reason.clone(),
},
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct ProviderStatus {
pub requested: String,
pub selected_ep: Option<String>,
pub fallback_to_cpu: Option<bool>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum OcrAdmission {
Run,
Queue { waiting_for: &'static str },
ModelsMissing { missing: Vec<String> },
Unavailable { reason: String },
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ModelReadiness {
Complete,
Incomplete { missing: Vec<String> },
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TransitionError {
from: &'static str,
to: &'static str,
allowed_from: String,
}
impl std::fmt::Display for TransitionError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"illegal engine state transition {} -> {}: {} may only be entered from [{}]",
self.from, self.to, self.to, self.allowed_from
)
}
}
impl std::error::Error for TransitionError {}
#[derive(Debug, Clone)]
pub struct EngineStateMachine {
state: EngineState,
}
impl EngineStateMachine {
pub fn start(readiness: ModelReadiness) -> Self {
match readiness {
ModelReadiness::Complete => Self {
state: EngineState::Loading,
},
ModelReadiness::Incomplete { missing } => Self {
state: EngineState::BlockedModelsMissing { missing },
},
}
}
pub fn state(&self) -> &EngineState {
&self.state
}
pub fn begin_loading(&mut self) -> Result<(), TransitionError> {
self.require(
&[
StateKind::BlockedModelsMissing,
StateKind::Ready,
StateKind::Failed,
],
StateKind::Loading,
)?;
self.state = EngineState::Loading;
Ok(())
}
pub fn load_succeeded(
&mut self,
requested: impl Into<String>,
selected_ep: impl Into<String>,
fallback_to_cpu: bool,
) -> Result<(), TransitionError> {
self.require(
&[StateKind::Loading, StateKind::Rebuilding],
StateKind::Ready,
)?;
self.state = EngineState::Ready {
requested: requested.into(),
selected_ep: selected_ep.into(),
fallback_to_cpu,
};
Ok(())
}
pub fn load_failed(&mut self, reason: impl Into<String>) -> Result<(), TransitionError> {
self.require(
&[StateKind::Loading, StateKind::Rebuilding],
StateKind::Failed,
)?;
self.state = EngineState::Failed {
reason: reason.into(),
};
Ok(())
}
pub fn models_still_missing(&mut self, missing: Vec<String>) -> Result<(), TransitionError> {
self.require(
&[StateKind::BlockedModelsMissing],
StateKind::BlockedModelsMissing,
)?;
self.state = EngineState::BlockedModelsMissing { missing };
Ok(())
}
#[allow(dead_code)]
pub fn begin_rebuild(&mut self) -> Result<(), TransitionError> {
self.require(&[StateKind::Ready], StateKind::Rebuilding)?;
self.state = EngineState::Rebuilding;
Ok(())
}
fn require(&self, allowed: &[StateKind], target: StateKind) -> Result<(), TransitionError> {
let current = self.state.kind();
if allowed.contains(¤t) {
return Ok(());
}
let allowed_from = allowed
.iter()
.map(|kind| kind.name())
.collect::<Vec<_>>()
.join(", ");
Err(TransitionError {
from: current.name(),
to: target.name(),
allowed_from,
})
}
}
#[derive(Debug)]
pub enum StartupConfigError {
Limit(ServeConfigError),
ConfigFile {
path: std::path::PathBuf,
source: RapidOcrError,
},
Engine(RapidOcrError),
Provider(RapidOcrError),
LogLevel(String),
}
impl std::fmt::Display for StartupConfigError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Limit(error) => write!(f, "{error}"),
Self::ConfigFile { path, source } => {
write!(f, "cannot load --config {}: {source}", path.display())
}
Self::Engine(error) => write!(f, "engine configuration rejected: {error}"),
Self::Provider(error) => write!(f, "provider configuration rejected: {error}"),
Self::LogLevel(error) => {
write!(
f,
"--log-level / RAPID_STRUCTURE_SERVE_LOG rejected: {error}"
)
}
}
}
}
impl std::error::Error for StartupConfigError {}
impl From<ServeConfigError> for StartupConfigError {
fn from(error: ServeConfigError) -> Self {
Self::Limit(error)
}
}
#[derive(Debug, Clone)]
pub struct ServeStartup {
pub limits: ServeLimits,
pub plan: ServeConfigPlan,
}
impl ServeStartup {
pub fn validate(
raw_limits: RawServeLimits,
engine: EngineConfig,
cli_provider: Option<ProviderPreference>,
cli_max_side: Option<usize>,
allow_provider_fallback: bool,
) -> Result<Self, StartupConfigError> {
let limits = raw_limits.validate()?;
let plan =
ServeConfigPlan::validate(engine, cli_provider, cli_max_side, allow_provider_fallback)?;
Ok(Self { limits, plan })
}
}
#[derive(Debug, Clone)]
pub struct ServeConfigPlan {
pub engine: EngineConfig,
pub requested: ProviderPreference,
pub fail_if_provider_unavailable: bool,
}
impl ServeConfigPlan {
pub fn validate(
mut engine: EngineConfig,
cli_provider: Option<ProviderPreference>,
cli_max_side: Option<usize>,
allow_provider_fallback: bool,
) -> Result<Self, StartupConfigError> {
if let Some(max_side) = cli_max_side {
engine.global.max_side_len = max_side;
}
if let Some(provider) = cli_provider {
engine.runtime.provider_preference = provider;
}
engine.validate().map_err(StartupConfigError::Engine)?;
let requested = engine.runtime.provider_preference;
if allow_provider_fallback {
engine.runtime.fail_if_provider_unavailable = false;
}
let fail_if_provider_unavailable = engine.runtime.fail_if_provider_unavailable;
match rapid_structure_rs::resolve_execution_providers(
&requested,
engine.runtime.enable_cpu_mem_arena,
false,
) {
Ok(_) => {}
Err(RapidOcrError::UnsupportedProvider(message)) => {
return Err(StartupConfigError::Provider(
RapidOcrError::UnsupportedProvider(message),
));
}
Err(_runtime_fact) => {
}
}
Ok(Self {
engine,
requested,
fail_if_provider_unavailable,
})
}
pub fn requested_label(&self) -> String {
rapid_structure_rs::format_provider_preference(self.requested)
}
pub fn with_provider(
&self,
requested: ProviderPreference,
allow_provider_fallback: bool,
) -> Result<Self, StartupConfigError> {
Self::validate(
self.engine.clone(),
Some(requested),
None,
allow_provider_fallback,
)
}
}
#[cfg(test)]
mod tests {
use rapid_structure_rs::{EngineConfig, ProviderPreference};
use super::{
EngineState, EngineStateMachine, ModelReadiness, OcrAdmission, ProviderStatus,
ServeConfigPlan, ServeStartup, ServiceState, StartupConfigError, TransitionError,
};
use crate::serve::error::ServeError;
use crate::serve::limits::RawServeLimits;
fn blocked() -> EngineStateMachine {
EngineStateMachine::start(ModelReadiness::Incomplete {
missing: vec!["PP-OCRv6_det_medium.onnx".to_string()],
})
}
fn ready() -> EngineStateMachine {
let mut machine = EngineStateMachine::start(ModelReadiness::Complete);
machine
.load_succeeded("cpu", "Cpu", false)
.expect("Loading -> Ready is legal");
machine
}
#[test]
fn service_state_starts_then_becomes_ready_once() {
let starting = ServiceState::Starting;
let ready = starting.listening_succeeded().expect("Starting -> Ready");
assert_eq!(ready, ServiceState::Ready);
assert!(
ready.listening_succeeded().is_err(),
"Ready -> Ready must be illegal"
);
}
#[test]
fn complete_models_start_in_loading_and_incomplete_models_start_blocked() {
assert_eq!(
EngineStateMachine::start(ModelReadiness::Complete).state(),
&EngineState::Loading
);
assert_eq!(
blocked().state(),
&EngineState::BlockedModelsMissing {
missing: vec!["PP-OCRv6_det_medium.onnx".to_string()]
}
);
}
#[test]
fn loading_transitions_to_ready_with_the_three_provider_fields() {
let mut machine = EngineStateMachine::start(ModelReadiness::Complete);
machine
.load_succeeded("directml(device_id=0)", "DirectMl", false)
.expect("Loading -> Ready");
assert_eq!(
machine.state(),
&EngineState::Ready {
requested: "directml(device_id=0)".to_string(),
selected_ep: "DirectMl".to_string(),
fallback_to_cpu: false,
}
);
}
#[test]
fn loading_transitions_to_failed_and_records_the_reason() {
let mut machine = EngineStateMachine::start(ModelReadiness::Complete);
machine
.load_failed("DirectML is unavailable in the loaded ONNX Runtime")
.expect("Loading -> Failed");
match machine.state() {
EngineState::Failed { reason } => {
assert!(reason.contains("unavailable"), "reason: {reason}");
}
other => panic!("expected Failed, got {other:?}"),
}
}
#[test]
fn failed_can_only_be_left_through_loading() {
let mut machine = EngineStateMachine::start(ModelReadiness::Complete);
machine.load_failed("boom").expect("Loading -> Failed");
let error = machine
.load_succeeded("cpu", "Cpu", false)
.expect_err("Failed -> Ready must be rejected");
assert_eq!(error.from, "failed");
assert_eq!(error.to, "ready");
assert!(error.allowed_from.contains("loading"), "error: {error}");
assert_eq!(machine.state().name(), "failed");
machine.begin_loading().expect("Failed -> Loading");
assert_eq!(machine.state().name(), "loading");
machine
.load_succeeded("cpu", "Cpu", false)
.expect("Loading -> Ready");
}
#[test]
fn ready_reload_goes_ready_loading_ready() {
let mut machine = ready();
machine.begin_loading().expect("Ready -> Loading (reload)");
assert_eq!(machine.state().name(), "loading");
machine
.load_succeeded("cpu", "Cpu", true)
.expect("Loading -> Ready");
match machine.state() {
EngineState::Ready {
fallback_to_cpu, ..
} => assert!(*fallback_to_cpu),
other => panic!("expected Ready, got {other:?}"),
}
}
#[test]
fn ready_reload_can_end_in_failed() {
let mut machine = ready();
machine.begin_loading().expect("Ready -> Loading");
machine
.load_failed("session creation failed")
.expect("Loading -> Failed");
assert_eq!(machine.state().name(), "failed");
}
#[test]
fn blocked_models_missing_can_be_refreshed_or_enter_loading() {
let mut machine = blocked();
machine
.models_still_missing(vec!["a.onnx".to_string(), "b.onnx".to_string()])
.expect("BlockedModelsMissing -> BlockedModelsMissing");
assert_eq!(
machine.state(),
&EngineState::BlockedModelsMissing {
missing: vec!["a.onnx".to_string(), "b.onnx".to_string()]
}
);
machine.begin_loading().expect("models are complete now");
assert_eq!(machine.state().name(), "loading");
let error = machine
.models_still_missing(vec!["c.onnx".to_string()])
.expect_err("Loading -> BlockedModelsMissing must be rejected");
assert_eq!(error.from, "loading");
assert_eq!(error.to, "blocked_models_missing");
}
#[test]
fn rebuilding_is_reachable_from_ready_and_back_to_ready_or_failed() {
let mut machine = ready();
machine.begin_rebuild().expect("Ready -> Rebuilding");
assert_eq!(
machine.state(),
&EngineState::Rebuilding,
"M3 runtime provider switch"
);
let mut restored = machine.clone();
restored
.load_succeeded("cpu", "Cpu", false)
.expect("Rebuilding -> Ready");
assert_eq!(restored.state().name(), "ready");
machine
.load_failed("the new provider cannot be created")
.expect("Rebuilding -> Failed");
assert_eq!(machine.state().name(), "failed");
}
#[test]
fn rebuilding_cannot_be_entered_from_loading_or_blocked() {
let mut loading = EngineStateMachine::start(ModelReadiness::Complete);
let error = loading.begin_rebuild().expect_err("Loading -> Rebuilding");
assert_eq!(error.from, "loading");
assert_eq!(error.to, "rebuilding");
assert!(error.allowed_from.contains("ready"), "error: {error}");
let mut blocked = blocked();
let error = blocked
.begin_rebuild()
.expect_err("BlockedModelsMissing -> Rebuilding");
assert_eq!(error.from, "blocked_models_missing");
let mut rebuilding = ready();
rebuilding.begin_rebuild().expect("Ready -> Rebuilding");
assert!(
rebuilding.begin_rebuild().is_err(),
"Rebuilding -> Rebuilding must be illegal"
);
}
#[test]
fn illegal_transitions_report_from_and_to_and_legal_predecessors() {
let mut blocked = blocked();
let error: TransitionError = blocked
.load_succeeded("cpu", "Cpu", false)
.expect_err("BlockedModelsMissing -> Ready must be rejected");
assert_eq!(error.from, "blocked_models_missing");
assert_eq!(error.to, "ready");
assert!(error.allowed_from.contains("loading"), "error: {error}");
assert!(error.allowed_from.contains("rebuilding"), "error: {error}");
let text = error.to_string();
assert!(text.contains("blocked_models_missing -> ready"), "{text}");
}
#[test]
fn ocr_admission_queues_while_loading_and_rebuilding_and_never_fails() {
let mut machine = EngineStateMachine::start(ModelReadiness::Complete);
match machine.state().ocr_admission() {
OcrAdmission::Queue { waiting_for } => assert_eq!(waiting_for, "loading"),
other => panic!("Loading must queue new OCR work, got {other:?}"),
}
assert!(ServeError::from_ocr_admission(machine.state().ocr_admission()).is_ok());
machine
.load_succeeded("cpu", "Cpu", false)
.expect("-> Ready");
machine.begin_rebuild().expect("Ready -> Rebuilding");
match machine.state().ocr_admission() {
OcrAdmission::Queue { waiting_for } => assert_eq!(waiting_for, "rebuilding"),
other => panic!("Rebuilding must queue new OCR work, got {other:?}"),
}
assert!(ServeError::from_ocr_admission(machine.state().ocr_admission()).is_ok());
}
#[test]
fn ocr_admission_reports_missing_models_and_engine_failure() {
let blocked = blocked();
match blocked.state().ocr_admission() {
OcrAdmission::ModelsMissing { missing } => {
assert_eq!(missing, vec!["PP-OCRv6_det_medium.onnx".to_string()]);
}
other => panic!("expected ModelsMissing, got {other:?}"),
}
let error = ServeError::from_ocr_admission(blocked.state().ocr_admission())
.expect_err("models missing must be an error");
assert_eq!(error.status_code(), 409);
assert_eq!(error.code(), "models_missing");
let mut failed = EngineStateMachine::start(ModelReadiness::Complete);
failed.load_failed("no EP available").expect("-> Failed");
let error = ServeError::from_ocr_admission(failed.state().ocr_admission())
.expect_err("failed engine must be an error");
assert_eq!(error.status_code(), 503);
assert_eq!(error.code(), "engine_unavailable");
assert_eq!(error.detail()["reason"], "no EP available");
}
#[test]
fn ready_admission_is_run() {
let machine = ready();
assert_eq!(machine.state().ocr_admission(), OcrAdmission::Run);
assert!(ServeError::from_ocr_admission(machine.state().ocr_admission()).is_ok());
}
#[test]
fn provider_status_never_disguises_unknown_as_false() {
let loading = EngineState::Loading;
let status = loading.provider_status("directml(device_id=0)");
assert_eq!(
status,
ProviderStatus {
requested: "directml(device_id=0)".to_string(),
selected_ep: None,
fallback_to_cpu: None,
}
);
let json = serde_json::to_value(&status).expect("ProviderStatus must serialize");
assert_eq!(json["requested"], "directml(device_id=0)");
assert!(json["selected_ep"].is_null(), "json: {json}");
assert!(
json["fallback_to_cpu"].is_null(),
"unknown must be null, not false: {json}"
);
let ready = EngineState::Ready {
requested: "directml(device_id=0)".to_string(),
selected_ep: "Cpu".to_string(),
fallback_to_cpu: true,
};
let json = serde_json::to_value(ready.provider_status("ignored")).expect("serializes");
assert_eq!(json["requested"], "directml(device_id=0)");
assert_eq!(json["selected_ep"], "Cpu");
assert_eq!(json["fallback_to_cpu"], true);
}
#[test]
fn engine_state_is_serializable_for_api_status() {
let cases: [(EngineState, &str); 5] = [
(
EngineState::BlockedModelsMissing {
missing: vec!["a.onnx".to_string()],
},
"blocked_models_missing",
),
(EngineState::Loading, "loading"),
(
EngineState::Ready {
requested: "cpu".to_string(),
selected_ep: "Cpu".to_string(),
fallback_to_cpu: false,
},
"ready",
),
(
EngineState::Failed {
reason: "boom".to_string(),
},
"failed",
),
(EngineState::Rebuilding, "rebuilding"),
];
for (state, expected) in cases {
let json = serde_json::to_value(&state).expect("EngineState must serialize");
assert_eq!(json["state"], expected, "json: {json}");
}
let json = serde_json::to_value(EngineState::BlockedModelsMissing {
missing: vec!["a.onnx".to_string()],
})
.expect("serializes");
assert_eq!(json["missing"][0], "a.onnx");
}
#[test]
fn startup_rejects_invalid_limits_before_touching_the_provider() {
let error = ServeStartup::validate(
RawServeLimits {
max_body_mb: 0,
..Default::default()
},
EngineConfig::default(),
Some(ProviderPreference::Cuda { device_id: 0 }),
None,
false,
)
.expect_err("a zero body limit must be fatal");
match error {
StartupConfigError::Limit(inner) => {
assert_eq!(inner.field(), "--max-body-mb");
}
other => panic!("expected a limit error, got {other}"),
}
}
#[test]
#[cfg(not(feature = "directml-provider"))]
fn startup_allows_uncompiled_directml_to_fall_back_when_non_strict() {
let plan = ServeStartup::validate(
RawServeLimits::default(),
EngineConfig::default(),
Some(ProviderPreference::DirectMl { device_id: 0 }),
None,
false,
)
.expect("uncompiled directml should use CPU fallback");
assert_eq!(
plan.requested,
ProviderPreference::DirectMl { device_id: 0 }
);
assert!(!plan.fail_if_provider_unavailable);
}
#[test]
fn provider_fallback_policy_is_frozen_by_default() {
let cpu = ServeConfigPlan::validate(EngineConfig::default(), None, None, false)
.expect("the default config must be valid");
assert_eq!(cpu.requested, ProviderPreference::default());
assert_eq!(
cpu.requested_label(),
rapid_structure_rs::format_provider_preference(ProviderPreference::default())
);
assert!(
!cpu.fail_if_provider_unavailable,
"the default provider must allow CPU fallback"
);
let plan = ServeConfigPlan::validate(
EngineConfig::default(),
Some(ProviderPreference::Cpu),
None,
true,
)
.expect("cpu is always resolvable");
assert!(!plan.fail_if_provider_unavailable);
}
#[test]
fn switching_a_provider_reuses_the_frozen_startup_rules() {
let mut engine = EngineConfig::default();
engine.global.max_side_len = 1600;
let base = ServeConfigPlan::validate(engine, Some(ProviderPreference::Cpu), None, false)
.expect("cpu is always resolvable");
let same = base
.with_provider(ProviderPreference::Cpu, false)
.expect("cpu stays resolvable");
assert_eq!(same.requested, ProviderPreference::Cpu);
assert_eq!(same.requested_label(), "cpu");
assert_eq!(same.engine.global.max_side_len, 1600);
assert!(!same.fail_if_provider_unavailable);
match base.with_provider(ProviderPreference::DirectMl { device_id: 0 }, false) {
Ok(plan) => assert!(
!plan.fail_if_provider_unavailable,
"a non-strict provider switch must allow CPU fallback"
),
Err(error) => assert!(
error.to_string().contains("provider"),
"the only allowed failure here is the provider configuration: {error}"
),
}
match base.with_provider(ProviderPreference::DirectMl { device_id: 0 }, true) {
Ok(plan) => assert!(
!plan.fail_if_provider_unavailable,
"--allow-provider-fallback must keep fallback enabled"
),
Err(error) => assert!(
error.to_string().contains("provider"),
"the only allowed failure here is the provider configuration: {error}"
),
}
}
#[test]
fn cli_overrides_beat_yaml_and_yaml_beats_the_builtin_default() {
let mut yaml = EngineConfig::default();
yaml.global.max_side_len = 1234;
yaml.runtime.provider_preference = ProviderPreference::Cpu;
let plan = ServeConfigPlan::validate(yaml.clone(), None, None, false)
.expect("the YAML config must be valid");
assert_eq!(plan.engine.global.max_side_len, 1234);
let plan =
ServeConfigPlan::validate(yaml, Some(ProviderPreference::Cpu), Some(2000), false)
.expect("the overridden config must be valid");
assert_eq!(plan.engine.global.max_side_len, 2000);
let plan = ServeConfigPlan::validate(EngineConfig::default(), None, None, false)
.expect("defaults must be valid");
assert_eq!(plan.engine.global.max_side_len, 2000);
}
#[test]
fn cli_max_side_zero_is_rejected_by_the_library_validation() {
let error = ServeConfigPlan::validate(EngineConfig::default(), None, Some(0), false)
.expect_err("--max-side=0 must be fatal");
let text = error.to_string();
assert!(text.contains("max_side_len"), "{text}");
assert!(text.contains("greater than zero"), "{text}");
}
}