use std::path::{Path, PathBuf};
use std::sync::Arc;
use image::RgbImage;
#[cfg(feature = "onnx")]
use rightkit_onnx::{configure_runtime, CandidateSource, RuntimeCandidate};
use serde::Serialize;
use crate::native;
use crate::types::{decode_encoded, is_backend_unavailable, OcrBackend, OcrLine};
use crate::worker::WorkerCommand;
#[cfg(feature = "onnx")]
use crate::OcrEngine;
use crate::{OcrArtifactManifest, OcrError};
#[cfg(feature = "onnx")]
type OnnxSlot = OcrEngine;
#[cfg(not(feature = "onnx"))]
type OnnxSlot = std::convert::Infallible;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum BackendPreference {
Auto,
Native,
Onnx,
}
#[derive(Debug, Clone, Default)]
pub enum Isolation {
#[default]
InProcess,
Worker(WorkerCommand),
}
pub trait ModelLeaseHook: Send + Sync + std::fmt::Debug {
fn acquire_model_lease(
&self,
paths: &[PathBuf],
) -> Result<Box<dyn std::any::Any + Send>, String>;
}
pub trait NativeHealthHook: Send + Sync {
fn cached_verdict(&self) -> Option<bool>;
fn record_verdict(&self, healthy: bool);
}
#[derive(Debug, Clone)]
pub struct OnnxConfig {
pub model_dir: PathBuf,
pub manifest: OcrArtifactManifest,
pub ort_runtime: Option<PathBuf>,
pub lease_hook: Option<Arc<dyn ModelLeaseHook>>,
}
impl OnnxConfig {
pub fn new(model_dir: impl Into<PathBuf>, manifest: OcrArtifactManifest) -> Self {
Self {
model_dir: model_dir.into(),
manifest,
ort_runtime: None,
lease_hook: None,
}
}
pub fn with_lease_hook(mut self, hook: Arc<dyn ModelLeaseHook>) -> Self {
self.lease_hook = Some(hook);
self
}
}
#[derive(Debug, Clone)]
pub struct OcrOptions {
pub preference: BackendPreference,
pub onnx: Option<OnnxConfig>,
pub isolation: Isolation,
}
impl OcrOptions {
pub fn new(preference: BackendPreference) -> Self {
Self {
preference,
onnx: None,
isolation: Isolation::InProcess,
}
}
pub fn with_onnx(mut self, onnx: OnnxConfig) -> Self {
self.onnx = Some(onnx);
self
}
pub fn with_isolation(mut self, isolation: Isolation) -> Self {
self.isolation = isolation;
self
}
}
impl Default for OcrOptions {
fn default() -> Self {
Self::new(BackendPreference::Auto)
}
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct OcrResult {
pub lines: Vec<OcrLine>,
pub backend: OcrBackend,
pub fallback_reason: Option<String>,
}
impl OcrResult {
pub fn text(&self) -> String {
self.lines
.iter()
.map(|l| l.text.as_str())
.collect::<Vec<_>>()
.join("\n")
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct OcrOutcome {
pub text: String,
pub backend: OcrBackend,
}
pub struct Ocr {
options: OcrOptions,
#[cfg_attr(not(feature = "onnx"), allow(dead_code))]
onnx: Option<OnnxSlot>,
health: Option<Arc<dyn NativeHealthHook>>,
}
impl Ocr {
pub fn new(options: OcrOptions) -> Self {
Self {
options,
onnx: None,
health: None,
}
}
pub fn with_health_hook(mut self, hook: Arc<dyn NativeHealthHook>) -> Self {
self.health = Some(hook);
self
}
pub fn recognize_file(&mut self, path: &Path) -> Result<OcrResult, OcrError> {
let bytes = std::fs::read(path)
.map_err(|e| OcrError::Runtime(format!("Cannot read image {path:?}: {e}")))?;
self.recognize_encoded(&bytes)
}
pub fn recognize_encoded(&mut self, bytes: &[u8]) -> Result<OcrResult, OcrError> {
let img = decode_encoded(bytes)?;
self.recognize_inner(&img, Some(bytes))
}
pub fn recognize(&mut self, img: &RgbImage) -> Result<OcrResult, OcrError> {
self.recognize_inner(img, None)
}
pub fn recognize_with_provenance(&mut self, img: &RgbImage) -> Result<OcrOutcome, OcrError> {
self.recognize(img).map(|r| OcrOutcome {
text: r.text(),
backend: r.backend,
})
}
pub fn is_available(&self) -> bool {
matches!(self.probe_native(), Ok(true))
}
#[cfg(feature = "onnx")]
pub fn validate_fallback(&self) -> Result<(), OcrError> {
let cfg = self.options.onnx.as_ref().ok_or_else(|| {
OcrError::NoFallbackConfigured("no ONNX model config to validate".into())
})?;
configure_ort_runtime(cfg)?;
OcrEngine::validate_sessions(&cfg.model_dir, &cfg.manifest)
}
pub fn probe_native(&self) -> Result<bool, String> {
if native::native_backend().is_none() {
return Ok(false);
}
if let Some(verdict) = self.health.as_ref().and_then(|hook| hook.cached_verdict()) {
return Ok(verdict);
}
let result = match &self.options.isolation {
Isolation::Worker(cmd) => cmd.for_canary().run(native::canary_png()),
Isolation::InProcess => {
decode_encoded(native::canary_png()).and_then(|i| native::recognize(&i))
}
};
let verdict = match result {
Ok(lines) if !lines.is_empty() => Ok(true),
Ok(_) => Err("native canary returned no observations".into()),
Err(e) if is_backend_unavailable(&e) => Ok(false),
Err(e) => Err(e),
};
if let (Some(hook), Ok(healthy)) = (self.health.as_ref(), verdict.as_ref()) {
hook.record_verdict(*healthy);
}
verdict
}
fn recognize_inner(
&mut self,
img: &RgbImage,
encoded: Option<&[u8]>,
) -> Result<OcrResult, OcrError> {
match self.options.preference {
BackendPreference::Onnx => self.run_onnx(img, None),
BackendPreference::Native => {
let lines = self.run_native(img, encoded).map_err(native_error)?;
Ok(native_result(lines, None))
}
BackendPreference::Auto => match self.run_native(img, encoded) {
Ok(lines) => Ok(native_result(lines, None)),
Err(e) if is_backend_unavailable(&e) => {
if self.options.onnx.is_none() {
return Err(OcrError::NoFallbackConfigured(e));
}
self.run_onnx(img, Some(e))
}
Err(e) => Err(OcrError::Runtime(e)),
},
}
}
fn run_native(&self, img: &RgbImage, encoded: Option<&[u8]>) -> Result<Vec<OcrLine>, String> {
let Some(backend) = native::native_backend() else {
return Err(format!(
"{} no native OCR backend on this platform",
crate::types::OCR_UNSUPPORTED_PLATFORM
));
};
match &self.options.isolation {
Isolation::InProcess => native::recognize(img),
Isolation::Worker(cmd) => {
let owned;
let bytes = match encoded {
Some(b) => b,
None => {
owned = encode_png(img)?;
&owned
}
};
let lines = cmd.run(bytes)?;
if !lines.is_empty() || backend != OcrBackend::Vision {
return Ok(lines);
}
match cmd.for_canary().run(native::canary_png()) {
Ok(c) if !c.is_empty() => Ok(lines),
Ok(_) => Err(format!(
"{} Vision returned no observations and the canary in a fresh worker \
also returned none, so text recognition is not working here",
crate::types::MACOS_VISION_UNAVAILABLE
)),
Err(e) => Err(e),
}
}
}
}
#[cfg(not(feature = "onnx"))]
fn run_onnx(
&mut self,
_img: &RgbImage,
_fallback_reason: Option<String>,
) -> Result<OcrResult, OcrError> {
Err(OcrError::BackendUnavailable(
"ONNX fallback not compiled in: rebuild rightkit-ocr with the `onnx` feature".into(),
))
}
#[cfg(feature = "onnx")]
fn run_onnx(
&mut self,
img: &RgbImage,
fallback_reason: Option<String>,
) -> Result<OcrResult, OcrError> {
let cfg = self.options.onnx.clone().ok_or_else(|| {
OcrError::NoFallbackConfigured("ONNX backend requested without model config".into())
})?;
if self.onnx.is_none() {
configure_ort_runtime(&cfg)?;
self.onnx = Some(OcrEngine::load_with_lease(
&cfg.model_dir,
&cfg.manifest,
cfg.lease_hook.as_deref(),
)?);
}
let lines = self.onnx.as_mut().expect("loaded above").run_lines(img)?;
Ok(OcrResult {
lines,
backend: OcrBackend::Onnx,
fallback_reason,
})
}
}
#[cfg(feature = "onnx")]
fn configure_ort_runtime(cfg: &OnnxConfig) -> Result<(), OcrError> {
if let Some(rt) = &cfg.ort_runtime {
configure_runtime([RuntimeCandidate::new(rt.clone(), CandidateSource::Explicit)])
.map_err(|e| OcrError::Runtime(format!("ONNX runtime: {e:?}")))?;
}
Ok(())
}
fn native_result(lines: Vec<OcrLine>, fallback_reason: Option<String>) -> OcrResult {
OcrResult {
lines,
backend: native::native_backend().expect("native result implies native backend"),
fallback_reason,
}
}
fn native_error(e: String) -> OcrError {
if is_backend_unavailable(&e) {
OcrError::BackendUnavailable(e)
} else {
OcrError::Runtime(e)
}
}
fn encode_png(img: &RgbImage) -> Result<Vec<u8>, String> {
let mut out = Vec::new();
img.write_to(&mut std::io::Cursor::new(&mut out), image::ImageFormat::Png)
.map_err(|e| format!("Cannot encode image for worker: {e}"))?;
Ok(out)
}