Skip to main content

rightkit_ocr/
router.rs

1//! Backend selection and native-first fallback (ScrapeRight's routing).
2//!
3//! `Auto` runs the OS-native backend and falls back to the two-stage ONNX path
4//! **only** when the native backend proves unusable on this machine
5//! ([`crate::is_backend_unavailable`]) — never on quality, never on a
6//! per-image fault (those propagate, so a bug is not masked by degraded text).
7//! The result always says which backend produced it and why a fallback fired.
8
9use std::path::{Path, PathBuf};
10use std::sync::Arc;
11
12use image::RgbImage;
13#[cfg(feature = "onnx")]
14use rightkit_ort::{configure_runtime, CandidateSource, RuntimeCandidate};
15use serde::Serialize;
16
17use crate::native;
18use crate::types::{decode_encoded, is_backend_unavailable, OcrBackend, OcrLine};
19use crate::worker::WorkerCommand;
20#[cfg(feature = "onnx")]
21use crate::OcrEngine;
22use crate::{OcrArtifactManifest, OcrError};
23
24/// The loaded ONNX engine slot. Without the `onnx` feature it is uninhabited, so the
25/// slot is always `None` and no ONNX runtime code is compiled in.
26#[cfg(feature = "onnx")]
27type OnnxSlot = OcrEngine;
28#[cfg(not(feature = "onnx"))]
29type OnnxSlot = std::convert::Infallible;
30
31#[derive(Debug, Clone, Copy, PartialEq, Eq)]
32pub enum BackendPreference {
33    /// Native first, ONNX fallback when native is unavailable.
34    Auto,
35    /// Native only; an unavailable backend is an error.
36    Native,
37    /// ONNX only (e.g. benchmarking or a deliberate cross-platform baseline).
38    Onnx,
39}
40
41/// Where native recognition runs.
42#[derive(Debug, Clone, Default)]
43pub enum Isolation {
44    /// In this process (Vision still self-checks blank results).
45    #[default]
46    InProcess,
47    /// One short-lived child per request, canary in a fresh child on blank
48    /// results (ViewRight's protocol; required for repeated accurate Vision
49    /// requests on affected Macs).
50    Worker(WorkerCommand),
51}
52
53/// An app's model-store lease/refs registration, held for an ONNX engine's lifetime.
54///
55/// Implement it in the app (no app crate is a dependency here). `acquire_model_lease` is called
56/// before the model files are verified; `paths` are the expected `det.onnx`, `rec.onnx` and
57/// `en_dict.txt` locations. Return the guard whose `Drop` releases the lease, or an error string
58/// (for example "model is in use by another app") to refuse the load. The engine drops the guard
59/// after its sessions.
60pub trait ModelLeaseHook: Send + Sync + std::fmt::Debug {
61    fn acquire_model_lease(
62        &self,
63        paths: &[PathBuf],
64    ) -> Result<Box<dyn std::any::Any + Send>, String>;
65}
66
67/// Verified ONNX model set used by the fallback / `Onnx` preference.
68#[derive(Debug, Clone)]
69pub struct OnnxConfig {
70    pub model_dir: PathBuf,
71    pub manifest: OcrArtifactManifest,
72    /// Absolute ONNX Runtime library; `None` keeps whatever the caller already
73    /// configured through `rightkit-ort` / `ORT_DYLIB_PATH`.
74    pub ort_runtime: Option<PathBuf>,
75    /// Optional app model-store lease, held while the engine is loaded.
76    pub lease_hook: Option<Arc<dyn ModelLeaseHook>>,
77}
78
79impl OnnxConfig {
80    pub fn new(model_dir: impl Into<PathBuf>, manifest: OcrArtifactManifest) -> Self {
81        Self {
82            model_dir: model_dir.into(),
83            manifest,
84            ort_runtime: None,
85            lease_hook: None,
86        }
87    }
88
89    /// Register a model-store lease hook (see [`ModelLeaseHook`]).
90    pub fn with_lease_hook(mut self, hook: Arc<dyn ModelLeaseHook>) -> Self {
91        self.lease_hook = Some(hook);
92        self
93    }
94}
95
96#[derive(Debug, Clone)]
97pub struct OcrOptions {
98    pub preference: BackendPreference,
99    pub onnx: Option<OnnxConfig>,
100    pub isolation: Isolation,
101}
102
103impl OcrOptions {
104    pub fn new(preference: BackendPreference) -> Self {
105        Self {
106            preference,
107            onnx: None,
108            isolation: Isolation::InProcess,
109        }
110    }
111    pub fn with_onnx(mut self, onnx: OnnxConfig) -> Self {
112        self.onnx = Some(onnx);
113        self
114    }
115    pub fn with_isolation(mut self, isolation: Isolation) -> Self {
116        self.isolation = isolation;
117        self
118    }
119}
120
121impl Default for OcrOptions {
122    fn default() -> Self {
123        Self::new(BackendPreference::Auto)
124    }
125}
126
127/// Text + geometry + confidence + provenance.
128#[derive(Debug, Clone, Serialize)]
129#[serde(rename_all = "camelCase")]
130pub struct OcrResult {
131    pub lines: Vec<OcrLine>,
132    pub backend: OcrBackend,
133    /// Set when `Auto` fell back: the native backend's unavailability error.
134    pub fallback_reason: Option<String>,
135}
136
137impl OcrResult {
138    /// Newline-joined text (ScrapeRight / CodeRight shape).
139    pub fn text(&self) -> String {
140        self.lines
141            .iter()
142            .map(|l| l.text.as_str())
143            .collect::<Vec<_>>()
144            .join("\n")
145    }
146}
147
148/// Newline-joined text and the backend that produced it (ScrapeRight's `OcrOutcome` shape).
149#[derive(Debug, Clone, PartialEq, Eq)]
150pub struct OcrOutcome {
151    pub text: String,
152    pub backend: OcrBackend,
153}
154
155/// A reusable OCR handle. The ONNX engine loads lazily on first need and is
156/// then kept warm.
157pub struct Ocr {
158    options: OcrOptions,
159    #[cfg_attr(not(feature = "onnx"), allow(dead_code))]
160    onnx: Option<OnnxSlot>,
161}
162
163impl Ocr {
164    pub fn new(options: OcrOptions) -> Self {
165        Self {
166            options,
167            onnx: None,
168        }
169    }
170
171    pub fn recognize_file(&mut self, path: &Path) -> Result<OcrResult, OcrError> {
172        let bytes = std::fs::read(path)
173            .map_err(|e| OcrError::Runtime(format!("Cannot read image {path:?}: {e}")))?;
174        self.recognize_encoded(&bytes)
175    }
176
177    /// Decode (with the pixel guard) and recognize PNG/JPEG bytes. In worker
178    /// isolation the original bytes go to the child unchanged.
179    pub fn recognize_encoded(&mut self, bytes: &[u8]) -> Result<OcrResult, OcrError> {
180        let img = decode_encoded(bytes)?;
181        self.recognize_inner(&img, Some(bytes))
182    }
183
184    pub fn recognize(&mut self, img: &RgbImage) -> Result<OcrResult, OcrError> {
185        self.recognize_inner(img, None)
186    }
187
188    /// Recognize and report only text + backend (the `OcrOutcome` ScrapeRight consumes).
189    pub fn recognize_with_provenance(&mut self, img: &RgbImage) -> Result<OcrOutcome, OcrError> {
190        self.recognize(img).map(|r| OcrOutcome {
191            text: r.text(),
192            backend: r.backend,
193        })
194    }
195
196    /// Is the native backend usable here? `true` only on a definite yes; a transient probe
197    /// failure is `false` (use [`Ocr::probe_native`] to tell them apart). Stays `false` on a
198    /// machine that would only succeed through the ONNX fallback.
199    pub fn is_available(&self) -> bool {
200        matches!(self.probe_native(), Ok(true))
201    }
202
203    /// Build and drop real ONNX sessions from the configured model set, verifying digests first.
204    /// No lease is taken: call it under the caller's own lease (migration, doctor).
205    #[cfg(feature = "onnx")]
206    pub fn validate_fallback(&self) -> Result<(), OcrError> {
207        let cfg = self.options.onnx.as_ref().ok_or_else(|| {
208            OcrError::NoFallbackConfigured("no ONNX model config to validate".into())
209        })?;
210        configure_ort_runtime(cfg)?;
211        OcrEngine::validate_sessions(&cfg.model_dir, &cfg.manifest)
212    }
213
214    /// Is the native backend usable here? Probes with the embedded known-text
215    /// canary; `Ok(false)` is a stable capability verdict, `Err` a transient
216    /// failure the caller should retry rather than cache.
217    pub fn probe_native(&self) -> Result<bool, String> {
218        if native::native_backend().is_none() {
219            return Ok(false);
220        }
221        let result = match &self.options.isolation {
222            Isolation::Worker(cmd) => cmd.run(native::canary_png()),
223            Isolation::InProcess => {
224                decode_encoded(native::canary_png()).and_then(|i| native::recognize(&i))
225            }
226        };
227        match result {
228            Ok(lines) if !lines.is_empty() => Ok(true),
229            Ok(_) => Err("native canary returned no observations".into()),
230            Err(e) if is_backend_unavailable(&e) => Ok(false),
231            Err(e) => Err(e),
232        }
233    }
234
235    fn recognize_inner(
236        &mut self,
237        img: &RgbImage,
238        encoded: Option<&[u8]>,
239    ) -> Result<OcrResult, OcrError> {
240        match self.options.preference {
241            BackendPreference::Onnx => self.run_onnx(img, None),
242            BackendPreference::Native => {
243                let lines = self.run_native(img, encoded).map_err(native_error)?;
244                Ok(native_result(lines, None))
245            }
246            BackendPreference::Auto => match self.run_native(img, encoded) {
247                Ok(lines) => Ok(native_result(lines, None)),
248                Err(e) if is_backend_unavailable(&e) => {
249                    if self.options.onnx.is_none() {
250                        return Err(OcrError::NoFallbackConfigured(e));
251                    }
252                    self.run_onnx(img, Some(e))
253                }
254                Err(e) => Err(OcrError::Runtime(e)),
255            },
256        }
257    }
258
259    fn run_native(&self, img: &RgbImage, encoded: Option<&[u8]>) -> Result<Vec<OcrLine>, String> {
260        let Some(backend) = native::native_backend() else {
261            return Err(format!(
262                "{} no native OCR backend on this platform",
263                crate::types::OCR_UNSUPPORTED_PLATFORM
264            ));
265        };
266        match &self.options.isolation {
267            Isolation::InProcess => native::recognize(img),
268            Isolation::Worker(cmd) => {
269                let owned;
270                let bytes = match encoded {
271                    Some(b) => b,
272                    None => {
273                        owned = encode_png(img)?;
274                        &owned
275                    }
276                };
277                let lines = cmd.run(bytes)?;
278                if !lines.is_empty() || backend != OcrBackend::Vision {
279                    return Ok(lines);
280                }
281                // Vision cannot tell "blank page" from "could not run": verify
282                // in a FRESH child before trusting an empty result.
283                match cmd.run(native::canary_png()) {
284                    Ok(c) if !c.is_empty() => Ok(lines),
285                    Ok(_) => Err(format!(
286                        "{} Vision returned no observations and the canary in a fresh worker \
287                         also returned none, so text recognition is not working here",
288                        crate::types::MACOS_VISION_UNAVAILABLE
289                    )),
290                    Err(e) => Err(e),
291                }
292            }
293        }
294    }
295
296    #[cfg(not(feature = "onnx"))]
297    fn run_onnx(
298        &mut self,
299        _img: &RgbImage,
300        _fallback_reason: Option<String>,
301    ) -> Result<OcrResult, OcrError> {
302        Err(OcrError::BackendUnavailable(
303            "ONNX fallback not compiled in: rebuild rightkit-ocr with the `onnx` feature".into(),
304        ))
305    }
306
307    #[cfg(feature = "onnx")]
308    fn run_onnx(
309        &mut self,
310        img: &RgbImage,
311        fallback_reason: Option<String>,
312    ) -> Result<OcrResult, OcrError> {
313        let cfg = self.options.onnx.clone().ok_or_else(|| {
314            OcrError::NoFallbackConfigured("ONNX backend requested without model config".into())
315        })?;
316        if self.onnx.is_none() {
317            configure_ort_runtime(&cfg)?;
318            self.onnx = Some(OcrEngine::load_with_lease(
319                &cfg.model_dir,
320                &cfg.manifest,
321                cfg.lease_hook.as_deref(),
322            )?);
323        }
324        let lines = self.onnx.as_mut().expect("loaded above").run_lines(img)?;
325        Ok(OcrResult {
326            lines,
327            backend: OcrBackend::Onnx,
328            fallback_reason,
329        })
330    }
331}
332
333#[cfg(feature = "onnx")]
334fn configure_ort_runtime(cfg: &OnnxConfig) -> Result<(), OcrError> {
335    if let Some(rt) = &cfg.ort_runtime {
336        configure_runtime([RuntimeCandidate::new(rt.clone(), CandidateSource::Explicit)])
337            .map_err(|e| OcrError::Runtime(format!("ONNX runtime: {e:?}")))?;
338    }
339    Ok(())
340}
341
342fn native_result(lines: Vec<OcrLine>, fallback_reason: Option<String>) -> OcrResult {
343    OcrResult {
344        lines,
345        backend: native::native_backend().expect("native result implies native backend"),
346        fallback_reason,
347    }
348}
349
350fn native_error(e: String) -> OcrError {
351    if is_backend_unavailable(&e) {
352        OcrError::BackendUnavailable(e)
353    } else {
354        OcrError::Runtime(e)
355    }
356}
357
358fn encode_png(img: &RgbImage) -> Result<Vec<u8>, String> {
359    let mut out = Vec::new();
360    img.write_to(&mut std::io::Cursor::new(&mut out), image::ImageFormat::Png)
361        .map_err(|e| format!("Cannot encode image for worker: {e}"))?;
362    Ok(out)
363}