Skip to main content

rightkit_ocr/
router.rs

1//! Backend selection and native-first fallback (ScrapeRight's routing).
2//!
3//! `Auto` runs the OS-native backend and falls back to the two-stage ONNX path
4//! **only** when the native backend proves unusable on this machine
5//! ([`crate::is_backend_unavailable`]) — never on quality, never on a
6//! per-image fault (those propagate, so a bug is not masked by degraded text).
7//! The result always says which backend produced it and why a fallback fired.
8
9use std::path::{Path, PathBuf};
10use std::sync::Arc;
11
12use image::RgbImage;
13#[cfg(feature = "onnx")]
14use rightkit_onnx::{configure_runtime, CandidateSource, RuntimeCandidate};
15use serde::Serialize;
16
17use crate::native;
18use crate::types::{decode_encoded, is_backend_unavailable, OcrBackend, OcrLine};
19use crate::worker::WorkerCommand;
20#[cfg(feature = "onnx")]
21use crate::OcrEngine;
22use crate::{OcrArtifactManifest, OcrError};
23
24/// The loaded ONNX engine slot. Without the `onnx` feature it is uninhabited, so the
25/// slot is always `None` and no ONNX runtime code is compiled in.
26#[cfg(feature = "onnx")]
27type OnnxSlot = OcrEngine;
28#[cfg(not(feature = "onnx"))]
29type OnnxSlot = std::convert::Infallible;
30
31#[derive(Debug, Clone, Copy, PartialEq, Eq)]
32pub enum BackendPreference {
33    /// Native first, ONNX fallback when native is unavailable.
34    Auto,
35    /// Native only; an unavailable backend is an error.
36    Native,
37    /// ONNX only (e.g. benchmarking or a deliberate cross-platform baseline).
38    Onnx,
39}
40
41/// Where native recognition runs.
42#[derive(Debug, Clone, Default)]
43pub enum Isolation {
44    /// In this process (Vision still self-checks blank results).
45    #[default]
46    InProcess,
47    /// One short-lived child per request, canary in a fresh child on blank
48    /// results (ViewRight's protocol; required for repeated accurate Vision
49    /// requests on affected Macs).
50    Worker(WorkerCommand),
51}
52
53/// An app's model-store lease/refs registration, held for an ONNX engine's lifetime.
54///
55/// Implement it in the app (no app crate is a dependency here). `acquire_model_lease` is called
56/// before the model files are verified; `paths` are the expected `det.onnx`, `rec.onnx` and
57/// `en_dict.txt` locations. Return the guard whose `Drop` releases the lease, or an error string
58/// (for example "model is in use by another app") to refuse the load. The engine drops the guard
59/// after its sessions.
60pub trait ModelLeaseHook: Send + Sync + std::fmt::Debug {
61    fn acquire_model_lease(
62        &self,
63        paths: &[PathBuf],
64    ) -> Result<Box<dyn std::any::Any + Send>, String>;
65}
66
67/// A host-maintained memory of the native backend's health verdict (no cache lives in this crate).
68///
69/// Attach one with [`Ocr::with_health_hook`]. [`Ocr::probe_native`] asks it first and returns a
70/// cached `Some(verdict)` without spawning a worker or touching Vision. After a probe finishes
71/// with a definite verdict, it reports that verdict back through
72/// [`NativeHealthHook::record_verdict`]. Transient probe errors are never reported, so they are
73/// retried rather than remembered.
74///
75/// Scope the cache to the process, and clear it when the host has reason to doubt it (for example
76/// after a backend failure). Recognition itself is not short-circuited: only the availability probe
77/// consults the hook.
78pub trait NativeHealthHook: Send + Sync {
79    /// The remembered verdict for this process: `Some(true)` or `Some(false)` short-circuits the
80    /// probe, and `None` means unknown, so the probe runs.
81    fn cached_verdict(&self) -> Option<bool>;
82    /// A definite verdict from a completed probe: `true` after a canary proved recognition works,
83    /// `false` when the backend is a stable capability verdict on this machine.
84    fn record_verdict(&self, healthy: bool);
85}
86
87/// Verified ONNX model set used by the fallback / `Onnx` preference.
88#[derive(Debug, Clone)]
89pub struct OnnxConfig {
90    pub model_dir: PathBuf,
91    pub manifest: OcrArtifactManifest,
92    /// Absolute ONNX Runtime library; `None` keeps whatever the caller already
93    /// configured through `rightkit-onnx` / `ORT_DYLIB_PATH`.
94    pub ort_runtime: Option<PathBuf>,
95    /// Optional app model-store lease, held while the engine is loaded.
96    pub lease_hook: Option<Arc<dyn ModelLeaseHook>>,
97}
98
99impl OnnxConfig {
100    pub fn new(model_dir: impl Into<PathBuf>, manifest: OcrArtifactManifest) -> Self {
101        Self {
102            model_dir: model_dir.into(),
103            manifest,
104            ort_runtime: None,
105            lease_hook: None,
106        }
107    }
108
109    /// Register a model-store lease hook (see [`ModelLeaseHook`]).
110    pub fn with_lease_hook(mut self, hook: Arc<dyn ModelLeaseHook>) -> Self {
111        self.lease_hook = Some(hook);
112        self
113    }
114}
115
116#[derive(Debug, Clone)]
117pub struct OcrOptions {
118    pub preference: BackendPreference,
119    pub onnx: Option<OnnxConfig>,
120    pub isolation: Isolation,
121}
122
123impl OcrOptions {
124    pub fn new(preference: BackendPreference) -> Self {
125        Self {
126            preference,
127            onnx: None,
128            isolation: Isolation::InProcess,
129        }
130    }
131    pub fn with_onnx(mut self, onnx: OnnxConfig) -> Self {
132        self.onnx = Some(onnx);
133        self
134    }
135    pub fn with_isolation(mut self, isolation: Isolation) -> Self {
136        self.isolation = isolation;
137        self
138    }
139}
140
141impl Default for OcrOptions {
142    fn default() -> Self {
143        Self::new(BackendPreference::Auto)
144    }
145}
146
147/// Text + geometry + confidence + provenance.
148#[derive(Debug, Clone, Serialize)]
149#[serde(rename_all = "camelCase")]
150pub struct OcrResult {
151    pub lines: Vec<OcrLine>,
152    pub backend: OcrBackend,
153    /// Set when `Auto` fell back: the native backend's unavailability error.
154    pub fallback_reason: Option<String>,
155}
156
157impl OcrResult {
158    /// Newline-joined text (ScrapeRight / CodeRight shape).
159    pub fn text(&self) -> String {
160        self.lines
161            .iter()
162            .map(|l| l.text.as_str())
163            .collect::<Vec<_>>()
164            .join("\n")
165    }
166}
167
168/// Newline-joined text and the backend that produced it (ScrapeRight's `OcrOutcome` shape).
169#[derive(Debug, Clone, PartialEq, Eq)]
170pub struct OcrOutcome {
171    pub text: String,
172    pub backend: OcrBackend,
173}
174
175/// A reusable OCR handle. The ONNX engine loads lazily on first need and is
176/// then kept warm.
177pub struct Ocr {
178    options: OcrOptions,
179    #[cfg_attr(not(feature = "onnx"), allow(dead_code))]
180    onnx: Option<OnnxSlot>,
181    health: Option<Arc<dyn NativeHealthHook>>,
182}
183
184impl Ocr {
185    pub fn new(options: OcrOptions) -> Self {
186        Self {
187            options,
188            onnx: None,
189            health: None,
190        }
191    }
192
193    /// Attach a host-maintained health verdict cache (see [`NativeHealthHook`]).
194    pub fn with_health_hook(mut self, hook: Arc<dyn NativeHealthHook>) -> Self {
195        self.health = Some(hook);
196        self
197    }
198
199    pub fn recognize_file(&mut self, path: &Path) -> Result<OcrResult, OcrError> {
200        let bytes = std::fs::read(path)
201            .map_err(|e| OcrError::Runtime(format!("Cannot read image {path:?}: {e}")))?;
202        self.recognize_encoded(&bytes)
203    }
204
205    /// Decode (with the pixel guard) and recognize PNG/JPEG bytes. In worker
206    /// isolation the original bytes go to the child unchanged.
207    pub fn recognize_encoded(&mut self, bytes: &[u8]) -> Result<OcrResult, OcrError> {
208        let img = decode_encoded(bytes)?;
209        self.recognize_inner(&img, Some(bytes))
210    }
211
212    pub fn recognize(&mut self, img: &RgbImage) -> Result<OcrResult, OcrError> {
213        self.recognize_inner(img, None)
214    }
215
216    /// Recognize and report only text + backend (the `OcrOutcome` ScrapeRight consumes).
217    pub fn recognize_with_provenance(&mut self, img: &RgbImage) -> Result<OcrOutcome, OcrError> {
218        self.recognize(img).map(|r| OcrOutcome {
219            text: r.text(),
220            backend: r.backend,
221        })
222    }
223
224    /// Is the native backend usable here? `true` only on a definite yes; a transient probe
225    /// failure is `false` (use [`Ocr::probe_native`] to tell them apart). Stays `false` on a
226    /// machine that would only succeed through the ONNX fallback.
227    pub fn is_available(&self) -> bool {
228        matches!(self.probe_native(), Ok(true))
229    }
230
231    /// Build and drop real ONNX sessions from the configured model set, verifying digests first.
232    /// No lease is taken: call it under the caller's own lease (migration, doctor).
233    #[cfg(feature = "onnx")]
234    pub fn validate_fallback(&self) -> Result<(), OcrError> {
235        let cfg = self.options.onnx.as_ref().ok_or_else(|| {
236            OcrError::NoFallbackConfigured("no ONNX model config to validate".into())
237        })?;
238        configure_ort_runtime(cfg)?;
239        OcrEngine::validate_sessions(&cfg.model_dir, &cfg.manifest)
240    }
241
242    /// Is the native backend usable here? Probes with the embedded known-text
243    /// canary; `Ok(false)` is a stable capability verdict, `Err` a transient
244    /// failure the caller should retry rather than cache.
245    pub fn probe_native(&self) -> Result<bool, String> {
246        if native::native_backend().is_none() {
247            return Ok(false);
248        }
249        if let Some(verdict) = self.health.as_ref().and_then(|hook| hook.cached_verdict()) {
250            return Ok(verdict);
251        }
252        let result = match &self.options.isolation {
253            // The canary gets its own deadline, not whatever budget a request left over.
254            Isolation::Worker(cmd) => cmd.for_canary().run(native::canary_png()),
255            Isolation::InProcess => {
256                decode_encoded(native::canary_png()).and_then(|i| native::recognize(&i))
257            }
258        };
259        let verdict = match result {
260            Ok(lines) if !lines.is_empty() => Ok(true),
261            Ok(_) => Err("native canary returned no observations".into()),
262            Err(e) if is_backend_unavailable(&e) => Ok(false),
263            Err(e) => Err(e),
264        };
265        if let (Some(hook), Ok(healthy)) = (self.health.as_ref(), verdict.as_ref()) {
266            hook.record_verdict(*healthy);
267        }
268        verdict
269    }
270
271    fn recognize_inner(
272        &mut self,
273        img: &RgbImage,
274        encoded: Option<&[u8]>,
275    ) -> Result<OcrResult, OcrError> {
276        match self.options.preference {
277            BackendPreference::Onnx => self.run_onnx(img, None),
278            BackendPreference::Native => {
279                let lines = self.run_native(img, encoded).map_err(native_error)?;
280                Ok(native_result(lines, None))
281            }
282            BackendPreference::Auto => match self.run_native(img, encoded) {
283                Ok(lines) => Ok(native_result(lines, None)),
284                Err(e) if is_backend_unavailable(&e) => {
285                    if self.options.onnx.is_none() {
286                        return Err(OcrError::NoFallbackConfigured(e));
287                    }
288                    self.run_onnx(img, Some(e))
289                }
290                Err(e) => Err(OcrError::Runtime(e)),
291            },
292        }
293    }
294
295    fn run_native(&self, img: &RgbImage, encoded: Option<&[u8]>) -> Result<Vec<OcrLine>, String> {
296        let Some(backend) = native::native_backend() else {
297            return Err(format!(
298                "{} no native OCR backend on this platform",
299                crate::types::OCR_UNSUPPORTED_PLATFORM
300            ));
301        };
302        match &self.options.isolation {
303            Isolation::InProcess => native::recognize(img),
304            Isolation::Worker(cmd) => {
305                let owned;
306                let bytes = match encoded {
307                    Some(b) => b,
308                    None => {
309                        owned = encode_png(img)?;
310                        &owned
311                    }
312                };
313                let lines = cmd.run(bytes)?;
314                if !lines.is_empty() || backend != OcrBackend::Vision {
315                    return Ok(lines);
316                }
317                // Vision cannot tell "blank page" from "could not run": verify
318                // in a FRESH child before trusting an empty result.
319                match cmd.for_canary().run(native::canary_png()) {
320                    Ok(c) if !c.is_empty() => Ok(lines),
321                    Ok(_) => Err(format!(
322                        "{} Vision returned no observations and the canary in a fresh worker \
323                         also returned none, so text recognition is not working here",
324                        crate::types::MACOS_VISION_UNAVAILABLE
325                    )),
326                    Err(e) => Err(e),
327                }
328            }
329        }
330    }
331
332    #[cfg(not(feature = "onnx"))]
333    fn run_onnx(
334        &mut self,
335        _img: &RgbImage,
336        _fallback_reason: Option<String>,
337    ) -> Result<OcrResult, OcrError> {
338        Err(OcrError::BackendUnavailable(
339            "ONNX fallback not compiled in: rebuild rightkit-ocr with the `onnx` feature".into(),
340        ))
341    }
342
343    #[cfg(feature = "onnx")]
344    fn run_onnx(
345        &mut self,
346        img: &RgbImage,
347        fallback_reason: Option<String>,
348    ) -> Result<OcrResult, OcrError> {
349        let cfg = self.options.onnx.clone().ok_or_else(|| {
350            OcrError::NoFallbackConfigured("ONNX backend requested without model config".into())
351        })?;
352        if self.onnx.is_none() {
353            configure_ort_runtime(&cfg)?;
354            self.onnx = Some(OcrEngine::load_with_lease(
355                &cfg.model_dir,
356                &cfg.manifest,
357                cfg.lease_hook.as_deref(),
358            )?);
359        }
360        let lines = self.onnx.as_mut().expect("loaded above").run_lines(img)?;
361        Ok(OcrResult {
362            lines,
363            backend: OcrBackend::Onnx,
364            fallback_reason,
365        })
366    }
367}
368
369#[cfg(feature = "onnx")]
370fn configure_ort_runtime(cfg: &OnnxConfig) -> Result<(), OcrError> {
371    if let Some(rt) = &cfg.ort_runtime {
372        configure_runtime([RuntimeCandidate::new(rt.clone(), CandidateSource::Explicit)])
373            .map_err(|e| OcrError::Runtime(format!("ONNX runtime: {e:?}")))?;
374    }
375    Ok(())
376}
377
378fn native_result(lines: Vec<OcrLine>, fallback_reason: Option<String>) -> OcrResult {
379    OcrResult {
380        lines,
381        backend: native::native_backend().expect("native result implies native backend"),
382        fallback_reason,
383    }
384}
385
386fn native_error(e: String) -> OcrError {
387    if is_backend_unavailable(&e) {
388        OcrError::BackendUnavailable(e)
389    } else {
390        OcrError::Runtime(e)
391    }
392}
393
394fn encode_png(img: &RgbImage) -> Result<Vec<u8>, String> {
395    let mut out = Vec::new();
396    img.write_to(&mut std::io::Cursor::new(&mut out), image::ImageFormat::Png)
397        .map_err(|e| format!("Cannot encode image for worker: {e}"))?;
398    Ok(out)
399}