Skip to main content

rightkit_ocr/
router.rs

1//! Backend selection and native-first fallback (ScrapeRight's routing).
2//!
3//! `Auto` runs the OS-native backend and falls back to the two-stage ONNX path
4//! **only** when the native backend proves unusable on this machine
5//! ([`crate::is_backend_unavailable`]) — never on quality, never on a
6//! per-image fault (those propagate, so a bug is not masked by degraded text).
7//! The result always says which backend produced it and why a fallback fired.
8
9use std::path::{Path, PathBuf};
10
11use image::RgbImage;
12#[cfg(feature = "onnx")]
13use rightkit_ort::{configure_runtime, CandidateSource, RuntimeCandidate};
14use serde::Serialize;
15
16use crate::native;
17use crate::types::{decode_encoded, is_backend_unavailable, OcrBackend, OcrLine};
18use crate::worker::WorkerCommand;
19#[cfg(feature = "onnx")]
20use crate::OcrEngine;
21use crate::{OcrArtifactManifest, OcrError};
22
23/// The loaded ONNX engine slot. Without the `onnx` feature it is uninhabited, so the
24/// slot is always `None` and no ONNX runtime code is compiled in.
25#[cfg(feature = "onnx")]
26type OnnxSlot = OcrEngine;
27#[cfg(not(feature = "onnx"))]
28type OnnxSlot = std::convert::Infallible;
29
30#[derive(Debug, Clone, Copy, PartialEq, Eq)]
31pub enum BackendPreference {
32    /// Native first, ONNX fallback when native is unavailable.
33    Auto,
34    /// Native only; an unavailable backend is an error.
35    Native,
36    /// ONNX only (e.g. benchmarking or a deliberate cross-platform baseline).
37    Onnx,
38}
39
40/// Where native recognition runs.
41#[derive(Debug, Clone, Default)]
42pub enum Isolation {
43    /// In this process (Vision still self-checks blank results).
44    #[default]
45    InProcess,
46    /// One short-lived child per request, canary in a fresh child on blank
47    /// results (ViewRight's protocol; required for repeated accurate Vision
48    /// requests on affected Macs).
49    Worker(WorkerCommand),
50}
51
52/// Verified ONNX model set used by the fallback / `Onnx` preference.
53#[derive(Debug, Clone)]
54pub struct OnnxConfig {
55    pub model_dir: PathBuf,
56    pub manifest: OcrArtifactManifest,
57    /// Absolute ONNX Runtime library; `None` keeps whatever the caller already
58    /// configured through `rightkit-ort` / `ORT_DYLIB_PATH`.
59    pub ort_runtime: Option<PathBuf>,
60}
61
62impl OnnxConfig {
63    pub fn new(model_dir: impl Into<PathBuf>, manifest: OcrArtifactManifest) -> Self {
64        Self {
65            model_dir: model_dir.into(),
66            manifest,
67            ort_runtime: None,
68        }
69    }
70}
71
72#[derive(Debug, Clone)]
73pub struct OcrOptions {
74    pub preference: BackendPreference,
75    pub onnx: Option<OnnxConfig>,
76    pub isolation: Isolation,
77}
78
79impl OcrOptions {
80    pub fn new(preference: BackendPreference) -> Self {
81        Self {
82            preference,
83            onnx: None,
84            isolation: Isolation::InProcess,
85        }
86    }
87    pub fn with_onnx(mut self, onnx: OnnxConfig) -> Self {
88        self.onnx = Some(onnx);
89        self
90    }
91    pub fn with_isolation(mut self, isolation: Isolation) -> Self {
92        self.isolation = isolation;
93        self
94    }
95}
96
97impl Default for OcrOptions {
98    fn default() -> Self {
99        Self::new(BackendPreference::Auto)
100    }
101}
102
103/// Text + geometry + confidence + provenance.
104#[derive(Debug, Clone, Serialize)]
105#[serde(rename_all = "camelCase")]
106pub struct OcrResult {
107    pub lines: Vec<OcrLine>,
108    pub backend: OcrBackend,
109    /// Set when `Auto` fell back: the native backend's unavailability error.
110    pub fallback_reason: Option<String>,
111}
112
113impl OcrResult {
114    /// Newline-joined text (ScrapeRight / CodeRight shape).
115    pub fn text(&self) -> String {
116        self.lines
117            .iter()
118            .map(|l| l.text.as_str())
119            .collect::<Vec<_>>()
120            .join("\n")
121    }
122}
123
124/// A reusable OCR handle. The ONNX engine loads lazily on first need and is
125/// then kept warm.
126pub struct Ocr {
127    options: OcrOptions,
128    #[cfg_attr(not(feature = "onnx"), allow(dead_code))]
129    onnx: Option<OnnxSlot>,
130}
131
132impl Ocr {
133    pub fn new(options: OcrOptions) -> Self {
134        Self {
135            options,
136            onnx: None,
137        }
138    }
139
140    pub fn recognize_file(&mut self, path: &Path) -> Result<OcrResult, OcrError> {
141        let bytes = std::fs::read(path)
142            .map_err(|e| OcrError::Runtime(format!("Cannot read image {path:?}: {e}")))?;
143        self.recognize_encoded(&bytes)
144    }
145
146    /// Decode (with the pixel guard) and recognize PNG/JPEG bytes. In worker
147    /// isolation the original bytes go to the child unchanged.
148    pub fn recognize_encoded(&mut self, bytes: &[u8]) -> Result<OcrResult, OcrError> {
149        let img = decode_encoded(bytes)?;
150        self.recognize_inner(&img, Some(bytes))
151    }
152
153    pub fn recognize(&mut self, img: &RgbImage) -> Result<OcrResult, OcrError> {
154        self.recognize_inner(img, None)
155    }
156
157    /// Is the native backend usable here? Probes with the embedded known-text
158    /// canary; `Ok(false)` is a stable capability verdict, `Err` a transient
159    /// failure the caller should retry rather than cache.
160    pub fn probe_native(&self) -> Result<bool, String> {
161        if native::native_backend().is_none() {
162            return Ok(false);
163        }
164        let result = match &self.options.isolation {
165            Isolation::Worker(cmd) => cmd.run(native::canary_png()),
166            Isolation::InProcess => {
167                decode_encoded(native::canary_png()).and_then(|i| native::recognize(&i))
168            }
169        };
170        match result {
171            Ok(lines) if !lines.is_empty() => Ok(true),
172            Ok(_) => Err("native canary returned no observations".into()),
173            Err(e) if is_backend_unavailable(&e) => Ok(false),
174            Err(e) => Err(e),
175        }
176    }
177
178    fn recognize_inner(
179        &mut self,
180        img: &RgbImage,
181        encoded: Option<&[u8]>,
182    ) -> Result<OcrResult, OcrError> {
183        match self.options.preference {
184            BackendPreference::Onnx => self.run_onnx(img, None),
185            BackendPreference::Native => {
186                let lines = self.run_native(img, encoded).map_err(native_error)?;
187                Ok(native_result(lines, None))
188            }
189            BackendPreference::Auto => match self.run_native(img, encoded) {
190                Ok(lines) => Ok(native_result(lines, None)),
191                Err(e) if is_backend_unavailable(&e) => {
192                    if self.options.onnx.is_none() {
193                        return Err(OcrError::NoFallbackConfigured(e));
194                    }
195                    self.run_onnx(img, Some(e))
196                }
197                Err(e) => Err(OcrError::Runtime(e)),
198            },
199        }
200    }
201
202    fn run_native(&self, img: &RgbImage, encoded: Option<&[u8]>) -> Result<Vec<OcrLine>, String> {
203        let Some(backend) = native::native_backend() else {
204            return Err(format!(
205                "{} no native OCR backend on this platform",
206                crate::types::OCR_UNSUPPORTED_PLATFORM
207            ));
208        };
209        match &self.options.isolation {
210            Isolation::InProcess => native::recognize(img),
211            Isolation::Worker(cmd) => {
212                let owned;
213                let bytes = match encoded {
214                    Some(b) => b,
215                    None => {
216                        owned = encode_png(img)?;
217                        &owned
218                    }
219                };
220                let lines = cmd.run(bytes)?;
221                if !lines.is_empty() || backend != OcrBackend::Vision {
222                    return Ok(lines);
223                }
224                // Vision cannot tell "blank page" from "could not run": verify
225                // in a FRESH child before trusting an empty result.
226                match cmd.run(native::canary_png()) {
227                    Ok(c) if !c.is_empty() => Ok(lines),
228                    Ok(_) => Err(format!(
229                        "{} Vision returned no observations and the canary in a fresh worker \
230                         also returned none, so text recognition is not working here",
231                        crate::types::MACOS_VISION_UNAVAILABLE
232                    )),
233                    Err(e) => Err(e),
234                }
235            }
236        }
237    }
238
239    #[cfg(not(feature = "onnx"))]
240    fn run_onnx(
241        &mut self,
242        _img: &RgbImage,
243        _fallback_reason: Option<String>,
244    ) -> Result<OcrResult, OcrError> {
245        Err(OcrError::BackendUnavailable(
246            "ONNX fallback not compiled in: rebuild rightkit-ocr with the `onnx` feature".into(),
247        ))
248    }
249
250    #[cfg(feature = "onnx")]
251    fn run_onnx(
252        &mut self,
253        img: &RgbImage,
254        fallback_reason: Option<String>,
255    ) -> Result<OcrResult, OcrError> {
256        let cfg = self.options.onnx.clone().ok_or_else(|| {
257            OcrError::NoFallbackConfigured("ONNX backend requested without model config".into())
258        })?;
259        if self.onnx.is_none() {
260            if let Some(rt) = &cfg.ort_runtime {
261                configure_runtime([RuntimeCandidate::new(rt.clone(), CandidateSource::Explicit)])
262                    .map_err(|e| OcrError::Runtime(format!("ONNX runtime: {e:?}")))?;
263            }
264            self.onnx = Some(OcrEngine::load_with_manifest(
265                &cfg.model_dir,
266                &cfg.manifest,
267            )?);
268        }
269        let lines = self.onnx.as_mut().expect("loaded above").run_lines(img)?;
270        Ok(OcrResult {
271            lines,
272            backend: OcrBackend::Onnx,
273            fallback_reason,
274        })
275    }
276}
277
278fn native_result(lines: Vec<OcrLine>, fallback_reason: Option<String>) -> OcrResult {
279    OcrResult {
280        lines,
281        backend: native::native_backend().expect("native result implies native backend"),
282        fallback_reason,
283    }
284}
285
286fn native_error(e: String) -> OcrError {
287    if is_backend_unavailable(&e) {
288        OcrError::BackendUnavailable(e)
289    } else {
290        OcrError::Runtime(e)
291    }
292}
293
294fn encode_png(img: &RgbImage) -> Result<Vec<u8>, String> {
295    let mut out = Vec::new();
296    img.write_to(&mut std::io::Cursor::new(&mut out), image::ImageFormat::Png)
297        .map_err(|e| format!("Cannot encode image for worker: {e}"))?;
298    Ok(out)
299}