Skip to main content

rightkit_ocr/
lib.rs

1//! Product-neutral two-stage ONNX OCR pipeline.
2//!
3//! Two-model pipeline:
4//!   1. Det model (`det.onnx`) — DB detector; finds text
5//!      bounding boxes in the image.
6//!   2. Rec model (`rec.onnx`) — multilingual recognizer
7//!      (50 languages + symbols + emoji, 18,708-char dict); reads text from each
8//!      cropped box. The vocab length is read from the model output at runtime,
9//!      so the dict is model-driven, not hardcoded.
10//!
11//! The caller configures the dynamic runtime through `rightkit-ort`, supplies
12//! an absolute model directory, and supplies resolved artifact provenance.
13
14mod native;
15mod router;
16#[cfg(feature = "onnx")]
17mod support;
18mod types;
19pub mod worker;
20
21#[cfg(all(target_os = "macos", feature = "native"))]
22pub use native::vision;
23#[cfg(all(windows, feature = "native"))]
24pub use native::win_native;
25pub use native::{canary_png, native_backend};
26pub use router::{BackendPreference, Isolation, Ocr, OcrOptions, OcrResult, OnnxConfig};
27pub use types::{
28    decode_encoded, is_backend_unavailable, OcrBackend, OcrLine, OcrRect, MACOS_VISION_UNAVAILABLE,
29    MAX_PIXELS, OCR_UNSUPPORTED_PLATFORM, WINDOWS_OCR_UNAVAILABLE,
30};
31pub use worker::WorkerCommand;
32
33#[cfg(feature = "onnx")]
34use image::{imageops, RgbImage};
35#[cfg(feature = "onnx")]
36use rightkit_ort::ort;
37#[cfg(feature = "onnx")]
38use rightkit_ort::ort::{session::Session, value::Tensor};
39#[cfg(feature = "onnx")]
40use rightkit_ort::ProviderDiagnostic;
41#[cfg(feature = "onnx")]
42use sha2::{Digest, Sha256};
43use std::fmt;
44#[cfg(feature = "onnx")]
45use std::path::Path;
46use std::path::PathBuf;
47#[cfg(all(test, feature = "onnx"))]
48use support::ctc_greedy_decode;
49#[cfg(feature = "onnx")]
50use support::{
51    build_session, crop_box, ctc_greedy_decode_scored, dedup_boxes, det_chw_data, det_net_size,
52    extract_boxes, group_into_lines, load_image, load_vocab, rec_chw_data, BBox, DET_BAND_OVERLAP,
53    DET_MAX_W, DET_TILE_H,
54};
55
56// ---------------------------------------------------------------------------
57// Public artifact and error contract
58// ---------------------------------------------------------------------------
59
60#[derive(Debug, Clone, PartialEq, Eq)]
61pub enum OcrProvenance {
62    Resolved {
63        label: String,
64        source: String,
65        license: String,
66    },
67    Conflicting {
68        labels: Vec<String>,
69    },
70}
71
72#[derive(Debug, Clone, PartialEq, Eq)]
73pub struct OcrArtifactManifest {
74    pub provenance: OcrProvenance,
75    pub detector_sha256: String,
76    pub recognizer_sha256: String,
77    pub dictionary_sha256: String,
78    pub fixture_sha256: String,
79    pub expected_fixture_output: String,
80}
81
82impl OcrArtifactManifest {
83    pub fn characterized_windows_contract() -> Self {
84        Self {
85            provenance: OcrProvenance::Conflicting {
86                labels: vec!["PP-OCRv5".to_string(), "PP-OCRv6".to_string()],
87            },
88            detector_sha256: "eb13b44b25bb36f89528b68720af8a61d9cf381176107f465db1757b65d086e1"
89                .to_string(),
90            recognizer_sha256: "5435fd747c9e0efe15a96d0b378d5bd157e9492ed8fd80edf08f30d02fa24634"
91                .to_string(),
92            dictionary_sha256: "b5f2bfe2bdd9448429e3e82b51c789775d9b42f2403d082b00662eb77e401c5d"
93                .to_string(),
94            fixture_sha256: "803e102100c91a077607cd2d4750d9d748d86ced48fabc0f77062f26dcba0a4b"
95                .to_string(),
96            expected_fixture_output: "RIGHTKIT OCR 2026\nShared parity fixture".to_string(),
97        }
98    }
99
100    /// Replace the characterized conflict only with an explicit authoritative
101    /// label and provenance source supplied by the signed artifact catalog.
102    pub fn with_resolved_provenance(
103        mut self,
104        label: impl Into<String>,
105        source: impl Into<String>,
106        license: impl Into<String>,
107    ) -> Result<Self, OcrError> {
108        self.provenance = OcrProvenance::Resolved {
109            label: label.into(),
110            source: source.into(),
111            license: license.into(),
112        };
113        self.validate_metadata()?;
114        Ok(self)
115    }
116
117    pub fn validate_metadata(&self) -> Result<(), OcrError> {
118        match &self.provenance {
119            OcrProvenance::Conflicting { labels } => Err(OcrError::UnresolvedProvenance {
120                labels: labels.clone(),
121            }),
122            OcrProvenance::Resolved {
123                label,
124                source,
125                license,
126            } if label.trim().is_empty()
127                || source.trim().is_empty()
128                || license.trim().is_empty() =>
129            {
130                Err(OcrError::InvalidProvenance)
131            }
132            OcrProvenance::Resolved { .. } => Ok(()),
133        }
134    }
135
136    #[cfg(feature = "onnx")]
137    pub fn validate_model_dir(&self, model_dir: &Path) -> Result<VerifiedOcrArtifacts, OcrError> {
138        self.validate_metadata()?;
139        if !model_dir.is_absolute() {
140            return Err(OcrError::UnsafeArtifactPath {
141                path: model_dir.to_path_buf(),
142            });
143        }
144        let root = model_dir
145            .canonicalize()
146            .map_err(|_| OcrError::MissingArtifact {
147                path: model_dir.to_path_buf(),
148            })?;
149        if !root.is_dir() {
150            return Err(OcrError::MissingArtifact { path: root });
151        }
152        Ok(VerifiedOcrArtifacts {
153            detector: verify_artifact(&root, "det.onnx", &self.detector_sha256)?,
154            recognizer: verify_artifact(&root, "rec.onnx", &self.recognizer_sha256)?,
155            dictionary: verify_artifact(&root, "en_dict.txt", &self.dictionary_sha256)?,
156        })
157    }
158}
159
160#[derive(Debug, Clone, PartialEq, Eq)]
161pub struct VerifiedOcrArtifacts {
162    pub detector: PathBuf,
163    pub recognizer: PathBuf,
164    pub dictionary: PathBuf,
165}
166
167#[derive(Debug, Clone, PartialEq, Eq)]
168pub enum OcrError {
169    UnresolvedProvenance {
170        labels: Vec<String>,
171    },
172    InvalidProvenance,
173    UnsafeArtifactPath {
174        path: PathBuf,
175    },
176    MissingArtifact {
177        path: PathBuf,
178    },
179    DigestMismatch {
180        path: PathBuf,
181        expected: String,
182        actual: String,
183    },
184    Runtime(String),
185    /// The requested backend cannot run on this machine (capability verdict).
186    BackendUnavailable(String),
187    /// A fallback ONNX engine was needed but no [`OnnxConfig`] was supplied.
188    NoFallbackConfigured(String),
189}
190
191impl fmt::Display for OcrError {
192    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
193        match self {
194            Self::UnresolvedProvenance { labels } => write!(
195                f,
196                "OCR provenance conflict is unresolved: {}",
197                labels.join(" vs ")
198            ),
199            Self::InvalidProvenance => {
200                write!(
201                    f,
202                    "OCR provenance label, source, and license must be non-empty"
203                )
204            }
205            Self::UnsafeArtifactPath { path } => {
206                write!(f, "OCR artifact path must be absolute: {}", path.display())
207            }
208            Self::MissingArtifact { path } => {
209                write!(f, "OCR artifact is missing: {}", path.display())
210            }
211            Self::DigestMismatch {
212                path,
213                expected,
214                actual,
215            } => write!(
216                f,
217                "OCR artifact digest mismatch for {}: expected {expected}, got {actual}",
218                path.display()
219            ),
220            Self::Runtime(message) => write!(f, "OCR runtime error: {message}"),
221            Self::BackendUnavailable(message) => write!(f, "OCR backend unavailable: {message}"),
222            Self::NoFallbackConfigured(message) => {
223                write!(
224                    f,
225                    "OCR native backend unavailable and no ONNX fallback configured: {message}"
226                )
227            }
228        }
229    }
230}
231
232impl std::error::Error for OcrError {}
233impl From<String> for OcrError {
234    fn from(value: String) -> Self {
235        Self::Runtime(value)
236    }
237}
238
239#[cfg(feature = "onnx")]
240fn verify_artifact(root: &Path, name: &str, expected: &str) -> Result<PathBuf, OcrError> {
241    let path = root.join(name);
242    let canonical = path
243        .canonicalize()
244        .map_err(|_| OcrError::MissingArtifact { path: path.clone() })?;
245    if !canonical.starts_with(root) || !canonical.is_file() {
246        return Err(OcrError::UnsafeArtifactPath { path: canonical });
247    }
248    let bytes = std::fs::read(&canonical).map_err(|_| OcrError::MissingArtifact {
249        path: canonical.clone(),
250    })?;
251    let actual = format!("{:x}", Sha256::digest(bytes));
252    if actual != expected {
253        return Err(OcrError::DigestMismatch {
254            path: canonical,
255            expected: expected.to_string(),
256            actual,
257        });
258    }
259    Ok(canonical)
260}
261
262#[cfg(feature = "onnx")]
263pub fn ocr_image_with_manifest(
264    path: &Path,
265    model_dir: &Path,
266    manifest: &OcrArtifactManifest,
267) -> Result<String, OcrError> {
268    let mut engine = OcrEngine::load_with_manifest(model_dir, manifest)?;
269    let img = load_image(path)?;
270    engine.run(&img)
271}
272
273// ---------------------------------------------------------------------------
274
275#[cfg(all(test, feature = "onnx"))]
276mod tests;
277// OcrEngine — load once, reuse across slides
278// ---------------------------------------------------------------------------
279
280/// A loaded, warm OCR engine.
281///
282/// Build once on a dedicated worker thread; reuse across slides.
283/// DirectML sessions are GPU-state-bound — keep on the same thread.
284#[cfg(feature = "onnx")]
285pub struct OcrEngine {
286    det: Session,
287    rec: Session,
288    /// CTC vocabulary: index 0 = blank '\0'; 1..=N = dict chars (N = en_dict.txt
289    /// line count for the characterized model); final index = ' '. The decoder indexes
290    /// by the model's own output `vocab_size`, so an extra trailing entry is
291    /// never reached if the model omits the space class.
292    vocab: Vec<char>,
293    provider_diagnostics: Vec<ProviderDiagnostic>,
294}
295
296#[cfg(feature = "onnx")]
297impl OcrEngine {
298    /// Load using the currently characterized manifest. This deliberately
299    /// fails closed until its conflicting model provenance label is resolved.
300    pub fn load(model_dir: &Path) -> Result<Self, OcrError> {
301        Self::load_with_manifest(
302            model_dir,
303            &OcrArtifactManifest::characterized_windows_contract(),
304        )
305    }
306
307    /// Load det + rec sessions from an explicit, verified model directory.
308    ///
309    /// Probes DirectML first on Windows and records any CPU fallback.
310    ///
311    /// Expected files:
312    /// - `<model_dir>/det.onnx`      detector ONNX
313    /// - `<model_dir>/rec.onnx`      recognizer ONNX
314    /// - `<model_dir>/en_dict.txt`   character dictionary (UTF-8, one char/line)
315    pub fn load_with_manifest(
316        model_dir: &Path,
317        manifest: &OcrArtifactManifest,
318    ) -> Result<Self, OcrError> {
319        let artifacts = manifest.validate_model_dir(model_dir)?;
320        let vocab = load_vocab(&artifacts.dictionary)?;
321        let (det, mut provider_diagnostics) = build_session(&artifacts.detector)?;
322        let (rec, rec_diagnostics) = build_session(&artifacts.recognizer)?;
323        provider_diagnostics.extend(rec_diagnostics);
324        Ok(OcrEngine {
325            det,
326            rec,
327            vocab,
328            provider_diagnostics,
329        })
330    }
331
332    pub fn provider_diagnostics(&self) -> &[ProviderDiagnostic] {
333        &self.provider_diagnostics
334    }
335
336    /// OCR a pre-loaded RGB image; returns newline-joined text.
337    pub fn run(&mut self, img: &RgbImage) -> Result<String, OcrError> {
338        let lines = self.run_lines(img)?;
339        Ok(lines
340            .into_iter()
341            .map(|l| l.text)
342            .collect::<Vec<_>>()
343            .join("\n"))
344    }
345
346    /// OCR with per-line geometry: one [`OcrLine`] per visual line, rect =
347    /// union of the line's detected boxes normalized to the image (top-left
348    /// origin), confidence = mean CTC character score when the recognizer
349    /// emits probabilities.
350    pub fn run_lines(&mut self, img: &RgbImage) -> Result<Vec<OcrLine>, OcrError> {
351        let boxes = self.detect(img)?;
352        if boxes.is_empty() {
353            return Ok(Vec::new());
354        }
355        let (iw, ih) = (img.width().max(1) as f32, img.height().max(1) as f32);
356        // Group boxes into visual lines (baseline-overlap), then read each line
357        // left-to-right. Without this, higher-resolution detection fragments big
358        // display text into per-word boxes that a naive top-down sort scrambles.
359        let rows = group_into_lines(boxes);
360        let mut out = Vec::new();
361        for row in rows {
362            let mut words = Vec::new();
363            let (mut x0, mut y0, mut x1, mut y1) = (u32::MAX, u32::MAX, 0u32, 0u32);
364            let (mut conf_sum, mut conf_n, mut conf_ok) = (0.0f32, 0u32, true);
365            for bbox in &row {
366                let crop = crop_box(img, bbox);
367                if crop.width() == 0 || crop.height() == 0 {
368                    continue;
369                }
370                let (text, conf) = self.recognize(&crop)?;
371                let t = text.trim().to_string();
372                if t.is_empty() {
373                    continue;
374                }
375                words.push(t);
376                x0 = x0.min(bbox.x0);
377                y0 = y0.min(bbox.y0);
378                x1 = x1.max(bbox.x1);
379                y1 = y1.max(bbox.y1);
380                match conf {
381                    Some(c) => {
382                        conf_sum += c;
383                        conf_n += 1;
384                    }
385                    None => conf_ok = false,
386                }
387            }
388            if words.is_empty() {
389                continue;
390            }
391            let (x0, y0) = (x0 as f32 / iw, y0 as f32 / ih);
392            let (x1, y1) = ((x1 as f32 / iw).min(1.0), (y1 as f32 / ih).min(1.0));
393            out.push(OcrLine {
394                text: words.join(" "),
395                rect: OcrRect {
396                    x: x0,
397                    y: y0,
398                    width: (x1 - x0).max(0.0),
399                    height: (y1 - y0).max(0.0),
400                },
401                confidence: (conf_ok && conf_n > 0).then(|| conf_sum / conf_n as f32),
402            });
403        }
404        Ok(out)
405    }
406
407    // -----------------------------------------------------------------------
408    // Detection
409    // -----------------------------------------------------------------------
410
411    /// Detect text boxes across the whole image.
412    ///
413    /// Caps working width at [`DET_MAX_W`] (downscale only — never upscale) so
414    /// horizontal text keeps near-native resolution, then splits tall images
415    /// into overlapping vertical bands each ≤ [`DET_TILE_H`] and detects per
416    /// band. Boxes are translated to original-image coordinates and
417    /// de-duplicated across band seams. Recognition always crops from the
418    /// full-resolution original, so higher detection resolution directly
419    /// improves read accuracy.
420    fn detect(&mut self, img: &RgbImage) -> Result<Vec<BBox>, String> {
421        let (orig_w, orig_h) = (img.width(), img.height());
422
423        // Width cap: downscale only. Most screenshots are ≤ DET_MAX_W → native.
424        let ws = if orig_w > DET_MAX_W {
425            DET_MAX_W as f32 / orig_w as f32
426        } else {
427            1.0
428        };
429        let work = if ws < 1.0 {
430            let nh = ((orig_h as f32 * ws).round() as u32).max(1);
431            imageops::resize(img, DET_MAX_W, nh, imageops::FilterType::Lanczos3)
432        } else {
433            img.clone()
434        };
435        let (work_w, work_h) = (work.width(), work.height());
436
437        // Overlapping vertical bands so tall pages keep native resolution
438        // instead of being crushed to a 960px long side.
439        let mut bands: Vec<(u32, u32)> = Vec::new();
440        if work_h <= DET_TILE_H {
441            bands.push((0, work_h));
442        } else {
443            let step = DET_TILE_H - DET_BAND_OVERLAP;
444            let mut y = 0u32;
445            loop {
446                let end = (y + DET_TILE_H).min(work_h);
447                bands.push((y, end));
448                if end >= work_h {
449                    break;
450                }
451                y += step;
452            }
453        }
454
455        let mut all: Vec<BBox> = Vec::new();
456        for (ys, ye) in bands {
457            let band = imageops::crop_imm(&work, 0, ys, work_w, ye - ys).to_image();
458            for mut b in self.detect_one(&band)? {
459                b.y0 += ys;
460                b.y1 += ys;
461                all.push(b);
462            }
463        }
464
465        // Map working → original coordinates.
466        if ws < 1.0 {
467            let inv = 1.0 / ws;
468            for b in all.iter_mut() {
469                b.x0 = (b.x0 as f32 * inv) as u32;
470                b.y0 = (b.y0 as f32 * inv) as u32;
471                b.x1 = ((b.x1 as f32 * inv) as u32).min(orig_w);
472                b.y1 = ((b.y1 as f32 * inv) as u32).min(orig_h);
473            }
474        }
475
476        let mut boxes = dedup_boxes(all);
477        boxes.sort_by(|a, b| a.y0.cmp(&b.y0).then(a.x0.cmp(&b.x0)));
478        Ok(boxes)
479    }
480
481    /// Run the detector on a single band; returns boxes in band-local pixels.
482    fn detect_one(&mut self, band: &RgbImage) -> Result<Vec<BBox>, String> {
483        let (bw, bh) = (band.width(), band.height());
484        let (net_w, net_h) = det_net_size(bw, bh);
485        let resized = imageops::resize(band, net_w, net_h, imageops::FilterType::Lanczos3);
486
487        let data = det_chw_data(&resized, net_w, net_h);
488        let shape = [1i64, 3, net_h as i64, net_w as i64];
489        let tensor =
490            Tensor::<f32>::from_array((shape, data)).map_err(|e| format!("det tensor: {e}"))?;
491
492        let outputs = self
493            .det
494            .run(ort::inputs![tensor])
495            .map_err(|e| format!("det run: {e}"))?;
496
497        // Output: [1, 1, H, W].  Extract as flat &[f32] + shape.
498        let (out_shape, prob_flat) = outputs[0]
499            .try_extract_tensor::<f32>()
500            .map_err(|e| format!("det extract: {e}"))?;
501        // Shape derefs to &[i64]; dims are [batch, channel, H, W].
502        let dims: &[i64] = out_shape;
503        let (oh, ow) = match dims {
504            [_, _, h, w] => (*h as usize, *w as usize),
505            s => return Err(format!("unexpected det output shape: {s:?}")),
506        };
507        let prob_2d: &[f32] = &prob_flat[0..oh * ow];
508
509        // Map prob-map → band pixels (handles any det output downsampling).
510        let scale_x = bw as f32 / ow as f32;
511        let scale_y = bh as f32 / oh as f32;
512        Ok(extract_boxes(
513            prob_2d, ow, oh, scale_x, scale_y, 0.3, bw, bh,
514        ))
515    }
516
517    // -----------------------------------------------------------------------
518    // Recognition
519    // -----------------------------------------------------------------------
520
521    fn recognize(&mut self, crop: &RgbImage) -> Result<(String, Option<f32>), String> {
522        let (h, w) = (crop.height(), crop.width());
523        let new_w = ((48.0 / h as f32) * w as f32).max(4.0) as u32;
524        let strip = imageops::resize(crop, new_w, 48, imageops::FilterType::Lanczos3);
525
526        let data = rec_chw_data(&strip, new_w);
527        let shape = [1i64, 3, 48, new_w as i64];
528        let tensor =
529            Tensor::<f32>::from_array((shape, data)).map_err(|e| format!("rec tensor: {e}"))?;
530
531        let outputs = self
532            .rec
533            .run(ort::inputs![tensor])
534            .map_err(|e| format!("rec run: {e}"))?;
535
536        // Output: [1, T, vocab_size].
537        let (out_shape, logits_flat) = outputs[0]
538            .try_extract_tensor::<f32>()
539            .map_err(|e| format!("rec extract: {e}"))?;
540        // Shape derefs to &[i64]; dims are [batch, T, vocab_size].
541        let dims: &[i64] = out_shape;
542        let (t_len, vocab_size) = match dims {
543            [_, t, v] => (*t as usize, *v as usize),
544            s => return Err(format!("unexpected rec output shape: {s:?}")),
545        };
546        // logits_flat: [1, T, V]; skip batch dim.
547        let logits = &logits_flat[0..t_len * vocab_size];
548
549        Ok(ctc_greedy_decode_scored(
550            logits,
551            t_len,
552            vocab_size,
553            &self.vocab,
554        ))
555    }
556}
557
558// ---------------------------------------------------------------------------