use std::borrow::Cow;
use std::ffi::c_void;
use std::ptr::NonNull;
use std::sync::OnceLock;
use image::{imageops, RgbImage};
use objc2::rc::Retained;
use objc2::runtime::AnyObject;
use objc2::AllocAnyThread;
use objc2_core_foundation::{CFRetained, CGRect};
use objc2_core_graphics::{
CGBitmapInfo, CGColorRenderingIntent, CGColorSpace, CGDataProvider, CGImage,
};
use objc2_foundation::{NSArray, NSDictionary, NSString};
use objc2_vision::{
VNErrorCode, VNErrorDomain, VNImageRequestHandler, VNRecognizeTextRequest,
VNRecognizeTextRequestRevision3, VNRequest, VNRequestTextRecognitionLevel,
};
use crate::types::{OcrLine, OcrRect};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum RequestErrorClass {
Unavailable,
Transient,
}
pub(crate) fn classify_request_error(domain: &str, code: VNErrorCode) -> RequestErrorClass {
if unsafe { VNErrorDomain }.is_none_or(|d| domain != d.to_string()) {
return RequestErrorClass::Transient;
}
match code {
VNErrorCode::NotImplemented
| VNErrorCode::InvalidModel
| VNErrorCode::UnsupportedRevision
| VNErrorCode::DataUnavailable
| VNErrorCode::UnsupportedRequest
| VNErrorCode::UnsupportedComputeStage
| VNErrorCode::UnsupportedComputeDevice => RequestErrorClass::Unavailable,
_ => RequestErrorClass::Transient,
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum HealthCheck {
Healthy,
Unavailable,
Transient,
}
pub fn recognize(img: &RgbImage) -> Result<Vec<OcrLine>, String> {
let lines = recognize_once(img)?;
if !lines.is_empty() {
return Ok(lines);
}
match vision_health() {
HealthCheck::Healthy => Ok(lines),
HealthCheck::Unavailable => Err(format!(
"{} Vision returned no observations and then failed its built-in \
self-check, so text recognition is not working on this machine \
(this is not a blank page). Common causes: the process is \
sandboxed away from Vision's XPC service, or the OCR model \
assets are unavailable.",
crate::types::MACOS_VISION_UNAVAILABLE
)),
HealthCheck::Transient => Err(
"Vision returned no observations and its self-check could not confirm \
the backend either — a transient failure, retry recognition"
.to_string(),
),
}
}
pub fn recognize_once(img: &RgbImage) -> Result<Vec<OcrLine>, String> {
recognize_lines(img)
}
fn vision_health() -> HealthCheck {
static HEALTHY: OnceLock<()> = OnceLock::new();
cached_health(&HEALTHY, recognize_lines)
}
pub(crate) fn cached_health(
slot: &'static OnceLock<()>,
recognize: fn(&RgbImage) -> Result<Vec<OcrLine>, String>,
) -> HealthCheck {
if slot.get().is_some() {
return HealthCheck::Healthy;
}
match check_health(recognize) {
HealthCheck::Healthy => {
let _ = slot.set(());
HealthCheck::Healthy
}
other => other,
}
}
pub(crate) fn check_health(
recognize: fn(&RgbImage) -> Result<Vec<OcrLine>, String>,
) -> HealthCheck {
match decode_canary() {
Some(canary) => match recognize(&canary) {
Ok(lines) if !lines.is_empty() => HealthCheck::Healthy,
Err(e) if crate::types::is_backend_unavailable(&e) => HealthCheck::Unavailable,
_ => HealthCheck::Transient,
},
None => HealthCheck::Healthy,
}
}
fn decode_canary() -> Option<RgbImage> {
let bytes = canary_png();
image::load_from_memory(bytes).ok().map(|i| i.into_rgb8())
}
pub(crate) fn to_top_left(b: CGRect) -> OcrRect {
let x = b.origin.x.clamp(0.0, 1.0);
let y = (1.0 - (b.origin.y + b.size.height)).clamp(0.0, 1.0);
let right = (b.origin.x + b.size.width).clamp(0.0, 1.0);
let bottom = (1.0 - b.origin.y).clamp(0.0, 1.0);
OcrRect {
x: x as f32,
y: y as f32,
width: (right - x).max(0.0) as f32,
height: (bottom - y).max(0.0) as f32,
}
}
fn recognize_lines(img: &RgbImage) -> Result<Vec<OcrLine>, String> {
let prepared = prepare_for_vision(img);
let cg = cg_image_from_rgb(&prepared)?;
unsafe {
let options: Retained<NSDictionary<NSString, AnyObject>> = NSDictionary::new();
let handler = VNImageRequestHandler::initWithCGImage_options(
VNImageRequestHandler::alloc(),
&cg,
&options,
);
let request = VNRecognizeTextRequest::init(VNRecognizeTextRequest::alloc());
request.setRecognitionLevel(VNRequestTextRecognitionLevel::Accurate);
request.setUsesLanguageCorrection(true);
request.setRevision(VNRecognizeTextRequestRevision3);
request.setAutomaticallyDetectsLanguage(true);
let requests =
NSArray::from_retained_slice(&[Retained::cast_unchecked::<VNRequest>(request.clone())]);
handler.performRequests_error(&requests).map_err(|e| {
match classify_request_error(&e.domain().to_string(), VNErrorCode(e.code())) {
RequestErrorClass::Unavailable => format!(
"{} Vision performRequests failed: {e}",
crate::types::MACOS_VISION_UNAVAILABLE
),
RequestErrorClass::Transient => {
format!("Vision performRequests failed: {e}")
}
}
})?;
let Some(results) = request.results() else {
return Ok(Vec::new());
};
let mut lines: Vec<OcrLine> = Vec::with_capacity(results.len());
for obs in results.iter() {
let candidates = obs.topCandidates(1);
if let Some(text) = candidates.iter().next() {
lines.push(OcrLine {
text: text.string().to_string(),
rect: to_top_left(obs.boundingBox()),
confidence: Some(text.confidence()),
});
}
}
Ok(lines)
}
}
pub(crate) const MAX_VISION_DIMENSION: u32 = 1_800;
pub(crate) fn prepare_for_vision(img: &RgbImage) -> Cow<'_, RgbImage> {
let longest = img.width().max(img.height());
if longest <= MAX_VISION_DIMENSION {
return Cow::Borrowed(img);
}
let scale = f64::from(MAX_VISION_DIMENSION) / f64::from(longest);
let width = (f64::from(img.width()) * scale).round().max(1.0) as u32;
let height = (f64::from(img.height()) * scale).round().max(1.0) as u32;
Cow::Owned(imageops::resize(
img,
width,
height,
imageops::FilterType::Lanczos3,
))
}
fn cg_image_from_rgb(img: &RgbImage) -> Result<CFRetained<CGImage>, String> {
let (w, h) = (img.width() as usize, img.height() as usize);
if w == 0 || h == 0 {
return Err("empty image".into());
}
let bytes: Box<[u8]> = img.as_raw().clone().into_boxed_slice();
let len = bytes.len();
let ptr = Box::into_raw(bytes) as *mut c_void;
unsafe extern "C-unwind" fn release(_info: *mut c_void, data: NonNull<c_void>, size: usize) {
let slice = std::ptr::slice_from_raw_parts_mut(data.as_ptr() as *mut u8, size);
drop(Box::from_raw(slice));
}
unsafe {
let provider = CGDataProvider::with_data(std::ptr::null_mut(), ptr, len, Some(release))
.ok_or_else(|| {
let slice = std::ptr::slice_from_raw_parts_mut(ptr as *mut u8, len);
drop(Box::from_raw(slice));
"CGDataProvider creation failed".to_string()
})?;
let space = CGColorSpace::new_device_rgb().ok_or("device RGB color space failed")?;
let bitmap_info = CGBitmapInfo(0);
let cg = CGImage::new(
w,
h,
8, 24, w * 3, Some(&space),
bitmap_info,
Some(&provider),
std::ptr::null(),
false,
CGColorRenderingIntent::RenderingIntentDefault,
)
.ok_or("CGImageCreate failed")?;
Ok(cg)
}
}
pub fn canary_png() -> &'static [u8] {
include_bytes!("assets/vision_canary.png")
}