use std::path::Path;
use std::time::{Duration, Instant};
use image::DynamicImage;
use ndarray::Axis;
use serde::{Deserialize, Serialize};
use crate::{
config::RuntimeConfig,
error::{RapidOcrError, Result},
formula::{
preprocess::FormulaPreprocessor,
session::FormulaSession,
tokenizer::{FormulaDecode, FormulaTokenizer},
tokenizer_metadata::FormulaTokenizerMetadata,
},
input::image_loader::{LoadImage, OcrInput},
model_verify::VerifiedModel,
runtime::provider::ProviderResolution,
};
pub const DEFAULT_MAX_FORMULA_BATCH_SIZE: usize = 16;
pub const DEFAULT_MAX_FORMULA_INPUT_PIXELS: u64 = 24_000_000;
pub const DEFAULT_MAX_FORMULA_SEQUENCE_LENGTH: usize = 4096;
pub const FORMULA_MODEL_LOOP_BOUND: usize = 2561;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FormulaRecognition {
pub latex: String,
pub token_ids: Vec<i64>,
pub eos_index: Option<usize>,
pub truncated: bool,
pub model_id: String,
pub elapsed_ms: f32,
pub batch_size: usize,
}
impl FormulaRecognition {
pub fn decode(&self) -> FormulaDecode {
FormulaDecode {
latex: self.latex.clone(),
token_ids: self.token_ids.clone(),
eos_index: self.eos_index,
truncated: self.truncated,
}
}
}
#[derive(Debug)]
pub struct FormulaRecognizer {
session: FormulaSession,
preprocessor: FormulaPreprocessor,
tokenizer: FormulaTokenizer,
model_id: String,
max_batch_size: usize,
max_input_pixels: u64,
max_sequence_length: usize,
}
impl FormulaRecognizer {
pub fn from_model(model: &VerifiedModel, runtime_cfg: &RuntimeConfig) -> Result<Self> {
let model_path = model.path();
let session = FormulaSession::new(model, runtime_cfg)?;
let character_metadata = session.character_metadata()?.ok_or_else(|| {
RapidOcrError::Tokenizer(format!(
"formula model is missing `character` metadata: {}",
model_path.display()
))
})?;
let metadata = FormulaTokenizerMetadata::from_character_metadata(&character_metadata)?;
let tokenizer = FormulaTokenizer::from_metadata(&metadata)?;
let model_id = model_path
.file_stem()
.and_then(|stem| stem.to_str())
.unwrap_or("formula-model")
.to_string();
Ok(Self {
session,
preprocessor: FormulaPreprocessor::new(),
tokenizer,
model_id,
max_batch_size: DEFAULT_MAX_FORMULA_BATCH_SIZE,
max_input_pixels: DEFAULT_MAX_FORMULA_INPUT_PIXELS,
max_sequence_length: DEFAULT_MAX_FORMULA_SEQUENCE_LENGTH,
})
}
pub fn model_id(&self) -> &str {
&self.model_id
}
pub fn max_batch_size(&self) -> usize {
self.max_batch_size
}
pub fn set_max_batch_size(&mut self, max_batch_size: usize) -> Result<()> {
if max_batch_size == 0 {
return Err(RapidOcrError::InvalidInput(
"formula max batch size must be greater than zero".to_string(),
));
}
self.max_batch_size = max_batch_size;
Ok(())
}
pub fn max_input_pixels(&self) -> u64 {
self.max_input_pixels
}
pub fn set_max_input_pixels(&mut self, max_input_pixels: u64) -> Result<()> {
if max_input_pixels == 0 {
return Err(RapidOcrError::InvalidInput(
"formula max input pixels must be greater than zero".to_string(),
));
}
self.max_input_pixels = max_input_pixels;
Ok(())
}
pub fn max_sequence_length(&self) -> usize {
self.max_sequence_length
}
pub fn set_max_sequence_length(&mut self, max_sequence_length: usize) -> Result<()> {
if max_sequence_length == 0 {
return Err(RapidOcrError::InvalidInput(
"formula max sequence length must be greater than zero".to_string(),
));
}
self.max_sequence_length = max_sequence_length;
Ok(())
}
pub fn provider_resolution(&self) -> ProviderResolution {
self.session.provider_resolution()
}
pub fn session_threads(&self) -> (Option<usize>, Option<usize>) {
self.session.session_threads()
}
pub fn recognize(&mut self, image: &DynamicImage) -> Result<FormulaRecognition> {
self.ensure_image_limit(image)?;
let start = Instant::now();
let input = self.preprocessor.preprocess(image)?;
let output = self.session.run(input.view())?;
if output.nrows() != 1 {
return Err(RapidOcrError::Decode(format!(
"formula model returned {} rows for single input (model={})",
output.nrows(),
self.model_id
)));
}
self.ensure_sequence_limit(output.ncols())?;
let token_ids = output.row(0).to_vec();
let decoded = self.tokenizer.decode_ids(&token_ids)?;
Ok(self.recognition(decoded, start.elapsed().as_secs_f32() * 1000.0, 1))
}
pub fn recognize_encoded(
&mut self,
bytes: &[u8],
max_decode_pixels: u64,
max_encoded_bytes: u64,
) -> Result<FormulaRecognition> {
let image = LoadImage::default().load_dynamic_with_limit(
OcrInput::Bytes(bytes.to_vec()),
max_decode_pixels,
max_encoded_bytes,
)?;
self.recognize(&image)
}
pub fn recognize_file(
&mut self,
path: &Path,
max_decode_pixels: u64,
max_encoded_bytes: u64,
) -> Result<FormulaRecognition> {
let image = LoadImage::default().load_dynamic_with_limit(
OcrInput::Path(path.to_path_buf()),
max_decode_pixels,
max_encoded_bytes,
)?;
self.recognize(&image)
}
pub fn recognize_url(
&mut self,
url: &str,
max_decode_pixels: u64,
max_encoded_bytes: u64,
timeout: Duration,
) -> Result<FormulaRecognition> {
let loader = LoadImage::with_http_timeouts(timeout, timeout);
let image = loader.load_dynamic_with_limit(
OcrInput::Url(url.to_string()),
max_decode_pixels,
max_encoded_bytes,
)?;
self.recognize(&image)
}
pub fn recognize_batch(&mut self, images: &[DynamicImage]) -> Result<Vec<FormulaRecognition>> {
if images.is_empty() {
return Ok(Vec::new());
}
for image in images {
self.ensure_image_limit(image)?;
}
let start = Instant::now();
let mut results = Vec::with_capacity(images.len());
for chunk in images.chunks(self.max_batch_size) {
results.extend(self.recognize_chunk(chunk)?);
}
let elapsed_ms = start.elapsed().as_secs_f32() * 1000.0;
for result in &mut results {
result.elapsed_ms = elapsed_ms;
result.batch_size = images.len();
}
Ok(results)
}
fn recognize_chunk(&mut self, images: &[DynamicImage]) -> Result<Vec<FormulaRecognition>> {
let input = self.preprocessor.preprocess_batch(images)?;
let output = self.session.run(input.view())?;
if output.nrows() != images.len() {
return Err(RapidOcrError::Decode(format!(
"formula model returned {} rows for batch size {} (model={})",
output.nrows(),
images.len(),
self.model_id
)));
}
self.ensure_sequence_limit(output.ncols())?;
let mut results = Vec::with_capacity(images.len());
for row in output.axis_iter(Axis(0)) {
let token_ids = row.to_vec();
let decoded = self.tokenizer.decode_ids(&token_ids)?;
results.push(self.recognition(decoded, 0.0, images.len()));
}
Ok(results)
}
fn ensure_image_limit(&self, image: &DynamicImage) -> Result<()> {
use image::GenericImageView;
let (width, height) = image.dimensions();
crate::input::image_loader::ensure_decode_pixels(
width as usize,
height as usize,
self.max_input_pixels,
)
}
fn ensure_sequence_limit(&self, sequence_length: usize) -> Result<()> {
if sequence_length > self.max_sequence_length {
return Err(RapidOcrError::Decode(format!(
"formula output length {sequence_length} exceeds limit {}",
self.max_sequence_length
)));
}
Ok(())
}
fn recognition(
&self,
decoded: FormulaDecode,
elapsed_ms: f32,
batch_size: usize,
) -> FormulaRecognition {
FormulaRecognition {
latex: crate::formula::postprocess::postprocess_latex(&decoded.latex),
token_ids: decoded.token_ids,
eos_index: decoded.eos_index,
truncated: decoded.truncated,
model_id: self.model_id.clone(),
elapsed_ms,
batch_size,
}
}
}
#[cfg(test)]
mod tests {
use std::path::PathBuf;
use image::{DynamicImage, Rgb, RgbImage};
use super::*;
fn fixture(name: &str) -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("tests/fixtures/formula-onnx")
.join(name)
}
fn rt() -> RuntimeConfig {
RuntimeConfig::default()
}
fn cred(name: &str) -> VerifiedModel {
VerifiedModel::verify(&fixture(name), None).expect("fixture credential")
}
fn gray(value: u8) -> DynamicImage {
DynamicImage::ImageRgb8(RgbImage::from_pixel(32, 32, Rgb([value, value, value])))
}
#[test]
fn single_image_happy_path() {
let mut recognizer =
FormulaRecognizer::from_model(&cred("formula_recognizer_ok.onnx"), &rt())
.expect("recognizer should load");
let result = recognizer.recognize(&gray(255)).expect("recognize");
assert_eq!(result.latex, "\\cdot");
assert_eq!(result.token_ids, vec![0, 82, 1769, 2]);
assert_eq!(result.eos_index, Some(3));
assert!(!result.truncated);
assert_eq!(result.model_id, "formula_recognizer_ok");
}
#[test]
fn batch_preserves_order_and_independence() {
let mut recognizer =
FormulaRecognizer::from_model(&cred("formula_recognizer_ok.onnx"), &rt())
.expect("recognizer should load");
let images = vec![gray(0), gray(255), gray(128), gray(64)];
let results = recognizer
.recognize_batch(&images)
.expect("batch recognize");
assert_eq!(results.len(), images.len());
for result in &results {
assert_eq!(result.latex, "\\cdot");
assert_eq!(result.token_ids, vec![0, 82, 1769, 2]);
assert_eq!(result.eos_index, Some(3));
assert!(!result.truncated);
}
}
#[test]
fn empty_batch_is_supported() {
let mut recognizer =
FormulaRecognizer::from_model(&cred("formula_recognizer_ok.onnx"), &rt())
.expect("recognizer should load");
assert!(
recognizer
.recognize_batch(&[])
.expect("empty batch")
.is_empty()
);
}
#[test]
fn batches_larger_than_max_batch_size_are_chunked() {
let mut recognizer =
FormulaRecognizer::from_model(&cred("formula_recognizer_ok.onnx"), &rt())
.expect("recognizer should load");
recognizer.set_max_batch_size(1).expect("set batch size");
let images = vec![gray(0), gray(255), gray(128)];
let results = recognizer
.recognize_batch(&images)
.expect("a batch larger than max_batch_size must be chunked, not rejected");
assert_eq!(results.len(), images.len());
for result in &results {
assert_eq!(result.latex, "\\cdot");
assert_eq!(result.token_ids, vec![0, 82, 1769, 2]);
assert_eq!(
result.batch_size, 3,
"batch_size must describe the whole call, not one chunk"
);
}
assert!(
(results[0].elapsed_ms - results[2].elapsed_ms).abs() < f32::EPSILON,
"every result of one call shares the call wall time"
);
}
#[test]
fn twenty_crops_are_chunked_and_match_explicit_chunks() {
let mut recognizer =
FormulaRecognizer::from_model(&cred("formula_recognizer_ok.onnx"), &rt())
.expect("recognizer should load");
assert_eq!(recognizer.max_batch_size(), DEFAULT_MAX_FORMULA_BATCH_SIZE);
const _: () = assert!(
20 > DEFAULT_MAX_FORMULA_BATCH_SIZE,
"this test must exercise more than one chunk"
);
let images: Vec<DynamicImage> = (0..20).map(|i| gray((i * 11) as u8)).collect();
let results = recognizer
.recognize_batch(&images)
.expect("20 crops must be recognized across chunks");
assert_eq!(results.len(), 20);
for (index, result) in results.iter().enumerate() {
assert_eq!(result.latex, "\\cdot", "crop {index}");
assert_eq!(result.token_ids, vec![0, 82, 1769, 2], "crop {index}");
assert_eq!(result.eos_index, Some(3), "crop {index}");
assert_eq!(result.batch_size, 20, "crop {index}");
}
let wall = results[0].elapsed_ms;
assert!(
results
.iter()
.all(|r| (r.elapsed_ms - wall).abs() < f32::EPSILON),
"the whole call's wall time is shared by all results"
);
let explicit: Vec<_> = recognizer
.recognize_batch(&images[..16])
.expect("first explicit chunk")
.into_iter()
.chain(
recognizer
.recognize_batch(&images[16..])
.expect("second explicit chunk"),
)
.collect();
assert_eq!(explicit.len(), results.len());
for (chunked, split) in results.iter().zip(&explicit) {
assert_eq!(chunked.latex, split.latex);
assert_eq!(chunked.token_ids, split.token_ids);
assert_eq!(chunked.eos_index, split.eos_index);
assert_eq!(chunked.model_id, split.model_id);
}
}
#[test]
fn single_image_chunking_covers_every_input() {
let mut recognizer =
FormulaRecognizer::from_model(&cred("formula_recognizer_ok.onnx"), &rt())
.expect("recognizer should load");
recognizer.set_max_batch_size(1).expect("set batch size");
let images: Vec<DynamicImage> = (0..5).map(|i| gray((i * 40) as u8)).collect();
let results = recognizer
.recognize_batch(&images)
.expect("chunked batch recognize");
assert_eq!(results.len(), 5);
for result in &results {
assert_eq!(result.batch_size, 5);
assert_eq!(result.latex, "\\cdot");
}
}
#[test]
fn missing_model_is_located() {
let error = VerifiedModel::verify(&fixture("does_not_exist.onnx"), None)
.expect_err("missing model must fail");
assert!(matches!(error, RapidOcrError::FileNotFound(_)));
}
#[test]
fn hash_mismatch_is_rejected() {
let expected = "0000000000000000000000000000000000000000000000000000000000000000";
let error = VerifiedModel::verify(&fixture("formula_recognizer_ok.onnx"), Some(expected))
.expect_err("hash mismatch must fail");
assert!(matches!(error, RapidOcrError::HashMismatch { .. }));
}
#[test]
fn corrupted_tokenizer_metadata_is_rejected() {
let error = FormulaRecognizer::from_model(&cred("formula_bad_metadata.onnx"), &rt())
.expect_err("bad metadata must fail");
assert!(
error.to_string().contains("metadata") || error.to_string().contains("tokenizer"),
"error: {error}"
);
}
#[test]
fn out_of_vocab_model_output_is_rejected() {
let mut recognizer =
FormulaRecognizer::from_model(&cred("formula_recognizer_bad_token.onnx"), &rt())
.expect("recognizer should load");
let error = recognizer
.recognize(&gray(255))
.expect_err("bad token must fail");
assert!(
error.to_string().contains("out of vocabulary"),
"error: {error}"
);
}
#[test]
fn empty_image_is_rejected_before_inference() {
let mut recognizer =
FormulaRecognizer::from_model(&cred("formula_recognizer_ok.onnx"), &rt())
.expect("recognizer should load");
let empty = DynamicImage::ImageRgb8(RgbImage::from_raw(0, 0, Vec::new()).unwrap());
let error = recognizer
.recognize(&empty)
.expect_err("empty image must fail");
assert!(
error.to_string().contains("greater than zero"),
"error: {error}"
);
}
#[test]
fn default_sequence_limit_exceeds_model_loop_bound() {
const _: () = assert!(DEFAULT_MAX_FORMULA_SEQUENCE_LENGTH > FORMULA_MODEL_LOOP_BOUND);
let recognizer = FormulaRecognizer::from_model(&cred("formula_recognizer_ok.onnx"), &rt())
.expect("recognizer should load");
assert_eq!(
recognizer.max_sequence_length(),
DEFAULT_MAX_FORMULA_SEQUENCE_LENGTH
);
assert!(
recognizer
.ensure_sequence_limit(FORMULA_MODEL_LOOP_BOUND)
.is_ok(),
"a row at the model's own loop bound must be accepted"
);
assert!(
recognizer
.ensure_sequence_limit(DEFAULT_MAX_FORMULA_SEQUENCE_LENGTH + 1)
.is_err(),
"the limit must still bound memory for pathological outputs"
);
}
#[test]
fn pixel_and_sequence_limits_are_enforced() {
let mut recognizer =
FormulaRecognizer::from_model(&cred("formula_recognizer_ok.onnx"), &rt())
.expect("recognizer should load");
recognizer
.set_max_input_pixels(100)
.expect("set pixel limit");
let error = recognizer
.recognize(&gray(255))
.expect_err("large input must fail");
assert!(error.to_string().contains("limit is 100"), "error: {error}");
let mut recognizer =
FormulaRecognizer::from_model(&cred("formula_recognizer_ok.onnx"), &rt())
.expect("recognizer should load");
recognizer
.set_max_sequence_length(2)
.expect("set sequence limit");
let error = recognizer
.recognize(&gray(255))
.expect_err("long output must fail");
assert!(
error.to_string().contains("exceeds limit"),
"error: {error}"
);
}
#[test]
fn encoded_input_enforces_encoded_and_pixel_limits() {
let mut recognizer =
FormulaRecognizer::from_model(&cred("formula_recognizer_ok.onnx"), &rt())
.expect("recognizer should load");
let encoded = {
let image = gray(255).to_rgb8();
let mut bytes = Vec::new();
image
.write_to(
&mut std::io::Cursor::new(&mut bytes),
image::ImageFormat::Png,
)
.expect("encode png");
bytes
};
let result = recognizer
.recognize_encoded(&encoded, 2000, 10_000)
.expect("encoded recognize");
assert_eq!(result.latex, "\\cdot");
let error = recognizer
.recognize_encoded(&encoded, 2000, 10)
.expect_err("encoded limit must fail");
assert!(error.to_string().contains("limit is 10"), "error: {error}");
let error = recognizer
.recognize_encoded(&encoded, 100, 10_000)
.expect_err("pixel limit must fail");
assert!(error.to_string().contains("limit is 100"), "error: {error}");
}
#[test]
fn file_input_uses_shared_loader_limits() {
let mut recognizer =
FormulaRecognizer::from_model(&cred("formula_recognizer_ok.onnx"), &rt())
.expect("recognizer should load");
let error = recognizer
.recognize_file(Path::new("does-not-exist.png"), 10_000, 10_000)
.expect_err("missing file must be reported as FileNotFound");
assert!(
matches!(error, RapidOcrError::FileNotFound(_)),
"error: {error}"
);
let directory = std::env::temp_dir().join(format!(
"rapid-structure-rs-formula-loader-{}",
std::process::id()
));
std::fs::create_dir_all(&directory).expect("temp dir");
let path = directory.join("formula.png");
let image = gray(255).to_rgb8();
let mut bytes = Vec::new();
image
.write_to(
&mut std::io::Cursor::new(&mut bytes),
image::ImageFormat::Png,
)
.expect("encode png");
std::fs::write(&path, &bytes).expect("write png");
let result = recognizer
.recognize_file(&path, 2000, 10_000)
.expect("file recognize");
assert_eq!(result.latex, "\\cdot");
let error = recognizer
.recognize_file(&path, 2000, 10)
.expect_err("encoded file limit must fail");
assert!(
error.to_string().contains("encoded image has")
&& error.to_string().contains("limit is 10"),
"error: {error}"
);
let error = recognizer
.recognize_file(&path, 100, 10_000)
.expect_err("pixel file limit must fail");
assert!(error.to_string().contains("limit is 100"), "error: {error}");
let _ = std::fs::remove_dir_all(&directory);
}
#[test]
#[cfg(not(feature = "cuda-provider"))]
fn unavailable_provider_is_rejected_not_silently_fallen_back() {
let runtime = RuntimeConfig {
provider_preference: crate::config::ProviderPreference::Cuda { device_id: 0 },
fail_if_provider_unavailable: true,
..RuntimeConfig::default()
};
let error = FormulaRecognizer::from_model(&cred("formula_recognizer_ok.onnx"), &runtime)
.expect_err("unavailable accelerator must fail");
assert!(
matches!(error, RapidOcrError::UnsupportedProvider(_)),
"error: {error}"
);
}
#[test]
fn batch_results_share_call_wall_time() {
let mut recognizer =
FormulaRecognizer::from_model(&cred("formula_recognizer_ok.onnx"), &rt())
.expect("recognizer should load");
let images = vec![gray(0), gray(255), gray(128)];
let results = recognizer
.recognize_batch(&images)
.expect("batch recognize");
assert_eq!(results.len(), 3);
for result in &results {
assert_eq!(result.batch_size, 3);
assert!(
(result.elapsed_ms - results[0].elapsed_ms).abs() < f32::EPSILON,
"batch results must share the same call wall time"
);
assert!(result.elapsed_ms >= 0.0);
}
let single = recognizer.recognize(&gray(255)).expect("single recognize");
assert_eq!(single.batch_size, 1);
}
}