use std::sync::Arc;
use std::time::Instant;
use crate::{
api::OcrEngine as _,
error::Result,
input::image_loader::{LoadImage, OcrInput, ensure_decode_pixels},
ocr::cls::classifier::Classifier,
ocr::config::RecognizeOptions,
ocr::det::detector::Detector,
ocr::pipeline::{
config::EngineConfig,
image_ops::{
PreprocessRecord, apply_vertical_padding, crop_text_regions, map_boxes_to_original,
map_img_to_original, resize_image_within_bounds,
},
types::{ExecutionOptions, ExecutionOutput},
},
ocr::rec::recognizer::Recognizer,
ocr::types::{LineResult, WordBox},
runtime::{profile::RuntimeProfile, provider::ProviderResolution},
};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct PipelineProviderResolutions {
pub det: ProviderResolution,
pub cls: Option<ProviderResolution>,
pub rec: ProviderResolution,
}
#[derive(Debug, Clone, Copy)]
struct RunSwitches {
use_det: bool,
use_cls: bool,
apply_cls_rotation: bool,
use_rec: bool,
need_stage_images: bool,
return_word_box: bool,
return_single_char_box: bool,
text_score: f32,
}
#[derive(Debug)]
struct PreparedImage {
ori_h: usize,
ori_w: usize,
ratio_h: f32,
ratio_w: f32,
preprocess_record: PreprocessRecord,
proc_img: crate::config::RecImage,
decode_ms: f32,
resize_ms: f32,
}
#[derive(Debug, Default)]
struct RunBuffers {
det_boxes: Vec<crate::Quad>,
det_scores: Vec<f32>,
stage_images: Vec<crate::config::RecImage>,
lines: Vec<LineResult>,
}
#[derive(Debug)]
pub struct RapidOcr {
config: EngineConfig,
profile: RuntimeProfile,
detector: Detector,
classifier: Option<Classifier>,
recognizer: Recognizer,
loader: LoadImage,
mem_probe_enabled: bool,
stage_mem_records: Vec<crate::runtime::memory::StageMemRecord>,
rec_batch_records: Vec<crate::ocr::rec::recognizer::BatchRecord>,
rec_batch_probe_enabled: bool,
last_det_input_size: Option<(u32, u32)>,
}
impl RapidOcr {
pub fn new(config: EngineConfig) -> Result<Self> {
let profile = RuntimeProfile::resolve(&config.runtime, config.global.use_cls)?;
let session_runtime = profile.session_runtime();
let det = Detector::new(config.det.clone(), &session_runtime)?;
let cls = if config.global.use_cls {
Some(Classifier::new(config.cls.clone(), &session_runtime)?)
} else {
None
};
let rec = Recognizer::new(config.rec.clone(), &session_runtime)?;
Ok(Self {
config,
profile,
detector: det,
classifier: cls,
recognizer: rec,
loader: LoadImage::default(),
mem_probe_enabled: false,
stage_mem_records: Vec::new(),
rec_batch_records: Vec::new(),
rec_batch_probe_enabled: false,
last_det_input_size: None,
})
}
pub fn runtime_profile(&self) -> &RuntimeProfile {
&self.profile
}
pub fn enable_mem_probe(&mut self) {
self.mem_probe_enabled = true;
}
pub fn take_stage_mem_records(&mut self) -> Vec<crate::runtime::memory::StageMemRecord> {
std::mem::take(&mut self.stage_mem_records)
}
pub(crate) fn run(
&mut self,
input: OcrInput,
opts: ExecutionOptions,
) -> Result<ExecutionOutput> {
let e2e_start = Instant::now();
let mut output = ExecutionOutput::default();
self.rec_batch_records.clear();
self.last_det_input_size = None;
let switches = self.resolve_run_switches(&opts);
if self.mem_probe_enabled {
crate::runtime::memory::tl_probe_begin();
}
crate::runtime::memory::tl_probe("input");
let mut prepared = self.prepare_image(input)?;
output.decode_ms = Some(prepared.decode_ms);
output.resize_ms = Some(prepared.resize_ms);
output.processed_size = Some((
prepared.proc_img.width() as u32,
prepared.proc_img.height() as u32,
));
let mut buffers = RunBuffers::default();
crate::runtime::memory::tl_probe("input_end");
crate::runtime::memory::tl_probe("detect_pre");
if !self.run_detection_stage(&opts, switches, &mut prepared, &mut buffers, &mut output)? {
output.e2e_ms = Some(e2e_start.elapsed().as_secs_f32() * 1000.0);
crate::runtime::memory::tl_probe("output_end");
if self.mem_probe_enabled {
self.stage_mem_records
.extend(crate::runtime::memory::tl_probe_end());
}
self.last_det_input_size = output.det_input_size;
return Ok(output);
}
crate::runtime::memory::tl_probe("classifier");
self.run_classification_stage(switches, &mut buffers, &mut output)?;
crate::runtime::memory::tl_probe("rec_pre");
self.run_recognition_stage(switches, &mut buffers, &mut output)?;
crate::runtime::memory::tl_probe("page_post");
let postprocess_start = Instant::now();
self.finalize_detection_outputs(switches, &prepared, &mut buffers, &mut output)?;
self.finalize_recognition_outputs(switches, &buffers.lines, &mut output);
output.postprocess_ms = Some(postprocess_start.elapsed().as_secs_f32() * 1000.0);
output.e2e_ms = Some(e2e_start.elapsed().as_secs_f32() * 1000.0);
crate::runtime::memory::tl_probe("output_end");
if self.mem_probe_enabled {
self.stage_mem_records
.extend(crate::runtime::memory::tl_probe_end());
}
self.rec_batch_records = output.rec_batch_records.clone();
self.last_det_input_size = output.det_input_size;
Ok(output)
}
fn resolve_run_switches(&self, opts: &ExecutionOptions) -> RunSwitches {
let use_det = opts.use_det;
let use_cls = opts.use_cls;
let use_rec = opts.use_rec;
RunSwitches {
use_det,
use_cls,
apply_cls_rotation: opts.apply_cls_rotation,
use_rec,
need_stage_images: use_cls || use_rec,
return_word_box: opts.return_word_box,
return_single_char_box: opts.return_single_char_box,
text_score: opts.text_score.unwrap_or(self.config.global.text_score),
}
}
fn prepare_image(&mut self, input: OcrInput) -> Result<PreparedImage> {
let decode_start = Instant::now();
let ori_img = self.loader.load(input)?;
let decode_ms = decode_start.elapsed().as_secs_f32() * 1000.0;
let ori_h = ori_img.height();
let ori_w = ori_img.width();
let resize_start = Instant::now();
let (proc_img, ratio_h, ratio_w) = resize_image_within_bounds(
ori_img,
self.config.global.min_side_len,
self.config.global.max_side_len,
)?;
let resize_ms = resize_start.elapsed().as_secs_f32() * 1000.0;
Ok(PreparedImage {
ori_h,
ori_w,
ratio_h,
ratio_w,
preprocess_record: PreprocessRecord {
ratio_h,
ratio_w,
..PreprocessRecord::default()
},
proc_img,
decode_ms,
resize_ms,
})
}
fn run_detection_stage(
&mut self,
opts: &ExecutionOptions,
switches: RunSwitches,
prepared: &mut PreparedImage,
buffers: &mut RunBuffers,
output: &mut ExecutionOutput,
) -> Result<bool> {
if switches.use_det {
let (padded, pad_top) = apply_vertical_padding(
prepared.proc_img.clone(),
self.config.global.width_height_ratio,
self.config.global.min_height,
)?;
prepared.proc_img = padded;
prepared.preprocess_record.pad_top = pad_top;
self.detector
.update_postprocess(opts.box_thresh, opts.unclip_ratio);
let det_out = self.detector.detect(&prepared.proc_img)?;
output.det_input_size = Some(det_out.input_size);
if det_out.boxes.is_empty() {
return Ok(false);
}
output.elapsed_ms[0] = Some(det_out.elapsed_ms);
output.det_breakdown_ms = det_out.breakdown;
buffers.det_boxes = det_out.boxes;
buffers.det_scores = det_out.scores;
if switches.need_stage_images {
let crop_start = Instant::now();
buffers.stage_images = crop_text_regions(&prepared.proc_img, &buffers.det_boxes)?;
output.crop_ms = Some(crop_start.elapsed().as_secs_f32() * 1000.0);
}
} else if switches.need_stage_images {
buffers.stage_images.push(prepared.proc_img.clone());
}
Ok(true)
}
fn run_classification_stage(
&mut self,
switches: RunSwitches,
buffers: &mut RunBuffers,
output: &mut ExecutionOutput,
) -> Result<()> {
if switches.use_cls {
let classifier = self.classifier.as_mut().ok_or_else(|| {
crate::error::RapidOcrError::Config(
"classification requested but no classifier model is configured".to_string(),
)
})?;
let cls_result = classifier
.classify_in_place(&mut buffers.stage_images, switches.apply_cls_rotation)?;
output.cls_res = Some(cls_result.cls_res);
output.elapsed_ms[1] = Some(cls_result.elapsed_ms);
output.cls_breakdown_ms = Some([
cls_result.preprocess_ms,
cls_result.infer_ms,
cls_result.postprocess_ms,
]);
}
Ok(())
}
fn run_recognition_stage(
&mut self,
switches: RunSwitches,
buffers: &mut RunBuffers,
output: &mut ExecutionOutput,
) -> Result<()> {
if !switches.use_rec {
return Ok(());
}
if self.rec_batch_probe_enabled {
self.recognizer.batch_log = Some(Vec::new());
}
let rec = self.recognizer.recognize(
&buffers.stage_images,
RecognizeOptions {
return_word_box: switches.return_word_box,
return_single_char_box: switches.return_single_char_box,
},
)?;
output.rec_batch_records = self.recognizer.batch_log.take().unwrap_or_default();
output.elapsed_ms[2] = Some(rec.elapsed.as_secs_f32() * 1000.0);
output.rec_breakdown_ms = Some([rec.preprocess_ms, rec.infer_ms, rec.postprocess_ms]);
buffers.lines = rec.lines;
Ok(())
}
fn finalize_detection_outputs(
&mut self,
switches: RunSwitches,
prepared: &PreparedImage,
buffers: &mut RunBuffers,
output: &mut ExecutionOutput,
) -> Result<()> {
if !switches.use_det {
return Ok(());
}
let mut mapped_boxes = std::mem::take(&mut buffers.det_boxes);
map_boxes_to_original(
&mut mapped_boxes,
prepared.preprocess_record,
prepared.ori_h,
prepared.ori_w,
);
if !switches.use_rec {
output.boxes = Some(mapped_boxes);
output.det_scores = Some(std::mem::take(&mut buffers.det_scores));
return Ok(());
}
let lines = std::mem::take(&mut buffers.lines);
let det_scores = std::mem::take(&mut buffers.det_scores);
let (filtered_boxes, filtered_scores, filtered_lines, kept_indices) =
filter_empty_lines_boxes_and_scores(mapped_boxes, det_scores, lines);
buffers.lines = filtered_lines;
let mut computed_word_boxes = None;
if switches.return_word_box && !filtered_boxes.is_empty() && !buffers.lines.is_empty() {
let mapped_crops =
map_img_to_original(&buffers.stage_images, prepared.ratio_h, prepared.ratio_w)?;
let filtered_crops = select_items_by_indices(mapped_crops, &kept_indices);
let word_boxes = crate::ocr::rec::word_boxes::compute_word_boxes(
&filtered_crops,
&filtered_boxes,
&buffers.lines,
switches.return_single_char_box,
)?;
computed_word_boxes = Some(word_boxes);
}
let (
score_filtered_boxes,
score_filtered_scores,
score_filtered_lines,
score_filtered_words,
) = filter_by_text_score_for_full(
filtered_boxes,
filtered_scores,
std::mem::take(&mut buffers.lines),
computed_word_boxes,
switches.text_score,
);
buffers.lines = score_filtered_lines;
output.boxes = Some(score_filtered_boxes);
output.det_scores = Some(score_filtered_scores);
output.word_boxes = score_filtered_words;
Ok(())
}
fn finalize_recognition_outputs(
&self,
switches: RunSwitches,
lines: &[LineResult],
output: &mut ExecutionOutput,
) {
if !switches.use_rec || lines.is_empty() {
return;
}
output.txts = Some(lines.iter().map(|v| v.text.clone()).collect());
output.scores = Some(lines.iter().map(|v| v.score).collect());
output.lines = Some(lines.to_vec());
}
pub fn provider_resolutions(&self) -> PipelineProviderResolutions {
PipelineProviderResolutions {
det: self.detector.provider_resolution(),
cls: self
.classifier
.as_ref()
.map(Classifier::provider_resolution),
rec: self.recognizer.provider_resolution(),
}
}
pub fn model_paths(
&self,
) -> (
Option<&std::path::Path>,
Option<&std::path::Path>,
Option<&std::path::Path>,
) {
(
Some(self.detector.model_path()),
self.classifier.as_ref().map(Classifier::model_path),
Some(self.recognizer.model_path()),
)
}
pub fn take_rec_batch_records(&mut self) -> Vec<crate::ocr::rec::recognizer::BatchRecord> {
std::mem::take(&mut self.rec_batch_records)
}
pub fn enable_rec_batch_probe(&mut self) {
self.rec_batch_probe_enabled = true;
}
pub fn last_detector_input_size(&self) -> Option<(u32, u32)> {
self.last_det_input_size
}
pub fn set_max_side_len(&mut self, max_side_len: u32) {
self.config.global.max_side_len = max_side_len.max(32) as usize;
}
pub fn set_image_bounds(&mut self, min_side_len: u32, max_side_len: u32) {
self.config.global.min_side_len = min_side_len.max(1) as usize;
self.config.global.max_side_len = max_side_len.max(32) as usize;
}
}
type FullFilterOutput = (
Vec<crate::Quad>,
Vec<f32>,
Vec<LineResult>,
Option<Vec<Vec<WordBox>>>,
);
fn filter_empty_lines_boxes_and_scores(
boxes: Vec<crate::Quad>,
det_scores: Vec<f32>,
lines: Vec<LineResult>,
) -> (Vec<crate::Quad>, Vec<f32>, Vec<LineResult>, Vec<usize>) {
if boxes.len() != lines.len() || det_scores.len() != lines.len() {
return (boxes, det_scores, lines, Vec::new());
}
let mut out_boxes = Vec::new();
let mut out_scores = Vec::new();
let mut out_lines = Vec::new();
let mut kept_indices = Vec::new();
for (idx, ((b, s), l)) in boxes.into_iter().zip(det_scores).zip(lines).enumerate() {
if l.text.trim().is_empty() {
continue;
}
out_boxes.push(b);
out_scores.push(s);
out_lines.push(l);
kept_indices.push(idx);
}
(out_boxes, out_scores, out_lines, kept_indices)
}
fn select_items_by_indices<T>(items: Vec<T>, indices: &[usize]) -> Vec<T> {
if indices.is_empty() {
return Vec::new();
}
let mut out = Vec::with_capacity(indices.len());
let mut keep_iter = indices.iter().copied().peekable();
for (idx, item) in items.into_iter().enumerate() {
let Some(target_idx) = keep_iter.peek().copied() else {
break;
};
if idx == target_idx {
out.push(item);
keep_iter.next();
}
}
out
}
fn filter_by_text_score_for_full(
boxes: Vec<crate::Quad>,
det_scores: Vec<f32>,
lines: Vec<LineResult>,
word_boxes: Option<Vec<Vec<WordBox>>>,
text_score: f32,
) -> FullFilterOutput {
let mut out_boxes = Vec::new();
let mut out_scores = Vec::new();
let mut out_lines = Vec::new();
let mut out_word_boxes = Vec::new();
for (idx, line) in lines.into_iter().enumerate() {
if idx >= boxes.len() || idx >= det_scores.len() {
break;
}
if line.score < text_score {
continue;
}
out_boxes.push(boxes[idx]);
out_scores.push(det_scores[idx]);
if let Some(word_line) = word_boxes.as_ref().and_then(|v| v.get(idx)) {
out_word_boxes.push(word_line.clone());
}
out_lines.push(line);
}
let out_word_boxes = if word_boxes.is_some() {
Some(out_word_boxes)
} else {
None
};
(out_boxes, out_scores, out_lines, out_word_boxes)
}
fn rec_image_input(
image: crate::config::RecImage,
roi: Option<crate::api::RectU32>,
max_decode_pixels: u64,
) -> Result<(OcrInput, crate::api::ImageSize, (u32, u32))> {
let original_size = crate::api::ImageSize {
width: image.width() as u32,
height: image.height() as u32,
};
ensure_decode_pixels(
original_size.width as usize,
original_size.height as usize,
max_decode_pixels,
)?;
let Some(roi) = roi else {
return Ok((OcrInput::Image(image), original_size, (0, 0)));
};
roi.validate_against(original_size.width, original_size.height)?;
let src = image.as_bgr_bytes();
let row_bytes = roi.width as usize * 3;
let mut out = vec![0_u8; row_bytes * roi.height as usize];
for y in 0..roi.height as usize {
let src_start = ((roi.y as usize + y) * image.width() + roi.x as usize) * 3;
let dst_start = y * row_bytes;
out[dst_start..dst_start + row_bytes]
.copy_from_slice(&src[src_start..src_start + row_bytes]);
}
let cropped =
crate::config::RecImage::from_bgr_u8(roi.width as usize, roi.height as usize, out)?;
Ok((OcrInput::Image(cropped), original_size, (roi.x, roi.y)))
}
struct MaterializedInput {
rec: crate::config::RecImage,
original_size: crate::api::ImageSize,
}
fn rec_image_to_rgba(image: &crate::config::RecImage) -> Result<image::RgbaImage> {
let bgr = image.as_bgr_cow();
let mut rgba = vec![0_u8; image.width() * image.height() * 4];
for (src, dst) in bgr
.as_chunks::<3>()
.0
.iter()
.zip(rgba.as_chunks_mut::<4>().0.iter_mut())
{
dst[0] = src[2];
dst[1] = src[1];
dst[2] = src[0];
dst[3] = 255;
}
image::RgbaImage::from_raw(image.width() as u32, image.height() as u32, rgba)
.ok_or_else(|| crate::error::RapidOcrError::InvalidImage("invalid image buffer".into()))
}
#[derive(Debug)]
pub struct RapidOcrEngine {
inner: RapidOcr,
model_id: String,
base_min_side_len: usize,
base_max_side_len: usize,
formula_recognizer: Option<(
SessionCacheKey,
crate::formula::recognizer::FormulaRecognizer,
)>,
formula_detector: Option<(SessionCacheKey, crate::formula::detect::FormulaDetector)>,
table_session: Option<(
SessionCacheKey,
crate::table::structure::TableStructureSession,
)>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct SessionCacheKey {
identity: crate::model_verify::FileIdentity,
declared: Option<String>,
config: SessionConfigKey,
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum SessionConfigKey {
EngineOnly,
Table(crate::table::structure::TableStructureConfig),
}
impl SessionCacheKey {
fn engine_only(identity: crate::model_verify::FileIdentity, declared: Option<&str>) -> Self {
Self {
identity,
declared: declared.map(crate::model_verify::normalize_expected_sha256),
config: SessionConfigKey::EngineOnly,
}
}
fn table(
identity: crate::model_verify::FileIdentity,
declared: Option<&str>,
config: &crate::table::structure::TableStructureConfig,
) -> Self {
Self {
identity,
declared: declared.map(crate::model_verify::normalize_expected_sha256),
config: SessionConfigKey::Table(config.clone()),
}
}
}
fn needs_rebuild(cached: Option<&SessionCacheKey>, current: &SessionCacheKey) -> bool {
cached != Some(current)
}
impl RapidOcrEngine {
pub fn new(config: EngineConfig) -> Result<Self> {
let model_id = format!(
"{}-{}-{}",
config.rec.model.ocr_version.as_str(),
config.rec.model.model_type.as_str(),
config.rec.model.lang.as_str()
);
let base_min_side_len = config.global.min_side_len;
let base_max_side_len = config.global.max_side_len;
Ok(Self {
inner: RapidOcr::new(config)?,
model_id,
base_min_side_len,
base_max_side_len,
formula_recognizer: None,
formula_detector: None,
table_session: None,
})
}
pub fn provider_resolutions(&self) -> PipelineProviderResolutions {
self.inner.provider_resolutions()
}
pub fn runtime_profile(&self) -> &RuntimeProfile {
self.inner.runtime_profile()
}
fn provider_info(&self) -> crate::api::ProviderInfo {
let resolutions = self.inner.provider_resolutions();
let convert = |resolution: ProviderResolution| crate::api::ProviderResolutionInfo {
requested: match resolution.requested {
crate::config::ProviderPreference::Cpu => crate::api::ProviderPreference::Cpu,
crate::config::ProviderPreference::DirectMl { device_id } => {
crate::api::ProviderPreference::DirectMl { device_id }
}
crate::config::ProviderPreference::Cuda { device_id } => {
crate::api::ProviderPreference::Cuda { device_id }
}
},
selected_ep: match resolution.selected_ep {
crate::runtime::provider::ResolvedExecutionProvider::Cpu => {
crate::api::ResolvedProvider::Cpu
}
crate::runtime::provider::ResolvedExecutionProvider::DirectMl => {
crate::api::ResolvedProvider::DirectMl
}
crate::runtime::provider::ResolvedExecutionProvider::Cuda => {
crate::api::ResolvedProvider::Cuda
}
},
fallback_to_cpu: resolution.fallback_used,
};
crate::api::ProviderInfo {
detector: convert(resolutions.det),
classifier: resolutions.cls.map(convert),
recognizer: convert(resolutions.rec),
}
}
}
impl crate::api::OcrEngine for RapidOcrEngine {
fn model_id(&self) -> &str {
&self.model_id
}
fn provider(&self) -> crate::api::ProviderInfo {
self.provider_info()
}
fn recognize(&mut self, request: crate::api::OcrRequest) -> Result<crate::api::OcrOutput> {
let planned = crate::api::ValidatedRequest::plan(request)?;
self.recognize_page(planned)
}
}
impl RapidOcrEngine {
pub fn enable_mem_probe(&mut self) {
self.inner.enable_mem_probe();
}
pub fn take_stage_mem_records(&mut self) -> Vec<crate::runtime::memory::StageMemRecord> {
self.inner.take_stage_mem_records()
}
pub fn model_paths(
&self,
) -> (
Option<&std::path::Path>,
Option<&std::path::Path>,
Option<&std::path::Path>,
) {
self.inner.model_paths()
}
pub fn take_rec_batch_records(&mut self) -> Vec<crate::ocr::rec::recognizer::BatchRecord> {
self.inner.take_rec_batch_records()
}
pub fn enable_rec_batch_probe(&mut self) {
self.inner.enable_rec_batch_probe();
}
pub fn last_detector_input_size(&self) -> Option<(u32, u32)> {
self.inner.last_detector_input_size()
}
}
impl RapidOcrEngine {
fn recognize_page(
&mut self,
planned: crate::api::ValidatedRequest,
) -> Result<crate::api::OcrOutput> {
let request = planned.request();
let mut output = match planned.route().formula() {
Some(policy) => self.recognize_text_with_formula(request, policy)?,
None => self.recognize_text(request.clone())?,
};
if let Some(policy) = planned.route().table() {
self.attach_table_regions(&mut output, request, policy)?;
}
output.validate()?;
Ok(output)
}
}
impl RapidOcrEngine {
fn recognize_text(&mut self, request: crate::api::OcrRequest) -> Result<crate::api::OcrOutput> {
use crate::api::{
ClassifierPolicy, OcrRegion, RegionSource, StageReport, StageState, StageTiming,
TextOrientation,
};
if let Some(tile) = request.preprocess.tile {
return self.recognize_tiled_request(request, tile);
}
let preprocess_start = Instant::now();
let roi_size = request.roi.map(|roi| crate::api::ImageSize {
width: roi.width,
height: roi.height,
});
let m = self.materialize(&request)?;
let (input, original_size, offset) =
rec_image_input(m.rec, request.roi, request.preprocess.max_decode_pixels)?;
let mut input = input;
if matches!(
request.preprocess.enhance,
crate::api::EnhancementPolicy::ScreenAdaptive
) {
let img = self.inner.loader.load(input)?;
input = OcrInput::Image(crate::ocr::pipeline::image_ops::enhance_screen_adaptive(
&img,
)?);
}
let max_side = request
.preprocess
.max_side
.unwrap_or(self.base_max_side_len as u32);
self.inner
.set_image_bounds(self.base_min_side_len as u32, max_side);
let effective_scale = match (request.scale_hint, request.preprocess.min_text_scale) {
(Some(a), Some(b)) => Some(a.max(b)),
(Some(a), None) | (None, Some(a)) => Some(a),
(None, None) => None,
};
if let Some(scale) = effective_scale {
let hinted = ((max_side as f32) * scale).round().clamp(32.0, 8192.0) as u32;
let min_hint = ((self.base_min_side_len as f32) * scale)
.round()
.clamp(1.0, hinted as f32) as u32;
self.inner.set_image_bounds(min_hint, hinted);
}
let mut use_cls = match request.stages.classify.policy {
ClassifierPolicy::Off => false,
ClassifierPolicy::IfAvailable => self.inner.classifier.is_some(),
ClassifierPolicy::Required => true,
};
if use_cls && self.inner.classifier.is_none() {
if matches!(request.stages.classify.policy, ClassifierPolicy::Required) {
return Err(crate::error::RapidOcrError::Config(
"classification requested but no classifier model is configured".into(),
));
}
use_cls = false;
}
if !request.stages.detect && !request.stages.recognize && !use_cls {
return Err(crate::error::RapidOcrError::InvalidInput(
"requested stages are unavailable".into(),
));
}
let preprocess_ms = preprocess_start.elapsed().as_secs_f32() * 1000.0;
let exec = self.inner.run(
input,
ExecutionOptions {
use_det: request.stages.detect,
use_cls,
use_rec: request.stages.recognize,
apply_cls_rotation: request.stages.classify.apply_rotation,
return_word_box: !matches!(
request.recognition.words,
crate::api::WordOutputMode::Off
),
return_single_char_box: matches!(
request.recognition.words,
crate::api::WordOutputMode::Chars
),
text_score: request.detection.text_score,
box_thresh: request.detection.box_thresh,
unclip_ratio: request.detection.unclip_ratio,
},
)?;
let local_source_size = roi_size.unwrap_or(original_size);
let boxes = exec.boxes.clone().unwrap_or_default();
let det_scores = exec.det_scores.clone().unwrap_or_default();
let lines = exec.lines.clone().unwrap_or_default();
let words = exec.word_boxes.clone().unwrap_or_default();
let cls = exec.cls_res.clone().unwrap_or_default();
let cls_count = cls.len();
let mut regions = Vec::new();
if request.stages.detect {
for (i, det_box) in boxes.iter().enumerate() {
let mut polygon = *det_box;
for p in &mut polygon {
p[0] += offset.0 as f32;
p[1] += offset.1 as f32;
}
let recognition = lines.get(i).map(|line| crate::api::RecognitionOutcome {
text: line.text.clone(),
score: line.score,
words: words.get(i).map(|items| {
items
.iter()
.map(|w| crate::api::OcrWord {
text: w.text.clone(),
score: w.score,
polygon: crate::api::Polygon {
points: w.bbox.map(|mut p| {
p[0] += offset.0 as f32;
p[1] += offset.1 as f32;
p
}),
},
kind: request.recognition.words.kind_for(),
})
.collect()
}),
});
let classification =
cls.get(i)
.map(|(orientation, score)| crate::api::ClassificationOutcome {
orientation: if orientation == "180" {
TextOrientation::Deg180
} else {
TextOrientation::Deg0
},
score: *score,
applied_rotation: request.stages.classify.apply_rotation
&& orientation == "180",
});
regions.push(OcrRegion {
source: RegionSource::Detected { detector_index: i },
kind: crate::api::RegionKind::Text,
polygon: Some(crate::api::Polygon { points: polygon }),
detection: Some(crate::api::DetectionOutcome {
score: det_scores.get(i).copied().unwrap_or_default(),
}),
classification,
recognition,
formula: None,
table: None,
});
}
} else if request.stages.recognize {
for (i, line) in lines.iter().enumerate() {
let classification = cls.get(i).map(|(o, s)| crate::api::ClassificationOutcome {
orientation: if o == "180" {
TextOrientation::Deg180
} else {
TextOrientation::Deg0
},
score: *s,
applied_rotation: request.stages.classify.apply_rotation && o == "180",
});
regions.push(OcrRegion {
source: RegionSource::Input,
kind: crate::api::RegionKind::Text,
polygon: None,
detection: None,
classification,
recognition: Some(crate::api::RecognitionOutcome {
text: line.text.clone(),
score: line.score,
words: words.get(i).map(|items| {
items
.iter()
.map(|w| crate::api::OcrWord {
text: w.text.clone(),
score: w.score,
polygon: crate::api::Polygon {
points: w.bbox.map(|mut p| {
let sx = exec
.processed_size
.map(|(w, _)| {
local_source_size.width as f32 / w.max(1) as f32
})
.unwrap_or(1.0);
let sy = exec
.processed_size
.map(|(_, h)| {
local_source_size.height as f32
/ h.max(1) as f32
})
.unwrap_or(1.0);
p[0] = p[0] * sx + offset.0 as f32;
p[1] = p[1] * sy + offset.1 as f32;
p
}),
},
kind: request.recognition.words.kind_for(),
})
.collect()
}),
}),
formula: None,
table: None,
});
}
} else if use_cls {
for (o, s) in &cls {
regions.push(OcrRegion {
source: RegionSource::Input,
kind: crate::api::RegionKind::Text,
polygon: None,
detection: None,
classification: Some(crate::api::ClassificationOutcome {
orientation: if o == "180" {
TextOrientation::Deg180
} else {
TextOrientation::Deg0
},
score: *s,
applied_rotation: request.stages.classify.apply_rotation && o == "180",
}),
recognition: None,
formula: None,
table: None,
});
}
}
let timing = crate::api::OcrTimings {
decode_ms: exec.decode_ms.unwrap_or_default(),
resize_ms: exec.resize_ms.unwrap_or_default(),
crop_ms: exec.crop_ms.unwrap_or_default(),
preprocess_ms,
detector_preprocess_ms: exec
.det_breakdown_ms
.map(|v| v.preprocess_ms)
.unwrap_or_default(),
detector_infer_ms: exec
.det_breakdown_ms
.map(|v| v.infer_ms)
.unwrap_or_default(),
detector_postprocess_ms: exec
.det_breakdown_ms
.map(|v| v.postprocess_ms)
.unwrap_or_default(),
detect_ms: exec.elapsed_ms[0].unwrap_or_default(),
classifier_preprocess_ms: exec.cls_breakdown_ms.map(|v| v[0]).unwrap_or_default(),
classifier_infer_ms: exec.cls_breakdown_ms.map(|v| v[1]).unwrap_or_default(),
classifier_postprocess_ms: exec.cls_breakdown_ms.map(|v| v[2]).unwrap_or_default(),
classify_ms: exec.elapsed_ms[1].unwrap_or_default(),
recognizer_preprocess_ms: exec.rec_breakdown_ms.map(|v| v[0]).unwrap_or_default(),
recognizer_infer_ms: exec.rec_breakdown_ms.map(|v| v[1]).unwrap_or_default(),
recognizer_postprocess_ms: exec.rec_breakdown_ms.map(|v| v[2]).unwrap_or_default(),
recognize_ms: exec.elapsed_ms[2].unwrap_or_default(),
formula_ms: 0.0,
table_detect_ms: 0.0,
table_structure_infer_ms: 0.0,
table_backfill_ms: 0.0,
postprocess_ms: exec.postprocess_ms.unwrap_or_default(),
total_ms: exec.e2e_ms.unwrap_or_default() + preprocess_ms,
};
let detector = if !request.stages.detect {
StageReport {
state: StageState::Disabled,
timing: None,
}
} else if boxes.is_empty() {
StageReport {
state: StageState::SkippedNoInput,
timing: Some(StageTiming {
preprocess_ms: timing.detector_preprocess_ms,
infer_ms: timing.detector_infer_ms,
postprocess_ms: timing.detector_postprocess_ms,
}),
}
} else {
StageReport {
state: StageState::Completed { items: boxes.len() },
timing: Some(StageTiming {
preprocess_ms: timing.detector_preprocess_ms,
infer_ms: timing.detector_infer_ms,
postprocess_ms: timing.detector_postprocess_ms,
}),
}
};
let classifier = if matches!(request.stages.classify.policy, ClassifierPolicy::Off) {
StageReport {
state: StageState::Disabled,
timing: None,
}
} else if self.inner.classifier.is_none() || !use_cls {
StageReport {
state: StageState::SkippedUnavailable,
timing: None,
}
} else {
StageReport {
state: StageState::Completed { items: cls_count },
timing: exec.cls_breakdown_ms.map(|v| StageTiming {
preprocess_ms: v[0],
infer_ms: v[1],
postprocess_ms: v[2],
}),
}
};
let recognizer = if !request.stages.recognize {
StageReport {
state: StageState::Disabled,
timing: None,
}
} else if lines.is_empty() {
StageReport {
state: StageState::SkippedNoInput,
timing: exec.rec_breakdown_ms.map(|v| StageTiming {
preprocess_ms: v[0],
infer_ms: v[1],
postprocess_ms: v[2],
}),
}
} else {
StageReport {
state: StageState::Completed { items: lines.len() },
timing: exec.rec_breakdown_ms.map(|v| StageTiming {
preprocess_ms: v[0],
infer_ms: v[1],
postprocess_ms: v[2],
}),
}
};
Ok(crate::api::OcrOutput {
schema_version: 1,
image: crate::api::ImageInfo {
original_size,
processed_size: exec
.processed_size
.map(|(width, height)| crate::api::ImageSize { width, height })
.unwrap_or(original_size),
coordinate_space: request.output.coordinate_space,
},
stages: crate::api::StageReports {
input: crate::api::InputTimings {
decode_ms: exec.decode_ms,
resize_ms: exec.resize_ms,
crop_ms: exec.crop_ms,
},
detector,
classifier,
recognizer,
formula: crate::api::StageReport::default(),
table: crate::api::StageReport::default(),
},
regions,
timings: timing,
engine: crate::api::EngineInfo {
model_id: self.model_id.clone(),
provider: self.provider_info(),
},
})
}
}
impl RapidOcrEngine {
fn recognize_text_with_formula(
&mut self,
request: &crate::api::OcrRequest,
policy: &crate::api::FormulaPolicy,
) -> Result<crate::api::OcrOutput> {
use crate::api::{FormulaOutcome, ImageSize, OcrRegion, Polygon, RegionKind, RegionSource};
let formula_start = Instant::now();
let original = self.decode_source_image(request)?;
let size = ImageSize {
width: original.width(),
height: original.height(),
};
let mut candidates = self.detect_formula_candidates(&original, policy)?;
for polygon in &policy.input_regions {
polygon.validate(size)?;
candidates.push(crate::formula::route::FormulaCandidate {
polygon: *polygon,
score: 1.0,
detected: false,
});
}
let regions = crate::formula::route::resolve_formula_regions(
&candidates,
size,
policy.min_area_ratio,
policy.iou_threshold,
policy.max_regions,
);
let formula_ms = formula_start.elapsed().as_secs_f32() * 1000.0;
let mut evidence_output = None;
let regions = if regions.iter().any(|candidate| candidate.detected) {
let original_output = self.recognize_text(request.clone())?;
let evidence: Vec<_> = original_output
.regions
.iter()
.filter_map(|region| {
let polygon = region.polygon?;
let text = region.recognition.as_ref()?.text.as_str();
Some(crate::formula::route::TextEvidence { polygon, text })
})
.collect();
let arbitrated = crate::formula::route::arbitrate_formula_regions(regions, &evidence);
evidence_output = Some(original_output);
arbitrated
} else {
regions
};
let mut text_request = request.clone();
if !regions.is_empty() {
text_request.input = crate::api::ImageInput::Image(
crate::formula::route::whiten_regions(&original, ®ions)?,
);
text_request.roi = None;
}
let mut output = match (regions.is_empty(), evidence_output) {
(true, Some(original_output)) => original_output,
(_, _) => self.recognize_text(text_request)?,
};
if regions.is_empty() {
output.stages.formula = crate::api::StageReport {
state: crate::api::StageState::SkippedNoInput,
timing: Some(crate::api::StageTiming {
preprocess_ms: formula_ms,
infer_ms: 0.0,
postprocess_ms: 0.0,
}),
};
return Ok(output);
}
let crops: Vec<image::DynamicImage> = regions
.iter()
.map(|candidate| crop_polygon_bounds(&original, candidate.polygon))
.collect();
let recognize_start = Instant::now();
let recognitions = self.formula_recognizer(policy)?.recognize_batch(&crops)?;
let recognize_ms = recognize_start.elapsed().as_secs_f32() * 1000.0;
output.timings.formula_ms = recognize_ms + formula_ms;
output.timings.total_ms += recognize_ms + formula_ms;
for (index, (candidate, recognition)) in regions.iter().zip(recognitions).enumerate() {
let detected = candidate.detected;
let source = if detected {
RegionSource::Detected {
detector_index: index,
}
} else {
RegionSource::Input
};
let polygon = Some(Polygon {
points: canonical_polygon_points(candidate.polygon),
});
output.regions.push(OcrRegion {
source,
kind: RegionKind::Formula,
polygon,
detection: detected.then_some(crate::api::DetectionOutcome {
score: candidate.score,
}),
classification: None,
recognition: None,
formula: Some(FormulaOutcome {
latex: recognition.latex,
eos_index: recognition.eos_index,
truncated: recognition.truncated,
model_id: recognition.model_id,
token_ids: policy.include_token_ids.then_some(recognition.token_ids),
}),
table: None,
});
}
output.stages.formula = crate::api::StageReport {
state: crate::api::StageState::Completed {
items: output.formula_count(),
},
timing: Some(crate::api::StageTiming {
preprocess_ms: formula_ms,
infer_ms: recognize_ms,
postprocess_ms: 0.0,
}),
};
output.validate()?;
Ok(output)
}
fn materialize(&self, request: &crate::api::OcrRequest) -> Result<MaterializedInput> {
use crate::api::ImageInput;
let pixels = request.preprocess.max_decode_pixels;
let encoded = request.preprocess.max_encoded_bytes;
let rec = match &request.input {
ImageInput::Pixels(view) => {
view.validate()?;
ensure_decode_pixels(view.width as usize, view.height as usize, pixels)?;
let (bgr, size, _) = view.to_bgr(None)?;
crate::config::RecImage::from_bgr_u8(
size.width as usize,
size.height as usize,
bgr,
)?
}
ImageInput::Encoded(bytes) => self.inner.loader.load_with_limit(
OcrInput::Bytes(bytes.to_vec()),
pixels,
encoded,
)?,
ImageInput::File(path) => {
self.inner
.loader
.load_with_limit(OcrInput::Path(path.clone()), pixels, encoded)?
}
ImageInput::Url(url) => {
self.inner
.loader
.load_with_limit(OcrInput::Url(url.clone()), pixels, encoded)?
}
ImageInput::Image(image) => {
ensure_decode_pixels(image.width(), image.height(), pixels)?;
image.clone()
}
};
let original_size = crate::api::ImageSize {
width: rec.width() as u32,
height: rec.height() as u32,
};
Ok(MaterializedInput { rec, original_size })
}
fn decode_source_image(&self, request: &crate::api::OcrRequest) -> Result<image::DynamicImage> {
use crate::api::ImageInput;
let pixels = request.preprocess.max_decode_pixels;
let encoded = request.preprocess.max_encoded_bytes;
let image = match &request.input {
ImageInput::Encoded(bytes) => self.inner.loader.load_dynamic_with_limit(
OcrInput::Bytes(bytes.to_vec()),
pixels,
encoded,
)?,
ImageInput::File(path) => self.inner.loader.load_dynamic_with_limit(
OcrInput::Path(path.clone()),
pixels,
encoded,
)?,
ImageInput::Url(url) => self.inner.loader.load_dynamic_with_limit(
OcrInput::Url(url.clone()),
pixels,
encoded,
)?,
ImageInput::Pixels(view) => {
view.validate()?;
ensure_decode_pixels(
view.width as usize,
view.height as usize,
request.preprocess.max_decode_pixels,
)?;
let (bgr, size, _) = view.to_bgr(None)?;
let rec = crate::config::RecImage::from_bgr_u8(
size.width as usize,
size.height as usize,
bgr,
)?;
rec_image_to_dynamic(&rec)?
}
ImageInput::Image(rec) => {
ensure_decode_pixels(
rec.width(),
rec.height(),
request.preprocess.max_decode_pixels,
)?;
rec_image_to_dynamic(rec)?
}
};
Ok(image)
}
fn detect_formula_candidates(
&mut self,
image: &image::DynamicImage,
policy: &crate::api::FormulaPolicy,
) -> Result<Vec<crate::formula::route::FormulaCandidate>> {
let Some(path) = policy.detector_path.clone() else {
return Ok(Vec::new());
};
let options = crate::formula::detect::FormulaDetectOptions {
confidence_threshold: policy.confidence_threshold,
iou_threshold: policy.iou_threshold,
max_detections: policy.max_regions,
};
let runtime = self.inner.runtime_profile().session_runtime();
let key = SessionCacheKey::engine_only(
crate::model_verify::FileIdentity::of(&path)?,
policy.expected_detector_sha256.as_deref(),
);
if needs_rebuild(
self.formula_detector.as_ref().map(|(cached, _)| cached),
&key,
) {
let detector_model = crate::model_verify::VerifiedModel::verify(
&path,
policy.expected_detector_sha256.as_deref(),
)?;
self.formula_detector = Some((
key,
crate::formula::detect::FormulaDetector::from_model(&detector_model, &runtime)?,
));
}
let detector = self
.formula_detector
.as_mut()
.map(|(_, detector)| detector)
.ok_or_else(|| {
crate::error::RapidOcrError::Config("formula detector is not initialized".into())
})?;
Ok(detector
.detect(image, &options)?
.into_iter()
.map(|detected| crate::formula::route::FormulaCandidate {
polygon: crate::api::Polygon {
points: detected.polygon,
},
score: detected.score,
detected: true,
})
.collect())
}
fn formula_recognizer(
&mut self,
policy: &crate::api::FormulaPolicy,
) -> Result<&mut crate::formula::recognizer::FormulaRecognizer> {
let path = policy
.model_path
.clone()
.ok_or_else(|| crate::model_verify::missing_model_path_error("formula"))?;
let runtime = self.inner.runtime_profile().session_runtime();
let key = SessionCacheKey::engine_only(
crate::model_verify::FileIdentity::of(&path)?,
policy.expected_model_sha256.as_deref(),
);
if needs_rebuild(
self.formula_recognizer.as_ref().map(|(cached, _)| cached),
&key,
) {
let recognizer_model = crate::model_verify::VerifiedModel::verify(
&path,
policy.expected_model_sha256.as_deref(),
)?;
let mut recognizer = crate::formula::recognizer::FormulaRecognizer::from_model(
&recognizer_model,
&runtime,
)?;
recognizer.set_max_batch_size(runtime.formula_batch)?;
self.formula_recognizer = Some((key, recognizer));
}
self.formula_recognizer
.as_mut()
.map(|(_, recognizer)| recognizer)
.ok_or_else(|| {
crate::error::RapidOcrError::Config("formula recognizer is not initialized".into())
})
}
fn table_session(
&mut self,
policy: &crate::api::TablePolicy,
) -> Result<&mut crate::table::structure::TableStructureSession> {
let path = policy
.model_path
.clone()
.ok_or_else(|| crate::model_verify::missing_model_path_error("table"))?;
let expected = crate::table::registration::resolve_registered_expected(
&path,
policy.expected_model_sha256.as_deref(),
)?;
let config = crate::table::structure::TableStructureConfig {
input_side_len: policy.input_side_len,
fix_input_shape: policy.fix_input_shape,
intra_threads: policy.intra_threads,
};
let key = SessionCacheKey::table(
crate::model_verify::FileIdentity::of(&path)?,
expected.as_deref(),
&config,
);
if needs_rebuild(self.table_session.as_ref().map(|(cached, _)| cached), &key) {
let runtime = self.inner.runtime_profile().session_runtime();
let credential =
crate::table::registration::verified_registered_model(&path, expected.as_deref())?;
let session = crate::table::structure::TableStructureSession::open(
&credential,
&runtime,
&config,
)?;
self.table_session = Some((key, session));
}
self.table_session
.as_mut()
.map(|(_, session)| session)
.ok_or_else(|| {
crate::error::RapidOcrError::Config("table session is not initialized".into())
})
}
fn attach_table_regions(
&mut self,
output: &mut crate::api::OcrOutput,
request: &crate::api::OcrRequest,
policy: &crate::api::TablePolicy,
) -> Result<()> {
use crate::api::{
OcrRegion, Polygon, RegionKind, RegionSource, StageReport, StageState, StageTiming,
TableDegradation, TableOutcome, TableStructureConfidence,
};
use crate::table::detection::TableDetector;
let original = self.decode_source_image(request)?;
let source = table_source_lines(&output.regions);
let image = rec_image_from_dynamic(&original)?;
let detect_start = Instant::now();
let mut detector = crate::table::detection::TextDistributionDetector::new(
crate::table::detection::TableDetectorConfig::default(),
)?;
let mut regions = detector.detect(&image, &source.lines)?;
regions.retain(|region| region.score >= policy.min_region_score);
regions.sort_by(|a, b| {
b.score
.partial_cmp(&a.score)
.unwrap_or(std::cmp::Ordering::Equal)
});
regions.truncate(policy.max_regions);
let detect_ms = detect_start.elapsed().as_secs_f32() * 1000.0;
if regions.is_empty() {
output.timings.table_detect_ms = detect_ms;
output.timings.total_ms += detect_ms;
output.stages.table = StageReport {
state: StageState::SkippedNoInput,
timing: Some(StageTiming {
preprocess_ms: detect_ms,
infer_ms: 0.0,
postprocess_ms: 0.0,
}),
};
return Ok(());
}
let session = self.table_session(policy)?;
let vocabulary = session.vocabulary().clone();
let mut infer_ms = 0.0_f32;
let mut backfill_ms = 0.0_f32;
let mut table_regions: Vec<OcrRegion> = Vec::new();
for (index, region) in regions.iter().enumerate() {
let (tensor, _canvas) = crate::table::preprocess::table_structure_input(
®ion.crop,
policy.input_side_len,
)?;
let infer_start = Instant::now();
let (boxes, logits) = session.run_with(tensor.view(), |outputs| {
Ok((outputs.boxes.to_owned(), outputs.tokens.to_owned()))
})?;
infer_ms += infer_start.elapsed().as_secs_f32() * 1000.0;
let backfill_start = Instant::now();
let decoded = table_decode_and_backfill(&TableDecodeInput {
vocabulary: &vocabulary,
boxes: &boxes,
logits: &logits,
side_len: policy.input_side_len,
region,
source: &source,
})?;
backfill_ms += backfill_start.elapsed().as_secs_f32() * 1000.0;
let text_policy = match policy.text_policy {
crate::api::TableTextPolicy::Escape => crate::table::emitter::TextPolicy::Escape,
crate::api::TableTextPolicy::Raw => crate::table::emitter::TextPolicy::Raw,
};
let rendered =
crate::table::emitter::emit_html(&decoded.result, &decoded.texts, text_policy)?;
let table_html = crate::table::emitter::emit_table_html(
&decoded.result,
&decoded.texts,
text_policy,
)?
.rendered;
let markdown =
crate::table::emitter::emit_markdown(&decoded.result, &decoded.texts)?.rendered;
let (html, table_html) =
finalize_table_html(rendered.rendered, table_html, policy.text_policy);
table_regions.push(OcrRegion {
source: RegionSource::Detected {
detector_index: index,
},
kind: RegionKind::Table,
polygon: Some(Polygon {
points: region.polygon,
}),
detection: Some(crate::api::DetectionOutcome {
score: region.score,
}),
classification: None,
recognition: None,
formula: None,
table: Some(TableOutcome {
html,
table_html,
markdown,
rows: decoded.result.grid.rows,
cols: decoded.result.grid.cols,
confidence: match rendered.confidence {
crate::table::emitter::StructureConfidence::High => {
TableStructureConfidence::High
}
crate::table::emitter::StructureConfidence::Low => {
TableStructureConfidence::Low
}
},
holes: decoded.result.coverage.holes(),
degradations: rendered
.degradations
.iter()
.map(|degradation| match degradation {
crate::table::emitter::StructureDegradation::GridHoles { holes } => {
TableDegradation::GridHoles { holes: *holes }
}
crate::table::emitter::StructureDegradation::Truncated => {
TableDegradation::Truncated
}
})
.collect(),
cells: decoded.cells,
source_line_indices: decoded.source_line_indices,
model_id: policy
.model_path
.as_ref()
.map(|path| path.display().to_string())
.unwrap_or_default(),
}),
});
}
output.timings.table_detect_ms = detect_ms;
output.timings.table_structure_infer_ms = infer_ms;
output.timings.table_backfill_ms = backfill_ms;
output.timings.total_ms += detect_ms + infer_ms + backfill_ms;
let items = table_regions.len();
output.regions.extend(table_regions);
output.stages.table = StageReport {
state: StageState::Completed { items },
timing: Some(StageTiming {
preprocess_ms: detect_ms,
infer_ms,
postprocess_ms: backfill_ms,
}),
};
Ok(())
}
}
fn crop_polygon_bounds(
image: &image::DynamicImage,
polygon: crate::api::Polygon,
) -> image::DynamicImage {
use image::GenericImageView;
let (width, height) = image.dimensions();
let (mut x0, mut y0) = (f32::INFINITY, f32::INFINITY);
let (mut x1, mut y1) = (f32::NEG_INFINITY, f32::NEG_INFINITY);
for [x, y] in polygon.points {
x0 = x0.min(x);
y0 = y0.min(y);
x1 = x1.max(x);
y1 = y1.max(y);
}
let x0 = x0.floor().clamp(0.0, width as f32) as u32;
let y0 = y0.floor().clamp(0.0, height as f32) as u32;
let x1 = x1.ceil().clamp(0.0, width as f32) as u32;
let y1 = y1.ceil().clamp(0.0, height as f32) as u32;
let w = (x1.saturating_sub(x0))
.max(1)
.min(width.saturating_sub(x0).max(1));
let h = (y1.saturating_sub(y0))
.max(1)
.min(height.saturating_sub(y0).max(1));
image.crop_imm(x0, y0, w, h)
}
fn canonical_polygon_points(polygon: crate::api::Polygon) -> [[f32; 2]; 4] {
let (mut x0, mut y0) = (f32::INFINITY, f32::INFINITY);
let (mut x1, mut y1) = (f32::NEG_INFINITY, f32::NEG_INFINITY);
for [x, y] in polygon.points {
x0 = x0.min(x);
y0 = y0.min(y);
x1 = x1.max(x);
y1 = y1.max(y);
}
[[x0, y0], [x1, y0], [x1, y1], [x0, y1]]
}
fn rec_image_to_dynamic(image: &crate::config::RecImage) -> Result<image::DynamicImage> {
use crate::config::ColorOrder;
let width = image.width() as u32;
let height = image.height() as u32;
let rgb: Vec<u8> = match image.color_order() {
ColorOrder::Rgb => image.as_bytes().to_vec(),
ColorOrder::Bgr => image
.as_bytes()
.as_chunks::<3>()
.0
.iter()
.flat_map(|pixel| [pixel[2], pixel[1], pixel[0]])
.collect(),
};
let buffer = image::RgbImage::from_raw(width, height, rgb).ok_or_else(|| {
crate::error::RapidOcrError::InvalidImage("invalid RGB buffer for formula routing".into())
})?;
Ok(image::DynamicImage::ImageRgb8(buffer))
}
fn rec_image_from_dynamic(image: &image::DynamicImage) -> Result<crate::config::RecImage> {
let rgb = image.to_rgb8();
let (width, height) = (rgb.width() as usize, rgb.height() as usize);
let bgr: Vec<u8> = rgb
.as_raw()
.as_chunks::<3>()
.0
.iter()
.flat_map(|pixel| [pixel[2], pixel[1], pixel[0]])
.collect();
crate::config::RecImage::from_bgr_u8(width, height, bgr)
}
struct TableSourceLines {
lines: Vec<crate::table::detection::PageLine>,
text_lines: Vec<crate::table::cell_content_backfill::TextLine>,
region_indices: Vec<usize>,
}
fn table_source_lines(regions: &[crate::api::OcrRegion]) -> TableSourceLines {
let mut canonical: Vec<crate::table::page_line::PageTextLine> = Vec::new();
for (index, region) in regions.iter().enumerate() {
if region.kind != crate::api::RegionKind::Text {
continue;
}
let (Some(polygon), Some(recognition)) = (region.polygon, region.recognition.as_ref())
else {
continue;
};
canonical.push(crate::table::page_line::PageTextLine::new(
index,
polygon.points,
recognition.text.clone(),
recognition.score,
));
}
TableSourceLines {
lines: canonical.iter().map(|line| line.to_page_line()).collect(),
text_lines: canonical.iter().map(|line| line.to_text_line()).collect(),
region_indices: canonical.iter().map(|line| line.index).collect(),
}
}
struct TableDecodeInput<'a> {
vocabulary: &'a crate::table::decode::StructureVocabulary,
boxes: &'a ndarray::Array3<f32>,
logits: &'a ndarray::Array3<f32>,
side_len: usize,
region: &'a crate::table::detection::TableRegion,
source: &'a TableSourceLines,
}
struct TableDecoded {
result: crate::table::decode::StructureResult,
texts: Vec<String>,
cells: Vec<crate::api::TableCellOutcome>,
source_line_indices: Vec<usize>,
}
fn finalize_table_html(
html: String,
table_html: String,
policy: crate::api::TableTextPolicy,
) -> (String, String) {
match policy {
crate::api::TableTextPolicy::Escape => {
debug_assert_eq!(html, format!("<html><body>{table_html}</body></html>"));
let cleaned = crate::output::html::sanitize_table_html(&table_html);
let shell = format!("<html><body>{cleaned}</body></html>");
(shell, cleaned)
}
crate::api::TableTextPolicy::Raw => (html, table_html),
}
}
fn table_decode_and_backfill(input: &TableDecodeInput<'_>) -> Result<TableDecoded> {
use crate::api::{Polygon, TableCellMatch, TableCellOutcome};
use crate::table::cell_content_backfill::CellMatchKind;
if input.logits.shape()[0] != 1 || input.boxes.shape()[0] != 1 {
return Err(crate::error::RapidOcrError::Decode(format!(
"the table structure model returned a batch of {} box(es) / {} structure(s), but the page \
pipeline runs one region at a time",
input.boxes.shape()[0],
input.logits.shape()[0]
)));
}
let steps_count = input.logits.shape()[1];
let classes = input.logits.shape()[2];
if classes != input.vocabulary.class_count() {
return Err(crate::error::RapidOcrError::Decode(format!(
"the table structure model's logits have {classes} classes but the vocabulary has {}",
input.vocabulary.class_count()
)));
}
let argmaxed = crate::table::decode::argmax_structure(
input.logits.as_slice().expect("contiguous"),
steps_count,
)?;
let (steps, termination) =
crate::table::decode::decode_token_steps(input.vocabulary, &argmaxed)?;
let height = input.region.crop.height() as f32;
let width = input.region.crop.width() as f32;
let side = input.side_len as f32;
let shape = crate::table::decode::TableShape::new(height, width, side)?;
let ratios = crate::table::decode::rescale_ratios(width, height, side);
let box_flat = input.boxes.as_slice().expect("contiguous");
let cell_polygons: Vec<[f32; 8]> = steps
.iter()
.filter(|step| step.is_cell_start)
.map(|step| {
let offset = step.step_index * 8;
let mut raw = [0.0_f32; 8];
raw.copy_from_slice(&box_flat[offset..offset + 8]);
crate::table::decode::rescale_polygon(&shape.denormalize_polygon(&raw), ratios)
})
.collect();
let result = crate::table::decode::tokens_to_structure(
input.vocabulary,
&steps,
termination,
&cell_polygons,
)?;
let outcome = crate::table::cell_content_backfill::backfill(
&result.cells,
crate::table::cell_content_backfill::BackfillSource::Reuse {
lines: &input.source.text_lines,
offset: input.region.origin,
},
)?;
debug_assert_eq!(outcome.recognizer_calls, 0);
let mut cells = Vec::with_capacity(result.cells.len());
let mut consumed: Vec<usize> = Vec::new();
for (index, slot) in result.cells.iter().enumerate() {
if let Some(matched) = outcome.matched_lines.get(index) {
for line_index in matched {
if let Some(region_index) = input.source.region_indices.get(*line_index) {
consumed.push(*region_index);
}
}
}
cells.push(TableCellOutcome {
row: slot.logic.row_start,
col: slot.logic.col_start,
rowspan: slot.rowspan,
colspan: slot.colspan,
text: outcome.texts.get(index).cloned().unwrap_or_default(),
score: outcome.text_scores.get(index).copied().flatten(),
match_kind: match outcome.kinds.get(index) {
Some(CellMatchKind::BlankBox) => TableCellMatch::BlankBox,
Some(CellMatchKind::Unmatched) => TableCellMatch::Unmatched,
Some(CellMatchKind::Single) => TableCellMatch::Single,
Some(CellMatchKind::Multiple) => TableCellMatch::Multiple,
None => {
return Err(crate::error::RapidOcrError::Decode(format!(
"the backfill returned {} cell kind(s) for {} slot(s)",
outcome.kinds.len(),
result.cells.len()
)));
}
},
polygon: slot.bbox.map(|bbox| {
let mut quad = crate::table::cell_content_backfill::bbox_to_quad(bbox);
for point in &mut quad {
point[0] += input.region.origin[0];
point[1] += input.region.origin[1];
}
Polygon { points: quad }
}),
});
}
consumed.sort_unstable();
consumed.dedup();
Ok(TableDecoded {
result,
texts: outcome.texts,
cells,
source_line_indices: consumed,
})
}
impl RapidOcrEngine {
fn recognize_tiled_request(
&mut self,
request: crate::api::OcrRequest,
tile: crate::api::TilePolicy,
) -> Result<crate::api::OcrOutput> {
if matches!(
request.stages.classify.policy,
crate::api::ClassifierPolicy::Required
) && self.inner.classifier.is_none()
{
return Err(crate::error::RapidOcrError::Config(
"classification requested but no classifier model is configured".into(),
));
}
if tile.max_width == 0
|| tile.max_height == 0
|| tile.overlap >= tile.max_width
|| tile.overlap >= tile.max_height
{
return Err(crate::error::RapidOcrError::InvalidInput(
"invalid tile policy".into(),
));
}
let m = self.materialize(&request)?;
let rgba = rec_image_to_rgba(&m.rec)?;
let original_size = m.original_size;
if let Some(roi) = request.roi {
roi.validate_against(original_size.width, original_size.height)?;
}
let roi = request.roi.unwrap_or(crate::api::RectU32 {
x: 0,
y: 0,
width: original_size.width,
height: original_size.height,
});
let view = image::imageops::crop_imm(&rgba, roi.x, roi.y, roi.width, roi.height).to_image();
let step_x = tile.max_width - tile.overlap;
let step_y = tile.max_height - tile.overlap;
let mut regions = Vec::new();
let mut timings = crate::api::OcrTimings::default();
let mut y = 0;
while y < view.height() {
let h = tile.max_height.min(view.height() - y);
let mut x = 0;
while x < view.width() {
let w = tile.max_width.min(view.width() - x);
let crop = image::imageops::crop_imm(&view, x, y, w, h).to_image();
let mut preprocess = request.preprocess.clone();
preprocess.tile = None;
let sub = crate::api::OcrRequest {
input: crate::api::ImageInput::Pixels(crate::api::OwnedPixelBuffer {
width: w,
height: h,
stride: w as usize * 4,
format: crate::api::PixelFormat::Rgba8,
bottom_up: false,
data: Arc::from(crop.into_raw()),
}),
roi: None,
scale_hint: request.scale_hint,
stages: request.stages,
preprocess,
detection: request.detection,
recognition: request.recognition,
output: request.output,
formula: crate::api::FormulaPolicy::default(),
table: crate::api::TablePolicy::default(),
};
let mut out = self.recognize(sub)?;
let region_offset = regions.len();
timings.decode_ms += out.timings.decode_ms;
timings.resize_ms += out.timings.resize_ms;
timings.crop_ms += out.timings.crop_ms;
timings.preprocess_ms += out.timings.preprocess_ms;
timings.detector_preprocess_ms += out.timings.detector_preprocess_ms;
timings.detector_infer_ms += out.timings.detector_infer_ms;
timings.detector_postprocess_ms += out.timings.detector_postprocess_ms;
timings.detect_ms += out.timings.detect_ms;
timings.classifier_preprocess_ms += out.timings.classifier_preprocess_ms;
timings.classifier_infer_ms += out.timings.classifier_infer_ms;
timings.classifier_postprocess_ms += out.timings.classifier_postprocess_ms;
timings.classify_ms += out.timings.classify_ms;
timings.recognizer_preprocess_ms += out.timings.recognizer_preprocess_ms;
timings.recognizer_infer_ms += out.timings.recognizer_infer_ms;
timings.recognizer_postprocess_ms += out.timings.recognizer_postprocess_ms;
timings.recognize_ms += out.timings.recognize_ms;
timings.postprocess_ms += out.timings.postprocess_ms;
for region in &mut out.regions {
if let Some(poly) = &mut region.polygon {
for p in &mut poly.points {
p[0] += (roi.x + x) as f32;
p[1] += (roi.y + y) as f32;
}
}
if let Some(rec) = &mut region.recognition
&& let Some(words) = &mut rec.words
{
for word in words {
for p in &mut word.polygon.points {
p[0] += (roi.x + x) as f32;
p[1] += (roi.y + y) as f32;
}
}
}
if let crate::api::RegionSource::Detected { detector_index } = region.source {
region.source = crate::api::RegionSource::Detected {
detector_index: detector_index + region_offset,
};
}
}
regions.extend(out.regions);
if x + w >= view.width() {
break;
}
x += step_x;
}
if y + h >= view.height() {
break;
}
y += step_y;
}
regions = deduplicate_regions(regions);
timings.total_ms = timings.preprocess_ms
+ timings.detect_ms
+ timings.classify_ms
+ timings.recognize_ms
+ timings.postprocess_ms;
Ok(crate::api::OcrOutput {
schema_version: 1,
image: crate::api::ImageInfo {
original_size,
processed_size: original_size,
coordinate_space: request.output.coordinate_space,
},
stages: crate::api::StageReports {
input: crate::api::InputTimings {
decode_ms: Some(timings.decode_ms),
resize_ms: Some(timings.resize_ms),
crop_ms: Some(timings.crop_ms),
},
detector: crate::api::StageReport {
state: if !request.stages.detect {
crate::api::StageState::Disabled
} else {
let items = regions.iter().filter(|r| r.detection.is_some()).count();
if items == 0 {
crate::api::StageState::SkippedNoInput
} else {
crate::api::StageState::Completed { items }
}
},
timing: request.stages.detect.then_some(crate::api::StageTiming {
preprocess_ms: timings.detector_preprocess_ms,
infer_ms: timings.detector_infer_ms,
postprocess_ms: timings.detector_postprocess_ms,
}),
},
classifier: crate::api::StageReport {
state: if matches!(
request.stages.classify.policy,
crate::api::ClassifierPolicy::Off
) {
crate::api::StageState::Disabled
} else if self.inner.classifier.is_none() {
crate::api::StageState::SkippedUnavailable
} else {
let items = regions
.iter()
.filter(|r| r.classification.is_some())
.count();
if items == 0 {
crate::api::StageState::SkippedNoInput
} else {
crate::api::StageState::Completed { items }
}
},
timing: if matches!(
request.stages.classify.policy,
crate::api::ClassifierPolicy::Off
) || self.inner.classifier.is_none()
{
None
} else {
Some(crate::api::StageTiming {
preprocess_ms: timings.classifier_preprocess_ms,
infer_ms: timings.classifier_infer_ms,
postprocess_ms: timings.classifier_postprocess_ms,
})
},
},
recognizer: crate::api::StageReport {
state: if !request.stages.recognize {
crate::api::StageState::Disabled
} else {
let items = regions.iter().filter(|r| r.recognition.is_some()).count();
if items == 0 {
crate::api::StageState::SkippedNoInput
} else {
crate::api::StageState::Completed { items }
}
},
timing: request.stages.recognize.then_some(crate::api::StageTiming {
preprocess_ms: timings.recognizer_preprocess_ms,
infer_ms: timings.recognizer_infer_ms,
postprocess_ms: timings.recognizer_postprocess_ms,
}),
},
formula: crate::api::StageReport::default(),
table: crate::api::StageReport::default(),
},
regions,
timings,
engine: crate::api::EngineInfo {
model_id: self.model_id.clone(),
provider: self.provider_info(),
},
})
}
}
fn deduplicate_regions(mut regions: Vec<crate::api::OcrRegion>) -> Vec<crate::api::OcrRegion> {
let mut unique = Vec::with_capacity(regions.len());
for region in regions.drain(..) {
let duplicate = region.recognition.as_ref().and_then(|recognition| {
region.polygon.and_then(|polygon| {
unique.iter().position(|existing: &crate::api::OcrRegion| {
existing
.recognition
.as_ref()
.is_some_and(|v| v.text == recognition.text)
&& existing.polygon.is_some_and(|other| {
crate::evaluation::ocr::polygon_iou(other.points, polygon.points) >= 0.5
})
})
})
});
if let Some(index) = duplicate {
let replace = region
.recognition
.as_ref()
.zip(unique[index].recognition.as_ref())
.is_some_and(|(a, b)| a.score > b.score);
if replace {
unique[index] = region;
}
} else {
unique.push(region);
}
}
unique.sort_by(|a, b| {
let key = |region: &crate::api::OcrRegion| {
region
.polygon
.map(|polygon| {
(
polygon
.points
.iter()
.map(|p| p[1])
.fold(f32::INFINITY, f32::min),
polygon
.points
.iter()
.map(|p| p[0])
.fold(f32::INFINITY, f32::min),
)
})
.unwrap_or((f32::INFINITY, f32::INFINITY))
};
key(a)
.partial_cmp(&key(b))
.unwrap_or(std::cmp::Ordering::Equal)
});
unique
}
#[cfg(test)]
mod formula_integration_tests {
use std::path::PathBuf;
use std::sync::Arc;
use crate::api::{
ClassifierPlan, ClassifierPolicy, DetectionPolicy, FormulaPolicy, ImageInput, OcrEngine,
OcrRequest, OutputPolicy, PreprocessPolicy, RecognitionPolicy, RegionKind, RegionSource,
StagePlan, StageState, TextOrder, WordOutputMode,
};
use crate::ocr::pipeline::config::EngineConfig;
use crate::ocr::pipeline::rapid_structure::RapidOcrEngine;
fn engine_config(model_root: &std::path::Path) -> EngineConfig {
let yaml = format!(
r#"
global:
use_det: true
use_cls: false
use_rec: true
max_side_len: 2000
min_side_len: 30
det:
lang: multi
ocr_version: PP-OCRv6
model_type: small
model_path: {root}/small/PP-OCRv6_det_small.onnx
allow_download: false
cls:
lang: ch
rec:
model:
lang: ch
ocr_version: PP-OCRv6
model_type: small
model_path: {root}/small/PP-OCRv6_rec_small.onnx
rec_keys_path: {root}/small/ppocrv6_dict.txt
allow_download: false
"#,
root = model_root.display().to_string().replace('\\', "/")
);
EngineConfig::from_yaml_str(&yaml).expect("engine config template must parse")
}
struct Assets {
engine: RapidOcrEngine,
formula_model: PathBuf,
detector: PathBuf,
page_with_formula: PathBuf,
page_without_detections: PathBuf,
page_with_false_positives: PathBuf,
}
fn assets() -> Option<Assets> {
let model_root = crate::test_support::ocr_model_root()?;
let formula_model = crate::test_support::formula_model_path()?;
let detector = crate::test_support::formula_detector_path()?;
let page_with_formula = crate::test_support::page_fixture("08数字公式与符号.png")?;
let page_without_detections = crate::test_support::page_fixture("09竖排文本.png")?;
let page_with_false_positives = crate::test_support::page_fixture("05代码与等宽字体.png")?;
let engine = RapidOcrEngine::new(engine_config(&model_root)).expect("engine should load");
Some(Assets {
engine,
formula_model,
detector,
page_with_formula,
page_without_detections,
page_with_false_positives,
})
}
fn request(bytes: Vec<u8>, formula: FormulaPolicy) -> OcrRequest {
OcrRequest {
input: ImageInput::Encoded(Arc::from(bytes)),
roi: None,
scale_hint: None,
stages: StagePlan {
detect: true,
classify: ClassifierPlan {
policy: ClassifierPolicy::Off,
apply_rotation: false,
},
recognize: true,
},
preprocess: PreprocessPolicy::default(),
detection: DetectionPolicy::default(),
recognition: RecognitionPolicy {
words: WordOutputMode::Off,
},
output: OutputPolicy::default(),
formula,
table: crate::api::TablePolicy::default(),
}
}
fn formula_policy(assets: &Assets) -> FormulaPolicy {
FormulaPolicy {
enabled: true,
model_path: Some(assets.formula_model.clone()),
detector_path: Some(assets.detector.clone()),
..FormulaPolicy::default()
}
}
#[test]
fn a_replaced_formula_detector_is_reverified_instead_of_reused() {
let Some(model_root) = crate::test_support::ocr_model_root() else {
return;
};
let Some(detector) = crate::test_support::formula_detector_path() else {
return;
};
let Some(formula_model) = crate::test_support::formula_model_path() else {
return;
};
let Some(page) = crate::test_support::page_fixture("09竖排文本.png") else {
return;
};
let dir = crate::test_support::TempDir::new("pipeline-detector-identity");
let copy = dir.path().join("pix2text-mfd-1.5.onnx");
std::fs::copy(&detector, ©).expect("copy the real detector");
let declared = crate::model_store::sha256_file(©).expect("hash the copy");
let mut engine =
RapidOcrEngine::new(engine_config(&model_root)).expect("engine should load");
let policy = FormulaPolicy {
enabled: true,
model_path: Some(formula_model),
expected_model_sha256: None,
detector_path: Some(copy.clone()),
expected_detector_sha256: Some(declared.clone()),
..FormulaPolicy::default()
};
let bytes = std::fs::read(&page).expect("page readable");
engine
.recognize(request(bytes.clone(), policy.clone()))
.expect("the first load must succeed");
let mut file = std::fs::OpenOptions::new()
.append(true)
.open(©)
.expect("open the copy for append");
std::io::Write::write_all(&mut file, b"x").expect("append one byte");
drop(file);
let error = engine
.recognize(request(bytes, policy))
.expect_err("a replaced detector must not be reused");
match error {
crate::error::RapidOcrError::HashMismatch {
path: reported,
expected,
actual,
} => {
assert_eq!(reported, copy, "the error must name the replaced file");
assert_eq!(expected, declared);
assert_ne!(actual, declared);
}
other => panic!("expected a hash mismatch from the re-verification, got {other:?}"),
}
}
#[test]
fn a_changed_detector_declaration_is_not_answered_by_a_hot_session() {
let Some(model_root) = crate::test_support::ocr_model_root() else {
return;
};
let Some(detector) = crate::test_support::formula_detector_path() else {
return;
};
let Some(formula_model) = crate::test_support::formula_model_path() else {
return;
};
let Some(page) = crate::test_support::page_fixture("09竖排文本.png") else {
return;
};
let dir = crate::test_support::TempDir::new("pipeline-detector-declaration");
let copy = dir.path().join("pix2text-mfd-1.5.onnx");
std::fs::copy(&detector, ©).expect("copy the real detector");
let declared = crate::model_store::sha256_file(©).expect("hash the copy");
let mut engine =
RapidOcrEngine::new(engine_config(&model_root)).expect("engine should load");
let policy = |expected: Option<String>| FormulaPolicy {
enabled: true,
model_path: Some(formula_model.clone()),
expected_model_sha256: None,
detector_path: Some(copy.clone()),
expected_detector_sha256: expected,
..FormulaPolicy::default()
};
let bytes = std::fs::read(&page).expect("page readable");
engine
.recognize(request(bytes.clone(), policy(Some(declared.clone()))))
.expect("the first request with the correct digest must pass");
let bogus = format!("{:064x}", 0xdead_beef_u32);
let error = engine
.recognize(request(bytes.clone(), policy(Some(bogus.clone()))))
.expect_err("a changed declaration must not be answered by the hot session");
match error {
crate::error::RapidOcrError::HashMismatch {
path: reported,
expected,
actual,
} => {
assert_eq!(reported, copy, "the error must name the declared file");
assert_eq!(expected, bogus);
assert_eq!(
actual, declared,
"the file never changed - only the declaration did"
);
}
other => panic!("expected a hash mismatch from the declaration, got {other:?}"),
}
engine
.recognize(request(bytes, policy(Some(declared.to_uppercase()))))
.expect("an equivalently spelled declaration must still be accepted");
}
#[test]
fn a_changed_recognizer_declaration_is_not_answered_by_a_hot_session() {
let Some(model_root) = crate::test_support::ocr_model_root() else {
return;
};
let Some(formula_model) = crate::test_support::formula_model_path() else {
return;
};
let Some(page) = crate::test_support::page_fixture("08数字公式与符号.png") else {
return;
};
let dir = crate::test_support::TempDir::new("pipeline-recognizer-declaration");
let copy = dir.path().join("pp_formulanet_plus_m.onnx");
std::fs::copy(&formula_model, ©).expect("copy the real recognizer");
let declared = crate::model_store::sha256_file(©).expect("hash the copy");
let mut engine =
RapidOcrEngine::new(engine_config(&model_root)).expect("engine should load");
let policy = |expected: Option<String>| FormulaPolicy {
enabled: true,
model_path: Some(copy.clone()),
expected_model_sha256: expected,
detector_path: None,
input_regions: vec![crate::api::Polygon {
points: [
[120.0, 420.0],
[900.0, 420.0],
[900.0, 560.0],
[120.0, 560.0],
],
}],
..FormulaPolicy::default()
};
let bytes = std::fs::read(&page).expect("page readable");
let output = engine
.recognize(request(bytes.clone(), policy(Some(declared.clone()))))
.expect("the first request with the correct digest must pass");
assert_eq!(
output.formula_count(),
1,
"the explicit region must reach the recognizer (otherwise this test proves nothing)"
);
let bogus = format!("{:064x}", 7u32);
let error = engine
.recognize(request(bytes, policy(Some(bogus.clone()))))
.expect_err("a changed declaration must not be answered by the hot session");
match error {
crate::error::RapidOcrError::HashMismatch {
path: reported,
expected,
actual,
} => {
assert_eq!(reported, copy, "the error must name the declared file");
assert_eq!(expected, bogus);
assert_eq!(
actual, declared,
"the file never changed - only the declaration did"
);
}
other => panic!("expected a hash mismatch from the declaration, got {other:?}"),
}
}
#[test]
fn formula_routing_rejects_roi_and_tile() {
let base = FormulaPolicy {
enabled: true,
model_path: Some(PathBuf::from("model.onnx")),
..FormulaPolicy::default()
};
let mut with_roi = request(Vec::new(), base.clone());
with_roi.roi = Some(crate::api::RectU32 {
x: 0,
y: 0,
width: 10,
height: 10,
});
let error = with_roi.validate().expect_err("roi must be rejected");
assert!(error.to_string().contains("roi"), "error: {error}");
let mut with_tile = request(Vec::new(), base);
with_tile.preprocess.tile = Some(crate::api::TilePolicy {
max_width: 100,
max_height: 100,
overlap: 10,
});
let error = with_tile.validate().expect_err("tile must be rejected");
assert!(error.to_string().contains("tiled"), "error: {error}");
}
#[test]
fn formula_policy_requires_a_model_path() {
let policy = FormulaPolicy {
enabled: true,
..FormulaPolicy::default()
};
assert!(policy.validate().is_err());
}
#[test]
fn disabled_formula_policy_produces_no_formula_regions() {
let Some(mut assets) = assets() else {
return;
};
let bytes = std::fs::read(&assets.page_with_formula).expect("page readable");
let output = assets
.engine
.recognize(request(bytes, FormulaPolicy::default()))
.expect("baseline recognize");
assert!(output.validate().is_ok());
assert_eq!(output.formula_count(), 0);
assert!(output.formula_latex(TextOrder::Reading).is_empty());
assert_eq!(output.stages.formula.state, StageState::Disabled);
assert!(
output
.regions
.iter()
.all(|region| region.kind == RegionKind::Text),
"disabled formula policy must not produce formula regions"
);
}
#[test]
fn formula_routing_without_detections_matches_the_baseline() {
let Some(mut assets) = assets() else {
return;
};
let bytes = std::fs::read(&assets.page_without_detections).expect("page readable");
let baseline = assets
.engine
.recognize(request(bytes.clone(), FormulaPolicy::default()))
.expect("baseline recognize");
let routed = assets
.engine
.recognize(request(bytes, formula_policy(&assets)))
.expect("routed recognize");
assert_eq!(routed.formula_count(), 0, "no formula on this page");
assert_eq!(routed.stages.formula.state, StageState::SkippedNoInput);
assert_eq!(
routed.plain_text(TextOrder::Reading),
baseline.plain_text(TextOrder::Reading),
"formula routing must not change text output when nothing is detected"
);
assert_eq!(
routed.regions.len(),
baseline.regions.len(),
"region count must not change when nothing is detected"
);
}
#[test]
fn code_text_evidence_rejects_formula_false_positives_in_the_pipeline() {
let Some(mut assets) = assets() else {
return;
};
let bytes = std::fs::read(&assets.page_with_false_positives).expect("page readable");
let baseline = assets
.engine
.recognize(request(bytes.clone(), FormulaPolicy::default()))
.expect("baseline recognize");
let routed = assets
.engine
.recognize(request(bytes.clone(), formula_policy(&assets)))
.expect("routed recognize");
assert!(
routed.formula_count() == 0,
"this page is known to produce false positives at the default threshold; \
text evidence arbitration must reject them"
);
assert_eq!(routed.stages.formula.state, StageState::SkippedNoInput);
assert_eq!(
routed.plain_text(TextOrder::Reading),
baseline.plain_text(TextOrder::Reading),
"rejecting a false positive must preserve the original OCR text"
);
}
#[test]
fn detected_formula_regions_are_typed_and_carry_latex() {
let Some(mut assets) = assets() else {
return;
};
let bytes = std::fs::read(&assets.page_with_formula).expect("page readable");
let output = assets
.engine
.recognize(request(bytes, formula_policy(&assets)))
.expect("routed recognize");
output.validate().expect("routed output must validate");
assert!(
output.formula_count() > 0,
"the formula page must produce at least one formula region"
);
assert_eq!(
output.stages.formula.state,
StageState::Completed {
items: output.formula_count()
},
"the formula stage must count formula regions, not all regions"
);
assert!(
output.formula_count() < output.regions.len(),
"this page has text regions too, so the two counts must differ"
);
for region in output
.regions
.iter()
.filter(|region| region.kind == RegionKind::Formula)
{
let formula = region.formula.as_ref().expect("formula outcome");
assert!(
region.recognition.is_none(),
"formula regions must not carry CTC text: {:?}",
region.recognition
);
assert!(!formula.model_id.is_empty());
assert!(
region.polygon.is_some(),
"formula regions must keep their crop polygon"
);
assert!(
formula.token_ids.is_none(),
"token ids must stay opt-in outside debug mode"
);
}
}
#[test]
fn explicit_input_regions_are_recognized_without_a_detector() {
let Some(mut assets) = assets() else {
return;
};
let bytes = std::fs::read(&assets.page_with_formula).expect("page readable");
let policy = FormulaPolicy {
enabled: true,
model_path: Some(assets.formula_model.clone()),
detector_path: None,
input_regions: vec![crate::api::Polygon {
points: [
[120.0, 420.0],
[900.0, 420.0],
[900.0, 560.0],
[120.0, 560.0],
],
}],
include_token_ids: true,
..FormulaPolicy::default()
};
let output = assets
.engine
.recognize(request(bytes, policy))
.expect("explicit region recognize");
output.validate().expect("output must validate");
assert_eq!(output.formula_count(), 1);
let region = output
.regions
.iter()
.find(|region| region.kind == RegionKind::Formula)
.expect("formula region");
assert_eq!(region.source, RegionSource::Input);
assert!(
region.detection.is_none(),
"explicit regions have no detector score"
);
let formula = region.formula.as_ref().expect("formula outcome");
assert!(
formula.token_ids.is_some(),
"include_token_ids must keep the raw token sequence"
);
}
#[test]
fn engine_applies_formula_batch_and_chunks_region_batches() {
let Some(model_root) = crate::test_support::ocr_model_root() else {
return;
};
let Some(formula_model) = crate::test_support::formula_model_path() else {
return;
};
let Some(page) = crate::test_support::page_fixture("08数字公式与符号.png") else {
return;
};
let mut config = engine_config(&model_root);
config.runtime.formula_batch = 4;
let mut engine = RapidOcrEngine::new(config).expect("engine should load");
assert_eq!(
engine.runtime_profile().formula_batch,
4,
"the resolved profile must carry the configured formula batch"
);
let page_image = image::open(&page).expect("page readable");
let (width, height) = (page_image.width() as f32, page_image.height() as f32);
let input_regions: Vec<crate::api::Polygon> = (0..5)
.map(|index| {
let x0 = width * (0.05 + index as f32 * 0.19);
let x1 = x0 + width * 0.15;
crate::api::Polygon {
points: [
[x0, height * 0.05],
[x1, height * 0.05],
[x1, height * 0.15],
[x0, height * 0.15],
],
}
})
.collect();
let policy = FormulaPolicy {
enabled: true,
model_path: Some(formula_model),
detector_path: None,
input_regions: input_regions.clone(),
..FormulaPolicy::default()
};
let bytes = std::fs::read(&page).expect("page readable");
let output = engine
.recognize(request(bytes, policy))
.expect("more regions than formula_batch must be chunked, not rejected");
output.validate().expect("output must validate");
assert_eq!(output.formula_count(), input_regions.len());
assert_eq!(
output.stages.formula.state,
StageState::Completed {
items: input_regions.len()
}
);
let applied_batch = engine
.formula_recognizer
.as_ref()
.expect("the formula recognizer is created on first use")
.1
.max_batch_size();
assert_eq!(
applied_batch, 4,
"the engine must apply the profile's formula_batch to the recognizer"
);
}
#[test]
fn formula_routing_enforces_the_pixel_limit_for_in_memory_inputs() {
let Some(mut assets) = assets() else {
return;
};
let page = image::open(&assets.page_with_formula).expect("page readable");
let (width, height) = (page.width(), page.height());
let tight_limit = u64::from(width) * u64::from(height) - 1;
let mut from_file = request(Vec::new(), formula_policy(&assets));
from_file.input = ImageInput::File(assets.page_with_formula.clone());
from_file.preprocess.max_decode_pixels = tight_limit;
let error = assets
.engine
.recognize(from_file)
.expect_err("encoded input must respect the pixel limit");
assert!(
matches!(error, crate::error::RapidOcrError::InvalidImage(_)),
"unexpected error: {error}"
);
let rgb = page.to_rgb8();
let mut from_pixels = request(Vec::new(), formula_policy(&assets));
from_pixels.input = ImageInput::Pixels(crate::api::OwnedPixelBuffer {
data: Arc::from(rgb.as_raw().as_slice()),
width,
height,
stride: (width * 3) as usize,
format: crate::api::PixelFormat::Rgb8,
bottom_up: false,
});
from_pixels.preprocess.max_decode_pixels = tight_limit;
let error = assets
.engine
.recognize(from_pixels)
.expect_err("in-memory pixel input must respect the pixel limit");
assert!(
matches!(error, crate::error::RapidOcrError::InvalidImage(_)),
"unexpected error: {error}"
);
let rec = crate::config::RecImage::from_bgr_u8(width as usize, height as usize, {
let mut bgr = Vec::with_capacity((width * height * 3) as usize);
for pixel in rgb.pixels() {
bgr.extend_from_slice(&[pixel[2], pixel[1], pixel[0]]);
}
bgr
})
.expect("rec image");
let mut from_image = request(Vec::new(), formula_policy(&assets));
from_image.input = ImageInput::Image(rec);
from_image.preprocess.max_decode_pixels = tight_limit;
let error = assets
.engine
.recognize(from_image)
.expect_err("in-memory image input must respect the pixel limit");
assert!(
matches!(error, crate::error::RapidOcrError::InvalidImage(_)),
"unexpected error: {error}"
);
let rgb_again = page.to_rgb8();
let mut allowed = request(Vec::new(), formula_policy(&assets));
allowed.input = ImageInput::Pixels(crate::api::OwnedPixelBuffer {
data: Arc::from(rgb_again.as_raw().as_slice()),
width,
height,
stride: (width * 3) as usize,
format: crate::api::PixelFormat::Rgb8,
bottom_up: false,
});
allowed.preprocess.max_decode_pixels = u64::from(width) * u64::from(height);
let output = assets
.engine
.recognize(allowed)
.expect("pixel input within the limit must succeed");
output.validate().expect("output must validate");
}
#[test]
fn whitening_removes_text_inside_the_region_and_keeps_distant_text() {
let Some(mut assets) = assets() else {
return;
};
let bytes = std::fs::read(&assets.page_with_formula).expect("page readable");
let baseline = assets
.engine
.recognize(request(bytes.clone(), FormulaPolicy::default()))
.expect("baseline recognize");
let text_regions: Vec<_> = baseline
.regions
.iter()
.filter(|region| region.kind == RegionKind::Text)
.filter_map(|region| {
let polygon = region.polygon?;
let text = region.recognition.as_ref()?.text.clone();
(!text.trim().is_empty()).then_some((polygon, text))
})
.collect();
assert!(
text_regions.len() >= 3,
"baseline must produce several non-empty text regions"
);
let centroid = |polygon: &crate::api::Polygon| {
let points = polygon.points;
let sum_x: f32 = points.iter().map(|point| point[0]).sum();
let sum_y: f32 = points.iter().map(|point| point[1]).sum();
(sum_x / points.len() as f32, sum_y / points.len() as f32)
};
let compact = |text: &str| {
text.chars()
.filter(|c| !c.is_whitespace())
.collect::<String>()
};
let (target_polygon, target_text) = text_regions[text_regions.len() / 2].clone();
let target_centroid = centroid(&target_polygon);
let (distant_polygon, distant_text) = text_regions
.iter()
.max_by(|left, right| {
let distance = |candidate: &(crate::api::Polygon, String)| {
let (x, y) = centroid(&candidate.0);
(x - target_centroid.0).hypot(y - target_centroid.1)
};
distance(left)
.partial_cmp(&distance(right))
.unwrap_or(std::cmp::Ordering::Equal)
})
.expect("at least one distant region")
.clone();
assert_ne!(distant_polygon, target_polygon);
let policy = FormulaPolicy {
enabled: true,
model_path: Some(assets.formula_model.clone()),
detector_path: None,
input_regions: vec![target_polygon],
..FormulaPolicy::default()
};
let routed = assets
.engine
.recognize(request(bytes, policy))
.expect("routed recognize");
let routed_text = compact(&routed.plain_text(TextOrder::Reading));
assert!(
!routed_text.contains(&compact(&target_text)),
"text covered by the whitened region must disappear: {target_text:?}"
);
assert!(
routed_text.contains(&compact(&distant_text)),
"content far from the whitened region must survive (whitespace-insensitive, because \
whitening can re-segment distant regions): {distant_text:?}"
);
assert_eq!(
routed.formula_count(),
1,
"the explicit region must still be recognized as a formula"
);
}
#[test]
fn engine_honours_auto_tune_threads_disabled() {
let Some(model_root) = crate::test_support::ocr_model_root() else {
return;
};
let mut config = engine_config(&model_root);
config.runtime.auto_tune_threads = false;
config.runtime.intra_threads = None;
config.runtime.inter_threads = None;
config.runtime.rayon_threads = None;
let engine = RapidOcrEngine::new(config).expect("engine should load");
let threads = &engine.runtime_profile().threads;
assert_eq!(threads.ort_intra, None);
assert_eq!(threads.ort_inter, None);
assert_eq!(threads.rayon, None);
assert_eq!(threads.source, crate::runtime::profile::ThreadSource::Auto);
assert_eq!(threads.sessions, 2, "det + rec = two sessions");
let session = engine.runtime_profile().session_runtime();
assert_eq!(session.intra_threads, None);
assert_eq!(session.inter_threads, None);
assert_eq!(session.rayon_threads, None);
assert!(!session.auto_tune_threads);
}
#[test]
fn materialization_preserves_the_decode_count_per_route() {
let Some(mut assets) = assets() else {
return;
};
let bytes = std::fs::read(&assets.page_without_detections).expect("page readable");
crate::input::image_loader::reset_decode_calls();
assets
.engine
.recognize(request(bytes.clone(), FormulaPolicy::default()))
.expect("plain text route must succeed");
assert_eq!(
crate::input::image_loader::decode_calls(),
1,
"the text channel must decode an Encoded input exactly once"
);
crate::input::image_loader::reset_decode_calls();
let policy = formula_policy(&assets);
assets
.engine
.recognize(request(bytes, policy))
.expect("page-level formula route must succeed");
assert_eq!(
crate::input::image_loader::decode_calls(),
2,
"the page-level route must still decode twice (source + text fallback)"
);
}
}
#[cfg(test)]
mod table_html_policy_tests {
use crate::api::TableTextPolicy;
#[test]
fn the_user_facing_path_strips_active_markup() {
let table_html = "<table><tr><td>ok</td></tr></table><script>alert(1)</script>";
let (html, cleaned) = super::finalize_table_html(
format!("<html><body>{table_html}</body></html>"),
table_html.to_string(),
TableTextPolicy::Escape,
);
assert!(!cleaned.contains("<script"), "table_html: {cleaned}");
assert!(cleaned.contains("<table>"), "table_html: {cleaned}");
assert!(!html.contains("<script"), "html: {html}");
assert_eq!(html, format!("<html><body>{cleaned}</body></html>"));
}
#[test]
fn the_evaluation_path_is_byte_identical() {
let raw =
"<html><body><table><tr><td><script>alert(1)</script></td></tr></table></body></html>";
let (html, table_html) =
super::finalize_table_html(raw.to_string(), raw.to_string(), TableTextPolicy::Raw);
assert_eq!(table_html, raw);
assert_eq!(html, raw);
}
#[test]
fn sanitizing_is_a_no_op_on_whitelisted_emitter_output() {
let table_html = "<table><thead><tr><td rowspan=\"2\" colspan=\"3\">a</td></tr></thead><tbody><tr><td colspan=\"2\">b</td></tr></tbody></table>";
let shell = format!("<html><body>{table_html}</body></html>");
let (escaped_html, escaped_table) = super::finalize_table_html(
shell.clone(),
table_html.to_string(),
TableTextPolicy::Escape,
);
let (raw_html, raw_table) =
super::finalize_table_html(shell, table_html.to_string(), TableTextPolicy::Raw);
assert_eq!(
escaped_table, raw_table,
"sanitizing changed emitter output"
);
assert_eq!(escaped_table, table_html);
assert_eq!(escaped_html, raw_html);
}
}
#[cfg(test)]
mod table_pipeline_tests {
use std::path::PathBuf;
use std::sync::Arc;
use crate::api::{
ClassifierPlan, ClassifierPolicy, DetectionPolicy, ImageInput, OcrEngine, OcrRequest,
OutputPolicy, PreprocessPolicy, RecognitionPolicy, RegionKind, StagePlan, StageState,
TablePolicy, TableTextPolicy, WordOutputMode,
};
use crate::ocr::pipeline::config::EngineConfig;
use crate::ocr::pipeline::rapid_structure::RapidOcrEngine;
fn engine_config(model_root: &std::path::Path) -> EngineConfig {
let yaml = format!(
r#"
global:
use_det: true
use_cls: false
use_rec: true
max_side_len: 2000
min_side_len: 30
det:
lang: multi
ocr_version: PP-OCRv6
model_type: small
model_path: {root}/small/PP-OCRv6_det_small.onnx
allow_download: false
cls:
lang: ch
rec:
model:
lang: ch
ocr_version: PP-OCRv6
model_type: small
model_path: {root}/small/PP-OCRv6_rec_small.onnx
rec_keys_path: {root}/small/ppocrv6_dict.txt
allow_download: false
"#,
root = model_root.display().to_string().replace('\\', "/")
);
EngineConfig::from_yaml_str(&yaml).expect("engine config template must parse")
}
struct Assets {
engine: RapidOcrEngine,
table_model: PathBuf,
page_with_table: PathBuf,
}
fn assets() -> Option<Assets> {
let model_root = crate::test_support::ocr_model_root()?;
let table_model = crate::test_support::table_structure_model_path()?;
let page_with_table = crate::test_support::page_fixture("04表格与键值对.png")?;
let engine = RapidOcrEngine::new(engine_config(&model_root)).expect("engine should load");
Some(Assets {
engine,
table_model,
page_with_table,
})
}
fn request(bytes: Vec<u8>, table: TablePolicy) -> OcrRequest {
OcrRequest {
input: ImageInput::Encoded(Arc::from(bytes)),
roi: None,
scale_hint: None,
stages: StagePlan {
detect: true,
classify: ClassifierPlan {
policy: ClassifierPolicy::Off,
apply_rotation: false,
},
recognize: true,
},
preprocess: PreprocessPolicy::default(),
detection: DetectionPolicy::default(),
recognition: RecognitionPolicy {
words: WordOutputMode::Off,
},
output: OutputPolicy::default(),
formula: crate::api::FormulaPolicy::default(),
table,
}
}
fn table_policy(assets: &Assets) -> TablePolicy {
TablePolicy {
enabled: true,
model_path: Some(assets.table_model.clone()),
text_policy: TableTextPolicy::Escape,
..TablePolicy::default()
}
}
fn table_regions(output: &crate::api::OcrOutput) -> Vec<&crate::api::OcrRegion> {
output
.regions
.iter()
.filter(|region| region.kind == RegionKind::Table)
.collect()
}
fn bounds(points: &[[f32; 2]; 4]) -> [f32; 4] {
let mut out = [
f32::INFINITY,
f32::INFINITY,
f32::NEG_INFINITY,
f32::NEG_INFINITY,
];
for point in points {
out[0] = out[0].min(point[0]);
out[1] = out[1].min(point[1]);
out[2] = out[2].max(point[0]);
out[3] = out[3].max(point[1]);
}
out
}
#[test]
fn the_page_pipeline_produces_a_table_region_with_timings() {
let Some(mut assets) = assets() else {
return;
};
let bytes = std::fs::read(&assets.page_with_table).expect("the page must be readable");
let output = assets
.engine
.recognize(request(bytes, table_policy(&assets)))
.expect("the table route must not fail");
let regions = table_regions(&output);
assert!(
!regions.is_empty(),
"the fixture page contains a table; the locator found none"
);
for (index, region) in regions.iter().enumerate() {
let outcome = region.table.as_ref().expect("table outcome");
println!(
"TABLE_REGION[{index}] rows={} cols={} cells={} confidence={:?} holes={} \
consumed_lines={}",
outcome.rows,
outcome.cols,
outcome.cells.len(),
outcome.confidence,
outcome.holes,
outcome.source_line_indices.len(),
);
}
println!(
"TABLE_TIMING detect_ms={:.3} infer_ms={:.3} backfill_ms={:.3} page_total_ms={:.3} \
regions={}",
output.timings.table_detect_ms,
output.timings.table_structure_infer_ms,
output.timings.table_backfill_ms,
output.timings.total_ms,
output.regions.len(),
);
match output.stages.table.state {
StageState::Completed { items } => assert_eq!(
items,
regions.len(),
"the table stage must report exactly the table regions it produced"
),
other => panic!("the table stage must be Completed, got {other:?}"),
}
let timing = output
.stages
.table
.timing
.expect("a completed table stage must carry a timing");
assert_eq!(
timing.infer_ms, output.timings.table_structure_infer_ms,
"the stage timing and the ledger must report the same structure inference time"
);
assert!(
output.timings.table_detect_ms > 0.0,
"the locator must report a non-zero cost"
);
assert!(
output.timings.table_structure_infer_ms > 0.0,
"the structure model must report a non-zero cost"
);
assert!(
output.timings.table_backfill_ms >= 0.0,
"the backfill window cannot be negative"
);
let table_total = output.timings.table_detect_ms
+ output.timings.table_structure_infer_ms
+ output.timings.table_backfill_ms;
assert!(
table_total <= output.timings.total_ms + 1e-3,
"table stages ({table_total} ms) must fit inside total_ms ({})",
output.timings.total_ms
);
for region in ®ions {
let outcome = region
.table
.as_ref()
.expect("a table region must carry a table outcome");
assert!(
outcome.html.starts_with("<html><body><table>"),
"the full HTML must carry the page shell: {}",
outcome.html
);
assert!(
outcome.table_html.starts_with("<table>"),
"the table-only variant must start at the table element: {}",
outcome.table_html
);
assert!(!outcome.cells.is_empty(), "a located table has cells");
assert!(
outcome.rows >= 1 && outcome.cols >= 1,
"a decoded table must have at least one row and column, got {}x{}",
outcome.rows,
outcome.cols
);
for cell in &outcome.cells {
assert!(
cell.row < outcome.rows && cell.col < outcome.cols,
"cell ({}, {}) falls outside the {}x{} grid",
cell.row,
cell.col,
outcome.rows,
outcome.cols
);
assert!(cell.rowspan >= 1 && cell.colspan >= 1);
}
for index in &outcome.source_line_indices {
assert!(
*index < output.regions.len(),
"consumed line index {index} is out of range"
);
}
assert!(
outcome
.source_line_indices
.windows(2)
.all(|pair| pair[0] < pair[1]),
"consumed line indices must be sorted and deduplicated: {:?}",
outcome.source_line_indices
);
}
}
#[test]
fn the_table_polygon_covers_its_cells_in_page_coordinates() {
let Some(mut assets) = assets() else {
return;
};
let bytes = std::fs::read(&assets.page_with_table).expect("the page must be readable");
let output = assets
.engine
.recognize(request(bytes, table_policy(&assets)))
.expect("the table route must not fail");
let mut checked = 0;
for region in table_regions(&output) {
let outcome = region.table.as_ref().expect("table outcome");
let Some(table_polygon) = region.polygon else {
panic!("a table region must carry a polygon");
};
let table_box = bounds(&table_polygon.points);
let mut union: Option<[f32; 4]> = None;
for cell in &outcome.cells {
let Some(polygon) = cell.polygon else {
continue;
};
let cell_box = bounds(&polygon.points);
union = Some(match union {
None => cell_box,
Some(current) => [
current[0].min(cell_box[0]),
current[1].min(cell_box[1]),
current[2].max(cell_box[2]),
current[3].max(cell_box[3]),
],
});
}
let Some(union) = union else {
continue;
};
checked += 1;
let tolerance = 1.0;
assert!(
union[0] >= table_box[0] - tolerance
&& union[1] >= table_box[1] - tolerance
&& union[2] <= table_box[2] + tolerance
&& union[3] <= table_box[3] + tolerance,
"cell union {union:?} escapes the table polygon {table_box:?}"
);
}
assert!(
checked > 0,
"no table region carried cell geometry, so this test proved nothing"
);
}
#[test]
fn the_page_path_never_asks_the_backfill_to_recognize() {
let source = include_str!("rapid_structure.rs");
let test_marker = ["#[cfg", "(test)]"].concat();
let production = source
.split(&test_marker)
.next()
.expect("the production section must come first");
assert!(
production.len() < source.len(),
"the split must actually cut the test modules away"
);
let crop_probe = ["Backfill", "Source::", "Crop"].concat();
assert_eq!(
production.matches(&crop_probe).count(),
0,
"the page pipeline must reuse page-level lines (G2/G7); crop self-recognition \
belongs to the offline evaluator, not the page path"
);
let reuse_probe = ["Backfill", "Source::", "Reuse"].concat();
assert_eq!(
production.matches(&reuse_probe).count(),
1,
"the page pipeline must build exactly one reuse backfill source"
);
let run_probe = ["table_session", "(policy)?"].concat();
assert_eq!(
production.matches(&run_probe).count(),
1,
"the structure session must be obtained once per request, not per cell"
);
}
#[test]
fn a_disabled_table_route_ignores_the_model_path_entirely() {
let Some(mut assets) = assets() else {
return;
};
let bytes = std::fs::read(&assets.page_with_table).expect("the page must be readable");
let policy = TablePolicy {
enabled: false,
model_path: Some(PathBuf::from("does/not/exist/slanet-plus.onnx")),
..TablePolicy::default()
};
let output = assets
.engine
.recognize(request(bytes, policy))
.expect("a disabled table route must not try to load the model");
assert!(
table_regions(&output).is_empty(),
"a disabled table route must not produce table regions"
);
assert_eq!(
output.stages.table.state,
StageState::Disabled,
"the table stage must be reported as disabled"
);
assert_eq!(output.timings.table_detect_ms, 0.0);
assert_eq!(output.timings.table_structure_infer_ms, 0.0);
assert_eq!(output.timings.table_backfill_ms, 0.0);
}
#[test]
fn a_changed_table_declaration_is_not_answered_by_a_hot_session() {
let Some(model_root) = crate::test_support::ocr_model_root() else {
return;
};
let Some(table_model) = crate::test_support::table_structure_model_path() else {
return;
};
let Some(page) = crate::test_support::page_fixture("04表格与键值对.png") else {
return;
};
let dir = crate::test_support::TempDir::new("pipeline-table-declaration");
let copy = dir.path().join("slanet-plus.onnx");
std::fs::copy(&table_model, ©).expect("copy the real table model");
let declared = crate::model_store::sha256_file(©).expect("hash the copy");
let mut engine =
RapidOcrEngine::new(engine_config(&model_root)).expect("engine should load");
let policy = |expected: Option<String>| TablePolicy {
enabled: true,
model_path: Some(copy.clone()),
expected_model_sha256: expected,
text_policy: TableTextPolicy::Escape,
..TablePolicy::default()
};
let bytes = std::fs::read(&page).expect("the page must be readable");
let output = engine
.recognize(request(bytes.clone(), policy(Some(declared.clone()))))
.expect("the first request with the correct digest must pass");
assert!(
!table_regions(&output).is_empty(),
"the session must actually have been built, or this test proves nothing"
);
let bogus = format!("{:064x}", 0xabcd_u32);
let error = engine
.recognize(request(bytes, policy(Some(bogus.clone()))))
.expect_err("a changed declaration must not be answered by the hot session");
match error {
crate::error::RapidOcrError::HashMismatch {
path: reported,
expected,
actual,
} => {
assert_eq!(reported, copy, "the error must name the declared file");
assert_eq!(expected, bogus);
assert_eq!(
actual, declared,
"the file never changed - only the declaration did"
);
}
other => panic!("expected a hash mismatch from the declaration, got {other:?}"),
}
}
#[test]
fn a_changed_table_config_is_not_answered_by_a_hot_session() {
let Some(model_root) = crate::test_support::ocr_model_root() else {
return;
};
let Some(table_model) = crate::test_support::table_structure_model_path() else {
return;
};
let Some(page) = crate::test_support::page_fixture("04表格与键值对.png") else {
return;
};
let dir = crate::test_support::TempDir::new("pipeline-table-config");
let copy = dir.path().join("slanet-plus.onnx");
std::fs::copy(&table_model, ©).expect("copy the real table model");
let declared = crate::model_store::sha256_file(©).expect("hash the copy");
let bytes = std::fs::read(&page).expect("the page must be readable");
let mut engine =
RapidOcrEngine::new(engine_config(&model_root)).expect("engine should load");
let default_side = TablePolicy::default().input_side_len;
let policy =
|input_side_len: usize, fix_input_shape: bool, intra_threads: Option<usize>| {
TablePolicy {
enabled: true,
model_path: Some(copy.clone()),
expected_model_sha256: Some(declared.clone()),
input_side_len,
fix_input_shape,
intra_threads,
..TablePolicy::default()
}
};
let output = engine
.recognize(request(bytes.clone(), policy(default_side, true, None)))
.expect("the first request must pass");
assert!(
!table_regions(&output).is_empty(),
"the session must actually have been used by a table region"
);
let session = engine
.table_session(&policy(default_side, true, None))
.expect("session with the default canvas");
assert_eq!(session.config().input_side_len, default_side);
assert!(session.config().fix_input_shape);
let default_dims = session.input_dims().to_vec();
let wider = default_side + 112;
engine
.recognize(request(bytes.clone(), policy(wider, true, None)))
.expect("a larger canvas must still infer");
let session = engine
.table_session(&policy(wider, true, None))
.expect("session with the wider canvas");
assert_eq!(
session.config().input_side_len,
wider,
"配置变化被旧会话吞掉了:键里没有带 input_side_len"
);
assert_ne!(
session.input_dims(),
default_dims.as_slice(),
"会话配置里写着新边长、实测维度却还是旧的:override 没有重新施加"
);
assert_eq!(
session.input_dims(),
[-1, 3, wider as i64, wider as i64],
"固定形状必须等于这次的 `input_side_len`"
);
let session = engine
.table_session(&policy(wider, false, None))
.expect("session with a dynamic input shape");
assert!(
!session.config().fix_input_shape,
"配置变化被旧会话吞掉了:键里没有带 fix_input_shape"
);
let session = engine
.table_session(&policy(wider, false, Some(1)))
.expect("session with a pinned intra budget");
assert_eq!(
session.session_threads().0,
Some(1),
"配置变化被旧会话吞掉了:键里没有带 intra_threads"
);
}
#[test]
fn a_changed_table_registration_is_not_answered_by_a_hot_session() {
let Some(model_root) = crate::test_support::ocr_model_root() else {
return;
};
let Some(table_model) = crate::test_support::table_structure_model_path() else {
return;
};
let Some(page) = crate::test_support::page_fixture("04表格与键值对.png") else {
return;
};
let dir = crate::test_support::TempDir::new("pipeline-table-registration");
let copy = dir.path().join("slanet-plus.onnx");
std::fs::copy(&table_model, ©).expect("copy the real table model");
let registered = crate::model_store::sha256_file(©).expect("hash the copy");
let size = std::fs::metadata(©).expect("stat the copy").len();
let bytes = std::fs::read(&page).expect("the page must be readable");
let policy = TablePolicy {
enabled: true,
model_path: Some(copy.clone()),
expected_model_sha256: None,
text_policy: TableTextPolicy::Escape,
..TablePolicy::default()
};
let write_manifest = |sha256: &str| {
dir.write(
"manifest.json",
format!(
r#"{{"schema_version":1,"id":"scratch","family":"SLANet","version":"v2.0.0","files":[
{{"name":"slanet-plus.onnx","role":"table_structure","sha256":"{sha256}","size_bytes":{size}}}
]}}"#
)
.as_bytes(),
);
};
write_manifest(®istered);
let mut engine =
RapidOcrEngine::new(engine_config(&model_root)).expect("engine should load");
let output = engine
.recognize(request(bytes.clone(), policy.clone()))
.expect("the first request must pass on the registered digest");
assert!(
!table_regions(&output).is_empty(),
"the session must actually have been used by a table region"
);
let bogus = format!("{:064x}", 0xabcd_u32);
write_manifest(&bogus);
let error = engine
.recognize(request(bytes.clone(), policy.clone()))
.expect_err("a changed registration must not be answered by the hot session");
match error {
crate::error::RapidOcrError::HashMismatch {
path: reported,
expected,
actual,
} => {
assert_eq!(reported, copy, "the error must name the model file");
assert_eq!(
expected, bogus,
"期望必须来自清单里的登记值,而不是策略里那个 `None`"
);
assert_eq!(
actual, registered,
"the file never changed - only the registration did"
);
}
other => panic!("expected a hash mismatch from the registration, got {other:?}"),
}
write_manifest(®istered);
let output = engine
.recognize(request(bytes, policy))
.expect("restoring the registration must pass again");
assert!(!table_regions(&output).is_empty());
}
#[test]
fn table_routing_rejects_roi_and_tile() {
let base = TablePolicy {
enabled: true,
model_path: Some(PathBuf::from("model.onnx")),
..TablePolicy::default()
};
let mut with_roi = request(Vec::new(), base.clone());
with_roi.roi = Some(crate::api::RectU32 {
x: 0,
y: 0,
width: 10,
height: 10,
});
let error = with_roi.validate().expect_err("roi must be rejected");
assert!(error.to_string().contains("roi"), "error: {error}");
let mut with_tile = request(Vec::new(), base);
with_tile.preprocess.tile = Some(crate::api::TilePolicy {
max_width: 100,
max_height: 100,
overlap: 10,
});
let error = with_tile.validate().expect_err("tile must be rejected");
assert!(error.to_string().contains("tiled"), "error: {error}");
}
}
#[cfg(test)]
mod session_cache_rule_tests {
use super::{SessionCacheKey, needs_rebuild};
use crate::model_verify::FileIdentity;
use crate::table::structure::TableStructureConfig;
use crate::test_support::TempDir;
const HELLO_SHA256: &str = "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824";
const BOGUS_SHA256: &str = "0000000000000000000000000000000000000000000000000000000000000000";
#[test]
fn a_changed_declaration_with_an_unchanged_identity_rebuilds() {
let dir = TempDir::new("cache-rule-declaration");
dir.write("model.onnx", b"hello");
let file = FileIdentity::of(&dir.path().join("model.onnx")).expect("identity");
let first = SessionCacheKey::engine_only(file.clone(), Some(HELLO_SHA256));
assert!(
!needs_rebuild(Some(&first), &first),
"the same declaration against the same identity is a hit"
);
assert!(
needs_rebuild(
Some(&first),
&SessionCacheKey::engine_only(file.clone(), Some(BOGUS_SHA256))
),
"a different declaration on an unchanged file must not be answered by the hot session"
);
assert!(
!needs_rebuild(
Some(&first),
&SessionCacheKey::engine_only(file.clone(), Some(&HELLO_SHA256.to_uppercase()))
),
"case is the only difference verification ignores, so it must not force a rebuild"
);
assert!(
needs_rebuild(
Some(&first),
&SessionCacheKey::engine_only(file.clone(), None)
),
"dropping the declaration is a different statement about the file"
);
let undeclared = SessionCacheKey::engine_only(file.clone(), None);
assert!(needs_rebuild(
Some(&undeclared),
&SessionCacheKey::engine_only(file, Some(BOGUS_SHA256))
));
assert!(needs_rebuild(None, &first), "an empty cache always builds");
}
#[test]
fn a_padded_declaration_is_a_different_declaration() {
let dir = TempDir::new("cache-rule-padded");
dir.write("model.onnx", b"hello");
let file = FileIdentity::of(&dir.path().join("model.onnx")).expect("identity");
let first = SessionCacheKey::engine_only(file.clone(), Some(HELLO_SHA256));
assert!(needs_rebuild(
Some(&first),
&SessionCacheKey::engine_only(file, Some(&format!(" {HELLO_SHA256}")))
));
}
#[test]
fn a_changed_file_or_a_changed_path_rebuilds() {
let dir = TempDir::new("cache-rule-file");
dir.write("model.onnx", b"hello");
let before = FileIdentity::of(&dir.path().join("model.onnx")).expect("identity");
dir.write("model.onnx", b"hello!");
let rewritten = FileIdentity::of(&dir.path().join("model.onnx")).expect("identity");
assert!(needs_rebuild(
Some(&SessionCacheKey::engine_only(
before.clone(),
Some(HELLO_SHA256)
)),
&SessionCacheKey::engine_only(rewritten, Some(HELLO_SHA256))
));
dir.write("other.onnx", b"hello");
let other_path = FileIdentity::of(&dir.path().join("other.onnx")).expect("identity");
assert!(
needs_rebuild(
Some(&SessionCacheKey::engine_only(
before.clone(),
Some(HELLO_SHA256)
)),
&SessionCacheKey::engine_only(other_path, Some(HELLO_SHA256))
),
"the same bytes under a different path are a different file"
);
}
#[test]
fn two_observations_of_an_untouched_file_are_the_same_key() {
let dir = TempDir::new("cache-rule-stable");
dir.write("model.onnx", b"hello");
let path = dir.path().join("model.onnx");
let first = SessionCacheKey::engine_only(
FileIdentity::of(&path).expect("identity"),
Some(HELLO_SHA256),
);
let second = SessionCacheKey::engine_only(
FileIdentity::of(&path).expect("identity"),
Some(HELLO_SHA256),
);
assert!(!needs_rebuild(Some(&first), &second));
}
#[test]
fn a_changed_table_config_rebuilds_without_any_file_change() {
let dir = TempDir::new("cache-rule-table-config");
dir.write("slanet-plus.onnx", b"hello");
let file = FileIdentity::of(&dir.path().join("slanet-plus.onnx")).expect("identity");
let base = TableStructureConfig::default();
let first = SessionCacheKey::table(file.clone(), Some(HELLO_SHA256), &base);
assert!(
!needs_rebuild(
Some(&first),
&SessionCacheKey::table(file.clone(), Some(HELLO_SHA256), &base)
),
"identity、声明、配置全都不变 -> 不应该重建"
);
let mut wider = base.clone();
wider.input_side_len += 112;
assert!(
needs_rebuild(
Some(&first),
&SessionCacheKey::table(file.clone(), Some(HELLO_SHA256), &wider)
),
"input_side_len 变了却不重建 = 新画布被旧会话吞掉"
);
let mut unfixed = base.clone();
unfixed.fix_input_shape = !unfixed.fix_input_shape;
assert!(
needs_rebuild(
Some(&first),
&SessionCacheKey::table(file.clone(), Some(HELLO_SHA256), &unfixed)
),
"fix_input_shape 变了却不重建 = 形状策略被旧会话吞掉"
);
let mut threaded = base.clone();
threaded.intra_threads = Some(base.intra_threads.map_or(1, |threads| threads + 1));
assert!(
needs_rebuild(
Some(&first),
&SessionCacheKey::table(file.clone(), Some(HELLO_SHA256), &threaded)
),
"intra_threads 变了却不重建 = 线程预算被旧会话吞掉"
);
assert_ne!(
first,
SessionCacheKey::engine_only(file, Some(HELLO_SHA256)),
"两个站点即使文件身份与声明完全相同,也不是同一把键"
);
}
}