use clap::Parser;
use rapid_structure_rs::evaluation::stats::{PERCENTILE_METHOD, percentile};
use rapid_structure_rs::{
ClassifierPlan, ClassifierPolicy, DetectionPolicy, EngineConfig, ImageInput, OcrEngine,
OcrRequest, OutputPolicy, PreprocessPolicy, RapidOcrEngine, RecognitionPolicy, StagePlan,
TimingLedger, WordOutputMode,
};
use serde_json::json;
use std::{fs, path::PathBuf, sync::Arc, time::Instant};
#[derive(Debug, Parser)]
struct Cli {
#[arg(long = "config")]
config_path: Option<PathBuf>,
#[arg(long, default_value = "datasets/text")]
images_dir: PathBuf,
#[arg(long, default_value_t = 1)]
rounds: usize,
#[arg(long, default_value_t = 0)]
warmup_rounds: usize,
#[arg(long)]
output: Option<PathBuf>,
#[arg(long)]
max_side_len: Option<usize>,
#[arg(long)]
intra_threads: Option<usize>,
#[arg(long)]
table_model: Option<PathBuf>,
#[arg(long)]
table_intra_threads: Option<usize>,
}
fn main() {
if let Err(e) = run() {
eprintln!("error: {e}");
std::process::exit(1);
}
}
fn run() -> Result<(), Box<dyn std::error::Error>> {
let cli = Cli::parse();
if cfg!(debug_assertions) {
eprintln!(
"warning: bench_warm_e2e is running a debug build; use `cargo run --release` for performance comparisons"
);
}
let mut images = Vec::new();
for e in fs::read_dir(&cli.images_dir)? {
let p = e?.path();
if p.is_file()
&& let Some(ext) = p.extension().and_then(|v| v.to_str())
&& matches!(
ext.to_ascii_lowercase().as_str(),
"jpg" | "jpeg" | "png" | "bmp" | "webp" | "tif" | "tiff"
)
{
images.push(p);
}
}
images.sort();
if images.is_empty() {
return Err(format!("no images found under {}", cli.images_dir.display()).into());
}
let mut cfg = match cli.config_path {
Some(p) => EngineConfig::from_yaml_file(p)?,
None => EngineConfig::default(),
};
if let Some(v) = cli.max_side_len {
cfg.global.max_side_len = v;
}
if let Some(v) = cli.intra_threads {
cfg.runtime.intra_threads = Some(v);
cfg.runtime.inter_threads = Some(1);
cfg.runtime.auto_tune_threads = false;
}
let effective_max_side = cfg.global.max_side_len as u32;
let runtime_config = serde_json::to_value(&cfg.runtime)?;
let benchmark_meta = json!({
"build_profile": if cfg!(debug_assertions) { "debug" } else { "release" },
"max_side_len": effective_max_side,
"runtime": runtime_config,
"timing_scope": {
"wall_ms": "file_read_plus_ocr",
"ocr_total_ms": "ocr_pipeline_only",
},
});
let engine_start = Instant::now();
let mut engine = RapidOcrEngine::new(cfg)?;
let init_ms = engine_start.elapsed().as_secs_f64() * 1000.0;
let thread_plan = serde_json::to_value(&engine.runtime_profile().threads)?;
let table_policy = match &cli.table_model {
Some(model_path) => rapid_structure_rs::TablePolicy {
enabled: true,
model_path: Some(model_path.clone()),
intra_threads: cli.table_intra_threads,
..rapid_structure_rs::TablePolicy::default()
},
None => rapid_structure_rs::TablePolicy::default(),
};
let table_meta = json!({
"enabled": table_policy.enabled,
"model_path": cli.table_model.as_ref().map(|p| p.display().to_string()),
"input_side_len": table_policy.input_side_len,
"intra_threads": table_policy.intra_threads,
});
let request = |bytes: Vec<u8>| OcrRequest {
input: ImageInput::Encoded(Arc::from(bytes)),
roi: None,
scale_hint: None,
stages: StagePlan {
detect: true,
classify: ClassifierPlan {
policy: ClassifierPolicy::IfAvailable,
apply_rotation: true,
},
recognize: true,
},
preprocess: PreprocessPolicy {
max_side: Some(effective_max_side),
..PreprocessPolicy::default()
},
detection: DetectionPolicy::default(),
recognition: RecognitionPolicy {
words: WordOutputMode::Off,
},
output: OutputPolicy::default(),
formula: rapid_structure_rs::FormulaPolicy::default(),
table: table_policy.clone(),
};
for _ in 0..cli.warmup_rounds {
for p in &images {
let _ = engine.recognize(request(fs::read(p)?))?;
}
}
let mut wall = Vec::new();
let mut total = Vec::new();
let mut regions = Vec::new();
let mut stages = StageSamples::default();
let mut ledgers: Vec<TimingLedger> = Vec::new();
let mut provider_resolution = None;
for _ in 0..cli.rounds {
for p in &images {
let start = Instant::now();
let out = engine.recognize(request(fs::read(p)?))?;
wall.push(start.elapsed().as_secs_f64() * 1000.0);
total.push(out.timings.total_ms as f64);
regions.push(out.regions.len() as f64);
stages.push(&out);
ledgers.push(TimingLedger::from_timings(&out.timings));
if provider_resolution.is_none() {
let describe = |info: &rapid_structure_rs::ProviderResolutionInfo| {
serde_json::json!({
"requested": format!("{:?}", info.requested),
"selected_ep": format!("{:?}", info.selected_ep),
"fallback_to_cpu": info.fallback_to_cpu,
})
};
provider_resolution = Some(serde_json::json!({
"model_id": out.engine.model_id,
"detector": describe(&out.engine.provider.detector),
"classifier": out.engine.provider.classifier.as_ref().map(&describe),
"recognizer": describe(&out.engine.provider.recognizer),
}));
}
}
}
let stage_report = stages.into_report(total.len());
let total_avg = if total.is_empty() {
0.0
} else {
total.iter().sum::<f64>() / total.len() as f64
};
let ledger_report = ledger_report(&ledgers);
let timing_split = inference_share(&ledgers, total_avg);
let report = json!({
"meta": {
"images_dir": cli.images_dir,
"image_count": images.len(),
"rounds": cli.rounds,
"warmup_rounds": cli.warmup_rounds,
"init_ms": init_ms,
"benchmark": benchmark_meta,
"thread_plan": thread_plan,
"table": table_meta,
"provider_resolution": provider_resolution,
"ort_runtime": rapid_structure_rs::ort_runtime_fingerprint(),
"ort_runtime_version": rapid_structure_rs::ort_runtime_version(),
},
"percentile_method": PERCENTILE_METHOD,
"stats": {
"wall_ms": stats(&wall),
"ocr_total_ms": stats(&total),
"regions": stats(®ions),
},
"stages": stage_report,
"timing_ledger": ledger_report,
"timing_split": timing_split,
"memory": {
"peak_working_set_bytes": rapid_structure_rs::peak_working_set_bytes(),
"source": rapid_structure_rs::peak_memory_source(),
},
});
let text = serde_json::to_string_pretty(&report)?;
if let Some(p) = cli.output {
fs::write(&p, &text)?;
}
println!("{text}");
Ok(())
}
fn stats(v: &[f64]) -> serde_json::Value {
if v.is_empty() {
return json!({"count":0});
}
let mut s = v.to_vec();
s.sort_by(|a, b| a.total_cmp(b));
let avg = s.iter().sum::<f64>() / s.len() as f64;
json!({"count":s.len(),"avg":avg,"p50":percentile(&s,0.50),"p90":percentile(&s,0.90),"min":s[0],"max":s[s.len()-1]})
}
#[derive(Default)]
struct StageSamples {
input_decode_ms: Vec<f64>,
input_resize_ms: Vec<f64>,
input_crop_ms: Vec<f64>,
detector_preprocess_ms: Vec<f64>,
detector_infer_ms: Vec<f64>,
detector_postprocess_ms: Vec<f64>,
classifier_preprocess_ms: Vec<f64>,
classifier_infer_ms: Vec<f64>,
classifier_postprocess_ms: Vec<f64>,
recognizer_preprocess_ms: Vec<f64>,
recognizer_infer_ms: Vec<f64>,
recognizer_postprocess_ms: Vec<f64>,
preprocess_ms: Vec<f64>,
postprocess_ms: Vec<f64>,
}
impl StageSamples {
fn push(&mut self, out: &rapid_structure_rs::OcrOutput) {
let push_opt = |dst: &mut Vec<f64>, value: Option<f32>| {
if let Some(value) = value {
dst.push(f64::from(value));
}
};
push_opt(&mut self.input_decode_ms, Some(out.timings.decode_ms));
push_opt(&mut self.input_resize_ms, Some(out.timings.resize_ms));
push_opt(&mut self.input_crop_ms, Some(out.timings.crop_ms));
push_opt(
&mut self.detector_preprocess_ms,
out.stages.detector.timing.map(|t| t.preprocess_ms),
);
push_opt(
&mut self.detector_infer_ms,
out.stages.detector.timing.map(|t| t.infer_ms),
);
push_opt(
&mut self.detector_postprocess_ms,
out.stages.detector.timing.map(|t| t.postprocess_ms),
);
push_opt(
&mut self.classifier_preprocess_ms,
out.stages.classifier.timing.map(|t| t.preprocess_ms),
);
push_opt(
&mut self.classifier_infer_ms,
out.stages.classifier.timing.map(|t| t.infer_ms),
);
push_opt(
&mut self.classifier_postprocess_ms,
out.stages.classifier.timing.map(|t| t.postprocess_ms),
);
push_opt(
&mut self.recognizer_preprocess_ms,
out.stages.recognizer.timing.map(|t| t.preprocess_ms),
);
push_opt(
&mut self.recognizer_infer_ms,
out.stages.recognizer.timing.map(|t| t.infer_ms),
);
push_opt(
&mut self.recognizer_postprocess_ms,
out.stages.recognizer.timing.map(|t| t.postprocess_ms),
);
self.preprocess_ms
.push(f64::from(out.timings.preprocess_ms));
self.postprocess_ms
.push(f64::from(out.timings.postprocess_ms));
}
fn into_report(self, samples: usize) -> serde_json::Value {
json!({
"unit": "ms",
"samples": samples,
"input": {
"decode_ms": stats(&self.input_decode_ms),
"resize_ms": stats(&self.input_resize_ms),
"crop_ms": stats(&self.input_crop_ms),
},
"detector": {
"preprocess_ms": stats(&self.detector_preprocess_ms),
"infer_ms": stats(&self.detector_infer_ms),
"postprocess_ms": stats(&self.detector_postprocess_ms),
},
"classifier": {
"preprocess_ms": stats(&self.classifier_preprocess_ms),
"infer_ms": stats(&self.classifier_infer_ms),
"postprocess_ms": stats(&self.classifier_postprocess_ms),
},
"recognizer": {
"preprocess_ms": stats(&self.recognizer_preprocess_ms),
"infer_ms": stats(&self.recognizer_infer_ms),
"postprocess_ms": stats(&self.recognizer_postprocess_ms),
},
"page_total": {
"preprocess_ms": stats(&self.preprocess_ms),
"postprocess_ms": stats(&self.postprocess_ms),
},
})
}
}
fn ledger_report(ledgers: &[TimingLedger]) -> serde_json::Value {
if ledgers.is_empty() {
return json!({
"unit": "ms",
"samples": 0,
"error": "no timing samples were collected",
});
}
let mean = TimingLedger::mean(ledgers);
let conservation = mean.conservation();
let shares = mean.shares();
json!({
"unit": "ms",
"samples": ledgers.len(),
"basis": "per-sample ledger, then per-field mean (each component counted exactly once)",
"components": mean,
"totals": {
"input_ms": mean.input_ms(),
"input_preprocess_ms": mean.input_preprocess_ms,
"input_overflow_ms": mean.input_overflow_ms,
"model_preprocess_ms": mean.model_preprocess_ms(),
"inference_ms": mean.inference_ms(),
"model_postprocess_ms": mean.model_postprocess_ms(),
"page_postprocess_ms": mean.page_postprocess_ms,
"formula_ms": mean.formula_ms,
"table_detect_ms": mean.table_detect_ms,
"table_structure_infer_ms": mean.table_structure_infer_ms,
"table_backfill_ms": mean.table_backfill_ms,
"rust_ms": mean.rust_ms(),
"attributed_ms": mean.attributed_ms(),
"unattributed_ms": mean.unattributed_ms,
},
"shares": shares,
"conservation": conservation,
"per_sample_unattributed_ms": stats(
&ledgers.iter().map(|ledger| ledger.unattributed_ms).collect::<Vec<_>>()
),
"per_sample_residual_ms": stats(
&ledgers
.iter()
.map(|ledger| ledger.conservation().residual_ms)
.collect::<Vec<_>>()
),
})
}
fn inference_share(ledgers: &[TimingLedger], total_avg: f64) -> serde_json::Value {
if ledgers.is_empty() {
return json!({"error": "no timing samples were collected"});
}
if total_avg <= 0.0 {
return json!({"error": "ocr_total_ms.avg is zero; cannot compute shares"});
}
let mean = TimingLedger::mean(ledgers);
let inference = mean.inference_ms();
let rust = mean.rust_ms();
let input_overflow = mean.input_overflow_ms;
let unattributed = mean.unattributed_ms;
let share = |value: f64| value / total_avg;
let identity_sum = inference + rust + input_overflow + unattributed;
json!({
"basis": "per-sample ledger mean; the same samples back stats.ocr_total_ms",
"identity": "ort_inference_ms + rust_ms + input_overflow_ms + unattributed_ms == denominator_ms",
"identity_residual_ms": total_avg - identity_sum,
"denominator_ms": total_avg,
"ort_inference_ms": inference,
"ort_inference_share": share(inference),
"detector_infer_share": share(mean.detector_infer_ms),
"classifier_infer_share": share(mean.classifier_infer_ms),
"recognizer_infer_share": share(mean.recognizer_infer_ms),
"table_structure_infer_share": share(mean.table_structure_infer_ms),
"rust_ms": rust,
"rust_share": share(rust),
"rust_input_ms": mean.input_ms(),
"rust_input_share": share(mean.input_ms()),
"rust_model_preprocess_ms": mean.model_preprocess_ms(),
"rust_model_preprocess_share": share(mean.model_preprocess_ms()),
"rust_model_postprocess_ms": mean.model_postprocess_ms(),
"rust_model_postprocess_share": share(mean.model_postprocess_ms()),
"rust_page_postprocess_ms": mean.page_postprocess_ms,
"rust_page_postprocess_share": share(mean.page_postprocess_ms),
"rust_table_detect_ms": mean.table_detect_ms,
"rust_table_detect_share": share(mean.table_detect_ms),
"rust_table_backfill_ms": mean.table_backfill_ms,
"rust_table_backfill_share": share(mean.table_backfill_ms),
"input_overflow_ms": input_overflow,
"input_overflow_share": share(input_overflow),
"attributed_ms": mean.attributed_ms(),
"attributed_share": share(mean.attributed_ms()),
"unattributed_ms": unattributed,
"unattributed_share": share(unattributed),
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn the_page_benchmark_stats_report_a_frozen_key_set() {
let full = stats(&[1.0, 2.0, 3.0, 4.0]);
let mut keys: Vec<&str> = full
.as_object()
.expect("non-empty stats must be an object")
.keys()
.map(String::as_str)
.collect();
keys.sort_unstable();
assert_eq!(
keys,
vec!["avg", "count", "max", "min", "p50", "p90"],
"the page benchmark must keep exactly these stats keys"
);
let empty = stats(&[]);
let empty_obj = empty.as_object().expect("empty stats must be an object");
let empty_keys: Vec<&str> = empty_obj.keys().map(String::as_str).collect();
assert_eq!(
empty_keys,
vec!["count"],
"an empty distribution reports only `count`"
);
assert_eq!(empty["count"], 0);
}
#[test]
fn the_page_benchmark_stats_use_the_shared_linear_percentile() {
use rapid_structure_rs::evaluation::stats::percentile;
let sorted = [1.0, 2.0, 3.0, 4.0];
let full = stats(&[1.0, 2.0, 3.0, 4.0]);
assert_eq!(full["p50"], percentile(&sorted, 0.50));
assert_eq!(full["p50"], 2.5);
assert!((full["p90"].as_f64().unwrap() - 3.7).abs() < 1e-12);
}
#[test]
fn the_page_benchmark_declares_the_linear_percentile_method() {
assert_eq!(
rapid_structure_rs::evaluation::stats::PERCENTILE_METHOD,
"linear"
);
}
fn ledger(
total_ms: f64,
infer_ms: f64,
input_named_within_ms: f64,
input_overflow_ms: f64,
) -> TimingLedger {
let mut ledger = TimingLedger {
total_ms,
input_preprocess_ms: input_named_within_ms,
input_named_within_ms,
input_overflow_ms,
detector_infer_ms: infer_ms,
..TimingLedger::default()
};
ledger.unattributed_ms = ledger.total_ms - ledger.attributed_ms();
ledger
}
fn number(split: &serde_json::Value, key: &str) -> f64 {
split[key]
.as_f64()
.unwrap_or_else(|| panic!("`timing_split.{key}` must be a number: {split}"))
}
fn assert_split_identity(split: &serde_json::Value) {
let total = number(split, "denominator_ms");
let sum = number(split, "ort_inference_ms")
+ number(split, "rust_ms")
+ number(split, "input_overflow_ms")
+ number(split, "unattributed_ms");
assert!(
(sum - total).abs() < 1e-9,
"the reported identity must hold: {sum} != {total}: {split}"
);
assert!(
number(split, "identity_residual_ms").abs() < 1e-9,
"the report must carry a zero identity residual: {split}"
);
let share_sum = number(split, "ort_inference_share")
+ number(split, "rust_share")
+ number(split, "input_overflow_share")
+ number(split, "unattributed_share");
assert!(
(share_sum - 1.0).abs() < 1e-9,
"the reported shares must sum to 1: {share_sum}: {split}"
);
assert!(
split["identity"]
.as_str()
.is_some_and(|value| value.contains("input_overflow_ms")),
"the report must name `input_overflow_ms` as part of the identity: {split}"
);
}
#[test]
fn timing_split_identity_holds_without_input_overflow() {
let sample = ledger(1000.0, 800.0, 100.0, 0.0);
assert_eq!(sample.unattributed_ms, 100.0);
let split = inference_share(&[sample], sample.total_ms);
assert_eq!(number(&split, "denominator_ms"), 1000.0);
assert_eq!(number(&split, "ort_inference_ms"), 800.0);
assert_eq!(number(&split, "rust_ms"), 100.0);
assert_eq!(number(&split, "input_overflow_ms"), 0.0);
assert_eq!(number(&split, "unattributed_ms"), 100.0);
assert_split_identity(&split);
}
#[test]
fn timing_split_identity_keeps_input_overflow_explicit() {
let timings = rapid_structure_rs::OcrTimings {
decode_ms: 2.0,
resize_ms: 7.0,
crop_ms: 3.0,
preprocess_ms: 5.0,
detector_preprocess_ms: 1.0,
detector_infer_ms: 10.0,
detector_postprocess_ms: 1.0,
detect_ms: 12.0,
classifier_preprocess_ms: 0.0,
classifier_infer_ms: 0.0,
classifier_postprocess_ms: 0.0,
classify_ms: 0.0,
recognizer_preprocess_ms: 1.0,
recognizer_infer_ms: 20.0,
recognizer_postprocess_ms: 2.0,
recognize_ms: 23.0,
formula_ms: 0.0,
table_detect_ms: 0.0,
table_structure_infer_ms: 0.0,
table_backfill_ms: 0.0,
postprocess_ms: 3.0,
total_ms: 43.0,
};
let sample = TimingLedger::from_timings(&timings);
assert_eq!(sample.input_overflow_ms, 7.0);
assert_eq!(sample.total_ms, 43.0);
let split = inference_share(&[sample], sample.total_ms);
assert_eq!(number(&split, "ort_inference_ms"), 30.0);
assert_eq!(number(&split, "rust_ms"), 13.0);
assert_eq!(number(&split, "input_overflow_ms"), 7.0);
assert_eq!(number(&split, "unattributed_ms"), -7.0);
assert!(
(number(&split, "attributed_ms")
- (number(&split, "rust_ms") + number(&split, "ort_inference_ms"))
- number(&split, "input_overflow_ms"))
.abs()
< 1e-9,
"the overflow must be the only gap between rust+inference and attributed: {split}"
);
let stale = number(&split, "ort_inference_ms")
+ number(&split, "rust_ms")
+ number(&split, "unattributed_ms");
assert!(
(stale - number(&split, "denominator_ms")).abs() > 1.0,
"this sample must really exercise the overflow branch: {split}"
);
assert_split_identity(&split);
}
}