use std::io::{Cursor, Read, Write};
use std::net::{SocketAddr, TcpStream};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use super::download::{DownloadJob, DownloadSink, DownloaderFactory, ModelDownloader};
use super::engine::{BackendProvider, EngineFactory, OcrBackend};
use super::evaluate::EvalRoot;
use super::flowlog::FlowSink;
use super::http::{BoundServer, ServeHandle};
use super::limits::RawServeLimits;
use super::model_plan::ModelPlan;
use super::queue::QueueClass;
use super::run::render_page;
use super::security::{LocalOrigin, ServeToken, generate_nonce};
use super::server::{FreeSpaceFactory, ServeContext, ServeShared};
use super::state::ServeStartup;
use rapid_structure_rs::{
CoordinateSpace, DetectionOutcome, DownloadError, EngineConfig, EngineInfo, FileIdentity,
FormulaOutcome, FormulaPolicy, GenericProviderPreference, ImageInfo, ImageSize, LangDet,
LangRec, ModelFileSpec, ModelType, OcrOutput, OcrRegion, OcrRequest, OcrTimings, OcrVersion,
OcrWord, Polygon, ProviderInfo, ProviderResolutionInfo, RapidOcrError, RecognitionOutcome,
RegionKind, RegionSource, ResolvedProvider, StageReports, TableCellMatch, TableCellOutcome,
TableDegradation, TableOutcome, TablePolicy, TableStructureConfidence, TableTextPolicy,
WordOutputMode, sha256_file,
};
use serde_json::{Value, json};
#[derive(Clone)]
struct Scripted {
delay: Duration,
regions: usize,
text_bytes: usize,
fail: bool,
calls: Arc<AtomicUsize>,
running: Arc<AtomicBool>,
ep: String,
served: Arc<Mutex<Vec<String>>>,
formulas: Arc<Mutex<Vec<(String, FormulaPolicy)>>>,
tables: Arc<Mutex<Vec<(String, TablePolicy)>>>,
gate: Option<Arc<Gate>>,
formula_regions: usize,
table_regions: usize,
word_modes: Arc<Mutex<Vec<(String, WordOutputMode)>>>,
}
impl Scripted {
fn fast() -> Self {
Self {
delay: Duration::from_millis(0),
regions: 3,
text_bytes: 0,
fail: false,
calls: Arc::new(AtomicUsize::new(0)),
running: Arc::new(AtomicBool::new(false)),
ep: "cpu".to_string(),
served: Arc::new(Mutex::new(Vec::new())),
formulas: Arc::new(Mutex::new(Vec::new())),
tables: Arc::new(Mutex::new(Vec::new())),
gate: None,
formula_regions: 0,
table_regions: 0,
word_modes: Arc::new(Mutex::new(Vec::new())),
}
}
fn with_formula_regions(mut self, count: usize) -> Self {
self.formula_regions = count;
self
}
fn with_table_regions(mut self, count: usize) -> Self {
self.table_regions = count;
self
}
fn slow(delay: Duration) -> Self {
Self {
delay,
..Self::fast()
}
}
fn with_gate(mut self, gate: Arc<Gate>) -> Self {
self.gate = Some(gate);
self
}
fn formulas(&self) -> Vec<(String, FormulaPolicy)> {
self.formulas
.lock()
.expect("the scripted recorder is not poisoned")
.clone()
}
fn formula_policy(&self, body: &str) -> FormulaPolicy {
self.formulas()
.iter()
.find(|(recorded, _)| recorded == body)
.unwrap_or_else(|| panic!("no formula policy recorded for request body {body:?}"))
.1
.clone()
}
fn tables(&self) -> Vec<(String, TablePolicy)> {
self.tables
.lock()
.expect("the scripted recorder is not poisoned")
.clone()
}
fn table_policy(&self, body: &str) -> TablePolicy {
self.tables()
.iter()
.find(|(recorded, _)| recorded == body)
.unwrap_or_else(|| panic!("no table policy recorded for request body {body:?}"))
.1
.clone()
}
fn word_modes(&self) -> Vec<(String, WordOutputMode)> {
self.word_modes
.lock()
.expect("the scripted recorder is not poisoned")
.clone()
}
fn word_mode(&self, body: &str) -> WordOutputMode {
self.word_modes()
.iter()
.find(|(recorded, _)| recorded == body)
.unwrap_or_else(|| panic!("no word mode recorded for request body {body:?}"))
.1
}
fn factory(self) -> EngineFactory {
Arc::new(
move |_config: &EngineConfig| -> Result<Box<dyn OcrBackend>, RapidOcrError> {
Ok(Box::new(ScriptedBackend {
state: self.clone(),
}))
},
)
}
}
struct ScriptedBackend {
state: Scripted,
}
impl OcrBackend for ScriptedBackend {
fn recognize(&mut self, request: OcrRequest) -> Result<OcrOutput, RapidOcrError> {
self.state.calls.fetch_add(1, Ordering::SeqCst);
let body = match &request.input {
rapid_structure_rs::ImageInput::Encoded(bytes) => {
String::from_utf8_lossy(bytes).to_string()
}
other => format!("<non-encoded input: {other:?}>"),
};
self.state
.formulas
.lock()
.expect("the scripted recorder is not poisoned")
.push((body.clone(), request.formula.clone()));
self.state
.word_modes
.lock()
.expect("the scripted recorder is not poisoned")
.push((body.clone(), request.recognition.words));
self.state
.tables
.lock()
.expect("the scripted recorder is not poisoned")
.push((body, request.table.clone()));
self.state
.served
.lock()
.expect("the scripted recorder is not poisoned")
.push(self.state.ep.clone());
if let Some(gate) = &self.state.gate {
gate.wait();
}
self.state.running.store(true, Ordering::SeqCst);
if !self.state.delay.is_zero() {
std::thread::sleep(self.state.delay);
}
self.state.running.store(false, Ordering::SeqCst);
if self.state.fail {
return Err(RapidOcrError::InvalidImage(
"scripted failure: the image cannot be decoded".to_string(),
));
}
Ok(scripted_output_modes(
self.state.regions,
self.state.text_bytes,
self.state.formula_regions,
self.state.table_regions,
request.recognition.words,
))
}
fn provider(&self) -> BackendProvider {
BackendProvider {
selected_ep: self.state.ep.clone(),
fallback_to_cpu: false,
}
}
}
pub(crate) fn scripted_output(regions: usize, text_bytes: usize) -> OcrOutput {
scripted_output_with(regions, text_bytes, 0, 0)
}
fn scripted_table_outcome() -> TableOutcome {
TableOutcome {
html: "<html><body><table><tr><td>a</td><td><script>alert(1)</script></td></tr></table></body></html>"
.to_string(),
table_html:
"<table><tr><td>a</td><td><script>alert(1)</script></td></tr></table>".to_string(),
markdown: "| a | <script>alert(1)</script> |\n| --- | --- |\n".to_string(),
rows: 1,
cols: 2,
confidence: TableStructureConfidence::Low,
degradations: vec![TableDegradation::GridHoles { holes: 1 }],
holes: 1,
cells: vec![
TableCellOutcome {
row: 0,
col: 0,
rowspan: 1,
colspan: 1,
text: "a".to_string(),
score: Some(0.9),
match_kind: TableCellMatch::Single,
polygon: Some(Polygon {
points: [[1.0, 1.0], [5.0, 1.0], [5.0, 3.0], [1.0, 3.0]],
}),
},
TableCellOutcome {
row: 0,
col: 1,
rowspan: 1,
colspan: 1,
text: "<script>alert(1)</script>".to_string(),
score: None,
match_kind: TableCellMatch::BlankBox,
polygon: None,
},
],
source_line_indices: vec![0],
model_id: "scripted-table".to_string(),
}
}
pub(crate) fn scripted_output_with(
regions: usize,
text_bytes: usize,
formula_regions: usize,
table_regions: usize,
) -> OcrOutput {
scripted_output_modes(
regions,
text_bytes,
formula_regions,
table_regions,
WordOutputMode::Off,
)
}
const SCRIPT_ROW: (f32, f32, f32, f32) = (1.0, 1.0, 10.0, 5.0);
fn scripted_words(text: &str, mode: WordOutputMode) -> Option<Vec<OcrWord>> {
let items: Vec<&str> = match mode {
WordOutputMode::Off => return None,
WordOutputMode::Words => text.split_whitespace().collect(),
WordOutputMode::Chars => text
.char_indices()
.map(|(start, ch)| &text[start..start + ch.len_utf8()])
.collect(),
};
let (x0, y0, x1, y1) = SCRIPT_ROW;
let kind = mode.kind_for();
let band = (x1 - x0) / items.len().max(1) as f32;
Some(
items
.iter()
.enumerate()
.map(|(index, item)| {
let left = x0 + band * index as f32;
let right = left + band;
OcrWord {
text: (*item).to_string(),
score: 0.9,
polygon: Polygon {
points: [[left, y0], [right, y0], [right, y1], [left, y1]],
},
kind,
}
})
.collect(),
)
}
pub(crate) fn scripted_output_modes(
regions: usize,
text_bytes: usize,
formula_regions: usize,
table_regions: usize,
mode: WordOutputMode,
) -> OcrOutput {
let provider = ProviderResolutionInfo {
requested: GenericProviderPreference::Cpu,
selected_ep: ResolvedProvider::Cpu,
fallback_to_cpu: false,
};
let built = (0..regions)
.map(|index| {
let text = if text_bytes == 0 {
format!("region-{index}")
} else {
"x".repeat(text_bytes)
};
let (x0, y0, x1, y1) = SCRIPT_ROW;
let words = scripted_words(&text, mode);
OcrRegion::text(
RegionSource::Detected {
detector_index: index,
},
Some(Polygon {
points: [[x0, y0], [x1, y0], [x1, y1], [x0, y1]],
}),
Some(DetectionOutcome { score: 0.9 }),
None,
Some(RecognitionOutcome {
text,
score: 0.9,
words,
}),
)
})
.chain((0..formula_regions).map(|index| OcrRegion {
source: RegionSource::Detected {
detector_index: regions + index,
},
kind: RegionKind::Formula,
polygon: Some(Polygon {
points: [[2.0, 2.0], [20.0, 2.0], [20.0, 9.0], [2.0, 9.0]],
}),
detection: Some(DetectionOutcome { score: 0.88 }),
classification: None,
recognition: None,
formula: Some(FormulaOutcome {
latex: format!("E = mc^{{{index}}}"),
eos_index: Some(3),
truncated: false,
model_id: "scripted-formula".to_string(),
token_ids: None,
}),
table: None,
}))
.chain((0..table_regions).map(|index| {
OcrRegion::table(
RegionSource::Detected {
detector_index: regions + formula_regions + index,
},
Some(Polygon {
points: [[1.0, 1.0], [30.0, 1.0], [30.0, 12.0], [1.0, 12.0]],
}),
Some(DetectionOutcome { score: 0.85 }),
scripted_table_outcome(),
)
}))
.collect();
OcrOutput {
schema_version: 1,
image: ImageInfo {
original_size: ImageSize {
width: 100,
height: 50,
},
processed_size: ImageSize {
width: 100,
height: 50,
},
coordinate_space: CoordinateSpace::Image,
},
stages: StageReports::default(),
regions: built,
timings: OcrTimings {
total_ms: 1.0,
..OcrTimings::default()
},
engine: EngineInfo {
model_id: "scripted".to_string(),
provider: ProviderInfo {
detector: provider,
classifier: None,
recognizer: provider,
},
},
}
}
struct TestOptions {
limits: RawServeLimits,
allow_download: bool,
allow_download_hosts: Vec<String>,
allow_provider_fallback: bool,
formula_detector: Option<PathBuf>,
table_model: Option<PathBuf>,
eval_root: Option<EvalRoot>,
engine_factory: EngineFactory,
downloader: DownloaderFactory,
free_space: FreeSpaceFactory,
model_dir: PathBuf,
engine_config: EngineConfig,
post_verify: Option<Box<dyn Fn() + Send + Sync>>,
flow: FlowSink,
}
impl TestOptions {
fn new(model_dir: PathBuf, scripted: Scripted) -> Self {
let eval_root = EvalRoot::new(&model_dir).ok();
Self {
limits: RawServeLimits::default(),
allow_download: false,
allow_download_hosts: Vec::new(),
allow_provider_fallback: false,
formula_detector: None,
table_model: None,
eval_root,
engine_factory: scripted.factory(),
downloader: ScriptedDownload::default().factory(),
free_space: Arc::new(|_dir: &Path| Ok(1 << 40)),
model_dir,
engine_config: test_engine_config(),
post_verify: None,
flow: FlowSink::Off,
}
}
fn with_flow_capture(mut self) -> (Self, Arc<Mutex<Vec<String>>>) {
let FlowSink::Capture(lines) = FlowSink::capture() else {
unreachable!("FlowSink::capture always yields a Capture sink");
};
self.flow = FlowSink::Capture(Arc::clone(&lines));
(self, lines)
}
fn with_formula_routing(mut self, detector: &Path) -> Self {
std::fs::write(detector, plausible_onnx_bytes("detector fixture"))
.expect("write the detector fixture");
self.formula_detector = Some(detector.to_path_buf());
self
}
fn with_table_routing(mut self, model: &Path) -> Self {
std::fs::write(model, plausible_onnx_bytes("table structure fixture"))
.expect("write the table structure fixture");
self.table_model = Some(model.to_path_buf());
self
}
fn without_eval_root(mut self) -> Self {
self.eval_root = None;
self
}
fn with_limits(mut self, limits: RawServeLimits) -> Self {
self.limits = limits;
self
}
}
fn test_engine_config() -> EngineConfig {
let mut config = EngineConfig::default();
config.det.ocr_version = OcrVersion::PPocrV6;
config.det.model_type = ModelType::Small;
config.det.lang = LangDet::Multi;
config.rec.model.ocr_version = OcrVersion::PPocrV6;
config.rec.model.model_type = ModelType::Small;
config.rec.model.lang = LangRec::Ch;
config
}
struct TestServer {
handle: Option<ServeHandle>,
addr: SocketAddr,
token: String,
host: String,
origin: String,
}
impl TestServer {
fn start(options: TestOptions) -> Self {
let bound = BoundServer::bind(0).expect("an ephemeral port must bind");
let addr = bound.local_addr();
let local = LocalOrigin::from_bound(addr).expect("loopback");
let token = ServeToken::generate().expect("the OS CSPRNG must be available");
let nonce = generate_nonce().expect("the OS CSPRNG must be available");
let page = render_page(&token, &nonce).expect("the page must inject cleanly");
let startup = ServeStartup::validate(
options.limits,
options.engine_config,
None,
None,
options.allow_provider_fallback,
)
.expect("the test limits must be valid");
let model_plan = ModelPlan::resolve(
&options.model_dir,
&startup.plan.engine,
options.formula_detector.as_deref(),
options.table_model.as_deref(),
)
.expect("model plan");
let snapshot = model_plan.snapshot();
let post_verify = options.post_verify;
let context = ServeContext {
limits: startup.limits,
plan: startup.plan,
model_plan,
snapshot,
token,
local,
page,
nonce,
allow_download: options.allow_download,
allow_download_hosts: options.allow_download_hosts,
allow_provider_fallback: options.allow_provider_fallback,
eval_root: options.eval_root,
flow: options.flow,
engine_factory: options.engine_factory,
downloader: options.downloader,
free_space: options.free_space,
post_verify,
};
let handle = ServeHandle::start(bound, context).expect("the runtime must start");
let token = handle.shared().token().as_str().to_string();
Self {
handle: Some(handle),
addr,
token,
host: format!("127.0.0.1:{}", addr.port()),
origin: format!("http://127.0.0.1:{}", addr.port()),
}
}
fn shared(&self) -> Arc<ServeShared> {
Arc::clone(self.handle.as_ref().expect("running").shared())
}
fn flow_lines(&self) -> Vec<String> {
self.shared().flow().lines()
}
fn get(&self, path: &str) -> RawResponse {
self.request(
"GET",
path,
&[
("Host", self.host.as_str()),
("X-RapidStructure-Token", self.token.as_str()),
],
None,
)
}
fn post(&self, path: &str, headers: &[(&str, &str)], body: &[u8]) -> RawResponse {
let mut all = vec![
("Host", self.host.as_str()),
("X-RapidStructure-Token", self.token.as_str()),
("Origin", self.origin.as_str()),
];
all.extend_from_slice(headers);
self.request("POST", path, &all, Some(body))
}
fn request(
&self,
method: &str,
path: &str,
headers: &[(&str, &str)],
body: Option<&[u8]>,
) -> RawResponse {
raw_request(self.addr, method, path, headers, body)
}
fn submit_ocr(&self, body: &[u8]) -> RawResponse {
self.post(
"/api/ocr",
&[("Content-Type", "application/octet-stream")],
body,
)
}
fn download(&self, set_id: &str) -> RawResponse {
self.post(
"/api/models/download",
&[("Content-Type", "application/json")],
format!("{{\"set_id\":\"{set_id}\"}}").as_bytes(),
)
}
fn wait_terminal(&self, id: &str, timeout: Duration) -> Value {
let deadline = Instant::now() + timeout;
loop {
let response = self.get(&format!("/api/jobs/{id}"));
assert_eq!(response.status, 200, "job view: {}", response.text());
let value = response.json();
let state = value["state"].as_str().unwrap_or_default().to_string();
if matches!(state.as_str(), "succeeded" | "failed" | "cancelled") {
return value;
}
assert!(
Instant::now() < deadline,
"job {id} did not finish within {timeout:?}: {value}"
);
std::thread::sleep(Duration::from_millis(5));
}
}
}
impl Drop for TestServer {
fn drop(&mut self) {
self.handle.take();
}
}
struct RawResponse {
status: u16,
headers: Vec<(String, String)>,
body: Vec<u8>,
}
impl RawResponse {
fn header(&self, name: &str) -> Option<&str> {
self.headers
.iter()
.find(|(field, _)| field.eq_ignore_ascii_case(name))
.map(|(_, value)| value.as_str())
}
fn json(&self) -> Value {
serde_json::from_slice(&self.body)
.unwrap_or_else(|error| panic!("body is not JSON ({error}): {}", self.text()))
}
fn text(&self) -> String {
String::from_utf8_lossy(&self.body).into_owned()
}
fn code(&self) -> String {
self.json()["code"].as_str().unwrap_or_default().to_string()
}
}
fn raw_request(
addr: SocketAddr,
method: &str,
path: &str,
headers: &[(&str, &str)],
body: Option<&[u8]>,
) -> RawResponse {
let mut raw = format!("{method} {path} HTTP/1.1\r\nConnection: close\r\n");
for (name, value) in headers {
raw.push_str(&format!("{name}: {value}\r\n"));
}
match body {
Some(body) => {
raw.push_str(&format!("Content-Length: {}\r\n", body.len()));
raw.push_str("\r\n");
let mut bytes = raw.into_bytes();
bytes.extend_from_slice(body);
send(addr, &bytes)
}
None => {
raw.push_str("\r\n");
send(addr, raw.as_bytes())
}
}
}
fn send(addr: SocketAddr, raw: &[u8]) -> RawResponse {
let mut stream = TcpStream::connect(addr).expect("connect");
stream
.set_read_timeout(Some(Duration::from_secs(20)))
.expect("read timeout");
stream.write_all(raw).expect("write request");
stream.flush().expect("flush");
let mut buffer = Vec::new();
let mut chunk = [0_u8; 8192];
loop {
match stream.read(&mut chunk) {
Ok(0) => break,
Ok(read) => buffer.extend_from_slice(&chunk[..read]),
Err(error) => panic!("reading the response failed: {error}"),
}
}
parse_response(&buffer)
}
fn parse_response(buffer: &[u8]) -> RawResponse {
let head_end = find(buffer, b"\r\n\r\n").expect("the response must contain a header block") + 4;
let head = String::from_utf8_lossy(&buffer[..head_end - 4]).into_owned();
let mut lines = head.split("\r\n");
let status_line = lines.next().unwrap_or_default();
let status = status_line
.split_whitespace()
.nth(1)
.and_then(|code| code.parse().ok())
.unwrap_or_else(|| panic!("cannot parse the status line: {status_line:?}"));
let headers: Vec<(String, String)> = lines
.filter_map(|line| line.split_once(':'))
.map(|(name, value)| (name.trim().to_string(), value.trim().to_string()))
.collect();
let raw_body = buffer.get(head_end..).unwrap_or_default();
let header = |name: &str| -> Option<String> {
headers
.iter()
.find(|(field, _)| field.eq_ignore_ascii_case(name))
.map(|(_, value)| value.clone())
};
let body = match header("Transfer-Encoding") {
Some(value) if value.to_ascii_lowercase().contains("chunked") => decode_chunked(raw_body),
_ => match header("Content-Length").and_then(|value| value.parse::<usize>().ok()) {
Some(length) => raw_body.get(..length).unwrap_or(raw_body).to_vec(),
None => raw_body.to_vec(),
},
};
RawResponse {
status,
headers,
body,
}
}
fn decode_chunked(body: &[u8]) -> Vec<u8> {
let mut out = Vec::new();
let mut rest = body;
while let Some(position) = find(rest, b"\r\n") {
let size_line = String::from_utf8_lossy(&rest[..position]).into_owned();
let size_text = size_line.split(';').next().unwrap_or_default().trim();
let Ok(size) = usize::from_str_radix(size_text, 16) else {
break;
};
rest = &rest[position + 2..];
if size == 0 {
break;
}
let Some(chunk) = rest.get(..size) else {
out.extend_from_slice(rest);
break;
};
out.extend_from_slice(chunk);
rest = rest.get(size + 2..).unwrap_or_default();
}
out
}
fn find(haystack: &[u8], needle: &[u8]) -> Option<usize> {
haystack
.windows(needle.len())
.position(|window| window == needle)
}
fn model_file_bytes(name: &str) -> Vec<u8> {
format!("M2 scripted model file {name}").into_bytes()
}
fn plausible_onnx_bytes(tag: &str) -> Vec<u8> {
let mut bytes = vec![0x08, 0x07];
bytes.extend_from_slice(tag.as_bytes());
bytes
}
fn manifest_model_dir(name: &str, present: &[&str], declared_size: Option<u64>) -> PathBuf {
let dir = m2_root().join(format!("manifest-{name}-{}", unique()));
std::fs::create_dir_all(&dir).expect("create the model dir");
let files = model_fixture_files();
let mut manifest = String::from(
"{\"schema_version\":1,\"id\":\"test-set\",\"family\":\"PP-OCR\",\"version\":\"v-test\",\
\"languages\":[\"en\"],\"files\":[",
);
for (index, (file, role)) in files.iter().enumerate() {
let bytes = model_file_bytes(file);
let path = dir.join(file);
std::fs::write(&path, &bytes).expect("write the model file");
let sha = sha256_file(&path).expect("hash the model file");
if index > 0 {
manifest.push(',');
}
let size = declared_size.unwrap_or(bytes.len() as u64);
manifest.push_str(&format!(
"{{\"name\":\"{file}\",\"role\":\"{role}\",\"sha256\":\"{sha}\",\"size_bytes\":{size},\
\"source_url\":\"https://www.modelscope.cn/models/{file}\"}}"
));
}
manifest.push_str("]}");
std::fs::write(dir.join("manifest.json"), manifest).expect("write the manifest");
for (file, _) in files {
if !present.contains(&file) {
std::fs::remove_file(dir.join(file)).expect("remove the absent file");
}
}
dir
}
fn model_fixture_files() -> [(&'static str, &'static str); 4] {
[
("det.onnx", "detector"),
("rec.onnx", "recognizer"),
("dict.txt", "dictionary"),
("fx.onnx", "formula_recognizer"),
]
}
fn text_fixture_files() -> [&'static str; 3] {
["det.onnx", "rec.onnx", "dict.txt"]
}
fn manifest_model_dir_text_only(
name: &str,
present: &[&str],
declared_size: Option<u64>,
) -> PathBuf {
let mut files: Vec<&str> = present.to_vec();
files.push("fx.onnx");
manifest_model_dir(name, &files, declared_size)
}
fn m2_root() -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("target/m2-serve-tests")
}
struct Gate {
reached: std::sync::mpsc::SyncSender<()>,
reached_rx: Mutex<std::sync::mpsc::Receiver<()>>,
release: std::sync::mpsc::SyncSender<()>,
release_rx: Mutex<std::sync::mpsc::Receiver<()>>,
}
impl Gate {
fn new() -> Arc<Self> {
let (reached, reached_rx) = std::sync::mpsc::sync_channel(0);
let (release, release_rx) = std::sync::mpsc::sync_channel(0);
Arc::new(Self {
reached,
reached_rx: Mutex::new(reached_rx),
release,
release_rx: Mutex::new(release_rx),
})
}
fn wait(&self) {
let _ = self.reached.send(());
let release = self.release_rx.lock().expect("the gate is not poisoned");
let _ = release.recv();
}
fn arrive(&self) {
let reached = self.reached_rx.lock().expect("the gate is not poisoned");
let _ = reached.recv();
}
fn release(&self) {
let _ = self.release.send(());
}
}
#[derive(Debug, Clone)]
enum Step {
Write,
HashMismatch,
Network,
Cancelled,
}
#[derive(Clone)]
struct ScriptedDownload {
plan: Vec<Step>,
seen_hosts: Arc<Mutex<Vec<Vec<String>>>>,
written: Arc<Mutex<Vec<String>>>,
gate: Option<Arc<Gate>>,
}
impl Default for ScriptedDownload {
fn default() -> Self {
Self {
plan: Vec::new(),
seen_hosts: Arc::new(Mutex::new(Vec::new())),
written: Arc::new(Mutex::new(Vec::new())),
gate: None,
}
}
}
impl ScriptedDownload {
fn with_plan(plan: Vec<Step>) -> Self {
Self {
plan,
..Self::default()
}
}
fn with_gate(gate: Arc<Gate>) -> Self {
Self {
gate: Some(gate),
..Self::default()
}
}
fn step(&self, index: usize) -> Step {
if self.plan.is_empty() {
return Step::Write;
}
self.plan
.get(index)
.cloned()
.unwrap_or_else(|| self.plan.last().cloned().expect("a non-empty plan"))
}
fn factory(self) -> DownloaderFactory {
Arc::new(move || {
Box::new(ScriptedDownloader {
state: self.clone(),
})
})
}
fn hosts_per_call(&self) -> Vec<Vec<String>> {
self.seen_hosts.lock().expect("not poisoned").clone()
}
fn written(&self) -> Vec<String> {
self.written.lock().expect("not poisoned").clone()
}
}
struct ScriptedDownloader {
state: ScriptedDownload,
}
impl ModelDownloader for ScriptedDownloader {
fn download(
&mut self,
job: &DownloadJob<'_>,
sink: &mut dyn DownloadSink,
) -> Result<Vec<PathBuf>, RapidOcrError> {
self.state
.seen_hosts
.lock()
.expect("not poisoned")
.push(job.allowed_hosts.clone());
let pending: Vec<ModelFileSpec> = job
.set
.files
.iter()
.filter(|file| !file.state_in(job.root).is_present())
.cloned()
.collect();
let total = pending.len();
let mut paths = Vec::with_capacity(total);
for (offset, file) in pending.iter().enumerate() {
let index = offset + 1;
if !sink.file_started(file, index, total, file.size_bytes) {
return Err(DownloadError::Cancelled.into());
}
if let Some(gate) = &self.state.gate {
gate.wait();
}
match self.state.step(offset) {
Step::Write => {
let bytes = model_file_bytes(&file.name);
std::fs::write(job.root.join(&file.name), &bytes)
.expect("the scripted download must be able to write the model file");
sink.bytes_written(bytes.len() as u64 / 2);
sink.bytes_written(bytes.len() as u64);
sink.file_finished(file, index, bytes.len() as u64);
self.state
.written
.lock()
.expect("not poisoned")
.push(file.name.clone());
paths.push(job.root.join(&file.name));
}
Step::HashMismatch => {
sink.bytes_written(1);
return Err(DownloadError::HashMismatch {
expected: file.sha256.clone(),
actual: "00".repeat(32),
}
.into());
}
Step::Network => {
return Err(DownloadError::Network {
detail: "scripted transport failure".to_string(),
}
.into());
}
Step::Cancelled => return Err(DownloadError::Cancelled.into()),
}
}
Ok(paths)
}
}
fn empty_model_dir(name: &str) -> PathBuf {
let dir = test_root().join(format!("empty-{name}-{}", unique()));
std::fs::create_dir_all(&dir).expect("create the empty model dir");
dir
}
fn complete_model_dir(name: &str) -> PathBuf {
let dir = test_root().join(format!("complete-{name}-{}", unique()));
std::fs::create_dir_all(&dir).expect("create the model dir");
let files = model_fixture_files();
let mut manifest = String::from(
"{\"schema_version\":1,\"id\":\"test-set\",\"family\":\"PP-OCR\",\"version\":\"v-test\",\
\"files\":[",
);
for (index, (file, role)) in files.iter().enumerate() {
let path = dir.join(file);
std::fs::write(&path, format!("test model file {file}")).expect("write the model file");
let sha = sha256_file(&path).expect("hash the model file");
if index > 0 {
manifest.push(',');
}
manifest.push_str(&format!(
"{{\"name\":\"{file}\",\"role\":\"{role}\",\"sha256\":\"{sha}\",\"size_bytes\":18}}"
));
}
manifest.push_str("]}");
std::fs::write(dir.join("manifest.json"), manifest).expect("write the manifest");
dir
}
fn test_root() -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("target/m1-serve-tests")
}
fn complete_model_dir_with_table(name: &str) -> PathBuf {
let dir = test_root().join(format!("complete-table-{name}-{}", unique()));
std::fs::create_dir_all(&dir).expect("create the model dir");
let mut files: Vec<(&'static str, &'static str)> = model_fixture_files().to_vec();
files.push(("slanet-plus.onnx", "table_structure"));
let mut manifest = String::from(
"{\"schema_version\":1,\"id\":\"test-set\",\"family\":\"PP-OCR\",\"version\":\"v-test\",\
\"files\":[",
);
for (index, (file, role)) in files.iter().enumerate() {
let path = dir.join(file);
std::fs::write(&path, format!("test model file {file}")).expect("write the model file");
let sha = sha256_file(&path).expect("hash the model file");
if index > 0 {
manifest.push(',');
}
manifest.push_str(&format!(
"{{\"name\":\"{file}\",\"role\":\"{role}\",\"sha256\":\"{sha}\",\"size_bytes\":18}}"
));
}
manifest.push_str("]}");
std::fs::write(dir.join("manifest.json"), manifest).expect("write the manifest");
dir
}
fn unique() -> u64 {
use std::sync::atomic::AtomicU64;
static COUNTER: AtomicU64 = AtomicU64::new(0);
COUNTER.fetch_add(1, Ordering::SeqCst)
}
fn cache_serial() -> std::sync::MutexGuard<'static, ()> {
static SERIAL: Mutex<()> = Mutex::new(());
SERIAL.lock().unwrap_or_else(|error| error.into_inner())
}
fn limits(text: usize, formula: usize) -> RawServeLimits {
RawServeLimits {
max_queue_text: text,
max_queue_formula: formula,
..RawServeLimits::default()
}
}
fn wait_until(timeout: Duration, mut condition: impl FnMut() -> bool) -> bool {
let deadline = Instant::now() + timeout;
while Instant::now() < deadline {
if condition() {
return true;
}
std::thread::sleep(Duration::from_millis(5));
}
false
}
#[test]
fn the_page_carries_the_nonce_csp_and_the_frozen_security_headers() {
let server = TestServer::start(TestOptions::new(empty_model_dir("page"), Scripted::fast()));
let response = server.request("GET", "/", &[("Host", server.host.as_str())], None);
assert_eq!(response.status, 200, "{}", response.text());
assert_eq!(
response.header("Content-Type"),
Some("text/html; charset=utf-8")
);
assert_eq!(response.header("X-Content-Type-Options"), Some("nosniff"));
assert_eq!(response.header("Referrer-Policy"), Some("no-referrer"));
assert_eq!(response.header("Cache-Control"), Some("no-store"));
assert!(
response.header("Access-Control-Allow-Origin").is_none(),
"CORS headers must never be sent"
);
let csp = response
.header("Content-Security-Policy")
.expect("CSP header");
assert!(!csp.contains("unsafe-inline"), "{csp}");
assert!(csp.contains("script-src 'nonce-"), "{csp}");
assert!(csp.contains("style-src 'nonce-"), "{csp}");
let nonce = csp
.split("script-src 'nonce-")
.nth(1)
.and_then(|rest| rest.split('\'').next())
.expect("a nonce in the CSP");
assert_eq!(nonce, server.shared().nonce());
let page = response.text();
assert!(
!page.contains("__CSP_NONCE__"),
"no placeholder may survive"
);
assert!(!page.contains("__SRV_TOKEN__"));
assert_eq!(page.matches(&format!("nonce=\"{nonce}\"")).count(), 3);
assert_eq!(page.matches(&server.token).count(), 3);
}
#[test]
fn the_token_is_required_on_every_api_route() {
let server = TestServer::start(TestOptions::new(empty_model_dir("token"), Scripted::fast()));
assert_eq!(
server
.request("GET", "/", &[("Host", server.host.as_str())], None)
.status,
200
);
for path in ["/api/status", "/api/models", "/api/jobs/job-0"] {
let response = server.request("GET", path, &[("Host", server.host.as_str())], None);
assert_eq!(response.status, 401, "{path}: {}", response.text());
assert_eq!(response.code(), "unauthorized", "{path}");
let body = response.json();
assert!(body.get("code").is_some() && body.get("message").is_some());
assert!(body.get("detail").is_some(), "detail must exist: {body}");
}
let response = server.request(
"GET",
"/api/status",
&[
("Host", server.host.as_str()),
("X-RapidStructure-Token", "not-the-token"),
],
None,
);
assert_eq!(response.status, 401);
}
#[test]
fn static_formula_assets_are_served_same_origin_without_the_token() {
let server = TestServer::start(TestOptions::new(
empty_model_dir("static"),
Scripted::fast(),
));
for (path, mime) in [
(
"/static/katex/katex.min.js",
"application/javascript; charset=utf-8",
),
("/static/katex/katex.min.css", "text/css; charset=utf-8"),
("/static/katex/fonts/KaTeX_Main-Regular.woff2", "font/woff2"),
(
"/static/mathjax/tex-svg.js",
"application/javascript; charset=utf-8",
),
] {
let response = server.request("GET", path, &[("Host", server.host.as_str())], None);
assert_eq!(response.status, 200, "{path}: {}", response.text());
assert_eq!(response.header("Content-Type"), Some(mime), "{path}");
assert_eq!(response.header("Cache-Control"), Some("no-store"), "{path}");
assert!(!response.body.is_empty(), "{path} must not be empty");
}
for path in [
"/static/nope.js",
"/static/",
"/static/../Cargo.toml",
"/static/katex/../../Cargo.toml",
"/static/%2e%2e%2f%2e%2e%2fCargo.toml",
] {
let response = server.request("GET", path, &[("Host", server.host.as_str())], None);
assert_eq!(response.status, 404, "{path}: {}", response.text());
assert_eq!(response.code(), "not_found", "{path}");
}
let response = server.request(
"POST",
"/static/katex/katex.min.js",
&[("Host", server.host.as_str())],
None,
);
assert_eq!(response.status, 405);
assert_eq!(response.header("Allow"), Some("GET"));
}
#[test]
fn the_host_header_is_validated_for_dns_rebinding() {
let server = TestServer::start(TestOptions::new(empty_model_dir("host"), Scripted::fast()));
let token = server.token.clone();
for host in [
"evil.example:8760",
"127.0.0.1:1",
"[::1]:8760",
"localhost",
"127.0.0.10:1",
] {
let response = server.request(
"GET",
"/api/status",
&[("Host", host), ("X-RapidStructure-Token", token.as_str())],
None,
);
assert_eq!(response.status, 421, "Host: {host} → {}", response.text());
assert_eq!(response.code(), "bad_host");
}
let response = server.request(
"GET",
"/api/status",
&[("X-RapidStructure-Token", token.as_str())],
None,
);
assert_eq!(response.status, 421);
}
#[test]
fn state_changing_requests_require_a_matching_origin() {
let server = TestServer::start(TestOptions::new(
empty_model_dir("origin"),
Scripted::fast(),
));
let token = server.token.clone();
let host = server.host.clone();
let body = b"not-an-image".as_slice();
let response = server.request(
"POST",
"/api/ocr",
&[
("Host", host.as_str()),
("X-RapidStructure-Token", token.as_str()),
("Content-Type", "application/octet-stream"),
],
Some(body),
);
assert_eq!(response.status, 403, "{}", response.text());
assert_eq!(response.code(), "bad_origin");
let response = server.request(
"POST",
"/api/ocr",
&[
("Host", host.as_str()),
("X-RapidStructure-Token", token.as_str()),
("Origin", "null"),
("Content-Type", "application/octet-stream"),
],
Some(body),
);
assert_eq!(response.status, 403);
let response = server.request(
"POST",
"/api/ocr",
&[
("Host", host.as_str()),
("X-RapidStructure-Token", token.as_str()),
("Origin", "http://127.0.0.1:1"),
("Content-Type", "application/octet-stream"),
],
Some(body),
);
assert_eq!(response.status, 403);
let response = server.request(
"POST",
"/api/ocr",
&[
("Host", host.as_str()),
("X-RapidStructure-Token", token.as_str()),
("Origin", "http://127.0.0.1:1/"),
("Content-Type", "application/octet-stream"),
],
Some(body),
);
assert_eq!(response.status, 403, "a trailing slash is still a mismatch");
}
#[test]
fn read_requests_do_not_require_an_origin() {
let server = TestServer::start(TestOptions::new(
empty_model_dir("get-origin"),
Scripted::fast(),
));
let response = server.get("/api/status");
assert_eq!(response.status, 200, "{}", response.text());
}
#[test]
fn an_over_long_content_length_is_rejected_before_the_body_is_read() {
let mut raw = limits(4, 2);
raw.max_body_mb = 1;
let server = TestServer::start(TestOptions {
limits: raw,
..TestOptions::new(empty_model_dir("413"), Scripted::fast())
});
let response = server.request(
"POST",
"/api/ocr",
&[
("Host", server.host.as_str()),
("X-RapidStructure-Token", server.token.as_str()),
("Origin", server.origin.as_str()),
("Content-Type", "application/octet-stream"),
("Content-Length", "4194304"),
],
None,
);
assert_eq!(response.status, 413, "{}", response.text());
assert_eq!(response.code(), "payload_too_large");
}
#[test]
fn unknown_paths_are_404_and_wrong_methods_are_405() {
let server = TestServer::start(TestOptions::new(empty_model_dir("route"), Scripted::fast()));
for path in [
"/favicon.ico",
"/api",
"/api/jobs/job-0/annotated",
"/api/nope",
] {
let response = server.get(path);
assert_eq!(response.status, 404, "{path}: {}", response.text());
assert_eq!(response.code(), "not_found", "{path}");
}
for path in [
"/api/jobs/job-0/annotated.png",
"/api/jobs/job-0/export?format=json",
] {
let response = server.get(path);
assert_eq!(response.status, 404, "{path}: {}", response.text());
assert_eq!(response.code(), "job_not_found", "{path}");
}
let response = server.get("/api/ocr");
assert_eq!(response.status, 405, "{}", response.text());
assert_eq!(response.code(), "method_not_allowed");
assert_eq!(response.header("Allow"), Some("POST"));
let response = server.get("/api/engine/reload");
assert_eq!(response.status, 405, "{}", response.text());
assert_eq!(response.code(), "method_not_allowed");
assert_eq!(response.header("Allow"), Some("POST"));
for path in ["/api/jobs/job-0/annotated.png", "/api/jobs/job-0/export"] {
let response = server.post(path, &[], b"");
assert_eq!(response.status, 405, "{path}: {}", response.text());
assert_eq!(response.header("Allow"), Some("GET"), "{path}");
}
}
#[test]
fn the_ocr_media_type_is_validated() {
let server = TestServer::start(TestOptions::new(empty_model_dir("ctype"), Scripted::fast()));
let response = server.post(
"/api/ocr",
&[("Content-Type", "application/x-www-form-urlencoded")],
b"x=1",
);
assert_eq!(response.status, 400, "{}", response.text());
assert_eq!(response.code(), "bad_request");
}
#[test]
fn the_max_side_query_parameter_is_validated() {
let server = TestServer::start(TestOptions::new(
empty_model_dir("maxside"),
Scripted::fast(),
));
for bad in ["0", "29", "abc", "99999999999999999999"] {
let response = server.post(
&format!("/api/ocr?max_side={bad}"),
&[("Content-Type", "application/octet-stream")],
b"image",
);
assert_eq!(response.status, 400, "max_side={bad}: {}", response.text());
}
}
#[test]
fn the_job_goes_from_202_to_succeeded_and_its_result_round_trips() {
let server = TestServer::start(TestOptions::new(
complete_model_dir("flow"),
Scripted::fast(),
));
let status = server.get("/api/status").json();
assert_eq!(status["state"], "ready");
assert_eq!(status["engine"]["state"], "ready");
assert_eq!(status["engine"]["fallback_to_cpu"], false);
assert_eq!(
status["provider"]["requested"],
rapid_structure_rs::format_provider_preference(
rapid_structure_rs::ProviderPreference::default()
)
);
assert_eq!(status["provider"]["selected_ep"], "cpu");
assert_eq!(status["provider"]["fallback_to_cpu"], false);
let response = server.submit_ocr(b"pretend png bytes");
assert_eq!(response.status, 202, "{}", response.text());
let accepted = response.json();
assert_eq!(accepted["state"], "queued");
assert_eq!(accepted["queue"], "text");
assert_eq!(accepted["kind"], "ocr");
assert!(accepted["job_id"].as_str().is_some());
assert!(accepted["position"].as_u64().is_some());
let id = accepted["job_id"].as_str().expect("job id").to_string();
let view = server.wait_terminal(&id, Duration::from_secs(20));
assert_eq!(view["state"], "succeeded", "{view}");
let result = server.get(&format!("/api/jobs/{id}/result"));
assert_eq!(result.status, 200, "{}", result.text());
assert_eq!(
result.header("Content-Type"),
Some("application/json; charset=utf-8")
);
let value = result.json();
assert_eq!(value["regions"].as_array().map(Vec::len), Some(3));
assert_eq!(value["regions"][0]["kind"], "text");
assert_eq!(value["regions"][0]["recognition"]["text"], "region-0");
assert!(value["regions"][0]["polygon"]["points"].is_array());
assert!(
value["text"]
.as_str()
.is_some_and(|text| text.contains("region-1")),
"plain text must be present: {value}"
);
assert_eq!(value["plain_text"], value["text"]);
}
#[test]
fn a_failed_job_replays_its_original_status_and_code_on_result() {
let server = TestServer::start(TestOptions::new(
complete_model_dir("fail"),
Scripted {
fail: true,
..Scripted::fast()
},
));
let accepted = server.submit_ocr(b"broken").json();
let id = accepted["job_id"].as_str().expect("job id").to_string();
let view = server.wait_terminal(&id, Duration::from_secs(20));
assert_eq!(view["state"], "failed");
assert!(
view["error"]
.as_str()
.is_some_and(|error| error.contains("cannot be decoded")),
"{view}"
);
let result = server.get(&format!("/api/jobs/{id}/result"));
assert_eq!(result.status, 422, "{}", result.text());
assert_eq!(result.code(), "unsupported_input");
}
#[test]
fn a_result_over_the_limit_is_a_413_result_too_large() {
let mut raw = limits(4, 2);
raw.max_result_mb = 1;
let server = TestServer::start(TestOptions {
limits: raw,
..TestOptions::new(
complete_model_dir("toobig"),
Scripted {
regions: 1,
text_bytes: 2 * 1024 * 1024,
..Scripted::fast()
},
)
});
let accepted = server.submit_ocr(b"huge result").json();
let id = accepted["job_id"].as_str().expect("job id").to_string();
let view = server.wait_terminal(&id, Duration::from_secs(30));
assert_eq!(view["state"], "failed", "{view}");
let result = server.get(&format!("/api/jobs/{id}/result"));
assert_eq!(result.status, 413, "{}", result.text());
assert_eq!(result.code(), "result_too_large");
}
#[test]
fn a_result_under_the_limit_is_served_intact() {
let mut raw = limits(4, 2);
raw.max_result_mb = 1;
let server = TestServer::start(TestOptions {
limits: raw,
..TestOptions::new(
complete_model_dir("under"),
Scripted {
regions: 1,
text_bytes: 64 * 1024,
..Scripted::fast()
},
)
});
let accepted = server.submit_ocr(b"small result").json();
let id = accepted["job_id"].as_str().expect("job id").to_string();
assert_eq!(
server.wait_terminal(&id, Duration::from_secs(20))["state"],
"succeeded"
);
let result = server.get(&format!("/api/jobs/{id}/result"));
assert_eq!(result.status, 200);
assert_eq!(
result.json()["regions"][0]["recognition"]["text"],
"x".repeat(64 * 1024)
);
}
#[test]
fn cancelling_a_queued_job_succeeds_and_a_running_job_is_409() {
let server = TestServer::start(TestOptions {
limits: limits(2, 1),
..TestOptions::new(
complete_model_dir("cancel"),
Scripted::slow(Duration::from_millis(400)),
)
});
let first = server.submit_ocr(b"a").json()["job_id"]
.as_str()
.expect("job id")
.to_string();
assert!(
wait_until(Duration::from_secs(10), || {
server.get(&format!("/api/jobs/{first}")).json()["state"] == "running"
}),
"the first job must reach running"
);
let second = server.submit_ocr(b"b").json()["job_id"]
.as_str()
.expect("job id")
.to_string();
let cancelled = server.post(&format!("/api/jobs/{second}/cancel"), &[], b"");
assert_eq!(cancelled.status, 200, "{}", cancelled.text());
assert_eq!(cancelled.json()["state"], "cancelled");
let running = server.post(&format!("/api/jobs/{first}/cancel"), &[], b"");
assert_eq!(running.status, 409, "{}", running.text());
assert_eq!(running.code(), "not_cancellable");
assert_eq!(
server.get(&format!("/api/jobs/{first}")).json()["state"],
"running",
"a refused cancel must not change the state"
);
server.wait_terminal(&first, Duration::from_secs(20));
let again = server.post(&format!("/api/jobs/{first}/cancel"), &[], b"");
assert_eq!(again.status, 409);
}
#[test]
fn a_full_queue_is_503_without_reading_the_body() {
let server = TestServer::start(TestOptions {
limits: limits(1, 1),
..TestOptions::new(
complete_model_dir("busy"),
Scripted::slow(Duration::from_millis(400)),
)
});
let first = server.submit_ocr(b"a").json()["job_id"]
.as_str()
.expect("job id")
.to_string();
assert!(wait_until(Duration::from_secs(10), || {
server.get(&format!("/api/jobs/{first}")).json()["state"] == "running"
}));
let second = server.submit_ocr(b"b");
assert_eq!(second.status, 202, "{}", second.text());
let third = server.submit_ocr(b"c");
assert_eq!(third.status, 503, "{}", third.text());
assert_eq!(third.code(), "busy");
server.wait_terminal(&first, Duration::from_secs(20));
}
#[test]
fn a_full_queue_rejects_every_concurrent_request_without_reading_a_body() {
let server = TestServer::start(TestOptions {
limits: limits(1, 1),
..TestOptions::new(
complete_model_dir("reserve-full"),
Scripted::slow(Duration::from_millis(600)),
)
});
let running = server.submit_ocr(b"a").json()["job_id"]
.as_str()
.expect("job id")
.to_string();
assert!(wait_until(Duration::from_secs(10), || {
server.get(&format!("/api/jobs/{running}")).json()["state"] == "running"
}));
let queued = server.submit_ocr(b"b");
assert_eq!(queued.status, 202, "{}", queued.text());
let addr = server.addr;
let host = server.host.clone();
let origin = server.origin.clone();
let token = server.token.clone();
let barrier = Arc::new(std::sync::Barrier::new(8));
let mut threads = Vec::new();
for _ in 0..8 {
let (host, origin, token) = (host.clone(), origin.clone(), token.clone());
let barrier = Arc::clone(&barrier);
threads.push(std::thread::spawn(move || {
barrier.wait();
raw_request(
addr,
"POST",
"/api/ocr",
&[
("Host", host.as_str()),
("X-RapidStructure-Token", token.as_str()),
("Origin", origin.as_str()),
("Content-Type", "application/octet-stream"),
("Content-Length", "1048576"),
],
None,
)
}));
}
let mut refused = 0;
for thread in threads {
let response = thread.join().expect("the request thread must finish");
assert_eq!(response.status, 503, "{}", response.text());
assert_eq!(response.code(), "busy");
refused += 1;
}
assert_eq!(refused, 8);
let view = server.wait_terminal(&running, Duration::from_secs(20));
assert_eq!(view["state"], "succeeded", "{view}");
}
#[test]
fn the_queue_reservation_is_atomic_and_never_leaks_capacity() {
let server = TestServer::start(TestOptions {
limits: limits(1, 1),
..TestOptions::new(complete_model_dir("reserve-atomic"), Scripted::fast())
});
let shared = server.shared();
let barrier = Arc::new(std::sync::Barrier::new(8));
let mut threads = Vec::new();
for _ in 0..8 {
let shared = Arc::clone(&shared);
let barrier = Arc::clone(&barrier);
threads.push(std::thread::spawn(move || {
barrier.wait();
let reservation = shared.reserve_queue_slot(QueueClass::Text);
let held = reservation.is_some();
std::thread::sleep(Duration::from_millis(50));
drop(reservation);
held
}));
}
let mut winners = 0;
for thread in threads {
if thread.join().expect("the contender must finish") {
winners += 1;
}
}
assert_eq!(winners, 1, "exactly one contender may hold the only slot");
let first = shared
.reserve_queue_slot(QueueClass::Text)
.expect("the released slot must be available again");
assert!(
shared.reserve_queue_slot(QueueClass::Text).is_none(),
"capacity 1 must admit exactly one reservation"
);
drop(first);
shared
.reserve_queue_slot(QueueClass::Text)
.expect("dropping a reservation must return the slot");
}
#[test]
fn evicted_jobs_are_410_and_after_the_ttl_they_become_404() {
let mut raw = limits(2, 1);
raw.max_retained = 1;
raw.job_ttl_secs = 1;
let server = TestServer::start(TestOptions {
limits: raw,
..TestOptions::new(complete_model_dir("evict"), Scripted::fast())
});
let first = server.submit_ocr(b"a").json()["job_id"]
.as_str()
.expect("job id")
.to_string();
server.wait_terminal(&first, Duration::from_secs(20));
let second = server.submit_ocr(b"b").json()["job_id"]
.as_str()
.expect("job id")
.to_string();
server.wait_terminal(&second, Duration::from_secs(20));
let evicted = server.get(&format!("/api/jobs/{first}"));
assert_eq!(evicted.status, 410, "{}", evicted.text());
assert_eq!(evicted.code(), "job_evicted");
assert_eq!(server.get(&format!("/api/jobs/{first}/result")).status, 410);
assert!(
wait_until(Duration::from_secs(15), || {
server.get(&format!("/api/jobs/{first}")).status == 404
}),
"the tombstone must expire back to 404"
);
assert_eq!(server.get("/api/jobs/never-existed").status, 404);
}
#[test]
fn status_reports_the_frozen_three_provider_fields_and_redacts_paths() {
let dir = complete_model_dir("status");
let server = TestServer::start(TestOptions::new(dir.clone(), Scripted::fast()));
let response = server.get("/api/status");
assert_eq!(response.status, 200);
let value = response.json();
for key in ["requested", "selected_ep", "fallback_to_cpu"] {
assert!(
value["provider"].get(key).is_some(),
"provider.{key} must always exist: {value}"
);
}
assert_eq!(value["model_dir"], "<redacted>");
assert_eq!(value["source"], "local_manifest");
assert_eq!(value["downloads_allowed"], false);
assert!(value["ort"]["version"].is_string() || value["ort"]["version"].is_null());
assert!(value["queues"]["text"]["capacity"].as_u64().unwrap_or(0) > 0);
assert!(value["queues"]["formula"]["capacity"].as_u64().unwrap_or(0) > 0);
assert!(value["limits"]["max_body_bytes"].as_u64().unwrap_or(0) > 0);
assert!(
value["memory"]["peak_working_set_bytes"]
.as_u64()
.unwrap_or(0)
> 0
);
assert!(value["retention"]["job_ttl_ms"].as_u64().unwrap_or(0) > 0);
let text = response.text();
assert!(!text.contains(&dir.display().to_string()), "{text}");
assert!(!text.contains("\\\\?\\"), "{text}");
assert!(
!server
.get("/api/models")
.text()
.contains(&dir.display().to_string())
);
}
#[test]
fn models_reports_every_file_state_and_matches_the_ocr_409() {
let server = TestServer::start(TestOptions::new(
empty_model_dir("models"),
Scripted::fast(),
));
let models = server.get("/api/models");
assert_eq!(models.status, 200);
let value = models.json();
assert_eq!(value["source"], "default_table");
assert_eq!(value["downloads_allowed"], false);
let sets = value["sets"].as_array().expect("sets");
assert_eq!(sets.len(), 2, "text set + formula set (M4): {value}");
let files = sets[0]["files"].as_array().expect("files");
assert_eq!(files.len(), 3);
for file in files {
assert_eq!(file["state"], "missing");
assert!(file["name"].as_str().is_some());
assert!(file["role"].as_str().is_some());
assert!(file["size_bytes"].as_u64().is_some());
assert!(file["sha256"].as_str().is_some());
assert!(file["source_url"].as_str().is_some());
}
let formula_set = &sets[1];
assert_eq!(formula_set["id"], "PP-FormulaNet_plus-M");
let formula_files = formula_set["files"].as_array().expect("formula files");
assert_eq!(formula_files.len(), 1);
assert_eq!(formula_files[0]["role"], "formula_recognizer");
assert_eq!(formula_files[0]["name"], "pp_formulanet_plus_m.onnx");
assert_eq!(formula_files[0]["state"], "missing");
assert_eq!(formula_files[0]["size_bytes"], 593_916_167u64);
assert_eq!(
formula_files[0]["sha256"],
"6e509ec4b7cda604dc9bf80a876ae6494693deea4c7f4157a4dc0eba0ebda745"
);
assert_eq!(
formula_files[0]["source_url"],
"https://www.modelscope.cn/seeyuer/pp-formulanet-plus-m-optimized/resolve/0bb8154e3583d24613eaa74448de43ecf4ae73ec/pp_formulanet_plus_m.onnx"
);
let missing: Vec<&str> = value["missing"]
.as_array()
.expect("missing")
.iter()
.filter_map(Value::as_str)
.collect();
assert_eq!(missing.len(), 3);
assert_eq!(value["complete"], false);
assert_eq!(value["blocked"], value["missing"]);
assert_eq!(
value["formula"]["missing"],
serde_json::json!(["pp_formulanet_plus_m.onnx"])
);
assert_eq!(value["formula"]["complete"], false);
assert_eq!(
value["formula"]["required_roles"],
serde_json::json!(["formula_recognizer"])
);
assert_eq!(value["formula"]["routing"], false);
assert!(
value["formula"]["disabled_reason"]
.as_str()
.is_some_and(|reason| reason.contains("--formula-detector")),
"{value}"
);
let status = server.get("/api/status").json();
assert_eq!(status["state"], "ready");
assert_eq!(status["engine"]["state"], "blocked_models_missing");
let engine_missing = status["engine"]["missing"]
.as_array()
.expect("engine missing");
assert_eq!(engine_missing.len(), 3, "{status}");
assert_eq!(status["formula"]["routing"], false);
assert!(status["limits"]["max_eval_cases"].as_u64().unwrap_or(0) > 0);
let rejected = server.submit_ocr(b"image bytes");
assert_eq!(rejected.status, 409, "{}", rejected.text());
assert_eq!(rejected.code(), "models_missing");
let detail = &rejected.json()["detail"];
assert_eq!(detail["missing"], value["missing"], "{detail}");
assert_eq!(detail["corrupt"], value["corrupt"]);
assert_eq!(detail["source"], value["source"]);
assert_eq!(detail["model_dir"], value["model_dir"]);
let formula = server.post(
"/api/ocr?queue=formula",
&[("Content-Type", "application/octet-stream")],
b"image bytes",
);
assert_eq!(formula.status, 400, "{}", formula.text());
assert_eq!(formula.code(), "bad_request");
}
#[test]
fn an_engine_that_fails_to_load_is_failed_with_a_reason_and_503() {
let dir = complete_model_dir("enginefail");
let server = TestServer::start(TestOptions {
engine_factory: Arc::new(|_config: &EngineConfig| {
Err(RapidOcrError::UnsupportedProvider(
"scripted: the requested provider is unavailable in this build".to_string(),
))
}),
..TestOptions::new(dir, Scripted::fast())
});
let status = server.get("/api/status").json();
assert_eq!(
status["state"], "ready",
"the service is ready even if the engine is not"
);
assert_eq!(status["engine"]["state"], "failed");
assert!(
status["engine"]["reason"]
.as_str()
.is_some_and(|reason| reason.contains("provider is unavailable")),
"{status}"
);
let rejected = server.submit_ocr(b"image");
assert_eq!(rejected.status, 503, "{}", rejected.text());
assert_eq!(rejected.code(), "engine_unavailable");
assert!(rejected.json()["detail"]["reason"].is_string());
}
#[test]
fn the_download_endpoint_refuses_disabled_unknown_and_url_bearing_requests() {
let dir = complete_model_dir("download");
let disabled = TestServer::start(TestOptions::new(dir.clone(), Scripted::fast()));
let response = disabled.post(
"/api/models/download",
&[("Content-Type", "application/json")],
br#"{"set_id":"test-set"}"#,
);
assert_eq!(response.status, 403, "{}", response.text());
assert_eq!(response.code(), "downloads_disabled");
let enabled = TestServer::start(TestOptions {
allow_download: true,
..TestOptions::new(dir, Scripted::fast())
});
let response = enabled.post(
"/api/models/download",
&[("Content-Type", "application/json")],
br#"{"set_id":"test-set"}"#,
);
assert_eq!(response.status, 202, "{}", response.text());
let accepted = response.json();
assert_eq!(accepted["kind"], "model_download");
assert_eq!(accepted["queue"], "download");
assert_eq!(accepted["download"]["files_total"], 0);
let id = accepted["job_id"].as_str().expect("job id").to_string();
let view = enabled.wait_terminal(&id, Duration::from_secs(20));
assert_eq!(view["state"], "succeeded", "{view}");
assert_eq!(view["kind"], "model_download");
assert_eq!(view["failure"], serde_json::Value::Null);
let unknown = enabled.post(
"/api/models/download",
&[("Content-Type", "application/json")],
br#"{"set_id":"nope"}"#,
);
assert_eq!(unknown.status, 404, "{}", unknown.text());
assert_eq!(unknown.code(), "model_set_not_found");
let with_url = enabled.post(
"/api/models/download",
&[("Content-Type", "application/json")],
br#"{"set_id":"test-set","url":"https://evil.example/x.onnx"}"#,
);
assert_eq!(with_url.status, 400, "{}", with_url.text());
}
#[test]
fn a_formula_flood_does_not_starve_text_jobs() {
let delay = Duration::from_millis(25);
let dir = complete_model_dir("fair-formula");
let scripted = Scripted::slow(delay);
let server = TestServer::start(
TestOptions::new(dir.clone(), scripted.clone())
.with_formula_routing(&dir.join("pix2text-mfd-1.5.onnx"))
.with_limits(limits(4, 2)),
);
let shared = server.shared();
let status = server.get("/api/status").json();
let bound = status["queues"]["text"]["wait_bound"]
.as_u64()
.expect("wait_bound") as usize;
assert!(bound > 0, "{status}");
assert_eq!(shared.service_state(), super::state::ServiceState::Ready);
let starved = run_flood(&server, "formula", QueueClass::Text, bound, delay);
assert!(
starved.is_none(),
"the text job was starved under a formula flood: {starved:?}"
);
}
#[test]
fn a_text_flood_does_not_starve_formula_jobs() {
let delay = Duration::from_millis(25);
let dir = complete_model_dir("fair-text");
let scripted = Scripted::slow(delay);
let server = TestServer::start(
TestOptions::new(dir.clone(), scripted.clone())
.with_formula_routing(&dir.join("pix2text-mfd-1.5.onnx"))
.with_limits(limits(4, 2)),
);
let shared = server.shared();
assert_eq!(shared.engine_state_name(), "ready");
let status = server.get("/api/status").json();
let bound = status["queues"]["formula"]["wait_bound"]
.as_u64()
.expect("wait_bound") as usize;
assert!(bound > 0, "{status}");
let starved = run_flood(&server, "text", QueueClass::Formula, bound, delay);
assert!(
starved.is_none(),
"the formula job was starved under a text flood: {starved:?}"
);
}
fn run_flood(
server: &TestServer,
flood: &str,
victim: QueueClass,
bound_jobs: usize,
delay: Duration,
) -> Option<u128> {
let flood = flood.to_string();
let url = format!("/api/ocr?queue={flood}");
let stop = Arc::new(AtomicBool::new(false));
let flooded = Arc::new(AtomicUsize::new(0));
let warmup = server.submit_ocr(b"warmup");
assert!(warmup.status == 202, "{}", warmup.text());
let warmup_id = warmup.json()["job_id"].as_str().expect("id").to_string();
server.wait_terminal(&warmup_id, Duration::from_secs(20));
let flooder = {
let stop = Arc::clone(&stop);
let flooded = Arc::clone(&flooded);
let addr = server.addr;
let host = server.host.clone();
let origin = server.origin.clone();
let token = server.token.clone();
let url = url.clone();
std::thread::spawn(move || {
while !stop.load(Ordering::SeqCst) {
let raw = format!(
"POST {url} HTTP/1.1\r\nConnection: close\r\nHost: {host}\r\nOrigin: {origin}\r\n\
X-RapidStructure-Token: {token}\r\nContent-Type: application/octet-stream\r\n\
Content-Length: 5\r\n\r\nimage"
);
let response = send(addr, raw.as_bytes());
if response.status == 202 {
flooded.fetch_add(1, Ordering::SeqCst);
} else {
std::thread::sleep(Duration::from_millis(2));
}
}
})
};
let saturated = wait_until(Duration::from_secs(10), || {
let status = server.get("/api/status").json();
let queues = &status["queues"][flood.as_str()];
let used = queues["used"].as_u64().unwrap_or(0);
let capacity = queues["capacity"].as_u64().unwrap_or(0);
used >= capacity
});
assert!(
saturated,
"the flood never saturated the {flood} queue (accepted {} job(s))",
flooded.load(Ordering::SeqCst)
);
let victim_path = match victim {
QueueClass::Text => "/api/ocr?queue=text",
QueueClass::Formula => "/api/ocr?queue=formula",
};
let started = Instant::now();
let mut accepted = None;
for _ in 0..200 {
let response = server.post(
victim_path,
&[("Content-Type", "application/octet-stream")],
b"victim",
);
if response.status == 202 {
accepted = Some(response.json());
break;
}
assert_eq!(response.status, 503, "{}", response.text());
std::thread::sleep(Duration::from_millis(2));
}
let accepted = accepted.expect("the victim job must eventually be accepted");
let id = accepted["job_id"].as_str().expect("id").to_string();
let view = server.wait_terminal(&id, Duration::from_secs(30));
let elapsed = started.elapsed();
let flooded_during_wait = flooded.load(Ordering::SeqCst);
stop.store(true, Ordering::SeqCst);
let _ = flooder.join();
assert_eq!(view["state"], "succeeded", "{view}");
assert!(
flooded_during_wait > 0,
"the {flood} queue was never served while the victim waited"
);
let budget = delay * u32::try_from(bound_jobs + 3).expect("small bound");
if elapsed > budget + Duration::from_millis(750) {
return Some(elapsed.as_millis());
}
None
}
#[test]
fn the_formula_queue_is_refused_when_formula_routing_is_off() {
let server = TestServer::start(TestOptions::new(
complete_model_dir("noformula"),
Scripted::fast(),
));
let response = server.post(
"/api/ocr?queue=formula",
&[("Content-Type", "application/octet-stream")],
b"image",
);
assert_eq!(response.status, 400, "{}", response.text());
assert_eq!(response.code(), "bad_request");
assert_eq!(server.submit_ocr(b"image").status, 202);
let models = server.get("/api/models").json();
assert_eq!(models["formula"]["routing"], false);
let reason = models["formula"]["disabled_reason"]
.as_str()
.expect("a textual reason");
assert!(reason.contains("--formula-detector"), "{reason}");
assert_eq!(models["formula"]["detector"]["configured"], false);
assert_eq!(models["formula"]["detector"]["file"], Value::Null);
let status = server.get("/api/status").json();
assert_eq!(status["formula"]["routing"], false);
assert_eq!(
status["formula"]["disabled_reason"],
models["formula"]["disabled_reason"]
);
}
#[test]
fn the_formula_queue_runs_the_formula_pipeline_from_the_model_set() {
let dir = complete_model_dir("m4-formula");
let detector = dir.join("pix2text-mfd-1.5.onnx");
let scripted = Scripted::fast();
let server = TestServer::start(
TestOptions::new(dir.clone(), scripted.clone()).with_formula_routing(&detector),
);
let models = server.get("/api/models").json();
assert_eq!(models["formula"]["complete"], true, "{models}");
assert_eq!(models["formula"]["routing"], true);
assert_eq!(models["formula"]["disabled_reason"], Value::Null);
assert_eq!(models["formula"]["detector"]["configured"], true);
assert_eq!(
models["formula"]["detector"]["file"], "pix2text-mfd-1.5.onnx",
"only the file name is exposed (§7.4)"
);
assert!(
!models.to_string().contains(&dir.display().to_string()),
"no absolute path may leak: {models}"
);
let text = server.submit_ocr(b"text image");
assert_eq!(text.status, 202, "{}", text.text());
assert_eq!(text.json()["queue"], "text");
let formula = server.post(
"/api/ocr?queue=formula",
&[("Content-Type", "application/octet-stream")],
b"formula image",
);
assert_eq!(formula.status, 202, "{}", formula.text());
let accepted = formula.json();
assert_eq!(accepted["queue"], "formula", "{accepted}");
assert_eq!(accepted["kind"], "ocr");
let text_id = text.json()["job_id"].as_str().expect("id").to_string();
let formula_id = accepted["job_id"].as_str().expect("id").to_string();
let text_view = server.wait_terminal(&text_id, Duration::from_secs(20));
let formula_view = server.wait_terminal(&formula_id, Duration::from_secs(20));
assert_eq!(text_view["state"], "succeeded", "{text_view}");
assert_eq!(formula_view["state"], "succeeded", "{formula_view}");
assert_eq!(formula_view["queue"], "formula");
let policies = scripted.formulas();
assert_eq!(policies.len(), 2, "one request per job");
let text_policy = scripted.formula_policy("text image");
assert!(
!text_policy.enabled,
"the text queue must not enable the formula pipeline: every recorded policy: {policies:?}"
);
assert_eq!(text_policy.model_path, None);
let formula = scripted.formula_policy("formula image");
assert!(
formula.enabled,
"the formula queue must enable it: every recorded policy: {policies:?}"
);
assert_eq!(
formula.model_path.as_deref(),
Some(dir.join("fx.onnx").as_path())
);
assert_eq!(
formula.detector_path.as_deref(),
Some(detector.as_path()),
"the detector comes from the startup-resolved path"
);
let declared = models["sets"][0]["files"]
.as_array()
.expect("files")
.iter()
.find(|file| file["role"] == "formula_recognizer")
.and_then(|file| file["sha256"].as_str())
.expect("the formula model's declared hash");
assert_eq!(formula.expected_model_sha256.as_deref(), Some(declared));
assert!(
!formula.include_token_ids,
"token ids stay out of the default"
);
}
#[test]
fn an_incomplete_formula_set_is_409_while_text_ocr_keeps_working() {
let dir = manifest_model_dir("m4-formula-incomplete", &text_fixture_files(), None);
let detector = dir.join("pix2text-mfd-1.5.onnx");
let server = TestServer::start(
TestOptions::new(dir.clone(), Scripted::fast()).with_formula_routing(&detector),
);
let models = server.get("/api/models").json();
assert_eq!(models["complete"], true, "{models}");
assert_eq!(models["missing"], serde_json::json!([]));
assert_eq!(models["formula"]["complete"], false);
assert_eq!(models["formula"]["missing"], serde_json::json!(["fx.onnx"]));
assert_eq!(models["formula"]["routing"], true);
let formula = server.post(
"/api/ocr?queue=formula",
&[("Content-Type", "application/octet-stream")],
b"formula image",
);
assert_eq!(formula.status, 409, "{}", formula.text());
assert_eq!(formula.code(), "models_missing");
let detail = &formula.json()["detail"];
assert_eq!(detail["scope"], "formula");
assert_eq!(detail["missing"], serde_json::json!(["fx.onnx"]));
assert_eq!(detail["detector"]["state"], "present");
assert_eq!(detail["detector"]["sha256"], Value::Null);
assert!(
detail.get("missing_on_disk").is_none(),
"the existence-only field is gone: {detail}"
);
let text = server.submit_ocr(b"text image");
assert_eq!(text.status, 202, "{}", text.text());
let id = text.json()["job_id"].as_str().expect("id").to_string();
let view = server.wait_terminal(&id, Duration::from_secs(20));
assert_eq!(view["state"], "succeeded", "{view}");
assert_eq!(view["queue"], "text");
}
#[test]
fn removing_the_formula_model_turns_the_route_into_a_409() {
let dir = complete_model_dir("m4-formula-removed");
let detector = dir.join("pix2text-mfd-1.5.onnx");
let server = TestServer::start(
TestOptions::new(dir.clone(), Scripted::fast()).with_formula_routing(&detector),
);
assert_eq!(
server.get("/api/models").json()["formula"]["complete"],
true
);
std::fs::remove_file(dir.join("fx.onnx")).expect("remove the formula model");
let models = server.get("/api/models").json();
assert_eq!(models["formula"]["complete"], false);
assert_eq!(models["formula"]["missing"], serde_json::json!(["fx.onnx"]));
assert_eq!(models["complete"], true, "{models}");
let formula = server.post(
"/api/ocr?queue=formula",
&[("Content-Type", "application/octet-stream")],
b"formula image",
);
assert_eq!(formula.status, 409, "{}", formula.text());
assert_eq!(formula.json()["detail"]["scope"], "formula");
assert_eq!(server.submit_ocr(b"text").status, 202);
}
#[test]
fn the_table_switch_is_refused_when_table_routing_is_off() {
let server = TestServer::start(TestOptions::new(
complete_model_dir("notable"),
Scripted::fast(),
));
for query in ["?table=1", "?queue=text&table=1", "?queue=formula&table=1"] {
let response = server.post(
&format!("/api/ocr{query}"),
&[("Content-Type", "application/octet-stream")],
b"image",
);
assert_eq!(response.status, 400, "{query}: {}", response.text());
assert_eq!(response.code(), "bad_request");
}
assert_eq!(server.submit_ocr(b"image").status, 202);
assert_eq!(
server
.post(
"/api/ocr?table=0",
&[("Content-Type", "application/octet-stream")],
b"image",
)
.status,
202
);
let models = server.get("/api/models").json();
assert_eq!(models["table"]["routing"], false);
let reason = models["table"]["disabled_reason"]
.as_str()
.expect("a textual reason");
assert!(reason.contains("--table-model"), "{reason}");
assert!(reason.contains("ordinary OCR"), "{reason}");
assert_eq!(models["table"]["structure"]["configured"], false);
assert_eq!(models["table"]["structure"]["file"], Value::Null);
assert_eq!(models["table"]["in_plan"], false);
assert_eq!(models["complete"], true, "{models}");
assert_eq!(models["pipelines"]["table"]["routing"], false);
assert_eq!(models["pipelines"]["table"]["in_plan"], false);
assert_eq!(models["pipelines"]["table"]["complete"], false);
let status = server.get("/api/status").json();
assert_eq!(status["table"]["routing"], false);
assert_eq!(
status["table"]["disabled_reason"],
models["table"]["disabled_reason"]
);
}
#[test]
fn the_table_switch_rejects_anything_but_one_and_zero() {
let dir = complete_model_dir("e16-table-values");
let model = dir.join("slanet-plus.onnx");
let scripted = Scripted::fast();
let server = TestServer::start(
TestOptions::new(dir.clone(), scripted.clone()).with_table_routing(&model),
);
for raw in ["2", "true", "yes", "", "TRUE", "-1"] {
let response = server.post(
&format!("/api/ocr?table={raw}"),
&[("Content-Type", "application/octet-stream")],
b"image",
);
assert_eq!(response.status, 400, "`?table={raw}`: {}", response.text());
assert_eq!(response.code(), "bad_request");
}
assert_eq!(
scripted.calls.load(Ordering::SeqCst),
0,
"a rejected request must not reach the backend"
);
}
#[test]
fn the_table_stage_runs_from_the_model_set_when_requested() {
let dir = complete_model_dir_with_table("e16-table-on");
let model = dir.join("slanet-plus.onnx");
let detector = dir.join("pix2text-mfd-1.5.onnx");
let scripted = Scripted::fast().with_table_regions(1);
let server = TestServer::start(
TestOptions::new(dir.clone(), scripted.clone())
.with_formula_routing(&detector),
);
let models = server.get("/api/models").json();
assert_eq!(models["table"]["routing"], true);
assert_eq!(models["table"]["disabled_reason"], Value::Null);
assert_eq!(models["table"]["complete"], true, "{models}");
assert_eq!(models["table"]["structure"]["configured"], true);
assert_eq!(models["table"]["structure"]["file"], "slanet-plus.onnx");
assert_eq!(models["table"]["in_plan"], true);
assert_eq!(
models["table"]["required_roles"],
json!(["table_structure"])
);
assert_eq!(models["pipelines"]["table"]["complete"], true);
assert_eq!(
models["pipelines"]["table"]["pipeline"], "table",
"the pipeline label is the documented one"
);
let plan = models["pipelines"]["table"]["plan"]
.as_array()
.expect("the table pipeline publishes its plan");
assert_eq!(plan.len(), 1, "{plan:?}");
assert_eq!(plan[0]["name"], "slanet-plus.onnx");
assert_eq!(plan[0]["role"], "table_structure");
assert_eq!(plan[0]["pipeline"], "table");
assert_eq!(plan[0]["state"], "present");
assert_eq!(plan[0]["has_declared_digest"], true);
let text_plan = models["pipelines"]["text"]["plan"]
.as_array()
.expect("the text pipeline publishes its plan");
assert_eq!(
text_plan
.iter()
.map(|file| file["role"].as_str().unwrap_or("").to_string())
.collect::<Vec<_>>(),
["detector", "recognizer", "dictionary"]
);
assert_eq!(models["sets"][0]["id"], "test-set");
assert_eq!(models["sets"][0]["family"], "PP-OCR");
assert_eq!(models["sets"][0]["version"], "v-test");
assert_eq!(models["sets"][0]["complete"], true);
assert!(
!models.to_string().contains(&dir.display().to_string()),
"no absolute path may leak: {models}"
);
let plain = server.submit_ocr(b"plain");
assert_eq!(plain.status, 202, "{}", plain.text());
assert_eq!(plain.json()["queue"], "text");
let text_table = server.post(
"/api/ocr?queue=text&table=1",
&[("Content-Type", "application/octet-stream")],
b"text+table",
);
assert_eq!(text_table.status, 202, "{}", text_table.text());
assert_eq!(text_table.json()["queue"], "text");
let formula_table = server.post(
"/api/ocr?queue=formula&table=1",
&[("Content-Type", "application/octet-stream")],
b"formula+table",
);
assert_eq!(formula_table.status, 202, "{}", formula_table.text());
assert_eq!(formula_table.json()["queue"], "formula");
let explicit_off = server.post(
"/api/ocr?table=0",
&[("Content-Type", "application/octet-stream")],
b"explicit off",
);
assert_eq!(explicit_off.status, 202, "{}", explicit_off.text());
for response in [&plain, &text_table, &formula_table, &explicit_off] {
let id = response.json()["job_id"].as_str().expect("id").to_string();
let view = server.wait_terminal(&id, Duration::from_secs(20));
assert_eq!(view["state"], "succeeded", "{view}");
}
let policies = scripted.tables();
assert_eq!(policies.len(), 4, "one request per job");
assert!(
!scripted.table_policy("plain").enabled,
"absent means off; every recorded policy: {policies:?}"
);
assert!(
!scripted.table_policy("explicit off").enabled,
"`?table=0` is an explicit off; every recorded policy: {policies:?}"
);
assert_eq!(
policies.iter().filter(|(_, policy)| policy.enabled).count(),
2,
"only the two `?table=1` requests may enable the stage: {policies:?}"
);
let declared = models["sets"][0]["files"]
.as_array()
.expect("files")
.iter()
.find(|file| file["role"] == "table_structure")
.and_then(|file| file["sha256"].as_str())
.expect("the fixture manifest declares the table structure model")
.to_string();
for (body, label) in [
("text+table", "?table=1"),
("formula+table", "?queue=formula&table=1"),
] {
let policy = scripted.table_policy(body);
assert!(policy.enabled, "{label} must enable the table stage");
assert_eq!(
policy.model_path.as_deref(),
Some(model.as_path()),
"{label}: the structure model comes from the startup-resolved path"
);
assert_eq!(
policy.expected_model_sha256.as_deref(),
Some(declared.as_str()),
"{label}: the declared hash comes from the model set"
);
assert_eq!(policy.text_policy, TableTextPolicy::Escape);
}
}
#[test]
fn an_incomplete_table_set_is_409_while_text_ocr_keeps_working() {
let dir = manifest_model_dir("e16-table-incomplete", &text_fixture_files(), None);
let model = dir.join("slanet-plus.onnx");
let server = TestServer::start(
TestOptions::new(dir.clone(), Scripted::fast()).with_table_routing(&model),
);
std::fs::remove_file(&model).expect("remove the structure model");
let models = server.get("/api/models").json();
assert_eq!(models["table"]["routing"], true);
assert_eq!(models["table"]["complete"], false, "{models}");
assert_eq!(models["table"]["missing"], json!(["slanet-plus.onnx"]));
assert_eq!(
models["pipelines"]["table"]["blocked"],
json!(["slanet-plus.onnx"])
);
assert_eq!(models["complete"], true, "the text pipeline is unaffected");
let refused = server.post(
"/api/ocr?table=1",
&[("Content-Type", "application/octet-stream")],
b"table image",
);
assert_eq!(refused.status, 409, "{}", refused.text());
assert_eq!(refused.code(), "models_missing");
assert_eq!(refused.json()["detail"]["scope"], "table");
assert_eq!(
refused.json()["detail"]["missing"],
json!(["slanet-plus.onnx"])
);
assert_eq!(server.submit_ocr(b"text").status, 202);
assert_eq!(
server
.post(
"/api/ocr?table=0",
&[("Content-Type", "application/octet-stream")],
b"text",
)
.status,
202
);
}
#[test]
fn the_table_region_round_trips_through_the_result_endpoint() {
let dir = complete_model_dir("e16-table-result");
let model = dir.join("slanet-plus.onnx");
let server = TestServer::start(
TestOptions::new(dir.clone(), Scripted::fast().with_table_regions(1))
.with_table_routing(&model),
);
let accepted = server.post(
"/api/ocr?table=1",
&[("Content-Type", "application/octet-stream")],
b"table image",
);
assert_eq!(accepted.status, 202, "{}", accepted.text());
let id = accepted.json()["job_id"].as_str().expect("id").to_string();
let view = server.wait_terminal(&id, Duration::from_secs(20));
assert_eq!(view["state"], "succeeded", "{view}");
let result = server.get(&format!("/api/jobs/{id}/result")).json();
let regions = result["regions"].as_array().expect("regions");
assert_eq!(regions.len(), 4, "three text regions plus one table");
let table = regions
.iter()
.find(|region| region["kind"] == "table")
.expect("a table region");
assert!(
table["recognition"].is_null(),
"a table region has no CTC text"
);
assert!(table["formula"].is_null());
let outcome = &table["table"];
assert_eq!(outcome["rows"], 1);
assert_eq!(outcome["cols"], 2);
assert_eq!(outcome["confidence"], "low");
assert_eq!(outcome["holes"], 1);
assert_eq!(outcome["degradations"][0]["kind"], "grid_holes");
assert_eq!(outcome["source_line_indices"], json!([0]));
assert_eq!(outcome["model_id"], "scripted-table");
assert_eq!(outcome["cells"].as_array().expect("cells").len(), 2);
assert!(
outcome["html"].as_str().expect("html").contains("<script>"),
"the wire format carries the model's own HTML"
);
}
fn submit_words_query(
server: &TestServer,
scripted: &Scripted,
query: &str,
body: &str,
expected: WordOutputMode,
) -> Value {
let accepted = server.post(
&format!("/api/ocr{query}"),
&[("Content-Type", "application/octet-stream")],
body.as_bytes(),
);
assert_eq!(accepted.status, 202, "{query}: {}", accepted.text());
let id = accepted.json()["job_id"].as_str().expect("id").to_string();
let view = server.wait_terminal(&id, Duration::from_secs(20));
assert_eq!(view["state"], "succeeded", "{view}");
assert_eq!(
scripted.word_mode(body),
expected,
"the query must reach the engine as a mode, not be dropped on the way: {:?}",
scripted.word_modes()
);
server.get(&format!("/api/jobs/{id}/result")).json()
}
fn text_regions(result: &Value) -> Vec<&Value> {
result["regions"]
.as_array()
.expect("regions")
.iter()
.filter(|region| region["kind"] == "text")
.collect()
}
fn polygon_bounds(points: &Value) -> (f64, f64, f64, f64) {
let rows = points.as_array().expect("polygon points");
assert_eq!(rows.len(), 4, "a polygon is exactly four points");
let mut xs = Vec::with_capacity(4);
let mut ys = Vec::with_capacity(4);
for row in rows {
let pair = row.as_array().expect("a point is (x, y)");
assert_eq!(pair.len(), 2, "a point is (x, y)");
xs.push(pair[0].as_f64().expect("x is a number"));
ys.push(pair[1].as_f64().expect("y is a number"));
}
(
xs.iter().copied().fold(f64::INFINITY, f64::min),
ys.iter().copied().fold(f64::INFINITY, f64::min),
xs.iter().copied().fold(f64::NEG_INFINITY, f64::max),
ys.iter().copied().fold(f64::NEG_INFINITY, f64::max),
)
}
#[test]
fn the_default_and_the_explicit_off_both_omit_recognition_words() {
let dir = complete_model_dir("words-off");
let scripted = Scripted::fast();
let server = TestServer::start(TestOptions::new(dir, scripted.clone()));
let omitted = submit_words_query(&server, &scripted, "", "words absent", WordOutputMode::Off);
let explicit = submit_words_query(
&server,
&scripted,
"?words=off",
"words explicit off",
WordOutputMode::Off,
);
for result in [&omitted, &explicit] {
for region in text_regions(result) {
let words = ®ion["recognition"]["words"];
assert!(
words.is_null(),
"`off` must omit the array entirely, not emit an empty one: {words}"
);
}
}
assert_eq!(omitted["regions"], explicit["regions"]);
}
#[test]
fn an_unknown_words_value_is_rejected_before_any_job_is_created() {
let dir = complete_model_dir("words-unknown");
let scripted = Scripted::fast();
let server = TestServer::start(TestOptions::new(dir, scripted.clone()));
for raw in ["char", "WORDS", "1", "true", ""] {
let response = server.post(
&format!("/api/ocr?words={raw}"),
&[("Content-Type", "application/octet-stream")],
b"unknown words value",
);
assert_eq!(response.status, 400, "?words={raw}: {}", response.text());
}
assert!(
scripted.word_modes().is_empty(),
"a rejected query must not reach the engine: {:?}",
scripted.word_modes()
);
}
#[test]
fn the_words_mode_emits_token_boxes() {
let dir = complete_model_dir("words-tokens");
let scripted = Scripted::fast();
let server = TestServer::start(TestOptions::new(dir, scripted.clone()));
let result = submit_words_query(
&server,
&scripted,
"?words=words",
"token mode",
WordOutputMode::Words,
);
for region in text_regions(&result) {
let text = region["recognition"]["text"].as_str().expect("text");
let words = region["recognition"]["words"]
.as_array()
.expect("`words` must emit an array");
assert!(!words.is_empty(), "no boxes for {text:?}");
assert_eq!(
words
.iter()
.map(|word| word["kind"].as_str().expect("kind"))
.collect::<Vec<_>>(),
vec!["token"; words.len()],
"`words` is the token granularity"
);
assert_eq!(
words
.iter()
.map(|word| word["text"].as_str().expect("word text"))
.collect::<Vec<_>>(),
text.split_whitespace().collect::<Vec<_>>(),
"tokens must appear in reading order, verbatim"
);
}
}
#[test]
fn the_chars_mode_emits_char_boxes_that_rebuild_the_text() {
let dir = complete_model_dir("words-chars");
let scripted = Scripted::fast();
let server = TestServer::start(TestOptions::new(dir, scripted.clone()));
let result = submit_words_query(
&server,
&scripted,
"?words=chars",
"char mode",
WordOutputMode::Chars,
);
for region in text_regions(&result) {
let text = region["recognition"]["text"].as_str().expect("text");
let words = region["recognition"]["words"]
.as_array()
.expect("`chars` must emit an array");
assert_eq!(
words.len(),
text.chars().count(),
"one box per char, nothing dropped"
);
let kinds: Vec<&str> = words
.iter()
.map(|word| word["kind"].as_str().expect("kind"))
.collect();
assert!(
kinds.iter().all(|kind| *kind == "char"),
"`chars` is the char granularity, got {kinds:?}"
);
let rejoined: String = words
.iter()
.map(|word| word["text"].as_str().expect("char text"))
.collect();
assert_eq!(rejoined, text, "char concatenation must rebuild the text");
}
}
#[test]
fn the_word_polygons_are_in_the_original_image_space() {
let dir = complete_model_dir("words-geometry");
let scripted = Scripted::fast();
let server = TestServer::start(TestOptions::new(dir, scripted.clone()));
let result = submit_words_query(
&server,
&scripted,
"?words=chars",
"char geometry",
WordOutputMode::Chars,
);
assert_eq!(result["image"]["coordinate_space"], "Image");
let width = result["image"]["original_size"]["width"]
.as_f64()
.expect("width");
let height = result["image"]["original_size"]["height"]
.as_f64()
.expect("height");
for region in text_regions(&result) {
let (row_x0, row_y0, row_x1, row_y1) = polygon_bounds(®ion["polygon"]["points"]);
let words = region["recognition"]["words"]
.as_array()
.expect("char boxes");
for word in words {
let (x0, y0, x1, y1) = polygon_bounds(&word["polygon"]["points"]);
assert!(x1 > x0 && y1 > y0, "degenerate word box: {word}");
assert!(
x0 >= 0.0 && y0 >= 0.0 && x1 <= width && y1 <= height,
"word box {word} is outside the {width}x{height} image"
);
assert!(
x0 >= row_x0 - 1e-4
&& x1 <= row_x1 + 1e-4
&& y0 >= row_y0 - 1e-4
&& y1 <= row_y1 + 1e-4,
"word box {word} escapes its own row box ({row_x0}, {row_y0}, {row_x1}, {row_y1})"
);
}
}
}
#[test]
fn the_word_sequence_is_identical_across_two_identical_submissions() {
let dir = complete_model_dir("words-stable");
let scripted = Scripted::fast();
let server = TestServer::start(TestOptions::new(dir, scripted.clone()));
let first = submit_words_query(
&server,
&scripted,
"?words=chars",
"stable order",
WordOutputMode::Chars,
);
let second = submit_words_query(
&server,
&scripted,
"?words=chars",
"stable order",
WordOutputMode::Chars,
);
let boxes_of = |result: &Value| {
serde_json::to_string(&result["regions"][0]["recognition"]["words"]).expect("serialize")
};
assert_eq!(
boxes_of(&first),
boxes_of(&second),
"the same image must yield the same word array: text, kind, coordinates and order"
);
let words = second["regions"][0]["recognition"]["words"]
.as_array()
.expect("char boxes");
let centers: Vec<f64> = words
.iter()
.map(|word| {
let (x0, _, x1, _) = polygon_bounds(&word["polygon"]["points"]);
(x0 + x1) / 2.0
})
.collect();
for pair in centers.windows(2) {
assert!(
pair[1] > pair[0],
"array order must follow left-to-right geometry, got {centers:?}"
);
}
}
#[test]
fn the_page_carries_the_mode_selector_and_the_table_renderer() {
let server = TestServer::start(TestOptions::new(
empty_model_dir("page-e16-markup"),
Scripted::fast(),
));
let page = server.get("/").text();
for anchor in [
"id=\"modeRow\"",
"id=\"modeOpts\"",
"id=\"modeCap\"",
"role=\"radiogroup\"",
] {
assert!(page.contains(anchor), "the mode selector needs {anchor}");
}
assert!(
!page.contains("id=\"fxToggle\""),
"the single formula checkbox is gone: the mode selector replaces it (E16.2.1)"
);
for mode in [
"\"text\"",
"\"formula\"",
"\"text-table\"",
"\"formula-table\"",
] {
assert!(
page.contains(mode),
"the mode table must carry the {mode} mode"
);
}
assert!(
page.contains("modeAvailability") && page.contains("renderModeRow"),
"the modes must be gated by readiness, not rendered unconditionally"
);
assert!(
page.contains(r#""/api/ocr?queue=" + queue + (mode.table ? "&table=1" : "")"#),
"the table switch must ride on the documented query parameter"
);
assert!(
page.contains("id=\"qReadyTable\"") && page.contains("表格队列 · "),
"the queue panel must report the table queue as a third line"
);
assert!(
page.contains("sanitizeTableHtml") && page.contains("DOMParser"),
"the table renderer must parse and rebuild, not paste (E16.3.2)"
);
assert!(
page.contains("TABLE_ALLOWED_TAGS") && page.contains("TABLE_ALLOWED_ATTRS"),
"the sanitizer must be driven by an explicit whitelist"
);
assert!(
page.contains(r#"wrap.innerHTML = sanitizeTableHtml("#),
"the only way a table reaches the DOM is through the sanitizer"
);
assert!(
page.contains(r#"data-table-render="1""#)
&& page.contains("渲染表格")
&& page.contains("收起表格"),
"each table region needs its own render toggle (E24)"
);
assert!(
page.contains(r#"<div class="rt-wrap" hidden></div>"#)
&& page.contains(r#"<pre class="rt-raw""#),
"a table must start collapsed, showing its HTML source instead (E24)"
);
assert!(
page.contains(r#"btn.getAttribute("data-table-html")"#)
&& page.contains(r#"data-table-html="${esc(html)}""#),
"the render input must be the same string the source line shows (E24)"
);
assert!(
page.contains(r#"const tbtn = e.target.closest("[data-table-render]")"#),
"the toggle must be intercepted before the region-selection handler (E24)"
);
assert!(
!page.contains("innerHTML = t.html")
&& !page.contains("innerHTML = table.html")
&& !page.contains("innerHTML = r.table"),
"the model's own HTML must never be assigned to innerHTML"
);
assert!(
!page.contains("replace(/</g") && !page.contains("replace(/<script"),
"a regex sanitizer cannot be correct for HTML (E16.3.2)"
);
assert!(
page.contains("低置信(网格空洞或截断)"),
"a low-confidence table must say so in words"
);
assert!(
!page.contains("onclick=") && !page.contains(" style=\""),
"no inline handlers or styles under the page's nonce CSP"
);
}
#[test]
fn the_page_gates_the_table_mode_on_the_server_route_block() {
let server = TestServer::start(TestOptions::new(
empty_model_dir("page-e22-table-gate"),
Scripted::fast(),
));
let page = server.get("/").text();
let marker = "服务端未返回表格路由状态(/api/models.table)";
let scan = "const need = QUEUE_ROLES[queue]";
let marker_at = page.find(marker).unwrap_or_else(|| {
panic!("the table readiness branch must exist (E22): {marker}");
});
let scan_at = page
.find(scan)
.unwrap_or_else(|| panic!("the shared role scan must still exist: {scan}"));
assert!(
marker_at < scan_at,
"the table branch must run **before** the shared role scan, otherwise the role lookup \
reports `表格结构` missing and greys the mode out (E22)"
);
assert!(
page.contains("if (queue === \"table\")"),
"the table queue needs its own readiness branch (E22)"
);
assert!(
!page.contains("table: [\"table_structure\"]"),
"QUEUE_ROLES must not require table_structure from a model set (E22)"
);
assert!(
page.contains("function planRow(")
&& page.contains("required_roles: [\"table_structure\"]"),
"the demo models need a table route block and plan rows (E22/E16.3.8)"
);
}
#[test]
fn the_page_carries_the_model_manifest_panel() {
let server = TestServer::start(TestOptions::new(
empty_model_dir("page-e16-manifest"),
Scripted::fast(),
));
let page = server.get("/").text();
assert!(
page.contains("function planRowsHtml") && page.contains("管线"),
"the panel must render the resolved model plan (E16.1.3)"
);
assert!(
page.contains("const pipes = m && m.pipelines") && page.contains("p.plan.forEach"),
"the plan must come from the server's own pipelines payload, not a path built in the page"
);
assert!(
page.contains("命令行指定"),
"a model given on the command line is not in any set: the panel must say so"
);
assert!(
page.contains("data-reverify=\"1\"") && page.contains("重新校验"),
"a complete set still needs a reverify button (E16.1.1)"
);
assert!(
page.contains(r#"e.target.closest("[data-reverify]")"#),
"the reverify button must be wired through delegation, not a per-set listener"
);
assert!(
page.contains("function fileStateCell") && page.contains("bf-sha-row"),
"every file needs a three-state icon and an expandable sha256 (E16.1.2)"
);
assert!(
page.contains("表格管线:"),
"the reverify verdict must carry the table pipeline as well"
);
assert!(
!page.contains("onclick=") && !page.contains(" style=\""),
"no inline handlers or styles under the page's nonce CSP"
);
}
#[test]
fn the_table_sanitizer_produces_no_executable_nodes() {
let Some(browser) = test_browser() else {
eprintln!(
"skipping: no headless browser found; set RAPID_STRUCTURE_TEST_BROWSER to run gate G8"
);
return;
};
let server = TestServer::start(TestOptions::new(
empty_model_dir("page-e16-g8"),
Scripted::fast(),
));
let page = server.get("/").text();
let script = main_inline_script(&page).expect("the page must carry its main inline script");
let exposed = script.replacen(
"\"use strict\";",
"\"use strict\";\n globalThis.__RROCR_SANITIZE__ = sanitizeTableHtml;",
1,
);
assert!(
exposed.contains("__RROCR_SANITIZE__"),
"the exposure seam must land: the main script must open with \"use strict\";"
);
let payloads = [
r#"<table><tr><td><script>alert(1)</script></td></tr></table>"#,
r#"<table><tr><td><img src=x onerror="alert(1)"></td></tr></table>"#,
r#"<table><tr><td><a href="javascript:alert(1)">click</a></td></tr></table>"#,
r#"<table><tr><td onclick="alert(1)">tap</td></tr></table>"#,
r#"<table><tr><td><iframe src="javascript:alert(1)"></iframe></td></tr></table>"#,
r#"<table><tr><td><svg/onload=alert(1)></td></tr></table>"#,
r#"<table><tr><td style="background:url(javascript:alert(1))">styled</td></tr></table>"#,
r#"<table><tr><td colspan="2" rowspan="3">spanned</td></tr></table>"#,
r#"<table><tr><td colspan="0">zero</td><td colspan="abc">nan</td><td colspan="1e9">huge</td></tr></table>"#,
r#"<table><tr><td><b>bold</b><sub>2</sub><sup>+</sup><br>next</td></tr></table>"#,
r#"<table><tr><td><script>alert(1)</script></td></tr></table>"#,
];
let payload_json = serde_json::to_string(&payloads)
.expect("payloads serialize")
.replace("</", "<\\/");
let driver = format!(
r#"<script>
const payloads = {payload_json};
const out = payloads.map((p) => {{
const fn = globalThis.__RROCR_SANITIZE__;
if (typeof fn !== "function") return "__MISSING__";
return fn(p);
}});
const bytes = new TextEncoder().encode(JSON.stringify(out));
let hex = "";
for (const b of bytes) hex += b.toString(16).padStart(2, "0");
const pre = document.createElement("pre");
pre.id = "sanitizer-out";
pre.textContent = hex;
document.body.appendChild(pre);
</script>"#
);
let html = format!(
"<!doctype html><html><head><meta charset=\"utf-8\"></head><body>\n\
<script>{exposed}</script>\n{driver}\n</body></html>"
);
let dir = std::env::temp_dir().join(format!("rrocr-g8-{}", std::process::id()));
std::fs::create_dir_all(&dir).expect("temp dir");
let file = dir.join("g8.html");
std::fs::write(&file, html).expect("write harness");
let stdout_path = dir.join("dom.txt");
let stderr_path = dir.join("browser.log");
let stdout_file = std::fs::File::create(&stdout_path).expect("stdout file");
let stderr_file = std::fs::File::create(&stderr_path).expect("stderr file");
let profile = dir.join("profile");
let mut child = match std::process::Command::new(&browser)
.args([
"--headless=new",
"--disable-gpu",
"--no-first-run",
"--no-default-browser-check",
"--disable-extensions",
"--disable-background-networking",
"--allow-file-access-from-files",
"--virtual-time-budget=4000",
"--dump-dom",
])
.arg(format!("--user-data-dir={}", profile.display()))
.arg(&file)
.stdin(std::process::Stdio::null())
.stdout(std::process::Stdio::from(stdout_file))
.stderr(std::process::Stdio::from(stderr_file))
.spawn()
{
Ok(child) => child,
Err(error) => {
eprintln!("skipping: cannot launch {browser:?}: {error}");
return;
}
};
let deadline = Instant::now() + Duration::from_secs(60);
let mut ready = false;
while Instant::now() < deadline {
let written = std::fs::metadata(&stdout_path)
.map(|meta| meta.len() > 0)
.unwrap_or(false);
if written {
ready = true;
break;
}
match child.try_wait() {
Ok(Some(_)) => break,
Ok(None) => std::thread::sleep(Duration::from_millis(200)),
Err(error) => {
eprintln!("skipping: cannot poll {browser:?}: {error}");
break;
}
}
}
let _ = child.kill();
let _ = child.wait();
if !ready {
let log = std::fs::read_to_string(&stderr_path).unwrap_or_default();
let _ = std::fs::remove_dir_all(&dir);
eprintln!(
"skipping: {browser:?} wrote no DOM within 60s; \
the sanitizer is not verified on this machine (browser log: {})",
log.trim()
);
return;
}
let dom = std::fs::read_to_string(&stdout_path).unwrap_or_default();
let browser_log = std::fs::read_to_string(&stderr_path).unwrap_or_default();
let _ = std::fs::remove_dir_all(&dir);
let hex = extract_pre_hex(&dom, "sanitizer-out").unwrap_or_else(|| {
panic!(
"the harness produced no output (browser log: {})",
browser_log.trim()
)
});
let json = decode_hex(&hex).expect("the harness output must be hex-encoded UTF-8");
let sanitized: Vec<String> =
serde_json::from_str(&json).expect("the harness output must be a JSON array");
assert_eq!(
sanitized.len(),
payloads.len(),
"every payload must come back"
);
for (input, output) in payloads.iter().zip(sanitized.iter()) {
assert_ne!(
output, "__MISSING__",
"the page's sanitizer must be reachable: {input}"
);
let lowered = output.to_lowercase();
for forbidden in [
"<script",
"<img",
"<iframe",
"<svg",
"<a ",
"onerror",
"onclick",
"onload",
"javascript:",
"style=",
] {
assert!(
!lowered.contains(forbidden),
"gate G8: {forbidden:?} survived sanitizing {input:?} -> {output:?}"
);
}
}
let spanned = &sanitized[7];
assert!(
spanned.contains("colspan=\"2\"") && spanned.contains("rowspan=\"3\""),
"a legal span must survive: {spanned:?}"
);
let inline = &sanitized[9];
for kept in ["<b>", "<sub>", "<sup>", "<br>"] {
assert!(
inline.contains(kept),
"{kept} is on the whitelist and must survive: {inline:?}"
);
}
let spans = &sanitized[8];
assert!(
!spans.contains("colspan"),
"out-of-range spans must be dropped, not clamped: {spans:?}"
);
assert!(
sanitized[10].contains("<script>"),
"already-escaped text must survive as text: {:?}",
sanitized[10]
);
}
#[test]
fn the_table_render_toggle_is_lazy_and_stays_sanitized() {
let Some(browser) = test_browser() else {
eprintln!(
"skipping: no headless browser found; set RAPID_STRUCTURE_TEST_BROWSER to run the E24 toggle"
);
return;
};
let server = TestServer::start(TestOptions::new(
empty_model_dir("page-e24-toggle"),
Scripted::fast(),
));
let page = server.get("/").text();
let script = main_inline_script(&page).expect("the page must carry its main inline script");
let exposed = script.replacen(
"\"use strict\";",
"\"use strict\";\n globalThis.__RROCR_SANITIZE__ = sanitizeTableHtml;\n \
globalThis.__RROCR_TABLE_HTML__ = tableRegionHtml;\n \
globalThis.__RROCR_TOGGLE__ = toggleTableRender;",
1,
);
assert!(
exposed.contains("__RROCR_TOGGLE__") && exposed.contains("__RROCR_TABLE_HTML__"),
"the exposure seam must land: the main script must open with \"use strict\";"
);
assert!(
page.contains("function tableRegionHtml(") && page.contains("function toggleTableRender("),
"the toggle harness drives page functions that must exist"
);
let payloads = [
r#"<table><tr><th>甲</th><th>乙</th></tr><tr><td>1</td><td colspan="2">2</td></tr></table>"#,
r#"<table><tr><td><script>alert(1)</script></td><td><img src=x onerror="alert(1)"></td></tr></table>"#,
];
let payload_json = serde_json::to_string(&payloads)
.expect("payloads serialize")
.replace("</", "<\\/");
let driver = format!(
r#"<script>
const payloads = {payload_json};
const make = globalThis.__RROCR_TABLE_HTML__;
const toggle = globalThis.__RROCR_TOGGLE__;
const out = [];
for (const payload of payloads) {{
const host = document.createElement("div");
document.body.appendChild(host);
if (typeof make !== "function" || typeof toggle !== "function") {{
out.push({{ missing: true }});
continue;
}}
host.innerHTML = make(
{{
kind: "table",
table: {{
html: payload,
rows: 2,
cols: 3,
holes: 1,
confidence: "low",
degradations: [{{ kind: "grid_holes", holes: 1 }}],
cells: [{{ match_kind: "single" }}, {{ match_kind: "unmatched" }}],
sourceLineIndices: [0, 1],
}},
}},
0,
"1,2 3,4 5,6 7,8",
"1,2 → 7,8",
);
const btn = host.querySelector("[data-table-render]");
const wrap = host.querySelector(".rt-wrap");
const raw = host.querySelector(".rt-raw");
const fold = host.querySelector(".ri-fold");
const row = {{}};
row.attr = btn ? (btn.getAttribute("data-table-html") === payload ? "same" : "diff") : "__NO_BUTTON__";
row.collapsed = wrap && wrap.hidden ? "yes" : "no";
row.rawShown = raw && !raw.hidden ? "yes" : "no";
row.label = btn ? btn.textContent : "";
row.expanded = btn ? String(btn.getAttribute("aria-expanded")) : "";
row.rendered = wrap ? wrap.innerHTML.length : -1;
if (btn) toggle(btn);
row.afterHidden = wrap && wrap.hidden ? "hidden" : "shown";
row.afterHtml = wrap ? wrap.innerHTML.toLowerCase() : "";
row.afterLabel = btn ? btn.textContent : "";
row.afterExpanded = btn ? String(btn.getAttribute("aria-expanded")) : "";
row.afterRaw = raw && raw.hidden ? "hidden" : "shown";
if (btn) toggle(btn);
row.backHidden = wrap && wrap.hidden ? "hidden" : "shown";
row.backCleared = wrap && wrap.innerHTML === "" ? "cleared" : "kept";
row.backLabel = btn ? btn.textContent : "";
row.backRaw = raw && !raw.hidden ? "shown" : "hidden";
row.span = payload.indexOf("colspan") >= 0 ? "yes" : "no";
row.folded = fold ? "yes" : "no";
out.push(row);
}}
const bytes = new TextEncoder().encode(JSON.stringify(out));
let hex = "";
for (const b of bytes) hex += b.toString(16).padStart(2, "0");
const pre = document.createElement("pre");
pre.id = "table-render-out";
pre.textContent = hex;
document.body.appendChild(pre);
</script>"#
);
let html = format!(
"<!doctype html><html><head><meta charset=\"utf-8\"></head><body>\n\
<script>{exposed}</script>\n{driver}\n</body></html>"
);
let (dom, browser_log) = match run_headless_dom(&browser, &html, "e24-toggle") {
Ok(result) => result,
Err(reason) => {
eprintln!(
"skipping: {reason}; the table render toggle is not verified on this machine"
);
return;
}
};
let hex = extract_pre_hex(&dom, "table-render-out").unwrap_or_else(|| {
panic!(
"the toggle harness produced no output (browser log: {})",
browser_log.trim()
)
});
let json = decode_hex(&hex).expect("the toggle harness output must be hex-encoded UTF-8");
let rows: Vec<serde_json::Value> =
serde_json::from_str(&json).expect("the toggle harness output must be a JSON array");
assert_eq!(rows.len(), payloads.len(), "every payload must come back");
for row in &rows {
assert!(
row["missing"].as_bool() != Some(true),
"the exposed page functions must be reachable: {row}"
);
assert_ne!(
row["attr"], "__NO_BUTTON__",
"every table card needs its own render button (E24)"
);
assert_eq!(
row["attr"], "same",
"esc() + getAttribute must round-trip the model HTML byte for byte"
);
assert_eq!(
row["collapsed"], "yes",
"a table must start collapsed (E24)"
);
assert_eq!(row["rawShown"], "yes", "the source must be shown instead");
assert_eq!(row["label"], "渲染表格");
assert_eq!(row["expanded"], "false");
assert_eq!(
row["rendered"], 0,
"nothing may be rendered before the click"
);
assert_eq!(row["afterHidden"], "shown", "the click renders the table");
assert_eq!(row["afterLabel"], "收起表格");
assert_eq!(row["afterExpanded"], "true");
assert_eq!(row["afterRaw"], "hidden");
let rendered = row["afterHtml"]
.as_str()
.expect("rendered html is a string");
assert!(
rendered.contains("<table"),
"the rendered DOM must be a real table: {rendered}"
);
if row["span"] == "yes" {
assert!(
rendered.contains("colspan"),
"a legal span in the source must survive sanitizing: {rendered}"
);
} else {
assert_eq!(
row["span"], "no",
"every payload must report whether it carries a span"
);
}
for forbidden in ["<script", "<img", "onerror", "onload"] {
assert!(
!rendered.contains(forbidden),
"gate G8 on the lazy path: {forbidden:?} survived -> {rendered}"
);
}
assert_eq!(
row["backHidden"], "hidden",
"the second click collapses again"
);
assert_eq!(
row["backCleared"], "cleared",
"collapsing must drop the rendered DOM (a big table must not stay resident)"
);
assert_eq!(row["backLabel"], "渲染表格");
assert_eq!(row["backRaw"], "shown");
assert_eq!(
row["folded"], "no",
"the table card is not a folded row; it must not gain a details element"
);
}
}
fn run_headless_dom(browser: &Path, html: &str, tag: &str) -> Result<(String, String), String> {
let dir = std::env::temp_dir().join(format!("rrocr-{tag}-{}", std::process::id()));
std::fs::create_dir_all(&dir).map_err(|error| format!("temp dir: {error}"))?;
let file = dir.join("harness.html");
std::fs::write(&file, html).map_err(|error| format!("write harness: {error}"))?;
let stdout_path = dir.join("dom.txt");
let stderr_path = dir.join("browser.log");
let stdout_file = std::fs::File::create(&stdout_path).map_err(|error| error.to_string())?;
let stderr_file = std::fs::File::create(&stderr_path).map_err(|error| error.to_string())?;
let profile = dir.join("profile");
let mut child = match std::process::Command::new(browser)
.args([
"--headless=new",
"--disable-gpu",
"--no-first-run",
"--no-default-browser-check",
"--disable-extensions",
"--disable-background-networking",
"--allow-file-access-from-files",
"--virtual-time-budget=4000",
"--dump-dom",
])
.arg(format!("--user-data-dir={}", profile.display()))
.arg(&file)
.stdin(std::process::Stdio::null())
.stdout(std::process::Stdio::from(stdout_file))
.stderr(std::process::Stdio::from(stderr_file))
.spawn()
{
Ok(child) => child,
Err(error) => {
let _ = std::fs::remove_dir_all(&dir);
return Err(format!("cannot launch {browser:?}: {error}"));
}
};
let deadline = Instant::now() + Duration::from_secs(60);
let mut ready = false;
while Instant::now() < deadline {
let written = std::fs::metadata(&stdout_path)
.map(|meta| meta.len() > 0)
.unwrap_or(false);
if written {
ready = true;
break;
}
match child.try_wait() {
Ok(Some(_)) => break,
Ok(None) => std::thread::sleep(Duration::from_millis(200)),
Err(error) => {
eprintln!("warning: cannot poll {browser:?}: {error}");
break;
}
}
}
let _ = child.kill();
let _ = child.wait();
let dom = std::fs::read_to_string(&stdout_path).unwrap_or_default();
let log = std::fs::read_to_string(&stderr_path).unwrap_or_default();
let _ = std::fs::remove_dir_all(&dir);
if !ready {
return Err(format!("wrote no DOM within 60s: {}", log.trim()));
}
Ok((dom, log))
}
fn test_browser() -> Option<PathBuf> {
if let Some(path) = std::env::var_os("RAPID_STRUCTURE_TEST_BROWSER") {
let path = PathBuf::from(path);
if path.is_file() {
return Some(path);
}
}
[
r"C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe",
r"C:\Program Files\Microsoft\Edge\Application\msedge.exe",
r"C:\Program Files\Google\Chrome\Application\chrome.exe",
r"C:\Program Files (x86)\Google\Chrome\Application\chrome.exe",
]
.into_iter()
.map(PathBuf::from)
.find(|path| path.is_file())
}
fn main_inline_script(page: &str) -> Option<String> {
let lines: Vec<&str> = page.lines().collect();
let open = lines
.iter()
.rposition(|line| line.contains("<script nonce="))?;
let close = lines
.iter()
.skip(open + 1)
.position(|line| line.trim() == "</script>")?
+ open
+ 1;
Some(lines[open + 1..close].join("\n"))
}
fn extract_pre_hex(dom: &str, id: &str) -> Option<String> {
let needle = format!("<pre id=\"{id}\">");
let start = dom.find(&needle)? + needle.len();
let rest = &dom[start..];
let end = rest.find("</pre>")?;
Some(rest[..end].trim().to_string())
}
fn decode_hex(hex: &str) -> Option<String> {
if !hex.len().is_multiple_of(2) {
return None;
}
let bytes = (0..hex.len())
.step_by(2)
.map(|index| u8::from_str_radix(&hex[index..index + 2], 16).ok())
.collect::<Option<Vec<u8>>>()?;
String::from_utf8(bytes).ok()
}
struct DetectorFixture {
dir: PathBuf,
detector: PathBuf,
detector_sha: String,
}
fn detector_model_dir(name: &str) -> DetectorFixture {
let dir = m2_root().join(format!("detector-{name}-{}", unique()));
std::fs::create_dir_all(&dir).expect("create the model dir");
let files = [
("det.onnx", "detector"),
("rec.onnx", "recognizer"),
("dict.txt", "dictionary"),
("fx.onnx", "formula_recognizer"),
("mfd.onnx", "formula_detector"),
];
let mut manifest = String::from(
"{\"schema_version\":1,\"id\":\"test-set\",\"family\":\"PP-OCR\",\"version\":\"v-test\",\
\"languages\":[\"en\"],\"files\":[",
);
let mut detector_sha = String::new();
for (index, (file, role)) in files.iter().enumerate() {
let bytes = model_file_bytes(file);
let path = dir.join(file);
std::fs::write(&path, &bytes).expect("write the model file");
let sha = sha256_file(&path).expect("hash the model file");
if *file == "mfd.onnx" {
detector_sha = sha.clone();
}
if index > 0 {
manifest.push(',');
}
manifest.push_str(&format!(
"{{\"name\":\"{file}\",\"role\":\"{role}\",\"sha256\":\"{sha}\",\"size_bytes\":{},\
\"source_url\":\"https://www.modelscope.cn/models/{file}\"}}",
bytes.len()
));
}
manifest.push_str("]}");
std::fs::write(dir.join("manifest.json"), manifest).expect("write the manifest");
DetectorFixture {
detector: dir.join("mfd.onnx"),
detector_sha,
dir,
}
}
#[test]
fn a_corrupt_formula_detector_is_reported_and_refused_before_the_body() {
let fixture = detector_model_dir("corrupt-detector");
std::fs::write(&fixture.detector, b"corrupted detector bytes").expect("corrupt the detector");
let server = TestServer::start(TestOptions::new(fixture.dir.clone(), Scripted::fast()));
let models = server.get("/api/models").json();
assert_eq!(models["formula"]["complete"], false, "{models}");
assert_eq!(
models["formula"]["corrupt"],
json!(["mfd.onnx"]),
"{models}"
);
assert_eq!(models["formula"]["detector"]["configured"], true);
assert_eq!(models["formula"]["detector"]["file"], "mfd.onnx");
assert_eq!(
models["formula"]["detector"]["state"], "corrupt",
"{models}"
);
assert_eq!(
models["formula"]["detector"]["sha256"], fixture.detector_sha,
"the declared hash must be reported (not dropped)"
);
assert_eq!(models["pipelines"]["formula"]["in_plan"], true, "{models}");
assert_eq!(
models["pipelines"]["formula"]["corrupt"],
json!(["mfd.onnx"]),
"{models}"
);
assert_eq!(
models["pipelines"]["formula"]["files"],
json!(["fx.onnx", "mfd.onnx"]),
"the run plan's formula pipeline is exactly the recognizer + the detector: {models}"
);
assert!(
!models
.to_string()
.contains(&fixture.dir.display().to_string()),
"no absolute path may leak: {models}"
);
let refused = server.request(
"POST",
"/api/ocr?queue=formula",
&[
("Host", server.host.as_str()),
("X-RapidStructure-Token", server.token.as_str()),
("Origin", server.origin.as_str()),
("Content-Type", "application/octet-stream"),
("Content-Length", "1048576"),
],
None,
);
assert_eq!(refused.status, 409, "{}", refused.text());
assert_eq!(refused.code(), "models_corrupt");
let detail = &refused.json()["detail"];
assert_eq!(detail["scope"], "formula");
assert_eq!(detail["corrupt"], json!(["mfd.onnx"]));
assert_eq!(detail["detector"]["state"], "corrupt");
assert_eq!(detail["in_plan"], true);
let text = server.submit_ocr(b"text image");
assert_eq!(text.status, 202, "{}", text.text());
}
#[test]
fn a_cli_formula_detector_is_verified_by_the_same_rule() {
let dir = manifest_model_dir_text_only("cli-detector", &text_fixture_files(), None);
let detector = dir.join("pix2text-mfd-1.5.onnx");
std::fs::write(
&detector,
plausible_onnx_bytes("a detector the manifest never declared"),
)
.expect("write detector");
let server = TestServer::start(
TestOptions::new(dir.clone(), Scripted::fast()).with_formula_routing(&detector),
);
let models = server.get("/api/models").json();
assert_eq!(models["formula"]["detector"]["state"], "present");
assert_eq!(
models["formula"]["detector"]["sha256"],
Value::Null,
"an external detector has no declared digest, and the service must not invent one: {models}"
);
assert_eq!(models["formula"]["routing"], true);
assert_eq!(
models["pipelines"]["formula"]["files"],
json!(["fx.onnx", "pix2text-mfd-1.5.onnx"]),
"{models}"
);
let accepted = server.post(
"/api/ocr?queue=formula",
&[("Content-Type", "application/octet-stream")],
b"formula image",
);
assert_eq!(accepted.status, 202, "{}", accepted.text());
std::fs::write(&detector, b"corrupted after the first verification").expect("corrupt");
let models = server.get("/api/models").json();
assert_eq!(
models["formula"]["detector"]["state"], "corrupt",
"{models}"
);
assert_eq!(
models["pipelines"]["formula"]["corrupt"],
json!(["pix2text-mfd-1.5.onnx"]),
"{models}"
);
let refused = server.request(
"POST",
"/api/ocr?queue=formula",
&[
("Host", server.host.as_str()),
("X-RapidStructure-Token", server.token.as_str()),
("Origin", server.origin.as_str()),
("Content-Type", "application/octet-stream"),
("Content-Length", "1048576"),
],
None,
);
assert_eq!(refused.status, 409, "{}", refused.text());
assert_eq!(refused.code(), "models_corrupt");
assert_eq!(
refused.json()["detail"]["corrupt"],
json!(["pix2text-mfd-1.5.onnx"]),
"the detector is the only blocker and must be named"
);
assert!(
refused.json()["detail"]["detector"]["sha256"].is_null(),
"no declared digest → null (never a fabricated one)"
);
let text = server.submit_ocr(b"text image");
assert_eq!(text.status, 202, "{}", text.text());
}
#[test]
fn models_reuses_the_verified_digest_until_the_file_identity_changes() {
let _serial = cache_serial();
let fixture = detector_model_dir("verify-cache");
let server = TestServer::start(TestOptions::new(fixture.dir.clone(), Scripted::fast()));
let first = server.get("/api/models").json();
assert_eq!(
first["verification"]["cold_this_call"], 0,
"the startup snapshot already verified every declared file: {first}"
);
assert_eq!(
first["verification"]["identity"],
"path + size + mtime + SHA-256 of the first and last 64 KiB",
"the identity must name all four components (B: the windows narrow the blind spot)"
);
assert_eq!(
first["verification"]["partial_window_bytes"], 65536,
"{first}"
);
assert!(
first["verification"]["partial_reads"].as_u64().unwrap_or(0) >= 5,
"every check reads the 128 KiB window: {first}"
);
assert!(
first["verification"]["residual_blind_spot"]
.as_str()
.unwrap_or_default()
.contains("not a security boundary"),
"the honest statement must say what the heuristic is not: {first}"
);
assert!(first["verification"]["last_cold_ms"].is_number(), "{first}");
assert!(
first["verification"]["cold_verifications"]
.as_u64()
.unwrap_or(0)
>= 5,
"the five declared files were verified at least once: {first}"
);
let hits = first["verification"]["cache_hits"].as_u64().unwrap_or(0);
let second = server.get("/api/models").json();
assert_eq!(
second["verification"]["cold_this_call"], 0,
"a cache hit must not re-hash: {second}"
);
assert!(
second["verification"]["cache_hits"].as_u64().unwrap_or(0) >= hits + 5,
"five files were answered from the cache: {second}"
);
std::fs::write(fixture.dir.join("fx.onnx"), b"replaced formula model").expect("replace");
let third = server.get("/api/models").json();
assert_eq!(
third["verification"]["cold_this_call"], 1,
"only the replaced file may be re-hashed: {third}"
);
assert_eq!(third["formula"]["complete"], false, "{third}");
assert_eq!(third["formula"]["corrupt"], json!(["fx.onnx"]));
}
#[test]
fn the_page_carries_the_reverify_button_without_inline_handlers_or_styles() {
let server = TestServer::start(TestOptions::new(
empty_model_dir("page-reverify"),
Scripted::fast(),
));
let page = server.get("/").text();
assert!(
page.contains("id=\"verifyBtn\"") && page.contains("重新校验"),
"the page must carry the reverify button"
);
assert!(
page.contains("/api/models/reverify"),
"the button must go through the documented endpoint"
);
assert!(
page.contains(r#"$("verifyBtn").addEventListener("click", reverifyModels)"#),
"the handler must be attached with addEventListener (no inline onclick under the nonce CSP)"
);
assert!(
page.contains("id=\"verifyOut\"") && page.contains("role=\"status\""),
"the outcome must be announced through a live region"
);
assert!(
!page.contains("onclick=") && !page.contains(" style=\""),
"no inline handlers or inline styles (docs/05 §9.5, WCAG: the page's own CSP forbids them)"
);
assert!(
page.contains(r#"xhrSend("POST", "/api/models/reverify", "")"#),
"the request body must be empty, exactly as the endpoint requires"
);
assert!(
page.contains("c.pipelines") && page.contains("公式管线:"),
"the page must render A2's per-pipeline verdict"
);
assert_eq!(
page.matches("__CSP_NONCE__").count(),
0,
"placeholders are injected"
);
assert_eq!(
page.matches("__SRV_TOKEN__").count(),
0,
"placeholders are injected"
);
}
#[test]
fn models_detects_a_same_size_same_mtime_swap_through_the_content_windows() {
let _serial = cache_serial();
let dir = complete_model_dir("verify-window-swap");
let path = dir.join("rec.onnx");
let server = TestServer::start(TestOptions::new(dir, Scripted::fast()));
let first = server.get("/api/models").json();
assert_eq!(first["complete"], true, "{first}");
let mismatches_before = first["verification"]["partial_mismatches"]
.as_u64()
.expect("partial_mismatches");
let identity_before = FileIdentity::of(&path).expect("identity before the swap");
let before = std::fs::metadata(&path)
.expect("metadata")
.modified()
.expect("mtime");
let original = std::fs::read(&path).expect("read the healthy model");
std::fs::write(&path, corrupt_bytes(original.len())).expect("swap the content");
let file = std::fs::OpenOptions::new()
.write(true)
.open(&path)
.expect("open to restore the timestamp");
file.set_modified(before).expect("write the old mtime back");
drop(file);
let identity_after = FileIdentity::of(&path).expect("identity after the swap");
assert_eq!(
identity_after.size(),
identity_before.size(),
"the size must be unchanged"
);
assert_eq!(
identity_after.modified(),
identity_before.modified(),
"the mtime must be written back"
);
assert_ne!(
identity_after.partial(),
identity_before.partial(),
"only the first/last 64 KiB windows can distinguish this swap"
);
let second = server.get("/api/models").json();
assert!(
second["verification"]["cold_this_call"]
.as_u64()
.expect("cold_this_call")
>= 1,
"the swap must force a re-hash (a stale hit would make this 0 and miss it): {second}"
);
assert!(
second["verification"]["partial_mismatches"]
.as_u64()
.expect("partial_mismatches")
> mismatches_before,
"the reason must be visible in the cost ledger: {second}"
);
assert_eq!(second["complete"], false, "{second}");
assert_eq!(second["corrupt"], json!(["rec.onnx"]), "{second}");
let response = server.post("/api/models/reverify", &[], b"");
assert_eq!(response.status, 200, "{}", response.text());
let body = response.json();
assert_eq!(body["corrupt"], json!(["rec.onnx"]), "{body}");
let culprit = body["files"]
.as_array()
.expect("files")
.iter()
.find(|file| file["name"] == "rec.onnx")
.expect("the swapped file must be listed");
assert_eq!(culprit["state"], "corrupt", "{body}");
assert_eq!(
culprit["cause"], "first_sight",
"a forced verification never consults the cache, so its cause is the cold path: {body}"
);
}
fn corrupt_bytes(size: usize) -> Vec<u8> {
let seed = b"corrupted model bytes";
(0..size).map(|index| seed[index % seed.len()]).collect()
}
#[test]
fn reverify_reports_a_corrupt_model_and_leaves_the_service_blocked_not_stale_ready() {
let _serial = cache_serial();
let dir = complete_model_dir("reverify-corrupt");
let path = dir.join("rec.onnx");
let before = std::fs::metadata(&path)
.expect("metadata")
.modified()
.expect("mtime");
let original = std::fs::read(&path).expect("read the healthy model");
assert!(
original.starts_with(b"test model file"),
"the fixture content must be the healthy one: {:?}",
String::from_utf8_lossy(&original)
);
std::fs::write(&path, corrupt_bytes(original.len())).expect("corrupt the recognizer");
let file = std::fs::OpenOptions::new()
.write(true)
.open(&path)
.expect("open to restore the timestamp");
file.set_modified(before).expect("write the old mtime back");
drop(file);
let built = Arc::new(AtomicUsize::new(0));
let server = TestServer::start(TestOptions {
engine_factory: counting_engine_factory(Arc::clone(&built)),
..TestOptions::new(dir.clone(), Scripted::fast())
});
assert_eq!(
server.get("/api/status").json()["engine"]["state"],
"blocked_models_missing",
"the startup readiness check must already see the corrupt recognizer"
);
assert_eq!(built.load(Ordering::SeqCst), 0, "no session was created");
let response = server.post("/api/models/reverify", &[], b"");
assert_eq!(response.status, 200, "{}", response.text());
let body = response.json();
assert_eq!(
body["computed"].as_u64(),
Some(3),
"every file **this run's plan** uses was re-hashed; the formula model is not planned \
(no --formula-detector), so the 566 MB file must not be hashed: {body}"
);
assert_eq!(body["corrupt"], json!(["rec.onnx"]), "{body}");
assert_eq!(
body["pipelines"]["formula"]["in_plan"], false,
"the formula pipeline is not part of this run: {body}"
);
assert_eq!(
body["pipelines"]["text"]["outcome"], "blocked_models_missing",
"the text engine's outcome is reported per pipeline: {body}"
);
assert_eq!(
body["outcome"], "blocked_models_missing",
"the state must be locatable, not a stale ready: {body}"
);
assert_eq!(body["engine"]["state"], "blocked_models_missing", "{body}");
assert_eq!(body["engine"]["missing"], json!(["rec.onnx"]), "{body}");
let culprit = body["files"]
.as_array()
.expect("files")
.iter()
.find(|file| file["name"] == "rec.onnx")
.expect("the corrupt file must be listed");
assert_eq!(culprit["state"], "corrupt", "{body}");
assert_eq!(
culprit["cause"], "first_sight",
"this call's cold verification has no earlier identity to compare against: {body}"
);
assert_eq!(culprit["digest_computed_this_call"], true, "{body}");
let ocr = server.submit_ocr(b"image");
assert_eq!(ocr.status, 409, "{}", ocr.text());
assert_eq!(ocr.code(), "models_corrupt", "{}", ocr.text());
assert_eq!(
ocr.json()["detail"]["corrupt"],
json!(["rec.onnx"]),
"the 409 must name the file"
);
assert_eq!(
built.load(Ordering::SeqCst),
0,
"a blocked model set must not create a session"
);
}
#[test]
fn reverify_restores_ready_and_rebuilds_the_engine_after_a_healthy_revert() {
let _serial = cache_serial();
let dir = complete_model_dir("reverify-revert");
let kept = dir.join("kept.onnx");
std::fs::write(&kept, std::fs::read(dir.join("rec.onnx")).expect("read")).expect("seed");
let built = Arc::new(AtomicUsize::new(0));
let server = TestServer::start(TestOptions {
engine_factory: counting_engine_factory(Arc::clone(&built)),
..TestOptions::new(dir.clone(), Scripted::fast())
});
assert_eq!(server.get("/api/status").json()["engine"]["state"], "ready");
assert_eq!(built.load(Ordering::SeqCst), 1);
std::fs::write(dir.join("rec.onnx"), b"corrupted same-len").expect("corrupt");
let broken = server.post("/api/models/reverify", &[], b"").json();
assert_eq!(broken["outcome"], "failed", "{broken}");
assert_eq!(broken["corrupt"], json!(["rec.onnx"]), "{broken}");
std::fs::copy(&kept, dir.join("rec.onnx")).expect("restore the model");
let response = server.post("/api/models/reverify", &[], b"");
assert_eq!(response.status, 200, "{}", response.text());
let body = response.json();
assert_eq!(body["outcome"], "ready", "{body}");
assert_eq!(body["engine"]["state"], "ready", "{body}");
assert!(body["load_ms"].as_u64().is_some(), "{body}");
assert_eq!(body["content_changed"], json!([]), "{body}");
assert_eq!(body["missing"], json!([]), "{body}");
assert_eq!(
built.load(Ordering::SeqCst),
2,
"the endpoint must rebuild the engine, not just clear a cache"
);
assert_eq!(server.get("/api/status").json()["engine"]["state"], "ready");
let id = server.submit_ocr(b"image").json()["job_id"]
.as_str()
.expect("job id")
.to_string();
assert_eq!(
server.wait_terminal(&id, Duration::from_secs(20))["state"],
"succeeded"
);
}
#[test]
fn reverify_recomputes_digests_that_the_cache_would_have_answered() {
let _serial = cache_serial();
let dir = complete_model_dir("reverify-cold");
let server = TestServer::start(TestOptions::new(dir, Scripted::fast()));
let first = server.get("/api/models").json();
let hits_before = first["verification"]["cache_hits"]
.as_u64()
.expect("cache_hits");
let second = server.get("/api/models").json();
assert_eq!(
second["verification"]["cold_this_call"], 0,
"a second poll must not re-hash anything: {second}"
);
assert!(
second["verification"]["cache_hits"]
.as_u64()
.expect("cache_hits")
> hits_before,
"the cache really answered the files this time: {second}"
);
assert_eq!(second["pipelines"]["formula"]["in_plan"], false, "{second}");
assert_eq!(
second["pipelines"]["formula"]["files"],
json!([]),
"{second}"
);
let response = server.post("/api/models/reverify", &[], b"");
assert_eq!(response.status, 200, "{}", response.text());
let body = response.json();
assert_eq!(
body["computed"].as_u64(),
Some(3),
"a cache hit must not survive the forced re-verification, and only this run's plan is \
hashed: {body}"
);
assert_eq!(body["verification"]["digests_computed"], 3, "{body}");
let files = body["files"].as_array().expect("files");
assert_eq!(files.len(), 3, "{body}");
let names: Vec<&str> = files
.iter()
.map(|file| file["name"].as_str().expect("name"))
.collect();
assert_eq!(names, vec!["det.onnx", "rec.onnx", "dict.txt"], "{body}");
assert!(
!names.contains(&"fx.onnx"),
"the 566 MB formula recognizer is not part of this run's plan: {body}"
);
for file in files {
assert_ne!(
file["cause"], "cache_hit",
"a forced verification never reports a cache hit: {body}"
);
assert_eq!(file["digest_computed_this_call"], true, "{body}");
}
let with_body = server.post("/api/models/reverify", &[], b"{}");
assert_eq!(with_body.status, 400, "{}", with_body.text());
}
#[test]
fn concurrent_reverifications_are_single_flight_and_never_run_on_the_accept_thread() {
let _serial = cache_serial();
let dir = complete_model_dir("reverify-busy");
let (tx, rx) = std::sync::mpsc::sync_channel::<()>(0);
let server = TestServer::start(TestOptions {
post_verify: Some(Box::new(move || {
let _ = tx.send(());
})),
..TestOptions::new(dir, Scripted::fast())
});
let mut stream = TcpStream::connect(server.addr).expect("connect");
stream
.set_read_timeout(Some(Duration::from_secs(60)))
.expect("read timeout");
let head = format!(
"POST /api/models/reverify HTTP/1.1\r\nConnection: close\r\nHost: {}\r\n\
X-RapidStructure-Token: {}\r\nOrigin: {}\r\nContent-Length: 0\r\n\r\n",
server.host, server.token, server.origin
);
stream
.write_all(head.as_bytes())
.expect("write the request");
stream.flush().expect("flush");
rx.recv_timeout(Duration::from_secs(30))
.expect("the first call must reach the post-verification hook");
let second = Instant::now();
let response = server.post("/api/models/reverify", &[], b"");
let elapsed = second.elapsed();
assert_eq!(response.status, 503, "{}", response.text());
assert_eq!(response.code(), "busy", "{}", response.text());
assert!(
elapsed < Duration::from_secs(5),
"the second call must not queue behind the first: {elapsed:?}"
);
let mut buffer = Vec::new();
stream
.read_to_end(&mut buffer)
.expect("read the first response");
let first = parse_response(&buffer);
assert_eq!(first.status, 200, "{}", first.text());
let body = first.json();
assert_eq!(
body["computed"].as_u64(),
Some(3),
"the formula model is not part of this run's plan: {}",
first.text()
);
assert_eq!(body["outcome"], "ready", "{}", first.text());
}
#[test]
fn reverify_cold_verifies_the_formula_detector_and_reports_it_per_pipeline() {
let _serial = cache_serial();
let fixture = detector_model_dir("plan-detector");
let original = std::fs::read(&fixture.detector).expect("read the detector");
std::fs::write(&fixture.detector, corrupt_bytes(original.len())).expect("corrupt the detector");
let actual = sha256_file(&fixture.detector).expect("hash the corrupted detector");
assert_ne!(actual, fixture.detector_sha);
let server = TestServer::start(TestOptions::new(fixture.dir.clone(), Scripted::fast()));
let response = server.post("/api/models/reverify", &[], b"");
assert_eq!(response.status, 200, "{}", response.text());
let body = response.json();
assert_eq!(
body["computed"].as_u64(),
Some(5),
"the formula detector must be cold-verified too: {body}"
);
let files = body["files"].as_array().expect("files");
let names: Vec<&str> = files
.iter()
.map(|file| file["name"].as_str().expect("name"))
.collect();
assert_eq!(
names,
vec!["det.onnx", "rec.onnx", "dict.txt", "fx.onnx", "mfd.onnx"],
"{body}"
);
let detector = files
.iter()
.find(|file| file["name"] == "mfd.onnx")
.expect("the detector must be listed");
assert_eq!(detector["role"], "formula_detector");
assert_eq!(detector["pipeline"], "formula");
assert_eq!(detector["state"], "corrupt", "{body}");
assert_eq!(
detector["declared_sha256"], fixture.detector_sha,
"the set's declared digest must be reported"
);
assert_eq!(
detector["sha256"], actual,
"this call must have re-read the file and reported the digest it computed: {body}"
);
assert_eq!(detector["digest_computed_this_call"], true, "{body}");
assert_eq!(detector["has_declared_digest"], true, "{body}");
assert_eq!(body["outcome"], "ready", "{body}");
assert_eq!(body["pipelines"]["text"]["outcome"], "ready", "{body}");
assert_eq!(body["pipelines"]["text"]["complete"], true, "{body}");
assert_eq!(body["pipelines"]["text"]["blocked"], json!([]), "{body}");
assert_eq!(
body["pipelines"]["text"]["files"],
json!(["det.onnx", "rec.onnx", "dict.txt"])
);
assert_eq!(body["pipelines"]["formula"]["in_plan"], true, "{body}");
assert_eq!(body["pipelines"]["formula"]["routing"], true, "{body}");
assert_eq!(body["pipelines"]["formula"]["complete"], false, "{body}");
assert_eq!(
body["pipelines"]["formula"]["corrupt"],
json!(["mfd.onnx"]),
"{body}"
);
assert_eq!(
body["pipelines"]["formula"]["files"],
json!(["fx.onnx", "mfd.onnx"]),
"{body}"
);
assert_eq!(
body["pipelines"]["formula"]["detector"]["sha256"], fixture.detector_sha,
"{body}"
);
assert_eq!(
body["pipelines"]["formula"]["detector"]["state"], "corrupt",
"{body}"
);
let refused = server.request(
"POST",
"/api/ocr?queue=formula",
&[
("Host", server.host.as_str()),
("X-RapidStructure-Token", server.token.as_str()),
("Origin", server.origin.as_str()),
("Content-Type", "application/octet-stream"),
("Content-Length", "1048576"),
],
None,
);
assert_eq!(refused.status, 409, "{}", refused.text());
assert_eq!(refused.code(), "models_corrupt");
assert_eq!(refused.json()["detail"]["scope"], "formula");
assert_eq!(
refused.json()["detail"]["corrupt"],
json!(["mfd.onnx"]),
"the detector is the only formula blocker and must be named"
);
let text = server.submit_ocr(b"text image");
assert_eq!(text.status, 202, "{}", text.text());
let text_id = text.json()["job_id"].as_str().expect("id").to_string();
assert_eq!(
server.wait_terminal(&text_id, Duration::from_secs(20))["state"],
"succeeded"
);
}
#[test]
fn reverify_reports_text_ready_and_formula_corrupt_explicitly() {
let _serial = cache_serial();
let fixture = detector_model_dir("plan-text-ready-formula-corrupt");
std::fs::write(fixture.dir.join("fx.onnx"), b"corrupted formula recognizer")
.expect("corrupt the formula recognizer");
let built = Arc::new(AtomicUsize::new(0));
let server = TestServer::start(TestOptions {
engine_factory: counting_engine_factory(Arc::clone(&built)),
..TestOptions::new(fixture.dir.clone(), Scripted::fast())
});
assert_eq!(
server.get("/api/status").json()["engine"]["state"],
"ready",
"the text engine must load: the formula model is not part of it"
);
let body = server.post("/api/models/reverify", &[], b"").json();
assert_eq!(
body["outcome"], "ready",
"the text engine outcome is ready (it really rebuilt): {body}"
);
assert_eq!(built.load(Ordering::SeqCst), 2, "startup + rebuild");
assert_eq!(body["missing"], json!([]), "{body}");
assert_eq!(
body["corrupt"],
json!([]),
"the top-level lists stay text-scoped: {body}"
);
assert_eq!(body["pipelines"]["text"]["outcome"], "ready", "{body}");
assert_eq!(body["pipelines"]["text"]["complete"], true, "{body}");
assert_eq!(body["pipelines"]["text"]["blocked"], json!([]), "{body}");
assert_eq!(body["pipelines"]["formula"]["in_plan"], true, "{body}");
assert_eq!(
body["pipelines"]["formula"]["corrupt"],
json!(["fx.onnx"]),
"'text ready, formula corrupt' must be explicit: {body}"
);
assert_eq!(body["pipelines"]["formula"]["complete"], false, "{body}");
assert_eq!(body["pipelines"]["formula"]["missing"], json!([]), "{body}");
let refused = server.post(
"/api/ocr?queue=formula",
&[("Content-Type", "application/octet-stream")],
b"formula image",
);
assert_eq!(refused.status, 409, "{}", refused.text());
assert_eq!(refused.code(), "models_corrupt");
assert_eq!(
refused.json()["detail"]["corrupt"],
json!(["fx.onnx"]),
"{}",
refused.text()
);
let text = server.submit_ocr(b"text image");
assert_eq!(text.status, 202, "{}", text.text());
let text_id = text.json()["job_id"].as_str().expect("id").to_string();
assert_eq!(
server.wait_terminal(&text_id, Duration::from_secs(20))["state"],
"succeeded"
);
}
#[test]
fn an_external_detector_is_reported_honestly_without_a_declared_digest() {
let _serial = cache_serial();
let dir = manifest_model_dir_text_only("plan-external-detector", &text_fixture_files(), None);
let outside = m2_root().join(format!("external-detector-{}.onnx", unique()));
let name = outside
.file_name()
.expect("file name")
.to_string_lossy()
.into_owned();
let server = TestServer::start(
TestOptions::new(dir.clone(), Scripted::fast()).with_formula_routing(&outside),
);
let expected_sha = sha256_file(&outside).expect("hash the external detector");
let models = server.get("/api/models").json();
assert_eq!(models["formula"]["routing"], true, "{models}");
assert_eq!(
models["pipelines"]["formula"]["files"],
json!(["fx.onnx", name.clone()]),
"{models}"
);
assert_eq!(
models["pipelines"]["formula"]["detector"]["sha256"],
Value::Null,
"no digest was declared, and the service must not invent one: {models}"
);
assert_eq!(
models["pipelines"]["formula"]["detector"]["state"], "present",
"exists + readable + plausible ONNX: {models}"
);
assert!(
!models.to_string().contains(&dir.display().to_string()),
"no absolute path may leak: {models}"
);
let body = server.post("/api/models/reverify", &[], b"").json();
assert_eq!(body["computed"].as_u64(), Some(5), "{body}");
let detector = body["files"]
.as_array()
.expect("files")
.iter()
.find(|file| file["name"] == name.as_str())
.expect("the external detector must be in the plan");
assert_eq!(detector["declared_sha256"], Value::Null, "{body}");
assert_eq!(detector["has_declared_digest"], false, "{body}");
assert_eq!(
detector["sha256"], expected_sha,
"the digest computed by this call must be reported: {body}"
);
assert_eq!(detector["digest_computed_this_call"], true, "{body}");
assert_eq!(detector["state"], "present", "{body}");
let accepted = server.post(
"/api/ocr?queue=formula",
&[("Content-Type", "application/octet-stream")],
b"formula image",
);
assert_eq!(accepted.status, 202, "{}", accepted.text());
std::fs::write(&outside, b"not a model at all").expect("corrupt the detector");
let models = server.get("/api/models").json();
assert_eq!(
models["pipelines"]["formula"]["detector"]["state"], "corrupt",
"{models}"
);
assert_eq!(
models["pipelines"]["formula"]["corrupt"],
json!([name.clone()]),
"{models}"
);
let refused = server.request(
"POST",
"/api/ocr?queue=formula",
&[
("Host", server.host.as_str()),
("X-RapidStructure-Token", server.token.as_str()),
("Origin", server.origin.as_str()),
("Content-Type", "application/octet-stream"),
("Content-Length", "1048576"),
],
None,
);
assert_eq!(refused.status, 409, "{}", refused.text());
assert_eq!(refused.code(), "models_corrupt");
assert_eq!(refused.json()["detail"]["corrupt"], json!([name.clone()]));
let text = server.submit_ocr(b"text image");
assert_eq!(text.status, 202, "{}", text.text());
std::fs::remove_file(&outside).ok();
}
#[test]
fn every_entry_point_reads_the_same_run_plan() {
let _serial = cache_serial();
let fixture = detector_model_dir("plan-identity");
let plan =
ModelPlan::resolve(&fixture.dir, &test_engine_config(), None, None).expect("model plan");
let expected: Vec<&str> = plan
.plan_files()
.iter()
.map(|file| file.name.as_str())
.collect();
assert_eq!(
expected,
vec!["det.onnx", "rec.onnx", "dict.txt", "fx.onnx", "mfd.onnx"],
"{expected:?}"
);
let text_expected: Vec<&str> = plan
.plan_files_in(super::model_plan::Pipeline::Text)
.iter()
.map(|file| file.name.as_str())
.collect();
let formula_expected: Vec<&str> = plan
.plan_files_in(super::model_plan::Pipeline::Formula)
.iter()
.map(|file| file.name.as_str())
.collect();
assert_eq!(text_expected, vec!["det.onnx", "rec.onnx", "dict.txt"]);
assert_eq!(formula_expected, vec!["fx.onnx", "mfd.onnx"]);
let server = TestServer::start(TestOptions::new(fixture.dir.clone(), Scripted::fast()));
let models = server.get("/api/models").json();
assert_eq!(models["pipelines"]["text"]["files"], json!(text_expected));
assert_eq!(
models["pipelines"]["formula"]["files"],
json!(formula_expected)
);
let body = server.post("/api/models/reverify", &[], b"").json();
let verified: Vec<&str> = body["files"]
.as_array()
.expect("files")
.iter()
.map(|file| file["name"].as_str().expect("name"))
.collect();
assert_eq!(verified, expected, "A2 must verify exactly this run's plan");
assert_eq!(
body["computed"].as_u64(),
Some(expected.len() as u64),
"{body}"
);
assert_eq!(
body["pipelines"]["text"]["files"],
json!(text_expected),
"the per-pipeline report is the same plan"
);
assert_eq!(
body["pipelines"]["formula"]["files"],
json!(formula_expected)
);
let policy = server
.shared()
.formula_policy()
.expect("formula is planned");
assert_eq!(
policy.model_path.as_deref(),
Some(
plan.plan_file(rapid_structure_rs::ModelRole::FormulaRecognizer)
.expect("planned")
.path
.as_path()
)
);
assert_eq!(
policy.expected_model_sha256,
plan.plan_file(rapid_structure_rs::ModelRole::FormulaRecognizer)
.expect("planned")
.declared_sha256
);
assert_eq!(
policy.detector_path.as_deref(),
Some(
plan.plan_file(rapid_structure_rs::ModelRole::FormulaDetector)
.expect("planned")
.path
.as_path()
)
);
assert_eq!(
policy.expected_detector_sha256,
plan.plan_file(rapid_structure_rs::ModelRole::FormulaDetector)
.expect("planned")
.declared_sha256
);
let original = std::fs::read(&fixture.detector).expect("read the detector");
std::fs::write(&fixture.detector, b"corrupted detector bytes").expect("corrupt");
let refused = server.post(
"/api/ocr?queue=formula",
&[("Content-Type", "application/octet-stream")],
b"formula image",
);
assert_eq!(refused.status, 409, "{}", refused.text());
assert_eq!(
refused.json()["detail"]["corrupt"],
json!(["mfd.onnx"]),
"admission names the same planned file the report does"
);
assert_eq!(server.submit_ocr(b"text image").status, 202);
std::fs::write(&fixture.detector, &original).expect("restore");
let accepted = server.post(
"/api/ocr?queue=formula",
&[("Content-Type", "application/octet-stream")],
b"formula image",
);
assert_eq!(accepted.status, 202, "{}", accepted.text());
}
#[test]
fn the_evaluate_endpoint_returns_the_library_summary() {
let dir = complete_model_dir("m4-evaluate");
let scripted = Scripted::fast();
let server = TestServer::start(TestOptions::new(dir.clone(), scripted));
let manifest = eval_manifest(
&dir,
"eval-ok",
&[
("01.png", "region-0\nregion-1\nregion-2"),
("02.png", "totally different"),
],
);
let response = server.post(
"/api/evaluate",
&[("Content-Type", "application/json")],
format!("{{\"manifest\":{:?}}}", manifest.display().to_string()).as_bytes(),
);
assert_eq!(response.status, 200, "{}", response.text());
let report = response.json();
let expected_second = cer("totally different", "region-0\nregion-1\nregion-2");
let expected_mean = (0.0_f32 + expected_second) / 2.0;
assert_eq!(report["cases"].as_array().expect("cases").len(), 2);
assert_eq!(report["cases"][0]["image"], "01.png");
assert_eq!(report["cases"][0]["cer"], 0.0);
assert_eq!(report["cases"][0]["exact_text"], true);
assert_eq!(report["cases"][1]["image"], "02.png");
assert!(
(report["cases"][1]["cer"].as_f64().expect("cer") - f64::from(expected_second)).abs()
< 1e-6,
"{report}"
);
assert!(
(report["mean_cer"].as_f64().expect("mean_cer") - f64::from(expected_mean)).abs() < 1e-6
);
assert_eq!(report["exact_match_rate"], 0.5);
assert_eq!(report["iou_threshold"], 0.5);
assert_eq!(report["manifest_file"], "manifest.json");
assert!(report["peak_working_set_bytes"].as_u64().unwrap_or(0) > 0);
assert!(report["memory_source"].is_string());
assert!(report["ort_runtime"].is_object());
assert!(report["ort_runtime_version"].is_string());
assert!(
!report.to_string().contains(&dir.display().to_string()),
"no absolute path in the report: {report}"
);
}
#[test]
fn an_invalid_evaluation_request_is_a_locating_400() {
let dir = complete_model_dir("m4-evaluate-refused");
let server = TestServer::start(TestOptions::new(dir.clone(), Scripted::fast()));
let no_body = server.post(
"/api/evaluate",
&[("Content-Type", "application/json")],
b"{}",
);
assert_eq!(no_body.status, 400, "{}", no_body.text());
let extra_key = server.post(
"/api/evaluate",
&[("Content-Type", "application/json")],
br#"{"manifest":"a.json","iou_threshold":0.9}"#,
);
assert_eq!(extra_key.status, 400);
let missing = dir.join("no-such-manifest.json");
let response = server.post(
"/api/evaluate",
&[("Content-Type", "application/json")],
format!("{{\"manifest\":{:?}}}", missing.display().to_string()).as_bytes(),
);
assert_eq!(response.status, 400, "{}", response.text());
assert_eq!(response.code(), "bad_request");
let reason = response.json()["detail"]["reason"]
.as_str()
.expect("reason")
.to_string();
assert!(reason.contains("no-such-manifest.json"), "{reason}");
let manifest = eval_manifest(
&dir,
"eval-too-many",
&[("a.png", "x"), ("b.png", "x"), ("c.png", "x")],
);
let server = TestServer::start(TestOptions::new(dir.clone(), Scripted::fast()).with_limits(
RawServeLimits {
max_eval_cases: 2,
..RawServeLimits::default()
},
));
let response = server.post(
"/api/evaluate",
&[("Content-Type", "application/json")],
format!("{{\"manifest\":{:?}}}", manifest.display().to_string()).as_bytes(),
);
assert_eq!(response.status, 400, "{}", response.text());
let reason = response.json()["detail"]["reason"]
.as_str()
.expect("reason")
.to_string();
assert!(reason.contains("--max-eval-cases"), "{reason}");
}
#[test]
fn evaluation_is_refused_without_an_explicit_eval_root() {
let dir = complete_model_dir("m4-evaluate-no-root");
let server =
TestServer::start(TestOptions::new(dir.clone(), Scripted::fast()).without_eval_root());
let manifest = eval_manifest(&dir, "eval-no-root", &[("01.png", "x")]);
let response = server.post(
"/api/evaluate",
&[("Content-Type", "application/json")],
format!("{{\"manifest\":{:?}}}", manifest.display().to_string()).as_bytes(),
);
assert_eq!(response.status, 400, "{}", response.text());
assert_eq!(response.code(), "bad_request");
let reason = response.json()["detail"]["reason"]
.as_str()
.expect("reason")
.to_string();
assert!(reason.contains("--eval-root"), "{reason}");
assert!(reason.contains("disabled"), "{reason}");
}
#[test]
fn an_evaluation_manifest_outside_the_eval_root_is_refused() {
let dir = complete_model_dir("m4-evaluate-escape");
let server = TestServer::start(TestOptions::new(dir.clone(), Scripted::fast()));
let outside = m2_root().join(format!("outside-manifest-{}", unique()));
std::fs::create_dir_all(&outside).expect("create the outside dir");
let manifest = outside.join("manifest.json");
std::fs::write(&manifest, br#"[{"image":"01.png","text":"x","boxes":[]}]"#).expect("write");
std::fs::write(outside.join("01.png"), b"image bytes").expect("write the image");
let response = server.post(
"/api/evaluate",
&[("Content-Type", "application/json")],
format!("{{\"manifest\":{:?}}}", manifest.display().to_string()).as_bytes(),
);
assert_eq!(response.status, 400, "{}", response.text());
let reason = response.json()["detail"]["reason"]
.as_str()
.expect("reason")
.to_string();
assert!(reason.contains("--eval-root sandbox"), "{reason}");
assert!(
reason.contains("manifest"),
"the offending path must be named: {reason}"
);
std::fs::remove_dir_all(&outside).ok();
}
#[test]
fn an_evaluation_image_outside_the_eval_root_is_refused() {
let dir = complete_model_dir("m4-evaluate-image-escape");
let server = TestServer::start(TestOptions::new(dir.clone(), Scripted::fast()));
let outside_dir = test_root().join(format!("outside-image-{}", unique()));
std::fs::create_dir_all(&outside_dir).expect("create the outside dir");
let outside_image = outside_dir.join("01.png");
std::fs::write(&outside_image, b"image bytes outside the sandbox").expect("write the image");
let outside_name = outside_dir
.file_name()
.expect("a directory name")
.to_string_lossy()
.into_owned();
let sandbox_dir = dir.join(format!("escape-images-{}", unique()));
std::fs::create_dir_all(&sandbox_dir).expect("create the sandbox subdir");
let absolute = sandbox_dir.join("absolute.json");
std::fs::write(
&absolute,
format!(
"[{{\"image\":{:?},\"text\":\"x\",\"boxes\":[]}}]",
outside_image.display().to_string()
),
)
.expect("write the absolute manifest");
let response = server.post(
"/api/evaluate",
&[("Content-Type", "application/json")],
format!("{{\"manifest\":{:?}}}", absolute.display().to_string()).as_bytes(),
);
assert_eq!(response.status, 400, "{}", response.text());
let reason = response.json()["detail"]["reason"]
.as_str()
.expect("reason")
.to_string();
assert!(
reason.contains("outside the --eval-root sandbox"),
"an escaping absolute path must be named as an escape, not as a missing file: {reason}"
);
assert!(reason.contains("01.png"), "{reason}");
let relative = sandbox_dir.join("relative.json");
std::fs::write(
&relative,
format!(
"[{{\"image\":{:?},\"text\":\"x\",\"boxes\":[]}}]",
format!("../../{outside_name}/01.png")
),
)
.expect("write the relative manifest");
let response = server.post(
"/api/evaluate",
&[("Content-Type", "application/json")],
format!("{{\"manifest\":{:?}}}", relative.display().to_string()).as_bytes(),
);
assert_eq!(response.status, 400, "{}", response.text());
let reason = response.json()["detail"]["reason"]
.as_str()
.expect("reason")
.to_string();
assert!(
reason.contains("outside the --eval-root sandbox"),
"a `..` escape must be refused as an escape: {reason}"
);
let link = sandbox_dir.join("link.png");
match std::os::windows::fs::symlink_file(&outside_image, &link) {
Ok(()) => {
let linked = sandbox_dir.join("symlink.json");
std::fs::write(&linked, br#"[{"image":"link.png","text":"x","boxes":[]}]"#)
.expect("write the symlink manifest");
let response = server.post(
"/api/evaluate",
&[("Content-Type", "application/json")],
format!("{{\"manifest\":{:?}}}", linked.display().to_string()).as_bytes(),
);
assert_eq!(response.status, 400, "{}", response.text());
assert!(
response.json()["detail"]["reason"]
.as_str()
.unwrap_or_default()
.contains("outside the --eval-root sandbox"),
"a symlink escape must be refused: {}",
response.text()
);
}
Err(error) => eprintln!(
"serve tests: cannot create a symlink to prove the symlink-escape case ({error}); \
the absolute and `..` cases exercise the same canonical-containment rule"
),
}
std::fs::remove_dir_all(&outside_dir).ok();
}
#[test]
fn evaluation_needs_the_same_model_admission_as_ocr() {
let dir = empty_model_dir("m4-evaluate-models");
let server = TestServer::start(TestOptions::new(dir.clone(), Scripted::fast()));
let manifest = eval_manifest(&dir, "admission", &[("01.png", "x")]);
let response = server.post(
"/api/evaluate",
&[("Content-Type", "application/json")],
format!("{{\"manifest\":{:?}}}", manifest.display().to_string()).as_bytes(),
);
assert_eq!(response.status, 409, "{}", response.text());
assert_eq!(response.code(), "models_missing");
assert_eq!(
response.json()["detail"]["missing"],
server.get("/api/models").json()["missing"]
);
}
#[test]
fn a_second_evaluation_is_refused_while_one_is_running() {
let dir = complete_model_dir("m4-evaluate-busy");
let gate = Gate::new();
let server = TestServer::start(TestOptions::new(
dir.clone(),
Scripted::fast().with_gate(Arc::clone(&gate)),
));
let manifest = eval_manifest(
&dir,
"eval-busy",
&[("01.png", "region-0\nregion-1\nregion-2")],
);
let body = format!("{{\"manifest\":{:?}}}", manifest.display().to_string()).into_bytes();
let addr = server.addr;
let host = server.host.clone();
let origin = server.origin.clone();
let token = server.token.clone();
let first_body = body.clone();
let first = std::thread::spawn(move || {
raw_request(
addr,
"POST",
"/api/evaluate",
&[
("Host", host.as_str()),
("X-RapidStructure-Token", token.as_str()),
("Origin", origin.as_str()),
("Content-Type", "application/json"),
],
Some(&first_body),
)
});
gate.arrive();
let second = server.post(
"/api/evaluate",
&[("Content-Type", "application/json")],
&body,
);
assert_eq!(second.status, 503, "{}", second.text());
assert_eq!(second.code(), "busy");
gate.release();
let first = first.join().expect("the evaluation thread must finish");
assert_eq!(first.status, 200, "{}", first.text());
let third = server.post(
"/api/evaluate",
&[("Content-Type", "application/json")],
br#"{"manifest":"definitely-not-here.json"}"#,
);
assert_eq!(third.status, 400, "{}", third.text());
}
fn cer(reference: &str, hypothesis: &str) -> f32 {
rapid_structure_rs::evaluation::ocr::character_error_rate(reference, hypothesis)
}
fn eval_manifest(dir: &Path, label: &str, cases: &[(&str, &str)]) -> PathBuf {
let root = dir.join(format!("{label}-{}", unique()));
std::fs::create_dir_all(&root).expect("create the manifest dir");
let mut manifest = String::from("[");
for (index, (image, text)) in cases.iter().enumerate() {
std::fs::write(root.join(image), format!("image bytes for {image}")).expect("write image");
if index > 0 {
manifest.push(',');
}
manifest.push_str(&format!(
"{{\"image\":{image:?},\"text\":{text:?},\"boxes\":[]}}"
));
}
manifest.push(']');
let path = root.join("manifest.json");
std::fs::write(&path, manifest).expect("write the manifest");
path
}
#[test]
fn the_documented_route_set_is_the_only_one_that_exists() {
assert!(
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("src/bin/web/index.html")
.is_file(),
"the inlined page must exist"
);
}
fn part_files(dir: &Path) -> Vec<String> {
let mut names: Vec<String> = std::fs::read_dir(dir)
.expect("the model dir must be readable")
.map(|entry| {
entry
.expect("a readable entry")
.file_name()
.to_string_lossy()
.into_owned()
})
.filter(|name| name.contains(".part"))
.collect();
names.sort();
names
}
fn counting_engine_factory(counter: Arc<AtomicUsize>) -> EngineFactory {
Arc::new(move |_config: &EngineConfig| {
counter.fetch_add(1, Ordering::SeqCst);
Ok(Box::new(ScriptedBackend {
state: Scripted::fast(),
}))
})
}
fn gated_engine_factory(gate: Arc<Gate>, counter: Arc<AtomicUsize>) -> EngineFactory {
Arc::new(move |_config: &EngineConfig| {
counter.fetch_add(1, Ordering::SeqCst);
gate.wait();
Ok(Box::new(ScriptedBackend {
state: Scripted::fast(),
}))
})
}
#[derive(Clone)]
struct SessionPlan {
calls: Arc<AtomicUsize>,
gate_at: Option<usize>,
fail_at: Vec<usize>,
delay: Duration,
}
impl SessionPlan {
fn new() -> Self {
Self {
calls: Arc::new(AtomicUsize::new(0)),
gate_at: None,
fail_at: Vec::new(),
delay: Duration::ZERO,
}
}
fn calls(&self) -> usize {
self.calls.load(Ordering::SeqCst)
}
fn factory(&self, gate: Option<Arc<Gate>>) -> EngineFactory {
let plan = self.clone();
Arc::new(
move |_config: &EngineConfig| -> Result<Box<dyn OcrBackend>, RapidOcrError> {
let index = plan.calls.fetch_add(1, Ordering::SeqCst) + 1;
if plan.gate_at == Some(index)
&& let Some(gate) = &gate
{
gate.wait();
}
if plan.fail_at.contains(&index) {
return Err(RapidOcrError::UnsupportedProvider(format!(
"scripted session {index} cannot be created"
)));
}
Ok(Box::new(ScriptedBackend {
state: Scripted {
ep: format!("scripted-{index}"),
delay: plan.delay,
..Scripted::fast()
},
}))
},
)
}
}
struct ReleaseOnDrop {
gate: Arc<Gate>,
released: AtomicBool,
}
impl ReleaseOnDrop {
fn new(gate: Arc<Gate>) -> Self {
Self {
gate,
released: AtomicBool::new(false),
}
}
fn release(&self) {
if !self.released.swap(true, Ordering::SeqCst) {
self.gate.release();
}
}
}
impl Drop for ReleaseOnDrop {
fn drop(&mut self) {
self.release();
}
}
#[test]
fn a_download_job_fetches_every_missing_file_and_makes_the_set_complete() {
let dir = manifest_model_dir_text_only("download-ok", &["det.onnx"], None);
let scripted = ScriptedDownload::default();
let server = TestServer::start(TestOptions {
allow_download: true,
downloader: scripted.clone().factory(),
..TestOptions::new(dir.clone(), Scripted::fast())
});
let before = server.get("/api/models").json();
assert_eq!(before["source"], "local_manifest");
assert_eq!(before["complete"], false);
assert_eq!(
before["missing"],
serde_json::json!(["rec.onnx", "dict.txt"])
);
let response = server.download("test-set");
assert_eq!(response.status, 202, "{}", response.text());
let accepted = response.json();
assert_eq!(accepted["kind"], "model_download");
assert_eq!(
accepted["queue"], "download",
"a download job is not in the text queue"
);
assert_eq!(accepted["state"], "queued");
assert_eq!(accepted["download"]["files_total"], 2);
assert_eq!(accepted["download"]["files_done"], 0);
let id = accepted["job_id"].as_str().expect("job id").to_string();
let view = server.wait_terminal(&id, Duration::from_secs(20));
assert_eq!(view["state"], "succeeded", "{view}");
assert_eq!(view["failure"], serde_json::Value::Null);
assert_eq!(view["queue"], "download");
assert_eq!(view["download"]["files_done"], 2);
assert_eq!(view["download"]["files_total"], 2);
assert_eq!(view["download"]["current_file"], serde_json::Value::Null);
let expected_bytes: u64 = ["rec.onnx", "dict.txt"]
.iter()
.map(|name| model_file_bytes(name).len() as u64)
.sum();
assert_eq!(view["download"]["bytes_done"], expected_bytes);
assert_eq!(view["download"]["bytes_total"], expected_bytes);
assert_eq!(
scripted.written(),
vec!["rec.onnx".to_string(), "dict.txt".to_string()]
);
let after = server.get("/api/models").json();
assert_eq!(after["complete"], true, "{after}");
assert_eq!(after["missing"], serde_json::json!([]));
assert_eq!(after["sets"][0]["complete"], true);
assert!(part_files(&dir).is_empty());
assert_eq!(
server.get("/api/status").json()["engine"]["state"],
"blocked_models_missing"
);
}
#[test]
fn a_hash_failure_fails_the_download_job_with_a_structured_error() {
let dir = manifest_model_dir("download-hash", &["det.onnx"], None);
let server = TestServer::start(TestOptions {
allow_download: true,
downloader: ScriptedDownload::with_plan(vec![Step::HashMismatch]).factory(),
..TestOptions::new(dir.clone(), Scripted::fast())
});
let id = server.download("test-set").json()["job_id"]
.as_str()
.expect("job id")
.to_string();
let view = server.wait_terminal(&id, Duration::from_secs(20));
assert_eq!(view["state"], "failed", "{view}");
assert_eq!(view["failure"]["status"], 502);
assert_eq!(view["failure"]["code"], "download_failed");
assert_eq!(view["failure"]["detail"]["kind"], "hash_mismatch");
assert!(
view["error"]
.as_str()
.is_some_and(|text| text.contains("SHA-256")),
"{view}"
);
assert!(!dir.join("rec.onnx").exists());
assert!(!dir.join("dict.txt").exists());
assert!(part_files(&dir).is_empty(), "no temp file may survive");
}
#[test]
fn a_download_budget_refusal_is_a_413_with_both_numbers() {
let declared = 10 * 1024 * 1024;
let dir = manifest_model_dir_text_only("download-budget", &["det.onnx"], Some(declared));
let scripted = ScriptedDownload::default();
let server = TestServer::start(TestOptions {
limits: RawServeLimits {
max_download_mb: 1,
..RawServeLimits::default()
},
allow_download: true,
downloader: scripted.clone().factory(),
..TestOptions::new(dir, Scripted::fast())
});
let response = server.download("test-set");
assert_eq!(response.status, 413, "{}", response.text());
assert_eq!(response.code(), "payload_too_large");
let detail = &response.json()["detail"];
assert_eq!(detail["limit_bytes"], 1024 * 1024);
assert_eq!(detail["observed_bytes"], 2 * declared);
assert!(
scripted.hosts_per_call().is_empty(),
"a refused task must not reach the downloader"
);
}
#[test]
fn a_disk_space_refusal_is_a_507_with_both_numbers() {
let declared = 10 * 1024 * 1024;
let dir = manifest_model_dir_text_only("download-disk", &["det.onnx"], Some(declared));
let scripted = ScriptedDownload::default();
let server = TestServer::start(TestOptions {
allow_download: true,
downloader: scripted.clone().factory(),
free_space: Arc::new(|_dir: &Path| Ok(10)),
..TestOptions::new(dir, Scripted::fast())
});
let response = server.download("test-set");
assert_eq!(response.status, 507, "{}", response.text());
assert_eq!(response.code(), "insufficient_disk_space");
let detail = &response.json()["detail"];
assert_eq!(detail["required_bytes"], 2 * declared);
assert_eq!(detail["available_bytes"], 10);
assert!(scripted.hosts_per_call().is_empty());
}
#[test]
fn a_per_file_failure_keeps_the_files_that_already_verified() {
let dir = manifest_model_dir("download-partial", &["det.onnx"], None);
let server = TestServer::start(TestOptions {
allow_download: true,
downloader: ScriptedDownload::with_plan(vec![Step::Write, Step::Network]).factory(),
..TestOptions::new(dir.clone(), Scripted::fast())
});
let id = server.download("test-set").json()["job_id"]
.as_str()
.expect("job id")
.to_string();
let view = server.wait_terminal(&id, Duration::from_secs(20));
assert_eq!(view["state"], "failed", "{view}");
assert_eq!(view["failure"]["code"], "download_failed");
assert_eq!(view["failure"]["detail"]["kind"], "network");
assert_eq!(view["download"]["files_done"], 1);
assert_eq!(
view["download"]["current_file"], "dict.txt",
"the failing file stays named in the progress"
);
assert!(dir.join("rec.onnx").is_file(), "the verified file stays");
assert!(!dir.join("dict.txt").exists());
assert!(part_files(&dir).is_empty());
}
#[test]
fn the_download_progress_is_observable_while_the_job_runs() {
let dir = manifest_model_dir_text_only("download-progress", &["det.onnx"], None);
let gate = Gate::new();
let server = TestServer::start(TestOptions {
allow_download: true,
downloader: ScriptedDownload::with_gate(Arc::clone(&gate)).factory(),
..TestOptions::new(dir, Scripted::fast())
});
let id = server.download("test-set").json()["job_id"]
.as_str()
.expect("job id")
.to_string();
let rec_len = model_file_bytes("rec.onnx").len() as u64;
let dict_len = model_file_bytes("dict.txt").len() as u64;
gate.arrive();
let view = server.get(&format!("/api/jobs/{id}")).json();
assert_eq!(view["state"], "running", "{view}");
assert_eq!(view["download"]["current_file"], "rec.onnx");
assert_eq!(view["download"]["files_done"], 0);
assert_eq!(view["download"]["files_total"], 2);
assert_eq!(view["download"]["bytes_done"], 0);
assert_eq!(view["download"]["bytes_total"], rec_len + dict_len);
gate.release();
gate.arrive();
let view = server.get(&format!("/api/jobs/{id}")).json();
assert_eq!(view["download"]["current_file"], "dict.txt");
assert_eq!(view["download"]["files_done"], 1);
assert_eq!(view["download"]["bytes_done"], rec_len);
gate.release();
let view = server.wait_terminal(&id, Duration::from_secs(20));
assert_eq!(view["state"], "succeeded", "{view}");
assert_eq!(view["download"]["files_done"], 2);
assert_eq!(view["download"]["bytes_done"], rec_len + dict_len);
assert_eq!(view["download"]["current_file"], serde_json::Value::Null);
}
#[test]
fn a_running_download_is_cancelled_at_a_file_boundary() {
let dir = manifest_model_dir("download-cancel", &["det.onnx"], None);
let gate = Gate::new();
let server = TestServer::start(TestOptions {
allow_download: true,
downloader: ScriptedDownload::with_gate(Arc::clone(&gate)).factory(),
..TestOptions::new(dir.clone(), Scripted::fast())
});
let id = server.download("test-set").json()["job_id"]
.as_str()
.expect("job id")
.to_string();
gate.arrive(); let cancelled = server.post(&format!("/api/jobs/{id}/cancel"), &[], b"");
assert_eq!(cancelled.status, 200, "{}", cancelled.text());
let view = cancelled.json();
assert_eq!(
view["state"], "running",
"the current file is still being downloaded"
);
assert_eq!(view["cancel_requested"], true);
gate.release();
let view = server.wait_terminal(&id, Duration::from_secs(20));
assert_eq!(view["state"], "cancelled", "{view}");
assert_eq!(view["cancel_requested"], true);
assert_eq!(
view["download"]["files_done"], 1,
"the current file was kept"
);
assert!(dir.join("rec.onnx").is_file(), "the verified file stays");
assert!(
!dir.join("dict.txt").exists(),
"the next file is never started"
);
assert!(
part_files(&dir).is_empty(),
"no temp file survives a cancellation"
);
}
#[test]
fn a_download_error_cancelled_lands_on_cancelled_not_running() {
let dir = manifest_model_dir("download-cancelled-error", &["det.onnx"], None);
let server = TestServer::start(TestOptions {
allow_download: true,
downloader: ScriptedDownload::with_plan(vec![Step::Cancelled]).factory(),
..TestOptions::new(dir.clone(), Scripted::fast())
});
let id = server.download("test-set").json()["job_id"]
.as_str()
.expect("job id")
.to_string();
let view = server.wait_terminal(&id, Duration::from_secs(20));
assert_eq!(view["state"], "cancelled", "{view}");
assert_eq!(
view["failure"],
serde_json::Value::Null,
"a cancellation is not a failure"
);
assert_eq!(view["cancel_requested"], true);
assert_eq!(view["download"]["current_file"], "rec.onnx");
assert!(!dir.join("rec.onnx").exists());
assert!(part_files(&dir).is_empty());
}
#[test]
fn cancelling_a_queued_download_is_immediate() {
let dir = manifest_model_dir_text_only("download-queued", &["det.onnx"], None);
let gate = Gate::new();
let server = TestServer::start(TestOptions {
allow_download: true,
downloader: ScriptedDownload::with_gate(Arc::clone(&gate)).factory(),
..TestOptions::new(dir, Scripted::fast())
});
let first = server.download("test-set").json()["job_id"]
.as_str()
.expect("job id")
.to_string();
gate.arrive();
let second = server.download("test-set").json()["job_id"]
.as_str()
.expect("job id")
.to_string();
let cancelled = server.post(&format!("/api/jobs/{second}/cancel"), &[], b"");
assert_eq!(cancelled.status, 200, "{}", cancelled.text());
let view = cancelled.json();
assert_eq!(
view["state"], "cancelled",
"a queued job cancels immediately"
);
assert_eq!(view["cancel_requested"], false);
gate.release();
gate.arrive();
gate.release();
let first_view = server.wait_terminal(&first, Duration::from_secs(20));
assert_eq!(first_view["state"], "succeeded", "{first_view}");
let second_view = server.get(&format!("/api/jobs/{second}")).json();
assert_eq!(second_view["state"], "cancelled");
assert_eq!(second_view["failure"], serde_json::Value::Null);
assert_eq!(second_view["download"]["files_done"], 0);
assert_eq!(
second_view["download"]["current_file"],
serde_json::Value::Null
);
}
#[test]
fn an_unknown_or_empty_set_id_is_refused_and_never_guessed() {
let dir = manifest_model_dir("download-unknown", &["det.onnx"], None);
let scripted = ScriptedDownload::default();
let server = TestServer::start(TestOptions {
allow_download: true,
downloader: scripted.clone().factory(),
..TestOptions::new(dir, Scripted::fast())
});
let unknown = server.download("nope");
assert_eq!(unknown.status, 404, "{}", unknown.text());
assert_eq!(unknown.code(), "model_set_not_found");
assert_eq!(unknown.json()["detail"]["set_id"], "nope");
assert_eq!(unknown.json()["detail"]["known_sets"][0], "test-set");
for bad in [
&br#"{"set_id":""}"#[..],
&br#"{}"#[..],
&br#"{"set_id":null}"#[..],
&br#"{"set_id":"test-set","url":"https://evil.example/x.onnx"}"#[..],
] {
let response = server.post(
"/api/models/download",
&[("Content-Type", "application/json")],
bad,
);
assert_eq!(response.status, 400, "{}", response.text());
}
assert!(
scripted.hosts_per_call().is_empty(),
"a refused request must not download anything"
);
}
#[test]
fn the_download_host_opt_in_is_passed_as_an_explicit_parameter() {
let dir = manifest_model_dir("download-hosts", &["det.onnx"], None);
let plain = ScriptedDownload::default();
let server = TestServer::start(TestOptions {
allow_download: true,
downloader: plain.clone().factory(),
..TestOptions::new(dir, Scripted::fast())
});
let id = server.download("test-set").json()["job_id"]
.as_str()
.expect("job id")
.to_string();
assert_eq!(
server.wait_terminal(&id, Duration::from_secs(20))["state"],
"succeeded"
);
assert_eq!(
plain.hosts_per_call(),
vec![vec![
"www.modelscope.cn".to_string(),
"cdn-lfs-cn-1.modelscope.cn".to_string()
]]
);
let dir = manifest_model_dir("download-hosts-optin", &["det.onnx"], None);
let extended = ScriptedDownload::default();
let server = TestServer::start(TestOptions {
allow_download: true,
allow_download_hosts: vec!["evil.example".to_string()],
downloader: extended.clone().factory(),
..TestOptions::new(dir, Scripted::fast())
});
let id = server.download("test-set").json()["job_id"]
.as_str()
.expect("job id")
.to_string();
assert_eq!(
server.wait_terminal(&id, Duration::from_secs(20))["state"],
"succeeded"
);
assert_eq!(
extended.hosts_per_call(),
vec![vec![
"www.modelscope.cn".to_string(),
"cdn-lfs-cn-1.modelscope.cn".to_string(),
"evil.example".to_string()
]]
);
let status = server.get("/api/status").json();
assert_eq!(status["download_hosts"][0], "www.modelscope.cn");
assert_eq!(status["download_hosts"][1], "cdn-lfs-cn-1.modelscope.cn");
assert_eq!(status["download_hosts"][2], "evil.example");
}
#[test]
fn the_download_endpoint_still_refuses_when_downloads_are_disabled() {
let dir = manifest_model_dir("download-disabled", &["det.onnx"], None);
let scripted = ScriptedDownload::default();
let server = TestServer::start(TestOptions {
downloader: scripted.clone().factory(),
..TestOptions::new(dir, Scripted::fast())
});
let response = server.download("test-set");
assert_eq!(response.status, 403, "{}", response.text());
assert_eq!(response.code(), "downloads_disabled");
assert!(scripted.hosts_per_call().is_empty());
}
#[test]
fn the_reload_endpoint_reports_blocked_models_with_the_missing_list() {
let built = Arc::new(AtomicUsize::new(0));
let server = TestServer::start(TestOptions {
engine_factory: counting_engine_factory(Arc::clone(&built)),
..TestOptions::new(empty_model_dir("reload-blocked"), Scripted::fast())
});
assert_eq!(
server.get("/api/status").json()["engine"]["state"],
"blocked_models_missing"
);
let reloaded = server.post("/api/engine/reload", &[], b"");
assert_eq!(reloaded.status, 200, "{}", reloaded.text());
let body = reloaded.json();
assert_eq!(body["outcome"], "blocked_models_missing");
assert_eq!(body["engine"]["state"], "blocked_models_missing");
let missing = body["missing"].as_array().expect("a missing list");
assert_eq!(missing.len(), 3, "{body}");
assert_eq!(
body["missing"],
server.get("/api/models").json()["missing"],
"the same computation as /api/models"
);
assert_eq!(body["engine"]["missing"], body["missing"]);
assert!(body["load_ms"].as_u64().is_some(), "{body}");
assert_eq!(built.load(Ordering::SeqCst), 0, "no session can be created");
assert_eq!(server.submit_ocr(b"image").status, 409);
}
#[test]
fn the_reload_endpoint_creates_the_engine_and_reports_the_elapsed_time() {
let dir = manifest_model_dir("reload-ready", &["det.onnx", "rec.onnx", "dict.txt"], None);
let built = Arc::new(AtomicUsize::new(0));
let server = TestServer::start(TestOptions {
engine_factory: counting_engine_factory(Arc::clone(&built)),
..TestOptions::new(dir, Scripted::fast())
});
assert_eq!(server.get("/api/status").json()["engine"]["state"], "ready");
assert_eq!(built.load(Ordering::SeqCst), 1);
let reloaded = server.post("/api/engine/reload", &[], b"");
assert_eq!(reloaded.status, 200, "{}", reloaded.text());
let body = reloaded.json();
assert_eq!(body["outcome"], "ready");
assert_eq!(body["engine"]["state"], "ready");
assert_eq!(body["engine"]["selected_ep"], "cpu");
assert!(body["load_ms"].as_u64().is_some(), "{body}");
assert_eq!(
built.load(Ordering::SeqCst),
2,
"an explicit reload rebuilds the session"
);
let id = server.submit_ocr(b"image").json()["job_id"]
.as_str()
.expect("job id")
.to_string();
assert_eq!(
server.wait_terminal(&id, Duration::from_secs(20))["state"],
"succeeded"
);
}
#[test]
fn the_engine_is_created_lazily_on_the_next_ocr_request_and_loading_is_visible() {
let dir = manifest_model_dir("lazy-engine", &["det.onnx"], None);
let gate = Gate::new();
let built = Arc::new(AtomicUsize::new(0));
let server = TestServer::start(TestOptions {
engine_factory: gated_engine_factory(Arc::clone(&gate), Arc::clone(&built)),
..TestOptions::new(dir.clone(), Scripted::fast())
});
assert_eq!(
server.get("/api/status").json()["engine"]["state"],
"blocked_models_missing"
);
assert_eq!(built.load(Ordering::SeqCst), 0);
for name in ["rec.onnx", "dict.txt"] {
std::fs::write(dir.join(name), model_file_bytes(name)).expect("write the model file");
}
assert_eq!(
server.get("/api/status").json()["engine"]["state"],
"blocked_models_missing",
"a completed download must not create the engine in the background"
);
assert_eq!(
server.get("/api/models").json()["complete"],
true,
"the files are complete on disk"
);
assert_eq!(built.load(Ordering::SeqCst), 0);
let accepted = server.submit_ocr(b"image");
assert_eq!(accepted.status, 202, "{}", accepted.text());
let id = accepted.json()["job_id"]
.as_str()
.expect("job id")
.to_string();
gate.arrive(); let status = server.get("/api/status").json();
assert_eq!(status["engine"]["state"], "loading", "{status}");
assert_eq!(built.load(Ordering::SeqCst), 1);
gate.release();
let view = server.wait_terminal(&id, Duration::from_secs(20));
assert_eq!(view["state"], "succeeded", "{view}");
let status = server.get("/api/status").json();
assert_eq!(status["engine"]["state"], "ready");
assert!(
status["engine_load_ms"].as_u64().is_some(),
"the session creation time must be reported: {status}"
);
}
#[test]
fn a_bodyless_reload_keeps_the_accept_loop_live() {
let dir = manifest_model_dir(
"m2-bodyless-reload",
&["det.onnx", "rec.onnx", "dict.txt"],
None,
);
let gate = Gate::new();
let plan = SessionPlan {
gate_at: Some(2),
..SessionPlan::new()
};
let server = TestServer::start(TestOptions {
engine_factory: plan.factory(Some(Arc::clone(&gate))),
..TestOptions::new(dir, Scripted::fast())
});
assert_eq!(server.get("/api/status").json()["engine"]["state"], "ready");
let addr = server.addr;
let host = server.host.clone();
let origin = server.origin.clone();
let token = server.token.clone();
let reload = std::thread::spawn(move || {
raw_request(
addr,
"POST",
"/api/engine/reload",
&[
("Host", host.as_str()),
("X-RapidStructure-Token", token.as_str()),
("Origin", origin.as_str()),
("Content-Length", "0"),
],
Some(b""),
)
});
let release = ReleaseOnDrop::new(Arc::clone(&gate));
gate.arrive(); assert!(
wait_until(Duration::from_secs(10), || {
server.get("/api/status").json()["engine"]["state"] == "loading"
}),
"the bodyless reload must be observable through /api/status while it blocks"
);
let queued = server.submit_ocr(b"during-bodyless-reload");
assert_eq!(queued.status, 202, "{}", queued.text());
assert_eq!(queued.json()["state"], "queued");
release.release();
drop(release);
let response = reload.join().expect("the reload thread must finish");
assert_eq!(response.status, 200, "{}", response.text());
let body = response.json();
assert_eq!(body["outcome"], "ready", "{body}");
assert!(body["load_ms"].as_u64().is_some(), "{body}");
assert_eq!(plan.calls(), 2, "an explicit reload rebuilds the session");
let status = server.get("/api/status").json();
assert_eq!(status["engine"]["state"], "ready", "{status}");
}
fn flat_source_png() -> Vec<u8> {
let image = image::RgbImage::from_fn(100, 50, |x, y| {
image::Rgb([(x % 256) as u8, (y % 256) as u8, 40])
});
let mut bytes = Vec::new();
image
.write_to(&mut Cursor::new(&mut bytes), image::ImageFormat::Png)
.expect("the fixture PNG encodes");
bytes
}
fn noise_png(dimension: u32) -> Vec<u8> {
let mut state = 0x1234_5678_9abc_def0_u64;
let image = image::RgbImage::from_fn(dimension, dimension, |_, _| {
state = state
.wrapping_mul(6364136223846793005)
.wrapping_add(1442695040888963407);
image::Rgb([
(state >> 33) as u8,
(state >> 41) as u8,
(state >> 49) as u8,
])
});
let mut bytes = Vec::new();
image
.write_to(&mut Cursor::new(&mut bytes), image::ImageFormat::Png)
.expect("the noise PNG encodes");
bytes
}
fn source_png_at_least(minimum: usize) -> Vec<u8> {
for dimension in [360_u32, 420, 480, 520, 560] {
let png = noise_png(dimension);
if png.len() >= minimum {
return png;
}
}
panic!("cannot build a PNG of at least {minimum} bytes");
}
#[test]
fn the_annotated_png_is_a_real_png_with_the_original_dimensions_and_drawn_boxes() {
let server = TestServer::start(TestOptions::new(
complete_model_dir("m3-annotated"),
Scripted {
regions: 1,
..Scripted::fast()
},
));
let source = flat_source_png();
let accepted = server.submit_ocr(&source).json();
let id = accepted["job_id"].as_str().expect("job id").to_string();
let view = server.wait_terminal(&id, Duration::from_secs(20));
assert_eq!(view["state"], "succeeded", "{view}");
assert_eq!(
view["original_retained"], true,
"the encoded original stays in the retention area after OCR (§4.5): {view}"
);
let result = server.get(&format!("/api/jobs/{id}/result")).json();
let expected = (
result["image"]["original_size"]["width"].as_u64(),
result["image"]["original_size"]["height"].as_u64(),
);
assert_eq!(expected, (Some(100), Some(50)), "{result}");
let response = server.get(&format!("/api/jobs/{id}/annotated.png"));
assert_eq!(response.status, 200, "{}", response.text());
assert_eq!(response.header("Content-Type"), Some("image/png"));
assert_eq!(response.header("X-Content-Type-Options"), Some("nosniff"));
assert_eq!(response.header("Cache-Control"), Some("no-store"));
assert_eq!(&response.body[..8], b"\x89PNG\r\n\x1a\n", "PNG signature");
let decoded = image::load_from_memory(&response.body).expect("the annotation is a real image");
assert_eq!(
(decoded.width() as u64, decoded.height() as u64),
(expected.0.unwrap(), expected.1.unwrap()),
"the canvas is the decoded original, not a guess"
);
assert_eq!(
decoded.to_rgb8().get_pixel(1, 1).0,
[255, 0, 0],
"the detection box must be drawn on the retained original"
);
}
#[test]
fn the_annotated_png_is_410_original_evicted_once_the_budget_releases_the_original() {
let mut raw = limits(4, 2);
raw.max_retained_mb = 1;
raw.max_result_mb = 8;
let scripted = || Scripted {
regions: 1,
text_bytes: 128 * 1024,
..Scripted::fast()
};
let probe = TestServer::start(TestOptions {
limits: raw,
..TestOptions::new(complete_model_dir("m3-annotated-evicted-probe"), scripted())
});
let accepted = probe.submit_ocr(&flat_source_png()).json();
let probe_id = accepted["job_id"].as_str().expect("job id").to_string();
assert_eq!(
probe.wait_terminal(&probe_id, Duration::from_secs(30))["state"],
"succeeded"
);
let result_bytes = probe
.get(&format!("/api/jobs/{probe_id}/result"))
.body
.len() as u64;
assert!(
result_bytes < 1 << 20,
"the result store's own budget is 1 MiB, so the fixture result must fit: {result_bytes}"
);
let needed = (1 << 20) - result_bytes + 64 * 1024;
let source = source_png_at_least(needed as usize);
assert!(
source.len() as u64 <= 1 << 20,
"the original alone must fit the 1 MiB budget: {} bytes",
source.len()
);
assert!(
source.len() as u64 + result_bytes > 1 << 20,
"the original plus the result must exceed it: {} + {result_bytes}",
source.len()
);
let server = TestServer::start(TestOptions {
limits: raw,
..TestOptions::new(complete_model_dir("m3-annotated-evicted"), scripted())
});
let accepted = server.submit_ocr(&source).json();
let id = accepted["job_id"].as_str().expect("job id").to_string();
let view = server.wait_terminal(&id, Duration::from_secs(30));
assert_eq!(view["state"], "succeeded", "{view}");
assert_eq!(
view["original_retained"], false,
"the byte budget released the original: {view}"
);
let response = server.get(&format!("/api/jobs/{id}/annotated.png"));
assert_eq!(response.status, 410, "{}", response.text());
assert_eq!(response.code(), "original_evicted");
assert!(
response
.json()
.get("detail")
.is_some_and(serde_json::Value::is_null),
"the body still carries the frozen three keys: {}",
response.text()
);
assert_eq!(server.get(&format!("/api/jobs/{id}")).status, 200);
let result = server.get(&format!("/api/jobs/{id}/result"));
assert_eq!(result.status, 200, "{}", result.text());
assert_eq!(result.body.len() as u64, result_bytes, "byte for byte");
let status = server.get("/api/status").json();
assert_eq!(status["retention"]["retained_originals"], 0, "{status}");
assert!(
status["retention"]["retained_bytes"].as_u64() == Some(result_bytes),
"the byte ledger follows the release: {status}"
);
}
#[test]
fn every_export_format_is_served_as_an_attachment_and_the_html_is_static_and_offline() {
let server = TestServer::start(TestOptions::new(
complete_model_dir("m3-export"),
Scripted {
regions: 2,
..Scripted::fast()
},
));
let accepted = server.submit_ocr(&flat_source_png()).json();
let id = accepted["job_id"].as_str().expect("job id").to_string();
assert_eq!(
server.wait_terminal(&id, Duration::from_secs(20))["state"],
"succeeded"
);
let result = server.get(&format!("/api/jobs/{id}/result"));
let json = server.get(&format!("/api/jobs/{id}/export?format=json"));
assert_eq!(json.status, 200, "{}", json.text());
assert_eq!(
json.header("Content-Type"),
Some("application/json; charset=utf-8")
);
assert_eq!(
json.header("Content-Disposition"),
Some(format!("attachment; filename=\"ocr-{id}.json\"").as_str())
);
assert_eq!(json.body, result.body, "the same bytes as /result");
assert_eq!(json.header("Content-Security-Policy"), None);
let markdown = server.get(&format!("/api/jobs/{id}/export?format=md"));
assert_eq!(markdown.status, 200, "{}", markdown.text());
assert_eq!(
markdown.header("Content-Type"),
Some("text/markdown; charset=utf-8")
);
assert_eq!(
markdown.header("Content-Disposition"),
Some(format!("attachment; filename=\"ocr-{id}.md\"").as_str())
);
let markdown_text = markdown.text();
assert!(markdown_text.contains("region-0"), "{markdown_text}");
let html = server.get(&format!("/api/jobs/{id}/export?format=html"));
assert_eq!(html.status, 200, "{}", html.text());
assert_eq!(
html.header("Content-Type"),
Some("text/html; charset=utf-8")
);
assert_eq!(
html.header("Content-Disposition"),
Some(format!("attachment; filename=\"ocr-{id}.html\"").as_str())
);
assert_eq!(
html.header("Content-Security-Policy"),
Some(super::http::EXPORT_CSP)
);
let html_text = html.text();
assert!(
!html_text.to_lowercase().contains("<script"),
"the export must not contain any <script: {html_text}"
);
assert!(
html_text.contains("data:image/png;base64,"),
"the annotated image must be embedded, not linked: {html_text}"
);
assert_eq!(
html_text.matches("src=\"").count(),
1,
"exactly one image source: {html_text}"
);
assert!(
html_text.contains("src=\"data:image/png;base64,"),
"the image must be embedded: {html_text}"
);
assert!(!html_text.contains("/api/"), "no service link: {html_text}");
assert!(html_text.contains("<style>"), "{html_text}");
let page = server.get("/");
assert!(
!page
.header("Content-Security-Policy")
.unwrap_or_default()
.contains("unsafe-inline")
);
for bad in ["", "?format=pdf", "?format=JSON"] {
let response = server.get(&format!("/api/jobs/{id}/export{bad}"));
assert_eq!(response.status, 400, "{bad}: {}", response.text());
assert_eq!(response.code(), "bad_request", "{bad}");
}
}
#[test]
fn an_export_over_the_limit_is_a_413_pointing_at_the_annotated_png() {
let mut raw = limits(4, 2);
raw.max_result_mb = 8;
raw.max_export_mb = 1;
let server = TestServer::start(TestOptions {
limits: raw,
..TestOptions::new(
complete_model_dir("m3-export-too-large"),
Scripted {
regions: 1,
text_bytes: 1400 * 1024,
..Scripted::fast()
},
)
});
let accepted = server.submit_ocr(&flat_source_png()).json();
let id = accepted["job_id"].as_str().expect("job id").to_string();
assert_eq!(
server.wait_terminal(&id, Duration::from_secs(30))["state"],
"succeeded"
);
let result_bytes = server.get(&format!("/api/jobs/{id}/result")).body.len() as u64;
assert!(
result_bytes > 1 << 20,
"the fixture must be over the export budget: {result_bytes}"
);
assert!(
result_bytes < 8 << 20,
"the fixture must stay inside --max-result-mb: {result_bytes}"
);
for format in ["json", "md", "html"] {
let response = server.get(&format!("/api/jobs/{id}/export?format={format}"));
assert_eq!(
response.status,
413,
"{format}: HTTP {} ({} bytes)",
response.status,
response.body.len()
);
assert_eq!(response.code(), "export_too_large", "{format}");
let body = response.json();
assert_eq!(body["detail"]["limit_bytes"], 1 << 20, "{format}: {body}");
assert_eq!(
body["detail"]["annotated"],
format!("/api/jobs/{id}/annotated.png"),
"{format}: {body}"
);
assert!(
body["message"]
.as_str()
.is_some_and(|message| message.contains("annotated.png")),
"{format}: {body}"
);
if format == "json" {
assert_eq!(body["detail"]["observed_bytes"], result_bytes, "{body}");
} else {
assert!(
body["detail"]["observed_bytes"]
.as_u64()
.is_some_and(|n| n > 1 << 20),
"{format}: {body}"
);
}
}
let annotated = server.get(&format!("/api/jobs/{id}/annotated.png"));
assert_eq!(annotated.status, 200, "{}", annotated.text());
assert_eq!(&annotated.body[..8], b"\x89PNG\r\n\x1a\n");
}
#[test]
fn an_export_under_the_limit_is_served_intact() {
let mut raw = limits(4, 2);
raw.max_export_mb = 1;
let server = TestServer::start(TestOptions {
limits: raw,
..TestOptions::new(
complete_model_dir("m3-export-under"),
Scripted {
regions: 1,
text_bytes: 64 * 1024,
..Scripted::fast()
},
)
});
let accepted = server.submit_ocr(&flat_source_png()).json();
let id = accepted["job_id"].as_str().expect("job id").to_string();
assert_eq!(
server.wait_terminal(&id, Duration::from_secs(20))["state"],
"succeeded"
);
for format in ["json", "md", "html"] {
let response = server.get(&format!("/api/jobs/{id}/export?format={format}"));
assert_eq!(response.status, 200, "{format}: {}", response.text());
assert!(!response.body.is_empty(), "{format}");
}
}
#[test]
fn exports_require_a_successful_job() {
let server = TestServer::start(TestOptions {
limits: limits(2, 1),
..TestOptions::new(
complete_model_dir("m3-export-not-finished"),
Scripted::slow(Duration::from_millis(500)),
)
});
let accepted = server.submit_ocr(b"image").json();
let id = accepted["job_id"].as_str().expect("job id").to_string();
for path in [
format!("/api/jobs/{id}/annotated.png"),
format!("/api/jobs/{id}/export?format=json"),
format!("/api/jobs/{id}/export?format=html"),
] {
let response = server.get(&path);
assert!(
response.status == 409 || response.status == 200,
"{path}: {}",
response.text()
);
}
let failing = TestServer::start(TestOptions::new(
complete_model_dir("m3-export-failed"),
Scripted {
fail: true,
..Scripted::fast()
},
));
let accepted = failing.submit_ocr(b"image").json();
let id = accepted["job_id"].as_str().expect("job id").to_string();
assert_eq!(
failing.wait_terminal(&id, Duration::from_secs(20))["state"],
"failed"
);
for path in [
format!("/api/jobs/{id}/annotated.png"),
format!("/api/jobs/{id}/export?format=md"),
] {
let response = failing.get(&path);
assert_eq!(response.status, 409, "{path}: {}", response.text());
assert_eq!(response.code(), "job_not_finished", "{path}");
}
}
#[test]
fn the_diagnostics_payload_reports_the_library_ledger_fingerprint_and_memory() {
let server = TestServer::start(TestOptions::new(
complete_model_dir("m3-diagnostics"),
Scripted::fast(),
));
let accepted = server.submit_ocr(&flat_source_png()).json();
let id = accepted["job_id"].as_str().expect("job id").to_string();
assert_eq!(
server.wait_terminal(&id, Duration::from_secs(20))["state"],
"succeeded"
);
let result = server.get(&format!("/api/jobs/{id}/result")).json();
let timings = result["timings"].as_object().expect("the stage timings");
for key in [
"total_ms",
"preprocess_ms",
"detector_infer_ms",
"recognizer_infer_ms",
"postprocess_ms",
] {
assert!(timings.get(key).is_some(), "missing {key}: {result}");
}
let ledger = &result["timing_ledger"];
assert!(ledger["total_ms"].is_number(), "{ledger}");
assert!(ledger["input_preprocess_ms"].is_number(), "{ledger}");
assert!(ledger["unattributed_ms"].is_number(), "{ledger}");
assert!(ledger["attributed_ms"].is_number(), "{ledger}");
assert!(ledger["inference_ms"].is_number(), "{ledger}");
assert!(ledger["rust_ms"].is_number(), "{ledger}");
assert_eq!(ledger["shares"]["inference_share"], 0.0, "{ledger}");
let conservation = &ledger["conservation"];
assert_eq!(conservation["residual_ms"], -1.0, "{conservation}");
assert_eq!(conservation["conserved"], false, "{conservation}");
assert_eq!(conservation["excess_ms"], 1.0, "{conservation}");
let interpretation = conservation["interpretation"]
.as_str()
.expect("the ledger explains itself");
assert!(
interpretation.contains("NOT a strict partition"),
"the panel must be able to show this wording verbatim: {interpretation}"
);
assert!(
interpretation.contains("residual_ms = -1.000000"),
"the interpretation quotes the actual residual: {interpretation}"
);
assert!(
interpretation.contains("SCOPE difference"),
"and says what the residual is: {interpretation}"
);
let status = server.get("/api/status").json();
assert!(status["ort"]["version"].is_string(), "{status}");
assert_eq!(status["ort"]["fingerprint"]["complete"], true, "{status}");
assert!(
status["ort"]["fingerprint"]["file"].is_string(),
"the fingerprint gives a file name, never a path: {status}"
);
assert!(
status["memory"]["peak_working_set_bytes"]
.as_u64()
.is_some(),
"{status}"
);
assert!(status["memory"]["source"].is_string(), "{status}");
assert_eq!(
status["provider"]["requested"],
rapid_structure_rs::format_provider_preference(
rapid_structure_rs::ProviderPreference::default()
)
);
assert_eq!(status["provider"]["selected_ep"], "cpu");
assert_eq!(status["provider"]["fallback_to_cpu"], false);
assert!(status["queues"]["text"]["wait_bound"].as_u64().is_some());
}
#[test]
fn switching_the_provider_rebuilds_the_session_and_queues_requests_meanwhile() {
let dir = manifest_model_dir(
"m3-provider-switch",
&["det.onnx", "rec.onnx", "dict.txt"],
None,
);
let gate = Gate::new();
let plan = SessionPlan {
gate_at: Some(2),
delay: Duration::from_millis(300),
..SessionPlan::new()
};
let server = TestServer::start(TestOptions {
engine_factory: plan.factory(Some(Arc::clone(&gate))),
..TestOptions::new(dir, Scripted::slow(Duration::from_millis(300)))
});
let status = server.get("/api/status").json();
assert_eq!(status["engine"]["state"], "ready", "{status}");
assert_eq!(status["provider"]["selected_ep"], "scripted-1", "{status}");
assert_eq!(plan.calls(), 1);
let running_id = server.submit_ocr(b"image").json()["job_id"]
.as_str()
.expect("job id")
.to_string();
assert!(
wait_until(Duration::from_secs(10), || {
server.get(&format!("/api/jobs/{running_id}")).json()["state"] == "running"
}),
"the first job must be running when the switch starts"
);
let addr = server.addr;
let host = server.host.clone();
let origin = server.origin.clone();
let token = server.token.clone();
let switch = std::thread::spawn(move || {
raw_request(
addr,
"POST",
"/api/engine/reload",
&[
("Host", host.as_str()),
("X-RapidStructure-Token", token.as_str()),
("Origin", origin.as_str()),
("Content-Type", "application/json"),
],
Some(br#"{"provider":"cpu"}"#),
)
});
let release = ReleaseOnDrop::new(Arc::clone(&gate));
gate.arrive(); assert!(
wait_until(Duration::from_secs(10), || {
server.get("/api/status").json()["engine"]["state"] == "rebuilding"
}),
"`Rebuilding` must be observable through /api/status"
);
assert_eq!(
server.get(&format!("/api/jobs/{running_id}")).json()["state"],
"succeeded"
);
let status = server.get("/api/status").json();
assert_eq!(status["engine"]["state"], "rebuilding", "{status}");
assert_eq!(status["provider"]["requested"], "cpu", "{status}");
assert_eq!(
status["provider"]["selected_ep"],
serde_json::Value::Null,
"{status}"
);
assert_eq!(
status["provider"]["fallback_to_cpu"],
serde_json::Value::Null,
"{status}"
);
let queued = server.submit_ocr(b"during-rebuild");
assert_eq!(queued.status, 202, "{}", queued.text());
assert_eq!(queued.json()["state"], "queued");
let queued_id = queued.json()["job_id"]
.as_str()
.expect("job id")
.to_string();
release.release();
drop(release);
let response = switch.join().expect("the switch thread must finish");
assert_eq!(response.status, 200, "{}", response.text());
let body = response.json();
assert_eq!(body["outcome"], "ready", "{body}");
assert_eq!(body["engine"]["selected_ep"], "scripted-2", "{body}");
assert_eq!(body["selected_ep"], "scripted-2", "{body}");
assert_eq!(body["fallback_to_cpu"], false, "{body}");
assert_eq!(body["error"], serde_json::Value::Null, "{body}");
assert_eq!(
plan.calls(),
2,
"the switch destroys the old session and builds one new"
);
let view = server.wait_terminal(&queued_id, Duration::from_secs(20));
assert_eq!(view["state"], "succeeded", "{view}");
let status = server.get("/api/status").json();
assert_eq!(status["engine"]["state"], "ready", "{status}");
assert_eq!(status["provider"]["selected_ep"], "scripted-2", "{status}");
assert!(
status["engine_load_ms"].as_u64().is_some(),
"the new session's creation time is reported: {status}"
);
}
#[test]
fn a_failed_switch_rolls_back_to_the_previous_engine() {
let dir = manifest_model_dir(
"m3-provider-rollback",
&["det.onnx", "rec.onnx", "dict.txt"],
None,
);
let plan = SessionPlan {
fail_at: vec![2],
..SessionPlan::new()
};
let server = TestServer::start(TestOptions {
engine_factory: plan.factory(None),
..TestOptions::new(dir, Scripted::fast())
});
assert_eq!(
server.get("/api/status").json()["provider"]["selected_ep"],
"scripted-1"
);
let response = server.post(
"/api/engine/reload",
&[("Content-Type", "application/json")],
br#"{"provider":"cpu"}"#,
);
assert_eq!(response.status, 200, "{}", response.text());
let body = response.json();
assert_eq!(body["outcome"], "rolled_back", "{body}");
assert!(
body["error"]
.as_str()
.is_some_and(|error| error.contains("scripted session 2")),
"{body}"
);
assert!(body["rollback_ms"].as_u64().is_some(), "{body}");
assert_eq!(body["engine"]["state"], "ready", "{body}");
assert_eq!(body["engine"]["selected_ep"], "scripted-3", "{body}");
assert_eq!(
plan.calls(),
3,
"one failed attempt + one successful restore"
);
let status = server.get("/api/status").json();
assert_eq!(status["engine"]["state"], "ready", "{status}");
assert_eq!(status["provider"]["selected_ep"], "scripted-3", "{status}");
assert_eq!(
status["provider"]["requested"],
rapid_structure_rs::format_provider_preference(
rapid_structure_rs::ProviderPreference::default()
),
"{status}"
);
let id = server.submit_ocr(b"image").json()["job_id"]
.as_str()
.expect("job id")
.to_string();
assert_eq!(
server.wait_terminal(&id, Duration::from_secs(20))["state"],
"succeeded"
);
}
#[test]
fn a_switch_that_cannot_restore_the_old_engine_lands_in_failed() {
let dir = manifest_model_dir(
"m3-provider-dead",
&["det.onnx", "rec.onnx", "dict.txt"],
None,
);
let plan = SessionPlan {
fail_at: vec![2, 3],
..SessionPlan::new()
};
let server = TestServer::start(TestOptions {
engine_factory: plan.factory(None),
..TestOptions::new(dir, Scripted::fast())
});
let response = server.post(
"/api/engine/reload",
&[("Content-Type", "application/json")],
br#"{"provider":"cpu"}"#,
);
assert_eq!(response.status, 200, "{}", response.text());
let body = response.json();
assert_eq!(body["outcome"], "failed", "{body}");
assert_eq!(body["engine"]["state"], "failed", "{body}");
let reason = body["engine"]["reason"].as_str().expect("a reason");
assert!(reason.contains("scripted session 2"), "{reason}");
assert!(reason.contains("restoring the previous engine"), "{reason}");
assert!(reason.contains("scripted session 3"), "{reason}");
let status = server.get("/api/status").json();
assert_eq!(status["engine"]["state"], "failed", "{status}");
assert_eq!(status["engine"]["reason"], reason, "{status}");
assert_eq!(status["provider"]["requested"], "cpu", "{status}");
assert_eq!(status["provider"]["selected_ep"], serde_json::Value::Null);
let response = server.submit_ocr(b"image");
assert_eq!(response.status, 503, "{}", response.text());
assert_eq!(response.code(), "engine_unavailable");
assert_eq!(plan.calls(), 3);
}
#[test]
fn requesting_a_provider_that_is_not_compiled_in_is_a_400_and_changes_nothing() {
let dir = manifest_model_dir(
"m3-provider-invalid",
&["det.onnx", "rec.onnx", "dict.txt"],
None,
);
let plan = SessionPlan::new();
let server = TestServer::start(TestOptions {
engine_factory: plan.factory(None),
..TestOptions::new(dir, Scripted::fast())
});
assert_eq!(
server.get("/api/status").json()["provider"]["selected_ep"],
"scripted-1"
);
let response = server.post(
"/api/engine/reload",
&[("Content-Type", "application/json")],
br#"{"provider":"directml"}"#,
);
if cfg!(feature = "directml-provider") {
assert_eq!(response.status, 200, "{}", response.text());
assert_eq!(response.json()["engine"]["state"], "ready");
assert_eq!(plan.calls(), 2, "this build can switch to DirectML");
return;
}
assert_eq!(response.status, 400, "{}", response.text());
assert_eq!(response.code(), "bad_request");
let body = response.json();
assert!(
body["detail"]["provider"]
.as_str()
.is_some_and(|provider| provider.starts_with("directml")),
"{body}"
);
assert!(
body["detail"]["reason"]
.as_str()
.is_some_and(|reason| reason.contains("directml-provider")),
"the library's own wording must survive: {body}"
);
assert_eq!(
plan.calls(),
1,
"a rejected request must not touch the engine"
);
let status = server.get("/api/status").json();
assert_eq!(status["engine"]["state"], "ready", "{status}");
assert_eq!(status["provider"]["selected_ep"], "scripted-1", "{status}");
assert_eq!(
status["provider"]["requested"],
rapid_structure_rs::format_provider_preference(
rapid_structure_rs::ProviderPreference::default()
),
"{status}"
);
let id = server.submit_ocr(b"image").json()["job_id"]
.as_str()
.expect("job id")
.to_string();
assert_eq!(
server.wait_terminal(&id, Duration::from_secs(20))["state"],
"succeeded"
);
}
fn network_tests_enabled() -> bool {
if std::env::var("RAPID_STRUCTURE_ALLOW_NETWORK").as_deref() == Ok("1") {
return true;
}
eprintln!("skipping the real-network test: set RAPID_STRUCTURE_ALLOW_NETWORK=1 to run it");
false
}
fn network_manifest_dir(name: &str, remote: &RemoteDefaultFile) -> PathBuf {
let dir = m2_root().join(format!("network-manifest-{name}-{}", unique()));
if dir.exists() {
std::fs::remove_dir_all(&dir).expect("clear the previous run's model dir");
}
std::fs::create_dir_all(&dir).expect("create the model dir");
let fixtures = [
("network_det_fixture.onnx", "detector"),
("network_rec_fixture.onnx", "recognizer"),
];
let mut entries: Vec<(String, String, String, u64)> = Vec::new();
for (file, role) in fixtures {
let path = dir.join(file);
std::fs::write(&path, format!("present fixture {file}\n")).expect("write the fixture");
let size = std::fs::metadata(&path).expect("metadata").len();
entries.push((
file.to_string(),
role.to_string(),
format!("https://www.modelscope.cn/models/{file}"),
size,
));
}
entries.push((
remote.file.to_string(),
remote.role.to_string(),
remote.url.to_string(),
remote.size_bytes,
));
let real_hashes = ["", "", remote.sha256];
let mut manifest = String::from(
"{\"schema_version\":1,\"id\":\"network-set\",\"family\":\"PP-OCR\",\"version\":\"v-real\",\
\"languages\":[\"en\"],\"files\":[",
);
for (index, (file, role, url, size)) in entries.iter().enumerate() {
let sha = if index < real_hashes.len() && !real_hashes[index].is_empty() {
real_hashes[index].to_string()
} else {
sha256_file(dir.join(file)).expect("hash the fixture")
};
if index > 0 {
manifest.push(',');
}
manifest.push_str(&format!(
"{{\"name\":\"{file}\",\"role\":\"{role}\",\"sha256\":\"{sha}\",\"size_bytes\":{size},\
\"source_url\":\"{url}\"}}"
));
}
manifest.push_str("]}");
std::fs::write(dir.join("manifest.json"), manifest).expect("write the manifest");
dir
}
struct RemoteDefaultFile {
file: &'static str,
role: &'static str,
url: &'static str,
sha256: &'static str,
size_bytes: u64,
}
const REAL_DICTIONARY_SHA256: &str =
"b5f2bfe2bdd9448429e3e82b51c789775d9b42f2403d082b00662eb77e401c5d";
const REMOTE_DICTIONARY: RemoteDefaultFile = RemoteDefaultFile {
file: "ppocrv6_dict.txt",
role: "dictionary",
url: "https://www.modelscope.cn/models/RapidAI/RapidOCR/resolve/v3.9.1/paddle/PP-OCRv6/rec/PP-OCRv6_rec_small/ppocrv6_dict.txt",
sha256: REAL_DICTIONARY_SHA256,
size_bytes: 74_947,
};
const REAL_WEIGHT_SHA256: &str = "f42c0fbd294d95eac1a550e131b277dac97462c8025fa4b6c3cec1b7894bd3d5";
const REMOTE_WEIGHT: RemoteDefaultFile = RemoteDefaultFile {
file: "PP-OCRv6_det_tiny.onnx",
role: "detector",
url: "https://www.modelscope.cn/models/RapidAI/RapidOCR/resolve/v3.9.1/onnx/PP-OCRv6/det/PP-OCRv6_det_tiny.onnx",
sha256: REAL_WEIGHT_SHA256,
size_bytes: 1_829_618,
};
#[test]
fn a_real_network_download_of_the_default_table_dictionary_lands_and_verifies() {
if !network_tests_enabled() {
return;
}
real_default_table_download_lands_and_verifies(
"dictionary",
&REMOTE_DICTIONARY,
"www.modelscope.cn",
false,
);
}
#[test]
fn a_real_network_weight_download_through_the_cdn_redirect_lands_and_verifies() {
if !network_tests_enabled() {
return;
}
let dir = m2_root().join(format!("network-weights-{}", unique()));
if dir.exists() {
std::fs::remove_dir_all(&dir).expect("clear the previous run's model dir");
}
std::fs::create_dir_all(&dir).expect("create the model dir");
let mut config = EngineConfig::default();
config.det.ocr_version = OcrVersion::PPocrV6;
config.det.model_type = ModelType::Tiny;
config.det.lang = LangDet::Multi;
config.rec.model.ocr_version = OcrVersion::PPocrV6;
config.rec.model.model_type = ModelType::Tiny;
config.rec.model.lang = LangRec::Ch;
let server = TestServer::start(TestOptions {
allow_download: true,
downloader: super::download::real_downloader_factory(),
engine_config: config,
..TestOptions::new(dir.clone(), Scripted::fast())
});
let models = server.get("/api/models").json();
assert_eq!(models["source"], "default_table", "{models}");
assert_eq!(models["complete"], false, "{models}");
let set_id = models["sets"][0]["id"]
.as_str()
.expect("set id")
.to_string();
let declared: Vec<(String, String, u64)> = models["sets"][0]["files"]
.as_array()
.expect("files")
.iter()
.map(|file| {
(
file["name"].as_str().expect("name").to_string(),
file["sha256"].as_str().expect("sha256").to_string(),
file["size_bytes"].as_u64().expect("size_bytes"),
)
})
.collect();
assert_eq!(
declared
.iter()
.filter(|(name, ..)| name.ends_with(".onnx"))
.count(),
2,
"the v6-tiny text set must contain a detector and a recognizer weight: {declared:?}"
);
assert!(
declared
.iter()
.any(|(name, sha, _)| name == REMOTE_WEIGHT.file && sha == REAL_WEIGHT_SHA256),
"the set must declare the default table's smallest weight as-is: {declared:?}"
);
let response = server.download(&set_id);
assert_eq!(response.status, 202, "{}", response.text());
let id = response.json()["job_id"]
.as_str()
.expect("job id")
.to_string();
let view = server.wait_terminal(&id, Duration::from_secs(300));
eprintln!("real network download (default-table v6 tiny): {view}");
assert_eq!(view["state"], "succeeded", "{view}");
let files_total = view["download"]["files_total"]
.as_u64()
.expect("files_total");
assert!(files_total >= 3, "det + rec + dict: {view}");
let cdn_host = "cdn-lfs-cn-1.modelscope.cn";
eprintln!(
"real network download (weights): files_total={files_total} bytes_total={} elapsed_ms={} \
max_hops={} expected_weight_host={cdn_host}",
view["download"]["bytes_total"],
view["elapsed_ms"],
rapid_structure_rs::MAX_REDIRECT_HOPS,
);
let after = server.get("/api/models").json();
assert_eq!(after["complete"], true, "{after}");
assert_eq!(after["missing"], serde_json::json!([]));
for (name, sha, size) in &declared {
let path = dir.join(name);
assert!(path.is_file(), "{} must be on disk", path.display());
let landed = sha256_file(&path).expect("hash the landed file");
assert_eq!(
std::fs::metadata(&path).expect("metadata").len(),
*size,
"{name} must have the declared size"
);
assert_eq!(&landed, sha, "{name} must match its declared SHA-256");
eprintln!("real network download (weights): {name} = {size} bytes, sha256 {landed}");
}
assert_eq!(
sha256_file(dir.join(REMOTE_WEIGHT.file)).expect("hash the landed weight"),
REAL_WEIGHT_SHA256,
"the landed weight must be the real default-table entry"
);
let leftovers: Vec<String> = std::fs::read_dir(&dir)
.expect("readable")
.map(|entry| {
entry
.expect("entry")
.file_name()
.to_string_lossy()
.into_owned()
})
.filter(|name| name.contains(".part-"))
.collect();
assert!(leftovers.is_empty(), "{leftovers:?}");
}
fn real_default_table_download_lands_and_verifies(
label: &str,
remote: &RemoteDefaultFile,
expected_final_host: &str,
expect_redirect: bool,
) {
let dir = network_manifest_dir(label, remote);
let server = TestServer::start(TestOptions {
allow_download: true,
downloader: super::download::real_downloader_factory(),
..TestOptions::new(dir.clone(), Scripted::fast())
});
let models = server.get("/api/models").json();
assert_eq!(models["source"], "local_manifest");
assert_eq!(models["complete"], false, "{models}");
assert_eq!(
models["missing"],
serde_json::json!([remote.file]),
"{models}"
);
assert_eq!(models["sets"][0]["download_bytes_total"], remote.size_bytes);
let response = server.download("network-set");
assert_eq!(response.status, 202, "{}", response.text());
let id = response.json()["job_id"]
.as_str()
.expect("job id")
.to_string();
let view = server.wait_terminal(&id, Duration::from_secs(300));
assert_eq!(view["state"], "succeeded", "{view}");
eprintln!(
"real network download ({label}): files_done={} files_total={} bytes_done={} \
bytes_total={} elapsed_ms={} expected_final_host={expected_final_host} \
max_hops={}",
view["download"]["files_done"],
view["download"]["files_total"],
view["download"]["bytes_done"],
view["download"]["bytes_total"],
view["elapsed_ms"],
rapid_structure_rs::MAX_REDIRECT_HOPS,
);
assert_eq!(
rapid_structure_rs::MAX_REDIRECT_HOPS,
5,
"the documented hop limit must not drift"
);
assert_eq!(
(expected_final_host, expect_redirect),
match label {
"dictionary" => ("www.modelscope.cn", false),
"weight" => ("cdn-lfs-cn-1.modelscope.cn", true),
other => panic!("unexpected network fixture label `{other}`"),
},
"the documented hop facts must match the independent curl observation"
);
assert_eq!(view["download"]["files_done"], 1);
assert_eq!(view["download"]["files_total"], 1);
assert_eq!(view["download"]["bytes_done"], remote.size_bytes);
assert_eq!(view["download"]["current_file"], serde_json::Value::Null);
let after = server.get("/api/models").json();
assert_eq!(after["complete"], true, "{after}");
assert_eq!(after["missing"], serde_json::json!([]));
for file in after["sets"][0]["files"].as_array().expect("files") {
let name = file["name"].as_str().expect("name");
let declared = file["sha256"].as_str().expect("sha256");
let path = dir.join(name);
assert!(path.is_file(), "{} must be on disk", path.display());
let landed = sha256_file(&path).expect("hash the landed file");
eprintln!(
"real network download ({label}): {name} = {} bytes, sha256 {}",
std::fs::metadata(&path).expect("metadata").len(),
landed
);
assert_eq!(landed, declared, "{name} must match its declared SHA-256");
}
let landed = dir.join(remote.file);
assert_eq!(
sha256_file(&landed).expect("hash"),
remote.sha256,
"the landed file must be the real default-table entry"
);
assert_eq!(
std::fs::metadata(&landed).expect("metadata").len(),
remote.size_bytes,
"the landed file must have the size the default table declares"
);
let leftovers: Vec<String> = std::fs::read_dir(&dir)
.expect("readable")
.map(|entry| {
entry
.expect("entry")
.file_name()
.to_string_lossy()
.into_owned()
})
.filter(|name| name.contains(".part-"))
.collect();
assert!(leftovers.is_empty(), "{leftovers:?}");
}
fn flow_lines_with(lines: &[String], needle: &str) -> Vec<String> {
lines
.iter()
.filter(|line| line.contains(needle))
.cloned()
.collect()
}
fn flow_field(line: &str, key: &str) -> Option<String> {
let prefix = format!("{key}=");
line.split(' ')
.find_map(|token| token.strip_prefix(&prefix).map(str::to_string))
}
#[test]
fn the_flow_log_is_silent_until_it_is_asked_for() {
let server = TestServer::start(TestOptions::new(
complete_model_dir("flow-off"),
Scripted::fast(),
));
let accepted = server.submit_ocr(b"image bytes").json();
let id = accepted["job_id"].as_str().expect("job id").to_string();
server.wait_terminal(&id, Duration::from_secs(20));
let _ = server.get(&format!("/api/jobs/{id}/result"));
assert!(
server.flow_lines().is_empty(),
"flow logging must be off unless --log-level/RAPID_STRUCTURE_SERVE_LOG asks for it: {:#?}",
server.flow_lines()
);
}
#[test]
fn one_upload_is_followable_end_to_end_by_its_request_id() {
let (options, lines) =
TestOptions::new(complete_model_dir("flow-on"), Scripted::fast()).with_flow_capture();
let server = TestServer::start(options);
let response = server.submit_ocr(b"image bytes");
assert_eq!(response.status, 202, "{}", response.text());
let accepted = response.json();
let id = accepted["job_id"].as_str().expect("job id").to_string();
let view = server.wait_terminal(&id, Duration::from_secs(20));
assert_eq!(view["state"], "succeeded", "{view}");
let result = server.get(&format!("/api/jobs/{id}/result"));
assert_eq!(result.status, 200, "{}", result.text());
let result_bytes = result.body.len();
let captured = lines.lock().expect("flow lines").clone();
assert!(
!captured.is_empty(),
"flow logging was asked for but wrote nothing"
);
let request_lines = flow_lines_with(&captured, "POST /api/ocr -> 202");
assert_eq!(request_lines.len(), 1, "{captured:#?}");
let request_line = &request_lines[0];
for field in ["req=", "> 202", "bytes=", "in ", "ms"] {
assert!(
request_line.contains(field),
"the request line must carry {field}: {request_line}"
);
}
let request_id = flow_field(request_line, "req").expect("a request id");
let admitted = flow_lines_with(&captured, "admission=accepted");
assert_eq!(admitted.len(), 1, "{captured:#?}");
assert!(
admitted[0].contains(&format!("req={request_id}"))
&& admitted[0].contains(&format!("job={id}"))
&& admitted[0].contains("queue=text")
&& admitted[0].contains("decision=run")
&& admitted[0].contains("queued position="),
"the admission line must name the request, the job, the queue, the reason and the \
position: {}",
admitted[0]
);
let running = flow_lines_with(&captured, "running wait_ms=");
assert_eq!(running.len(), 1, "{captured:#?}");
assert!(
running[0].contains(&format!("req={request_id}"))
&& running[0].contains(&format!("job={id}")),
"{}",
running[0]
);
let terminal = flow_lines_with(&captured, "terminal state=succeeded");
assert_eq!(terminal.len(), 1, "{captured:#?}");
assert!(
terminal[0].contains(&format!("req={request_id}"))
&& terminal[0].contains(&format!("job={id}"))
&& terminal[0].contains(&format!("result_bytes={result_bytes}"))
&& terminal[0].contains("backend_ms="),
"the terminal line must carry the request id, the job, the **measured** result size \
({result_bytes}) and the backend duration: {}",
terminal[0]
);
assert_eq!(
flow_lines_with(&captured, &format!("req={request_id} ")).len(),
4,
"{captured:#?}"
);
}
#[test]
fn a_rejected_upload_leaves_its_reason_in_the_flow_log() {
let (options, lines) =
TestOptions::new(empty_model_dir("flow-reject"), Scripted::fast()).with_flow_capture();
let server = TestServer::start(options);
let response = server.submit_ocr(b"image bytes");
assert_eq!(response.status, 409, "{}", response.text());
let body = response.json();
let code = body["code"].as_str().expect("a code");
assert_eq!(code, "models_missing", "{body}");
let captured = lines.lock().expect("flow lines").clone();
let rejected = flow_lines_with(&captured, "admission=rejected");
assert_eq!(rejected.len(), 1, "{captured:#?}");
assert!(
rejected[0].contains("status=409")
&& rejected[0].contains(&format!("code={code}"))
&& rejected[0].contains("detail="),
"a refusal must carry the status, the code and the detail: {}",
rejected[0]
);
assert_eq!(flow_lines_with(&captured, "POST /api/ocr -> 409").len(), 1);
assert!(
flow_lines_with(&captured, "terminal ").is_empty(),
"{captured:#?}"
);
}
#[test]
fn a_failed_job_carries_its_code_and_detail_in_the_flow_log() {
let (options, lines) = TestOptions::new(
complete_model_dir("flow-failed"),
Scripted {
fail: true,
..Scripted::fast()
},
)
.with_flow_capture();
let server = TestServer::start(options);
let accepted = server.submit_ocr(b"broken").json();
let id = accepted["job_id"].as_str().expect("job id").to_string();
let view = server.wait_terminal(&id, Duration::from_secs(20));
assert_eq!(view["state"], "failed", "{view}");
let captured = lines.lock().expect("flow lines").clone();
let terminal = flow_lines_with(&captured, "terminal state=failed");
assert_eq!(terminal.len(), 1, "{captured:#?}");
assert!(
terminal[0].contains("status=")
&& terminal[0].contains("code=")
&& terminal[0].contains("detail="),
"a failed job's terminal line must carry status, code and detail: {}",
terminal[0]
);
let status = view["failure"]["status"].as_u64().expect("a status");
let code = view["failure"]["code"].as_str().expect("a code");
assert!(
terminal[0].contains(&format!("status={status}"))
&& terminal[0].contains(&format!("code={code}")),
"the log and the job view must tell the same story: {} vs {view}",
terminal[0]
);
}
const PAGE_RESULT_CONTRACT: [(&str, &str); 11] = [
("regions", "Array.isArray(r.regions)"),
("kind", r#"pget(x, "kind")"#),
("polygon.points", r#"pget(r.polygon, "points")"#),
(
"polygon.points 恰好四点",
"!Array.isArray(pts) || pts.length !== 4",
),
(
"polygon.points[k] 恰好两个数",
"!Array.isArray(p) || p.length !== 2",
),
("recognition", "const rec = r && r.recognition;"),
("recognition.text", r#"pget(rec, "text")"#),
("recognition.score", r#"pget(rec, "score")"#),
("formula.latex", r#"pget(f, "latex")"#),
("plain_text", "r.plain_text"),
("timing_ledger", "r.timing_ledger"),
];
const PAGE_WORDS_CONTRACT: [(&str, &str); 5] = [
("recognition.words", r#"pget(rec, "words")"#),
("words[k].text", r#"pget(w, "text")"#),
("words[k].kind", r#"pget(w, "kind")"#),
("words[k].polygon.points", r#"pget(w.polygon, "points")"#),
(
"缺席合法:`off` 档不得报错",
"words === undefined || words === null",
),
];
fn assert_quad_points(polygon: &Value, path: &str) {
let points = polygon["points"].as_array().unwrap_or_else(|| {
panic!("{path}.points must be an array (exactly four pairs): {polygon}")
});
assert_eq!(
points.len(),
4,
"{path}.points must be exactly four points, got {}",
points.len()
);
for (corner, point) in points.iter().enumerate() {
let pair = point
.as_array()
.unwrap_or_else(|| panic!("{path}.points[{corner}] must be a pair: {point}"));
assert_eq!(pair.len(), 2, "{path}.points[{corner}] must be [x, y]");
for (axis, raw) in pair.iter().enumerate() {
assert!(
raw.as_f64().is_some_and(f64::is_finite),
"{path}.points[{corner}][{axis}] must be a finite number, got {raw}"
);
}
}
}
fn assert_optional_word_boxes(region: &Value, index: usize) {
let words = ®ion["recognition"]["words"];
if words.is_null() {
return;
}
let items = words.as_array().unwrap_or_else(|| {
panic!(
"regions[{index}].recognition.words must be null (the `off` mode) or an array of \
OcrWord, got {words}"
)
});
for (word_index, word) in items.iter().enumerate() {
let path = format!("regions[{index}].recognition.words[{word_index}]");
assert!(
word["text"].is_string(),
"{path}.text must be a string: {word}"
);
assert!(
word["score"].as_f64().is_some_and(f64::is_finite),
"{path}.score must be a finite number: {word}"
);
assert!(
matches!(word["kind"].as_str(), Some("token") | Some("char")),
"{path}.kind must be `token` or `char`, got {word}"
);
assert!(
!word["polygon"].is_null(),
"{path}.polygon must exist: a word box cannot be substituted by the row box, \
which covers the whole line: {word}"
);
assert_quad_points(&word["polygon"], &path);
}
}
fn assert_result_satisfies_page_contract(value: &Value) -> usize {
let regions = value["regions"]
.as_array()
.unwrap_or_else(|| panic!("the page reads `regions` as an array: {value}"));
for (index, region) in regions.iter().enumerate() {
let kind = region["kind"].as_str().unwrap_or_else(|| {
panic!(
"regions[{index}].kind must be a string (`text` | `formula` | `table`): {region}"
)
});
assert!(
matches!(kind, "text" | "formula" | "table"),
"regions[{index}].kind must be `text`, `formula` or `table`, got {kind}"
);
assert_quad_points(®ion["polygon"], &format!("regions[{index}]"));
if kind == "text" {
assert!(
region["recognition"]["text"].is_string(),
"regions[{index}].recognition.text must be a string for a text region: {region}"
);
assert!(
region["recognition"]["score"]
.as_f64()
.is_some_and(f64::is_finite),
"regions[{index}].recognition.score must be a finite number: {region}"
);
assert_optional_word_boxes(region, index);
} else if kind == "formula" {
assert!(
region["formula"]["latex"].is_string(),
"regions[{index}].formula.latex must be a string for a formula region: {region}"
);
} else {
assert!(
region["table"]["html"].is_string(),
"regions[{index}].table.html must be a string for a table region: {region}"
);
assert!(
region["table"]["cells"].is_array(),
"regions[{index}].table.cells must be an array: {region}"
);
assert!(
region["recognition"].is_null(),
"a table region must not carry a CTC recognition result: {region}"
);
}
}
assert!(
value["plain_text"].is_string(),
"the page's copy-all reads `plain_text` as a string: {value}"
);
assert!(
value["timings"].is_object(),
"the page's timing table reads `timings` as an object"
);
assert!(
value["timing_ledger"].is_object(),
"the page's ledger section reads `timing_ledger` as an object"
);
regions.len()
}
#[test]
fn the_served_result_carries_every_path_the_page_reads() {
let server = TestServer::start(TestOptions::new(
complete_model_dir("contract-result"),
Scripted::fast().with_formula_regions(1),
));
let accepted = server.submit_ocr(b"image bytes").json();
let id = accepted["job_id"].as_str().expect("job id").to_string();
assert_eq!(
server.wait_terminal(&id, Duration::from_secs(20))["state"],
"succeeded"
);
let result = server.get(&format!("/api/jobs/{id}/result"));
assert_eq!(result.status, 200, "{}", result.text());
let value = result.json();
let regions = assert_result_satisfies_page_contract(&value);
assert_eq!(
regions, 4,
"three text regions + one formula region: {value}"
);
assert!(
value["regions"]
.as_array()
.expect("regions")
.iter()
.any(|region| region["kind"] == "formula" && region["formula"]["latex"] == "E = mc^{0}"),
"{value}"
);
}
#[test]
fn the_served_char_boxes_satisfy_the_optional_page_contract() {
let dir = complete_model_dir("contract-words-chars");
let scripted = Scripted::fast();
let server = TestServer::start(TestOptions::new(dir, scripted.clone()));
let value = submit_words_query(
&server,
&scripted,
"?words=chars",
"page words contract",
WordOutputMode::Chars,
);
let regions = assert_result_satisfies_page_contract(&value);
assert_eq!(regions, 3, "three scripted text regions: {value}");
for (index, region) in text_regions(&value).into_iter().enumerate() {
let words = region["recognition"]["words"]
.as_array()
.unwrap_or_else(|| panic!("text region {index} must carry word boxes: {region}"));
assert!(
!words.is_empty(),
"text region {index} got an empty word array; `chars` must give one box per char: {region}"
);
assert!(
words
.iter()
.all(|word| word["kind"].as_str() == Some("char")),
"text region {index} must be char-granular: {region}"
);
}
}
#[test]
fn the_web_demo_text_layer_uses_the_shared_mode_vocabulary_and_an_affine_box() {
let page = std::fs::read_to_string(
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("src/bin/web/index.html"),
)
.expect("the inlined page must be readable");
for mode in WordOutputMode::ALL {
assert!(
page.contains(&format!("id: \"{}\"", mode.as_str())),
"the demo must offer the `{}` tier verbatim: src/api.rs holds the only word list",
mode.as_str()
);
}
assert!(
page.contains(r#"state.words === "off" ? "" : "&words=" + state.words"#),
"the demo must express the tier as `&words=` + the mode name, and omit it entirely for `off`"
);
assert!(
page.contains("user-select: text"),
"the text layer must opt back into selection: `user-select: text`"
);
assert!(
page.contains(r#"class="text-layer" id="textLayer""#),
"the stage must carry the transparent text layer element"
);
assert!(
page.contains("[ux / boxW, uy / boxW, dx / boxH, dy / boxH, x0, y0]"),
"the layer must map the span with an affine built from the word quad's two edges"
);
let at = page
.find("function layoutTextLayer() {")
.expect("the page must keep `layoutTextLayer`");
let end = page[at..]
.find("\n }")
.map(|offset| at + offset)
.expect("`layoutTextLayer` must have a closing brace");
let body = &page[at..end];
assert!(
body.contains("s.transform"),
"the extracted `layoutTextLayer` body must be the one that writes the transform"
);
assert!(
!body.contains("Math.min(") && !body.contains("Math.max("),
"the text layer must not fall back to an axis-aligned bounding box: position the span \
on the word quad's own edges instead"
);
}
#[test]
fn the_web_demo_copy_exits_follow_the_registered_semantics() {
let page = std::fs::read_to_string(
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("src/bin/web/index.html"),
)
.expect("the inlined page must be readable");
fn body<'a>(page: &'a str, start: &str, end: &str) -> &'a str {
let at = page
.find(start)
.unwrap_or_else(|| panic!("the page must keep `{start}`"));
let stop = at
+ page[at..]
.find(end)
.unwrap_or_else(|| panic!("`{start}` must be closed by `{end}`"));
&page[at..stop]
}
let full = body(&page, "function fullTextForCopy() {", "\n }");
assert!(
full.contains("regions.forEach((r, i)"),
"复制全文 must merge in `regions[]` order; the region index is what that order rule reads"
);
assert!(
full.contains("r.table.markdown") && full.contains("parts.push(md.trim())"),
"a table region must reach the plain-text exit through `table.markdown` (the lossy flat \
form the page documents), never through its HTML markup"
);
assert!(
full.contains("sourceLineIndices.forEach") && full.contains("consumed.has(i)"),
"the page-level lines a table consumed must be skipped, or the same text is copied twice, \
the second time outside its table context"
);
assert!(
full.contains("parts.join(\"\\n\")"),
"the cross-region merge rule must be an explicit newline between regions, not an accident \
of concatenation"
);
assert!(
!full.contains(".html"),
"the plain-text exit must not paste table markup (or any html field) as ordinary text"
);
let dbl = body(
&page,
"$(\"regionList\").addEventListener(\"dblclick\"",
"\n });",
);
assert!(
dbl.contains("slot.getAttribute(\"data-latex\")"),
"double-clicking a formula row must copy the LaTeX the server sent (`formula.latex`), which \
the row keeps in `data-latex`; the rendered slot's own textContent is empty by design"
);
assert!(
dbl.contains("copyTextToClipboard(latex)"),
"the formula exit must route that LaTeX to the clipboard, not to a re-render of it"
);
let layer = body(&page, "function renderTextLayer() {", "\n }");
assert!(
layer.contains("r.kind !== \"text\""),
"the transparent layer must be built from text regions only"
);
assert!(
!layer.contains("markdown") && !layer.contains("latex"),
"formula LaTeX and table markdown must never be laid over the picture: they are not what \
the reader's eyes see, and selecting them would copy something the image does not show"
);
let table = body(&page, "function toggleTableRender(btn) {", "\n }");
assert!(
table.contains("wrap.innerHTML = sanitizeTableHtml("),
"the rendered table must come from the sanitizing entry point; cell text is model output"
);
assert!(
table.contains(".rt-wrap"),
"the rendered cells must live in the page's own `.rt-wrap` container, which is where the \
browser-native selection reads them from"
);
}
#[test]
fn the_page_reads_exactly_the_pinned_result_paths() {
let page = std::fs::read_to_string(
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("src/bin/web/index.html"),
)
.expect("the inlined page must be readable");
for (path, accessor) in PAGE_RESULT_CONTRACT {
assert!(
page.contains(accessor),
"the page must keep reading `{path}` as `{accessor}`: the result contract is pinned \
to these exact names (rename it here **and** in src/api.rs, never add an alias)"
);
}
for (path, accessor) in PAGE_WORDS_CONTRACT {
assert!(
page.contains(accessor),
"the page must keep reading the optional `{path}` as `{accessor}`: the word boxes \
are a **tier**, not a required field — absent/null is the `off` mode and must \
parse, while a present array must have exactly this shape"
);
}
assert!(
page.contains(r#"typeof r.plain_text === "string""#),
"the page's copy-all must keep reading `plain_text`"
);
assert!(
page.contains("state.result.timings"),
"the page's timing table must keep reading `timings`"
);
for alias in [
"r.boxes",
"r.items",
"region.text",
"polygon.box",
"polygon.coords",
] {
assert!(
!page.contains(alias),
"the page must not re-introduce the alias `{alias}`: a rename has to fail visibly"
);
}
assert!(
page.contains(r#""/api/ocr?queue=" + queue"#),
"the page must express the pipeline choice as `?queue=` on POST /api/ocr"
);
assert!(
!page.contains(r#""/api/ocr","#),
"the page must not post to /api/ocr without the documented queue parameter"
);
}
#[test]
fn the_captured_real_result_body_still_satisfies_the_page_contract() {
let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("tests/fixtures/serve/result-real-42-regions.json");
let bytes = std::fs::read(&path).unwrap_or_else(|error| {
panic!(
"the captured real result body must exist at {}: {error}",
path.display()
)
});
assert_eq!(
rapid_structure_rs::sha256_file(&path).expect("hash the captured body"),
"fda2f71337aa8b03f147d5fb4b0065981fe0444703c30dfb74573439f691fd27",
"the captured body must be exactly the bytes the reporter's server returned"
);
let value: Value = serde_json::from_slice(&bytes).expect("the captured body is JSON");
let regions = assert_result_satisfies_page_contract(&value);
assert_eq!(regions, 42, "the captured response carried 42 regions");
for key in [
"schema_version",
"image",
"stages",
"regions",
"timings",
"engine",
"plain_text",
"timing_ledger",
] {
assert!(
value.get(key).is_some(),
"the captured body must still carry `{key}`: {}",
value
.as_object()
.map(|map| map.keys().cloned().collect::<Vec<_>>())
.unwrap_or_default()
.join(", ")
);
}
}
#[test]
fn the_queue_is_expressed_as_the_documented_query_parameter() {
let server = TestServer::start(TestOptions::new(
complete_model_dir("contract-queue"),
Scripted::fast(),
));
let text = server.post(
"/api/ocr?queue=text",
&[("Content-Type", "application/octet-stream")],
b"image bytes",
);
assert_eq!(text.status, 202, "{}", text.text());
let id = text.json()["job_id"].as_str().expect("job id").to_string();
server.wait_terminal(&id, Duration::from_secs(20));
let formula = server.post(
"/api/ocr?queue=formula",
&[("Content-Type", "application/octet-stream")],
b"image bytes",
);
assert_eq!(formula.status, 400, "{}", formula.text());
assert_eq!(formula.code(), "bad_request");
}