use super::*;
#[test]
fn quoted_multiline_table_matches_model_and_middle_shapes() {
let result = analyze(b"name;value\r\n\"a;b\n<c>\";7\r\n", "csv").unwrap();
assert!(
result.middle["pages"][0]["blocks"][0]["content"][0]["content"]
.as_str()
.unwrap()
.contains("a;b<br><c>")
);
assert!(result.markdown.contains("| name | value |"));
assert_eq!(result.middle["metadata"]["producer"]["name"], "mineru-rs");
assert_eq!(result.model["schema"], "docvortex.model");
assert_eq!(result.model["page_index_map"], json!([]));
assert!(result.model["pages"][0][0]["content"].is_string());
assert_eq!(
result.model["pages"][0][0]["content"],
result.middle["pages"][0]["blocks"][0]["content"][0]["content"]
);
assert_eq!(result.middle["pages"][0]["blocks"][0]["index"], 0);
assert_eq!(
result.middle["pages"][0]["blocks"][0]["content"][0]["index"],
0
);
assert_eq!(
result.structured["pages"][0]["blocks"][0]["captions"],
json!([])
);
assert!(result.structured.get("schema").is_none());
}
#[test]
fn pinned_csv_fixture_produces_raw_model_html_and_postprocessed_middle() {
let output = analyze(
include_bytes!("../../../tests/fixtures/legacy/minimal.csv"),
"csv",
)
.unwrap();
let model_block = &output.model["pages"][0][0];
let html = model_block["content"].as_str().unwrap();
assert_eq!(model_block["type"], "table");
assert_eq!(output.model["page_index_map"], json!([]));
assert!(output.middle["is_full_document"].as_bool().unwrap());
assert!(html.starts_with("<table>\n <tr>\n <th>a</th>"));
assert_eq!(
output.middle["pages"][0]["blocks"][0]["content"][0]["type"],
"table_body"
);
assert_eq!(output.middle["pages"][0]["blocks"][0]["index"], 0);
assert_eq!(
output.middle["pages"][0]["blocks"][0]["content"][0]["index"],
0
);
assert_eq!(
output.middle["pages"][0]["blocks"][0]["content"][0]["content"],
html
);
}
#[test]
fn delimiter_sniff_skips_single_column_modes_and_samples_long_files() {
let ragged = analyze(b"a,b\n1,2,3\n4,5,6", "csv").unwrap();
let html = ragged.middle["pages"][0]["blocks"][0]["content"][0]["content"]
.as_str()
.unwrap();
assert!(html.contains("<td>1</td>") && html.contains("<td>3</td>"));
let long = (0..24)
.map(|n| format!("{n};{}", n + 1))
.collect::<Vec<_>>()
.join("\n");
let output = analyze(long.as_bytes(), "csv").unwrap();
assert!(
output.middle["pages"][0]["blocks"][0]["content"][0]["content"]
.as_str()
.unwrap()
.contains("<td>24</td>")
);
}
#[test]
fn quotes_inside_unquoted_fields_are_data_but_closed_quotes_remain_strict() {
let result = analyze(b"name,value\nfoo\"bar,2", "csv").unwrap();
let html = result.model["pages"][0][0]["content"].as_str().unwrap();
assert!(html.contains("foo"bar"));
assert_eq!(
analyze(b"name,value\n\"closed\"junk,2", "csv")
.unwrap_err()
.kind,
OfficialErrorKind::InvalidInput
);
assert_eq!(
analyze(b"name,value\n\"unfinished,2", "csv")
.unwrap_err()
.kind,
OfficialErrorKind::InvalidInput
);
}
#[test]
fn c1_controls_are_preserved_while_disallowed_c0_controls_are_replaced() {
let input = "head\n\"a\u{85}b\u{9f}c\0d\u{000b}e\"";
let result = analyze(input.as_bytes(), "csv").unwrap();
let html = result.model["pages"][0][0]["content"].as_str().unwrap();
let expected = "a\u{85}b\u{9f}c\u{fffd}d\u{fffd}e";
assert!(html.contains(expected));
assert!(result.markdown.contains(expected));
assert!(
result.structured["pages"][0]["blocks"][0]["content"]
.as_str()
.unwrap()
.contains(expected)
);
}
#[test]
fn unicode_directive_numeric_data_controls_and_empty_tables_are_handled() {
let (text, delimiter) = directive("ێ\nx,y");
assert_eq!(text, "ێ\nx,y");
assert_eq!(delimiter, None);
let numeric = analyze(b"1,2\n3,4", "csv").unwrap();
let html = numeric.middle["pages"][0]["blocks"][0]["content"][0]["content"]
.as_str()
.unwrap();
assert!(html.contains("<td>1</td>") && !html.contains("<th>"));
let control = analyze(b"head\n\"a\0b\r\nc\"", "csv").unwrap();
assert!(control.markdown.contains("a\u{fffd}b<br>c"));
assert!(!control.markdown.contains("<br>"));
let whitespace = analyze(b"head\n\" a\t b\x0c c\x0b d \"", "csv").unwrap();
assert!(whitespace.markdown.contains("a b\u{fffd} c\u{fffd} d"));
let empty = analyze(b"", "csv").unwrap();
assert_eq!(empty.middle["pages"][0]["blocks"], json!([]));
assert_eq!(empty.structured["pages"][0]["blocks"], json!([]));
}
#[test]
fn header_inference_uses_unicode_folding_and_date_shapes() {
for input in ["ß,SS\n1,2", "ff,ff\n1,2"] {
let output = analyze(input.as_bytes(), "csv").unwrap();
assert!(
!output.model["pages"][0][0]["content"]
.as_str()
.unwrap()
.contains("<th>")
);
}
let output = analyze(b"x,amount,match\n1234-t,1,match\n5678-t,2,match", "csv").unwrap();
let html = output.model["pages"][0][0]["content"].as_str().unwrap();
assert!(!html.contains("<th>"));
let output = analyze(b"x,amount,match\n1/2/3,1,match\n4/5/6,2,match", "csv").unwrap();
let html = output.model["pages"][0][0]["content"].as_str().unwrap();
assert!(html.contains("<th>x</th>"));
}
#[test]
fn numeric_inference_accepts_python_unicode_decimal_float_tokens() {
for (first, second) in [("Ù¡2", "Ù£4"), ("Ù¡2e2", "Ù£4e2"), ("Ù¡2%", "Ù£4%")] {
let input = format!("x,amount,match\n{first},1,match\n{second},2,match");
let output = analyze(input.as_bytes(), "csv").unwrap();
let html = output.model["pages"][0][0]["content"].as_str().unwrap();
assert!(html.contains("<th>x</th>"), "{input}");
}
for (first, second) in [("١x2", "٣x4"), ("١٢", "٣٤")] {
let input = format!("x,amount,match\n{first},1,match\n{second},2,match");
let output = analyze(input.as_bytes(), "csv").unwrap();
let html = output.model["pages"][0][0]["content"].as_str().unwrap();
assert!(!html.contains("<th>"), "{input}");
}
}
#[test]
fn restore_requires_matching_nonnegative_table_indices() {
let result = analyze(b"head\nvalue", "csv").unwrap();
let valid = result.middle;
let cancelled = AtomicBool::new(false);
for (path, value) in [
("/pages/0/blocks/0/index", None),
("/pages/0/blocks/0/content/0/index", None),
("/pages/0/blocks/0/index", Some(json!(-1))),
("/pages/0/blocks/0/content/0/index", Some(json!(1))),
] {
let mut invalid = valid.clone();
if let Some(value) = value {
invalid.pointer_mut(path).unwrap().clone_from(&value);
} else {
let parent = path.rsplit_once('/').unwrap().0;
let key = path.rsplit('/').next().unwrap();
invalid
.pointer_mut(parent)
.unwrap()
.as_object_mut()
.unwrap()
.remove(key);
}
assert_eq!(
csv_restore::restore(invalid, Default::default(), &cancelled)
.unwrap_err()
.kind,
OfficialErrorKind::InvalidInput
);
}
}
#[test]
fn output_expansion_budget_and_invalid_encodings_fail_explicitly() {
assert_eq!(
analyze_with_cancel(&[b'<'; 128], "csv", None, 1_000)
.unwrap_err()
.kind,
OfficialErrorKind::LimitExceeded
);
assert_eq!(
analyze(b"a,\"unfinished", "csv").unwrap_err().kind,
OfficialErrorKind::InvalidInput
);
assert_eq!(
analyze(&[0xff, 0xfe, b'a', 0], "csv").unwrap_err().kind,
OfficialErrorKind::UnsupportedOperation
);
assert_eq!(
analyze(b"left\tright\n1\t2", "tsv").unwrap().middle["pages"][0]["blocks"]
.as_array()
.unwrap()
.len(),
1
);
}
#[tokio::test]
async fn dropping_restore_future_releases_its_blocking_worker() {
use std::sync::{
Arc, Barrier,
atomic::{AtomicBool, Ordering},
};
let started = Arc::new(Barrier::new(2));
let stopped = Arc::new(Barrier::new(2));
let started_worker = started.clone();
let stopped_worker = stopped.clone();
let task = tokio::spawn(async move {
let cancelled = Arc::new(AtomicBool::new(false));
let _guard = RestoreCancelGuard(Some(cancelled.clone()));
let worker_cancelled = cancelled.clone();
tokio::task::spawn_blocking(move || {
started_worker.wait();
while !worker_cancelled.load(Ordering::Relaxed) {
std::thread::yield_now();
}
stopped_worker.wait();
})
.await
.unwrap();
});
tokio::task::spawn_blocking(move || started.wait())
.await
.unwrap();
task.abort();
assert!(task.await.unwrap_err().is_cancelled());
tokio::time::timeout(
std::time::Duration::from_secs(2),
tokio::task::spawn_blocking(move || stopped.wait()),
)
.await
.expect("dropped restore future did not cancel blocking work")
.unwrap();
}