use std::path::Path;
use image::{DynamicImage, GrayImage, ImageBuffer, RgbImage};
use lopdf::xobject::PdfImage;
use lopdf::{Document, Object, ObjectId};
use crate::error::{FocrError, FocrResult};
const PDF_MAGIC: &[u8] = b"%PDF-";
#[must_use]
pub fn looks_like_pdf(path: &Path) -> bool {
if path
.extension()
.and_then(|e| e.to_str())
.is_some_and(|e| e.eq_ignore_ascii_case("pdf"))
{
return true;
}
let Ok(mut file) = std::fs::File::open(path) else {
return false;
};
let mut head = [0u8; 5];
use std::io::Read;
matches!(file.read_exact(&mut head), Ok(())) && head == PDF_MAGIC
}
#[must_use]
pub fn looks_like_pdf_bytes(bytes: &[u8]) -> bool {
bytes.starts_with(PDF_MAGIC)
}
pub struct PdfPages {
doc: Document,
pages: Vec<ObjectId>,
}
impl PdfPages {
pub fn open(path: &Path) -> FocrResult<Self> {
let doc = Document::load(path)
.map_err(|e| FocrError::InputDecode(format!("parse PDF {}: {e}", path.display())))?;
Self::from_document(doc, &path.display().to_string())
}
pub fn from_bytes(bytes: &[u8]) -> FocrResult<Self> {
let doc = Document::load_mem(bytes)
.map_err(|e| FocrError::InputDecode(format!("parse PDF bytes: {e}")))?;
Self::from_document(doc, "bytes")
}
fn from_document(doc: Document, what: &str) -> FocrResult<Self> {
let pages: Vec<ObjectId> = doc.get_pages().into_values().collect();
if pages.is_empty() {
return Err(FocrError::InputDecode(format!("PDF {what} has no pages")));
}
Ok(Self { doc, pages })
}
#[must_use]
pub fn len(&self) -> usize {
self.pages.len()
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.pages.is_empty()
}
pub fn render(&self, idx: usize) -> FocrResult<DynamicImage> {
let page_id = *self.pages.get(idx).ok_or_else(|| {
FocrError::InputDecode(format!(
"PDF page index {idx} out of range ({})",
self.len()
))
})?;
let images = self.doc.get_page_images(page_id).map_err(|e| {
FocrError::InputDecode(format!("read images on PDF page {}: {e}", idx + 1))
})?;
let main = images
.iter()
.max_by_key(|im| (im.width as i128) * (im.height as i128))
.ok_or_else(|| {
FocrError::InputDecode(format!(
"PDF page {} has no image XObject (vector/text PDFs are not supported by the \
native fast path; rasterize the PDF out of band, e.g. with pdftoppm, and pass \
the page images)",
idx + 1
))
})?;
let decoded = decode_image_xobject(&self.doc, main)
.map_err(|e| FocrError::InputDecode(format!("PDF page {}: {e}", idx + 1)))?;
let total_rotation = (page_rotation(&self.doc, page_id)
+ content_rotation(&self.doc, page_id))
.rem_euclid(360);
Ok(apply_rotation(decoded, total_rotation))
}
}
fn content_rotation(doc: &Document, page_id: ObjectId) -> i64 {
let Ok(content) = doc.get_and_decode_page_content(page_id) else {
return 0;
};
let mut cm: Option<[f64; 4]> = None;
for op in &content.operations {
match op.operator.as_ref() {
"cm" => {
let v: Vec<f64> = op
.operands
.iter()
.filter_map(|o| o.as_float().ok().map(f64::from))
.collect();
if v.len() >= 4 {
cm = Some([v[0], v[1], v[2], v[3]]);
}
}
"Do" => break,
_ => {}
}
}
let Some([a, b, c, d]) = cm else { return 0 };
if b.abs() > a.abs() && c.abs() > d.abs() {
if b > 0.0 { 270 } else { 90 }
} else if a < 0.0 && d < 0.0 {
180
} else {
0
}
}
#[must_use]
pub fn split_spread(img: &DynamicImage) -> Option<(DynamicImage, DynamicImage, u32)> {
let (w, h) = (img.width(), img.height());
if h == 0 || (f64::from(w) / f64::from(h)) < 1.25 {
return None;
}
let gray = img.to_luma8();
let ink_threshold = 160u8; let (lo, hi) = (w * 2 / 5, w * 3 / 5); let center = i64::from(w / 2);
let mut best: Option<(i64, u32)> = None; for x in lo..hi {
let mut dark = 0u32;
for y in 0..h {
if gray.get_pixel(x, y).0[0] < ink_threshold {
dark += 1;
}
}
let dark_frac_pct_x10 = u64::from(dark) * 1000 / u64::from(h);
let is_gutter = dark_frac_pct_x10 <= 5 || dark_frac_pct_x10 >= 600;
if is_gutter {
let dist = (i64::from(x) - center).abs();
if best.is_none_or(|(d, _)| dist < d) {
best = Some((dist, x));
}
}
}
let (_, gutter_x) = best?;
let left = img.crop_imm(0, 0, gutter_x, h);
let right = img.crop_imm(gutter_x, 0, w - gutter_x, h);
Some((left, right, gutter_x))
}
fn decode_image_xobject(doc: &Document, img: &PdfImage) -> Result<DynamicImage, String> {
let width = u32::try_from(img.width).map_err(|_| "negative image width".to_string())?;
let height = u32::try_from(img.height).map_err(|_| "negative image height".to_string())?;
if width == 0 || height == 0 {
return Err("zero image dimension".to_string());
}
const MAX_PIXELS: u64 = 1 << 30; if u64::from(width) * u64::from(height) > MAX_PIXELS {
return Err(format!(
"image dimensions {width}x{height} exceed the {MAX_PIXELS}-pixel maximum"
));
}
let bpc = img.bits_per_component.unwrap_or(8);
let color_space = img.color_space.as_deref().unwrap_or("DeviceRGB");
let filters = img.filters.clone().unwrap_or_default();
let terminal = filters.last().map(String::as_str).unwrap_or("");
let chained = filters.len() > 1;
match terminal {
"DCTDecode" if chained => Err(format!(
"image filter chain {filters:?} ending in DCTDecode is unsupported (only a \
sole DCTDecode filter); rasterize this PDF out of band and retry"
)),
"DCTDecode" => image::load_from_memory_with_format(img.content, image::ImageFormat::Jpeg)
.map_err(|e| format!("JPEG (DCTDecode) decode failed: {e}")),
"JPXDecode" => Err(
"image uses JPXDecode (JPEG 2000), which has no pure-Rust decoder; \
rasterize this PDF out of band and retry"
.to_string(),
),
"JBIG2Decode" => Err("image uses JBIG2Decode, which has no pure-Rust decoder; \
rasterize this PDF out of band and retry"
.to_string()),
"CCITTFaxDecode" if chained => Err(format!(
"image filter chain {filters:?} ending in CCITTFaxDecode is unsupported (only a \
sole CCITTFaxDecode filter); rasterize this PDF out of band and retry"
)),
"CCITTFaxDecode" => decode_ccitt_g4(doc, img, width, height),
"FlateDecode" | "LZWDecode" | "ASCII85Decode" | "" => {
let cap = expected_sample_cap(width, height, bpc, color_space);
let sole_flate = !chained && terminal == "FlateDecode";
let samples = decompressed_stream(doc, img.id, img.content, sole_flate, cap)?;
if color_space == "Indexed" || color_space == "I" {
return indexed_to_image(doc, img.id, &samples, width, height, bpc);
}
raw_samples_to_image(samples, width, height, bpc, color_space)
}
other => Err(format!("unsupported image filter {other}")),
}
}
fn decompressed_stream(
doc: &Document,
id: ObjectId,
raw: &[u8],
sole_flate: bool,
cap: u64,
) -> Result<Vec<u8>, String> {
let stream = doc
.get_object(id)
.and_then(Object::as_stream)
.map_err(|e| format!("read image stream: {e}"))?;
if sole_flate
&& stream_predictor(stream) <= 1
&& let Some(out) = bounded_inflate(raw, cap)?
{
return Ok(out);
}
stream
.decompressed_content()
.map_err(|e| format!("inflate image stream: {e}"))
}
fn stream_predictor(stream: &lopdf::Stream) -> i64 {
stream
.dict
.get(b"DecodeParms")
.or_else(|_| stream.dict.get(b"DP"))
.and_then(Object::as_dict)
.ok()
.and_then(|p| p.get(b"Predictor").ok())
.and_then(|o| o.as_i64().ok())
.unwrap_or(1)
}
fn bounded_inflate(raw: &[u8], cap: u64) -> Result<Option<Vec<u8>>, String> {
use std::io::Read;
let mut out = Vec::new();
if flate2::read::ZlibDecoder::new(raw)
.take(cap.saturating_add(1))
.read_to_end(&mut out)
.is_err()
{
return Ok(None);
}
if out.len() as u64 > cap {
return Err(format!(
"decompressed image stream exceeds the {cap}-byte cap \
(4x the expected sample size; possible decompression bomb)"
));
}
Ok(Some(out))
}
fn expected_sample_cap(width: u32, height: u32, bpc: i64, color_space: &str) -> u64 {
let comps: u64 = match color_space {
"DeviceGray" | "CalGray" => 1,
"DeviceRGB" | "CalRGB" => 3,
_ => 4, };
let bytes_per_comp = (bpc.clamp(1, 16) as u64).div_ceil(8);
u64::from(width)
.saturating_mul(u64::from(height))
.saturating_mul(comps)
.saturating_mul(bytes_per_comp)
.saturating_mul(4)
}
fn raw_samples_to_image(
samples: Vec<u8>,
width: u32,
height: u32,
bpc: i64,
color_space: &str,
) -> Result<DynamicImage, String> {
let comps = match color_space {
"DeviceRGB" | "CalRGB" => 3usize,
"DeviceGray" | "CalGray" => 1,
"DeviceCMYK" => 4,
other => return Err(format!("unsupported color space {other}")),
};
match bpc {
8 => match comps {
3 => from_raw_rgb(width, height, samples),
1 => from_raw_gray(width, height, samples),
4 => Ok(DynamicImage::ImageRgb8(cmyk8_to_rgb(
&samples, width, height,
)?)),
_ => Err(format!("unsupported component count {comps}")),
},
1 => bilevel_to_gray(&samples, width, height),
16 => {
let high: Vec<u8> = samples.as_chunks::<2>().0.iter().map(|c| c[0]).collect();
raw_samples_to_image(high, width, height, 8, color_space)
}
other => Err(format!("unsupported bits-per-component {other}")),
}
}
fn from_raw_rgb(width: u32, height: u32, samples: Vec<u8>) -> Result<DynamicImage, String> {
let buf: RgbImage = ImageBuffer::from_raw(width, height, samples)
.ok_or_else(|| "RGB sample count does not match image dimensions".to_string())?;
Ok(DynamicImage::ImageRgb8(buf))
}
fn from_raw_gray(width: u32, height: u32, samples: Vec<u8>) -> Result<DynamicImage, String> {
let buf: GrayImage = ImageBuffer::from_raw(width, height, samples)
.ok_or_else(|| "gray sample count does not match image dimensions".to_string())?;
Ok(DynamicImage::ImageLuma8(buf))
}
fn cmyk8_to_rgb(samples: &[u8], width: u32, height: u32) -> Result<RgbImage, String> {
let pixels = (width as usize) * (height as usize);
if samples.len() < pixels * 4 {
return Err("CMYK sample count does not match image dimensions".to_string());
}
let mut out = Vec::with_capacity(pixels * 3);
for px in samples.as_chunks::<4>().0.iter().take(pixels) {
let (c, m, y, k) = (
u16::from(px[0]),
u16::from(px[1]),
u16::from(px[2]),
u16::from(px[3]),
);
out.push((255 - (c + k).min(255)) as u8);
out.push((255 - (m + k).min(255)) as u8);
out.push((255 - (y + k).min(255)) as u8);
}
ImageBuffer::from_raw(width, height, out).ok_or_else(|| "CMYK->RGB pack failed".to_string())
}
fn bilevel_to_gray(samples: &[u8], width: u32, height: u32) -> Result<DynamicImage, String> {
let row_bytes = (width as usize).div_ceil(8);
if samples.len() < row_bytes * height as usize {
return Err("bilevel sample count does not match image dimensions".to_string());
}
let mut out = Vec::with_capacity((width as usize) * (height as usize));
for y in 0..height as usize {
let row = &samples[y * row_bytes..];
for x in 0..width as usize {
let bit = (row[x / 8] >> (7 - (x % 8))) & 1;
out.push(if bit == 1 { 255 } else { 0 });
}
}
from_raw_gray(width, height, out)
}
fn indexed_to_image(
doc: &Document,
id: ObjectId,
samples: &[u8],
width: u32,
height: u32,
bpc: i64,
) -> Result<DynamicImage, String> {
let (comps, palette) = indexed_palette(doc, id)?;
let indices = unpack_indices(samples, width, height, bpc)?;
let last = (palette.len() / comps).saturating_sub(1);
match comps {
1 => {
let out: Vec<u8> = indices
.iter()
.map(|&i| palette[(usize::from(i)).min(last)])
.collect();
from_raw_gray(width, height, out)
}
3 => {
let mut out = Vec::with_capacity(indices.len() * 3);
for &i in &indices {
let at = (usize::from(i)).min(last) * 3;
out.extend_from_slice(&palette[at..at + 3]);
}
from_raw_rgb(width, height, out)
}
other => Err(format!("unsupported Indexed component count {other}")),
}
}
fn indexed_palette(doc: &Document, id: ObjectId) -> Result<(usize, Vec<u8>), String> {
let deref = |obj: &'_ Object| -> Result<Object, String> {
doc.dereference(obj)
.map(|(_, o)| o.clone())
.map_err(|e| format!("resolve Indexed color space: {e}"))
};
let stream = doc
.get_object(id)
.and_then(Object::as_stream)
.map_err(|e| format!("read image stream: {e}"))?;
let cs = stream
.dict
.get(b"ColorSpace")
.map_err(|e| format!("Indexed image without /ColorSpace: {e}"))?;
let cs = deref(cs)?;
let arr = cs
.as_array()
.map_err(|_| "Indexed /ColorSpace is not an array".to_string())?;
if arr.len() < 4 {
return Err(format!(
"Indexed color space array has {} elements, expected 4",
arr.len()
));
}
let base = deref(&arr[1])?;
let (base_name, base_arr): (Vec<u8>, Option<&[Object]>) = match &base {
Object::Name(n) => (n.clone(), None),
Object::Array(a) => {
let head = a
.first()
.and_then(|o| o.as_name().ok())
.ok_or_else(|| "Indexed base color-space array has no name".to_string())?;
(head.to_vec(), Some(a.as_slice()))
}
_ => return Err("unsupported Indexed base color space object".to_string()),
};
let comps: usize = match base_name.as_slice() {
b"DeviceGray" | b"CalGray" | b"G" => 1,
b"DeviceRGB" | b"CalRGB" | b"RGB" => 3,
b"DeviceCMYK" | b"CMYK" => 4,
b"ICCBased" => {
let profile = base_arr
.and_then(|a| a.get(1))
.ok_or_else(|| "ICCBased Indexed base without a profile stream".to_string())?;
let profile = deref(profile)?;
let n = profile
.as_stream()
.ok()
.and_then(|s| s.dict.get(b"N").ok())
.and_then(|o| o.as_i64().ok())
.ok_or_else(|| "ICCBased Indexed base without /N".to_string())?;
usize::try_from(n)
.ok()
.filter(|n| [1, 3, 4].contains(n))
.ok_or_else(|| {
format!("ICCBased Indexed base with unsupported component count {n}")
})?
}
other => {
return Err(format!(
"unsupported Indexed base color space {}",
String::from_utf8_lossy(other)
));
}
};
let hival = deref(&arr[2])?
.as_i64()
.map_err(|_| "Indexed hival is not an integer".to_string())?;
if !(0..=255).contains(&hival) {
return Err(format!("Indexed hival {hival} outside 0..=255"));
}
#[allow(clippy::cast_sign_loss)] let entries = hival as usize + 1;
let lookup = deref(&arr[3])?;
let mut palette: Vec<u8> = match &lookup {
Object::String(bytes, _) => bytes.clone(),
Object::Stream(s) => s
.decompressed_content()
.map_err(|e| format!("inflate Indexed palette stream: {e}"))?,
_ => return Err("Indexed palette is neither a string nor a stream".to_string()),
};
palette.resize(entries * comps, 0);
if comps == 4 {
let rgb = cmyk8_to_rgb(&palette, u32::try_from(entries).unwrap_or(1), 1)?;
return Ok((3, rgb.into_raw()));
}
Ok((comps, palette))
}
fn unpack_indices(samples: &[u8], width: u32, height: u32, bpc: i64) -> Result<Vec<u8>, String> {
let bits: usize = match bpc {
1 | 2 | 4 | 8 => usize::try_from(bpc).expect("bpc in 1..=8"),
other => {
return Err(format!(
"unsupported bits-per-component {other} for Indexed"
));
}
};
let (w, h) = (width as usize, height as usize);
let row_bytes = (w * bits).div_ceil(8);
if samples.len() < row_bytes * h {
return Err("indexed sample count does not match image dimensions".to_string());
}
let mask = if bits == 8 { 0xFF } else { (1u8 << bits) - 1 };
let mut out = Vec::with_capacity(w * h);
for y in 0..h {
let row = &samples[y * row_bytes..];
for x in 0..w {
let bit = x * bits;
out.push((row[bit / 8] >> (8 - bits - bit % 8)) & mask);
}
}
Ok(out)
}
fn decode_ccitt_g4(
doc: &Document,
img: &PdfImage,
width: u32,
height: u32,
) -> Result<DynamicImage, String> {
use fax::Color;
use fax::decoder::{decode_g4, pels};
let stream = doc
.get_object(img.id)
.and_then(Object::as_stream)
.map_err(|e| format!("read CCITT stream: {e}"))?;
let parms = stream
.dict
.get(b"DecodeParms")
.or_else(|_| stream.dict.get(b"DP"))
.and_then(Object::as_dict)
.ok();
let param_i64 = |key: &[u8], default: i64| -> i64 {
parms
.and_then(|p| p.get(key).ok())
.and_then(|o| o.as_i64().ok())
.unwrap_or(default)
};
let k = param_i64(b"K", 0);
let columns = u16::try_from(param_i64(b"Columns", 1728)).unwrap_or(1728);
let black_is_1 = parms
.and_then(|p| p.get(b"BlackIs1").ok())
.and_then(|o| o.as_bool().ok())
.unwrap_or(false);
if k >= 0 {
return Err(
"CCITTFaxDecode K>=0 (Group 3) is not supported; only Group 4 (K<0)".to_string(),
);
}
let cols = if columns == 0 {
u16::try_from(width).unwrap_or(1728)
} else {
columns
};
let (black, white) = if black_is_1 {
(255u8, 0u8)
} else {
(0u8, 255u8)
};
let rows_hint = u16::try_from(height).ok().filter(|&h| h != 0);
let mut out: Vec<u8> = Vec::new();
decode_g4(img.content.iter().copied(), cols, rows_hint, |line| {
out.extend(pels(line, cols).map(|c| match c {
Color::Black => black,
Color::White => white,
}));
})
.ok_or_else(|| "CCITT Group 4 decode failed".to_string())?;
let decoded_rows = u32::try_from(out.len() / usize::from(cols).max(1)).unwrap_or(0);
from_raw_gray(u32::from(cols), decoded_rows, out)
}
fn page_rotation(doc: &Document, page_id: ObjectId) -> i64 {
inherited(doc, page_id, b"Rotate")
.and_then(|o| o.as_i64().ok())
.unwrap_or(0)
.rem_euclid(360)
}
fn apply_rotation(img: DynamicImage, degrees: i64) -> DynamicImage {
match degrees {
90 => DynamicImage::ImageRgba8(image::imageops::rotate90(&img)),
180 => DynamicImage::ImageRgba8(image::imageops::rotate180(&img)),
270 => DynamicImage::ImageRgba8(image::imageops::rotate270(&img)),
_ => img,
}
}
fn inherited<'a>(doc: &'a Document, mut id: ObjectId, key: &[u8]) -> Option<&'a Object> {
for _ in 0..64 {
let dict = doc.get_dictionary(id).ok()?;
if let Ok(value) = dict.get(key) {
return Some(value);
}
id = dict.get(b"Parent").and_then(Object::as_reference).ok()?;
}
None
}
pub fn select_pages(spec: Option<&str>, page_count: usize) -> FocrResult<Vec<usize>> {
let Some(spec) = spec else {
return Ok((0..page_count).collect());
};
let usage = |what: &str| {
FocrError::Usage(format!(
"--pages {spec:?}: {what} (expected 1-based pages/ranges like \"1,5-9\"; \
this document has {page_count} page(s))"
))
};
let parse_one = |tok: &str| -> FocrResult<usize> {
let n: usize = tok
.trim()
.parse()
.map_err(|_| usage(&format!("unparseable page {tok:?}")))?;
if n == 0 {
return Err(usage("page 0 (pages are 1-based)"));
}
if n > page_count {
return Err(usage(&format!("page {n} is out of range")));
}
Ok(n - 1)
};
let mut selected = Vec::new();
let mut seen = vec![false; page_count];
for part in spec.split(',') {
let part = part.trim();
if part.is_empty() {
return Err(usage("empty element"));
}
let range = match part.split_once('-') {
Some((a, b)) => {
let (a, b) = (parse_one(a)?, parse_one(b)?);
if a > b {
return Err(usage(&format!("reversed range {part:?}")));
}
a..=b
}
None => {
let n = parse_one(part)?;
n..=n
}
};
for idx in range {
if !seen[idx] {
seen[idx] = true;
selected.push(idx);
}
}
}
selected.sort_unstable();
Ok(selected)
}
#[must_use]
pub fn is_fatal_to_document(err: &FocrError) -> bool {
matches!(
err,
FocrError::ModelNotFound(_) | FocrError::Cancelled | FocrError::FormatMismatch(_)
)
}
#[derive(Debug, Clone)]
pub struct DocumentPage {
pub page: usize,
pub markdown: String,
pub layout: Vec<crate::native_engine::LayoutSpan>,
pub duration: std::time::Duration,
}
#[derive(Debug, Clone)]
pub struct SkippedPage {
pub page: usize,
pub reason: String,
}
#[derive(Debug, Clone, Default)]
pub struct DocumentOutcome {
pub pages: Vec<DocumentPage>,
pub skipped: Vec<SkippedPage>,
pub total_pages: usize,
}
impl DocumentOutcome {
#[must_use]
pub fn markdown(&self) -> String {
self.pages
.iter()
.map(|p| p.markdown.trim_end())
.collect::<Vec<_>>()
.join("\n\n")
}
#[must_use]
pub fn duration(&self) -> std::time::Duration {
self.pages.iter().map(|p| p.duration).sum()
}
}
#[derive(Debug, Clone, Copy)]
pub enum DocumentEvent<'a> {
PageStarted {
page: usize,
index: usize,
selected: usize,
},
PageDone(&'a DocumentPage),
PageSkipped(&'a SkippedPage),
}
pub fn walk_document<R>(
pages: &PdfPages,
selected: &[usize],
mut recognize: R,
observer: &mut dyn FnMut(DocumentEvent<'_>),
) -> FocrResult<DocumentOutcome>
where
R: FnMut(usize, DynamicImage) -> FocrResult<(String, Vec<crate::native_engine::LayoutSpan>)>,
{
let mut outcome = DocumentOutcome {
total_pages: pages.len(),
..Default::default()
};
let mut first_reason: Option<String> = None;
for (index, &idx) in selected.iter().enumerate() {
let page = idx + 1;
observer(DocumentEvent::PageStarted {
page,
index,
selected: selected.len(),
});
let started = std::time::Instant::now();
let attempt = pages
.render(idx)
.and_then(|image| recognize(page, image))
.map(|(markdown, layout)| DocumentPage {
page,
markdown,
layout,
duration: started.elapsed(),
});
match attempt {
Ok(done) => {
outcome.pages.push(done);
observer(DocumentEvent::PageDone(
outcome.pages.last().expect("just pushed"),
));
}
Err(err) if is_fatal_to_document(&err) => return Err(err),
Err(err) => {
let reason = err.to_string();
if first_reason.is_none() {
first_reason = Some(reason.clone());
}
outcome.skipped.push(SkippedPage { page, reason });
observer(DocumentEvent::PageSkipped(
outcome.skipped.last().expect("just pushed"),
));
}
}
}
if outcome.pages.is_empty() {
return Err(FocrError::InputDecode(first_reason.unwrap_or_else(|| {
"the page selection produced no decodable pages".to_string()
})));
}
Ok(outcome)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn no_spec_selects_every_page() {
assert_eq!(select_pages(None, 4).expect("ok"), vec![0, 1, 2, 3]);
}
#[test]
fn spec_is_1_based_sorted_and_deduped() {
assert_eq!(select_pages(Some("3,1"), 5).expect("ok"), vec![0, 2]);
assert_eq!(select_pages(Some("2-4"), 5).expect("ok"), vec![1, 2, 3]);
assert_eq!(
select_pages(Some("1-3,2-4"), 5).expect("ok"),
vec![0, 1, 2, 3]
);
}
#[test]
fn out_of_range_is_a_usage_error_naming_the_page_count() {
let err = select_pages(Some("9"), 3).expect_err("must reject");
assert!(matches!(err, FocrError::Usage(_)), "got {err:?}");
let text = err.to_string();
assert!(text.contains("out of range"), "{text}");
assert!(text.contains("3 page(s)"), "{text}");
}
#[test]
fn zero_reversed_and_empty_elements_are_rejected() {
for bad in ["0", "3-1", "1,,2", "x"] {
let err = select_pages(Some(bad), 5).expect_err(bad);
assert!(matches!(err, FocrError::Usage(_)), "{bad}: {err:?}");
}
}
#[test]
fn only_document_level_failures_are_fatal() {
assert!(!is_fatal_to_document(&FocrError::InputDecode(
"JPXDecode: no pure-Rust decoder".into()
)));
assert!(!is_fatal_to_document(&FocrError::NotImplemented(
"x".into()
)));
assert!(!is_fatal_to_document(&FocrError::Timeout("x".into())));
assert!(is_fatal_to_document(&FocrError::ModelNotFound("x".into())));
assert!(is_fatal_to_document(&FocrError::Cancelled));
assert!(is_fatal_to_document(&FocrError::FormatMismatch("x".into())));
}
#[test]
fn markdown_joins_pages_with_a_blank_line_and_trims() {
let outcome = DocumentOutcome {
pages: vec![
DocumentPage {
page: 1,
markdown: "one\n\n".into(),
layout: Vec::new(),
duration: std::time::Duration::from_millis(10),
},
DocumentPage {
page: 2,
markdown: "two".into(),
layout: Vec::new(),
duration: std::time::Duration::from_millis(20),
},
],
skipped: vec![SkippedPage {
page: 3,
reason: "JBIG2Decode".into(),
}],
total_pages: 3,
};
assert_eq!(outcome.markdown(), "one\n\ntwo");
assert_eq!(outcome.duration(), std::time::Duration::from_millis(30));
}
fn synth_page(w: u32, h: u32, text_cols: &[(u32, u32)]) -> DynamicImage {
let mut img = image::GrayImage::from_pixel(w, h, image::Luma([255u8]));
for &(x0, x1) in text_cols {
for x in x0..x1 {
for y in (10..h.saturating_sub(10)).step_by(3) {
img.put_pixel(x, y, image::Luma([20u8]));
}
}
}
DynamicImage::ImageLuma8(img)
}
#[test]
fn split_spread_positive_and_negatives() {
let spread = synth_page(1600, 1000, &[(100, 700), (900, 1500)]);
let (left, right, gx) = split_spread(&spread).expect("spread splits");
assert!((700..=900).contains(&gx), "gutter near center: {gx}");
assert_eq!(left.width() + right.width(), 1600);
assert_eq!(left.height(), 1000);
assert!(split_spread(&synth_page(1000, 1600, &[(100, 900)])).is_none());
assert!(split_spread(&synth_page(1600, 1000, &[(100, 1500)])).is_none());
assert!(split_spread(&synth_page(1600, 1000, &[(600, 1000)])).is_none());
let mut bound = synth_page(1600, 1000, &[(100, 700), (900, 1500)]).to_luma8();
for x in 780..820 {
for y in 0..1000 {
bound.put_pixel(x, y, image::Luma([30u8]));
}
}
let bound = DynamicImage::ImageLuma8(bound);
let (_, _, gx) = split_spread(&bound).expect("binding shadow splits");
assert!((780..=820).contains(&gx), "split inside the shadow: {gx}");
}
#[test]
fn looks_like_pdf_by_extension() {
assert!(looks_like_pdf(Path::new("/x/y/scan.pdf")));
assert!(looks_like_pdf(Path::new("/x/y/scan.PDF")));
assert!(!looks_like_pdf(Path::new("/x/y/page.png")));
assert!(!looks_like_pdf(Path::new("/no/such/file.bin")));
}
#[test]
fn looks_like_pdf_bytes_by_magic() {
assert!(looks_like_pdf_bytes(b"%PDF-1.5\n..."));
assert!(!looks_like_pdf_bytes(b"\x89PNG\r\n\x1a\n"));
assert!(!looks_like_pdf_bytes(b"%PDF")); assert!(!looks_like_pdf_bytes(b""));
}
#[test]
fn from_bytes_renders_identically_to_open() {
use image::{ImageBuffer, Rgb};
use lopdf::{Stream, dictionary};
use std::io::Cursor;
let (w, h) = (24u32, 18u32);
let src = DynamicImage::ImageRgb8(ImageBuffer::from_fn(w, h, |x, y| {
Rgb([(x * 10) as u8, (y * 13) as u8, 200])
}));
let mut jpeg = Vec::new();
src.write_to(&mut Cursor::new(&mut jpeg), image::ImageFormat::Jpeg)
.expect("encode jpeg");
let image = Stream::new(
dictionary! {
"Type" => "XObject",
"Subtype" => "Image",
"Width" => i64::from(w),
"Height" => i64::from(h),
"ColorSpace" => "DeviceRGB",
"BitsPerComponent" => 8,
"Filter" => "DCTDecode",
},
jpeg,
)
.with_compression(false);
let path = build_single_page_pdf(Some(image));
let by_path = PdfPages::open(&path).expect("open by path");
let bytes = std::fs::read(&path).expect("read pdf bytes");
assert!(looks_like_pdf_bytes(&bytes));
let by_bytes = PdfPages::from_bytes(&bytes).expect("open from bytes");
assert_eq!(by_path.len(), by_bytes.len());
let a = by_path.render(0).expect("render by path");
let b = by_bytes.render(0).expect("render from bytes");
assert_eq!((a.width(), a.height()), (b.width(), b.height()));
assert_eq!(
a.to_rgb8().into_raw(),
b.to_rgb8().into_raw(),
"open() and from_bytes() rasters must be byte-identical"
);
let _ = std::fs::remove_file(&path);
}
#[test]
fn from_bytes_rejects_junk_with_named_error() {
let Err(err) = PdfPages::from_bytes(b"not a pdf at all") else {
panic!("junk must error");
};
assert!(err.to_string().contains("parse PDF bytes"), "got: {err}");
}
#[test]
fn bilevel_unpacks_msb_first() {
let img = bilevel_to_gray(&[0b1010_0000], 8, 1).expect("bilevel");
let gray = img.to_luma8();
assert_eq!(gray.get_pixel(0, 0).0[0], 255);
assert_eq!(gray.get_pixel(1, 0).0[0], 0);
assert_eq!(gray.get_pixel(2, 0).0[0], 255);
assert_eq!(gray.get_pixel(3, 0).0[0], 0);
}
#[test]
fn cmyk_pure_black_and_white() {
let rgb = cmyk8_to_rgb(&[0, 0, 0, 255, 0, 0, 0, 0], 2, 1).expect("cmyk");
assert_eq!(rgb.get_pixel(0, 0).0, [0, 0, 0]);
assert_eq!(rgb.get_pixel(1, 0).0, [255, 255, 255]);
}
#[test]
fn rgb_dimension_mismatch_errors() {
assert!(from_raw_rgb(2, 2, vec![1, 2, 3]).is_err());
}
fn build_multi_page_pdf(pages_spec: &[Option<lopdf::Stream>]) -> std::path::PathBuf {
use lopdf::{Object, dictionary};
let mut doc = lopdf::Document::with_version("1.5");
let pages_id = doc.new_object_id();
let mut kids: Vec<Object> = Vec::new();
for image_xobject in pages_spec {
let resources = match image_xobject.clone() {
Some(stream) => {
let image_id = doc.add_object(stream);
dictionary! { "XObject" => dictionary! { "Im0" => image_id } }
}
None => dictionary! {},
};
let resources_id = doc.add_object(resources);
let page_id = doc.add_object(dictionary! {
"Type" => "Page",
"Parent" => pages_id,
"Resources" => resources_id,
"MediaBox" => vec![0_i64.into(), 0_i64.into(), 100_i64.into(), 100_i64.into()],
});
kids.push(page_id.into());
}
let count = i64::try_from(pages_spec.len()).expect("page count fits");
doc.objects.insert(
pages_id,
Object::Dictionary(dictionary! {
"Type" => "Pages",
"Kids" => kids,
"Count" => count,
}),
);
let catalog_id = doc.add_object(dictionary! {
"Type" => "Catalog",
"Pages" => pages_id,
});
doc.trailer.set("Root", catalog_id);
use std::sync::atomic::{AtomicU32, Ordering};
static SEQ: AtomicU32 = AtomicU32::new(0);
let path = std::env::temp_dir().join(format!(
"focr_pdf_walk_{}_{}.pdf",
std::process::id(),
SEQ.fetch_add(1, Ordering::Relaxed)
));
doc.save(&path).expect("save synthesized pdf");
path
}
fn jpeg_xobject() -> lopdf::Stream {
use image::{ImageBuffer, Rgb};
use lopdf::{Stream, dictionary};
use std::io::Cursor;
let (w, h) = (16u32, 12u32);
let src = DynamicImage::ImageRgb8(ImageBuffer::from_fn(w, h, |x, _| {
Rgb([(x * 16) as u8, 64, 128])
}));
let mut jpeg = Vec::new();
src.write_to(&mut Cursor::new(&mut jpeg), image::ImageFormat::Jpeg)
.expect("encode jpeg");
Stream::new(
dictionary! {
"Type" => "XObject",
"Subtype" => "Image",
"Width" => i64::from(w),
"Height" => i64::from(h),
"ColorSpace" => "DeviceRGB",
"BitsPerComponent" => 8,
"Filter" => "DCTDecode",
},
jpeg,
)
.with_compression(false)
}
#[test]
fn walk_reads_every_readable_page_and_skips_the_rest_with_reasons() {
let path = build_multi_page_pdf(&[Some(jpeg_xobject()), None, Some(jpeg_xobject())]);
let pages = PdfPages::open(&path).expect("open");
let selected = select_pages(None, pages.len()).expect("all pages");
let mut events: Vec<String> = Vec::new();
let outcome = walk_document(
&pages,
&selected,
|page, image| {
assert!(image.width() > 0, "page {page} rasterized");
Ok((format!("text of page {page}"), Vec::new()))
},
&mut |event| match event {
DocumentEvent::PageStarted {
page,
index,
selected,
} => {
events.push(format!("start {page} ({index}/{selected})"));
}
DocumentEvent::PageDone(p) => events.push(format!("done {}", p.page)),
DocumentEvent::PageSkipped(s) => events.push(format!("skip {}", s.page)),
},
)
.expect("walk succeeds when at least one page reads");
assert_eq!(outcome.total_pages, 3);
assert_eq!(
outcome.pages.iter().map(|p| p.page).collect::<Vec<_>>(),
vec![1, 3]
);
assert_eq!(outcome.skipped.len(), 1);
assert_eq!(outcome.skipped[0].page, 2);
assert!(
!outcome.skipped[0].reason.is_empty(),
"a skip must carry the engine's reason"
);
assert_eq!(outcome.markdown(), "text of page 1\n\ntext of page 3");
assert_eq!(
events,
vec![
"start 1 (0/3)".to_string(),
"done 1".into(),
"start 2 (1/3)".into(),
"skip 2".into(),
"start 3 (2/3)".into(),
"done 3".into(),
]
);
let _ = std::fs::remove_file(&path);
}
#[test]
fn walk_aborts_immediately_on_a_document_level_failure() {
let path = build_multi_page_pdf(&[Some(jpeg_xobject()), Some(jpeg_xobject())]);
let pages = PdfPages::open(&path).expect("open");
let selected = select_pages(None, pages.len()).expect("all pages");
let mut attempts = 0usize;
let err = walk_document(
&pages,
&selected,
|_page, _image| {
attempts += 1;
Err(FocrError::ModelNotFound("no model".into()))
},
&mut |_| {},
)
.expect_err("a missing model ends the run");
assert!(matches!(err, FocrError::ModelNotFound(_)), "got {err:?}");
assert_eq!(
attempts, 1,
"aborted on the first page, not after all of them"
);
let _ = std::fs::remove_file(&path);
}
#[test]
fn walk_with_no_readable_page_errors_with_the_first_reason() {
let path = build_multi_page_pdf(&[None, None]);
let pages = PdfPages::open(&path).expect("open");
let selected = select_pages(None, pages.len()).expect("all pages");
let err = walk_document(
&pages,
&selected,
|_page, _image| Ok((String::new(), Vec::new())),
&mut |_| {},
)
.expect_err("nothing readable");
assert!(matches!(err, FocrError::InputDecode(_)), "got {err:?}");
assert!(!err.to_string().is_empty());
let _ = std::fs::remove_file(&path);
}
fn build_single_page_pdf(image_xobject: Option<lopdf::Stream>) -> std::path::PathBuf {
use lopdf::{Object, dictionary};
let mut doc = lopdf::Document::with_version("1.5");
let pages_id = doc.new_object_id();
let resources = match image_xobject {
Some(stream) => {
let image_id = doc.add_object(stream);
dictionary! { "XObject" => dictionary! { "Im0" => image_id } }
}
None => dictionary! {},
};
let resources_id = doc.add_object(resources);
let page_id = doc.add_object(dictionary! {
"Type" => "Page",
"Parent" => pages_id,
"Resources" => resources_id,
"MediaBox" => vec![0_i64.into(), 0_i64.into(), 100_i64.into(), 100_i64.into()],
});
doc.objects.insert(
pages_id,
Object::Dictionary(dictionary! {
"Type" => "Pages",
"Kids" => vec![page_id.into()],
"Count" => 1,
}),
);
let catalog_id = doc.add_object(dictionary! {
"Type" => "Catalog",
"Pages" => pages_id,
});
doc.trailer.set("Root", catalog_id);
use std::sync::atomic::{AtomicU32, Ordering};
static SEQ: AtomicU32 = AtomicU32::new(0);
let path = std::env::temp_dir().join(format!(
"focr_pdf_test_{}_{}.pdf",
std::process::id(),
SEQ.fetch_add(1, Ordering::Relaxed)
));
doc.save(&path).expect("save synthesized pdf");
path
}
#[test]
fn render_dctdecode_pdf_page_decodes_jpeg_xobject() {
use image::{ImageBuffer, Rgb};
use lopdf::{Stream, dictionary};
use std::io::Cursor;
let (w, h) = (16u32, 12u32);
let src = DynamicImage::ImageRgb8(ImageBuffer::from_fn(w, h, |x, _| {
Rgb([(x * 16) as u8, 64, 128])
}));
let mut jpeg = Vec::new();
src.write_to(&mut Cursor::new(&mut jpeg), image::ImageFormat::Jpeg)
.expect("encode jpeg");
let image = Stream::new(
dictionary! {
"Type" => "XObject",
"Subtype" => "Image",
"Width" => i64::from(w),
"Height" => i64::from(h),
"ColorSpace" => "DeviceRGB",
"BitsPerComponent" => 8,
"Filter" => "DCTDecode",
},
jpeg,
)
.with_compression(false);
let path = build_single_page_pdf(Some(image));
let pages = PdfPages::open(&path).expect("open synthesized pdf");
assert_eq!(pages.len(), 1);
let page = pages.render(0).expect("render dct page");
assert_eq!((page.width(), page.height()), (w, h));
let _ = std::fs::remove_file(&path);
}
#[test]
fn render_image_free_page_errors_clearly() {
let path = build_single_page_pdf(None);
let pages = PdfPages::open(&path).expect("open synthesized pdf");
assert_eq!(pages.len(), 1);
let err = pages.render(0).expect_err("vector page must error");
let msg = err.to_string();
assert!(
msg.contains("no image XObject"),
"expected an actionable no-image message, got: {msg}"
);
let _ = std::fs::remove_file(&path);
}
#[test]
fn oversized_pdf_image_is_rejected_before_allocation() {
use lopdf::{Stream, dictionary};
let image = Stream::new(
dictionary! {
"Type" => "XObject",
"Subtype" => "Image",
"Width" => 100_000_i64,
"Height" => 100_000_i64, "ColorSpace" => "DeviceRGB",
"BitsPerComponent" => 8,
"Filter" => "DCTDecode",
},
vec![0u8; 16], )
.with_compression(false);
let path = build_single_page_pdf(Some(image));
let err = PdfPages::open(&path)
.expect("open")
.render(0)
.expect_err("oversized image must error");
assert!(err.to_string().contains("exceed"), "got: {err}");
let _ = std::fs::remove_file(&path);
}
#[test]
fn chained_filter_image_is_rejected() {
use lopdf::{Object, Stream, dictionary};
let image = Stream::new(
dictionary! {
"Type" => "XObject",
"Subtype" => "Image",
"Width" => 4_i64,
"Height" => 4_i64,
"ColorSpace" => "DeviceRGB",
"BitsPerComponent" => 8,
"Filter" => Object::Array(vec![
Object::Name(b"ASCII85Decode".to_vec()),
Object::Name(b"DCTDecode".to_vec()),
]),
},
vec![0u8; 16],
)
.with_compression(false);
let path = build_single_page_pdf(Some(image));
let err = PdfPages::open(&path)
.expect("open")
.render(0)
.expect_err("chained filter must error");
assert!(err.to_string().contains("chain"), "got: {err}");
let _ = std::fs::remove_file(&path);
}
#[test]
fn render_indexed_1bpc_rgb_pdf_page_expands_palette() {
use lopdf::{Object, Stream, StringFormat, dictionary};
let image = Stream::new(
dictionary! {
"Type" => "XObject",
"Subtype" => "Image",
"Width" => 8_i64,
"Height" => 2_i64,
"ColorSpace" => Object::Array(vec![
Object::Name(b"Indexed".to_vec()),
Object::Name(b"DeviceRGB".to_vec()),
Object::Integer(1),
Object::String(vec![0, 0, 0, 255, 255, 255], StringFormat::Hexadecimal),
]),
"BitsPerComponent" => 1,
},
vec![0b1010_0000, 0b0101_0000],
)
.with_compression(false);
let path = build_single_page_pdf(Some(image));
let page = PdfPages::open(&path)
.expect("open")
.render(0)
.expect("indexed 1-bpc page renders");
assert_eq!((page.width(), page.height()), (8, 2));
let rgb = page.to_rgb8();
assert_eq!(rgb.get_pixel(0, 0).0, [255, 255, 255]);
assert_eq!(rgb.get_pixel(1, 0).0, [0, 0, 0]);
assert_eq!(rgb.get_pixel(2, 0).0, [255, 255, 255]);
assert_eq!(rgb.get_pixel(0, 1).0, [0, 0, 0]);
assert_eq!(rgb.get_pixel(1, 1).0, [255, 255, 255]);
let _ = std::fs::remove_file(&path);
}
#[test]
fn render_indexed_8bpc_gray_palette_stream_clamps_out_of_range() {
use lopdf::{Object, Stream, dictionary};
let mut doc = lopdf::Document::with_version("1.5");
let palette_id = doc
.add_object(Stream::new(dictionary! {}, vec![10u8, 128, 250]).with_compression(false));
let image = Stream::new(
dictionary! {
"Type" => "XObject",
"Subtype" => "Image",
"Width" => 4_i64,
"Height" => 1_i64,
"ColorSpace" => Object::Array(vec![
Object::Name(b"Indexed".to_vec()),
Object::Name(b"DeviceGray".to_vec()),
Object::Integer(2),
Object::Reference(palette_id),
]),
"BitsPerComponent" => 8,
},
vec![0u8, 1, 2, 9],
)
.with_compression(false);
let image_id = doc.add_object(image);
let pages_id = doc.new_object_id();
let resources_id = doc.add_object(dictionary! {
"XObject" => dictionary! { "Im0" => image_id },
});
let page_id = doc.add_object(dictionary! {
"Type" => "Page",
"Parent" => pages_id,
"Resources" => resources_id,
"MediaBox" => vec![0_i64.into(), 0_i64.into(), 100_i64.into(), 100_i64.into()],
});
doc.objects.insert(
pages_id,
Object::Dictionary(dictionary! {
"Type" => "Pages",
"Kids" => vec![page_id.into()],
"Count" => 1,
}),
);
let catalog_id = doc.add_object(dictionary! { "Type" => "Catalog", "Pages" => pages_id });
doc.trailer.set("Root", catalog_id);
let path =
std::env::temp_dir().join(format!("focr_pdf_indexed_gray_{}.pdf", std::process::id()));
doc.save(&path).expect("save synthesized pdf");
let page = PdfPages::open(&path)
.expect("open")
.render(0)
.expect("indexed gray page renders");
let gray = page.to_luma8();
assert_eq!(
[
gray.get_pixel(0, 0).0[0],
gray.get_pixel(1, 0).0[0],
gray.get_pixel(2, 0).0[0],
gray.get_pixel(3, 0).0[0],
],
[10, 128, 250, 250], );
let _ = std::fs::remove_file(&path);
}
#[test]
fn unpack_indices_handles_all_legal_depths() {
assert_eq!(
unpack_indices(&[0xAB, 0xC0], 3, 1, 4).expect("4 bpc"),
vec![0xA, 0xB, 0xC]
);
assert_eq!(
unpack_indices(&[0b1110_0100, 0b0100_0000], 5, 1, 2).expect("2 bpc"),
vec![3, 2, 1, 0, 1]
);
assert_eq!(
unpack_indices(&[7, 0, 255], 3, 1, 8).expect("8 bpc"),
vec![7, 0, 255]
);
assert!(unpack_indices(&[0, 0], 1, 1, 16).is_err());
assert!(unpack_indices(&[0xFF], 8, 2, 1).is_err());
}
#[test]
fn expected_sample_cap_clamps_a_hostile_bit_depth() {
assert_eq!(
expected_sample_cap(1024, 1024, 16, "DeviceRGB"),
4 * 1024 * 1024 * 3 * 2
);
assert_eq!(
expected_sample_cap(1024, 1024, i64::MAX, "DeviceRGB"),
expected_sample_cap(1024, 1024, 16, "DeviceRGB")
);
assert_eq!(expected_sample_cap(8, 8, -5, "DeviceGray"), 4 * 8 * 8);
assert_eq!(expected_sample_cap(2, 2, 8, "Indexed"), 4 * 2 * 2 * 4);
}
#[test]
fn bounded_inflate_passes_small_streams_and_rejects_a_bomb() {
use std::io::Write;
let zlib_of = |n: usize| -> Vec<u8> {
let mut enc = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::best());
enc.write_all(&vec![0u8; n]).expect("encode");
enc.finish().expect("finish")
};
let small = zlib_of(1000);
let out = bounded_inflate(&small, 4096)
.expect("no error")
.expect("inflated");
assert_eq!(out.len(), 1000);
let bomb = zlib_of(1_000_000);
let err = bounded_inflate(&bomb, 4096).expect_err("bomb must be rejected");
assert!(err.contains("cap"), "got: {err}");
assert!(
bounded_inflate(b"not a zlib stream", 4096)
.expect("no error")
.is_none()
);
}
}