use pdf_content::{ContentOp, ContentProgram};
use spdfdiff_types::{Diagnostic, LineSegment, ObjectId, Point, Provenance, Rect};
use std::collections::{BTreeMap, BTreeSet};
#[derive(Debug, Clone, PartialEq)]
pub struct GlyphToken {
pub id: String,
pub unicode: Option<String>,
pub raw_bytes: Vec<u8>,
pub page_index: usize,
pub bbox: Rect,
pub baseline: LineSegment,
pub source: Provenance,
}
#[derive(Debug, Clone, PartialEq)]
pub struct TextRun {
pub id: String,
pub text: String,
pub normalized_text: String,
pub glyphs: Vec<GlyphToken>,
pub bbox: Rect,
pub source: Provenance,
pub marked_content: Option<MarkedContentRef>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct MarkedContentRef {
pub tag: String,
pub mcid: Option<usize>,
}
#[derive(Debug, Clone, PartialEq)]
pub struct TextExtraction {
pub runs: Vec<TextRun>,
pub diagnostics: Vec<Diagnostic>,
}
#[derive(Debug, Clone, PartialEq)]
pub struct FontResourceSet {
pub fonts: BTreeMap<String, FontResource>,
pub diagnostics: Vec<Diagnostic>,
}
#[derive(Debug, Clone, PartialEq)]
pub struct FontResource {
pub resource_name: String,
pub object_id: ObjectId,
pub subtype: Option<String>,
pub base_font: Option<String>,
pub encoding: Option<String>,
pub to_unicode: Option<ObjectId>,
pub first_char: Option<u16>,
pub widths: Vec<f32>,
pub descendant_font_object_ids: Vec<ObjectId>,
pub descendant_subtypes: Vec<String>,
}
impl FontResource {
#[must_use]
pub fn is_cid_or_type0(&self) -> bool {
self.subtype.as_deref() == Some("Type0")
|| self
.descendant_subtypes
.iter()
.any(|subtype| subtype.starts_with("CIDFont"))
}
#[must_use]
pub fn glyph_width_em_for_code(&self, code: u8) -> Option<f32> {
if self.is_cid_or_type0() {
return None;
}
let first_char = usize::from(self.first_char?);
let index = usize::from(code).checked_sub(first_char)?;
self.widths.get(index).map(|width| width / 1000.0)
}
fn supports_base14_latin_fallback(&self) -> bool {
if self.to_unicode.is_some()
|| self.is_cid_or_type0()
|| self.subtype.as_deref() != Some("Type1")
{
return false;
}
if !matches!(
self.encoding.as_deref(),
None | Some("/StandardEncoding" | "/WinAnsiEncoding")
) {
return false;
}
self.base_font
.as_deref()
.is_some_and(is_base14_latin_font_name)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ToUnicodeApplyResult {
pub applied: bool,
pub diagnostics: Vec<Diagnostic>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ToUnicodeMaps {
pub maps: BTreeMap<String, ToUnicodeCMap>,
pub diagnostics: Vec<Diagnostic>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ToUnicodeCMap {
pub map: BTreeMap<Vec<u8>, String>,
pub diagnostics: Vec<Diagnostic>,
}
#[must_use]
pub fn font_resources_from_document(document: &pdf_core::PdfDocument) -> FontResourceSet {
let objects_by_id = document
.objects
.iter()
.map(|object| (object.id, object))
.collect::<BTreeMap<_, _>>();
let mut fonts = BTreeMap::new();
let mut diagnostics = Vec::new();
for object in &document.objects {
if is_font_object_body(&object.body) {
continue;
}
let scoped_font_references = font_resource_references(&object.body);
for (resource_name, object_id) in named_references(&object.body) {
let Some(font_object) = objects_by_id.get(&object_id) else {
continue;
};
if !is_font_object_body(&font_object.body) {
continue;
}
fonts.insert(
resource_name.clone(),
font_resource_from_object(
resource_name,
object_id,
font_object.body.as_str(),
&objects_by_id,
),
);
}
let mut diagnosed_scoped_references = BTreeSet::new();
for (resource_name, object_id) in scoped_font_references {
if !diagnosed_scoped_references.insert((resource_name.clone(), object_id)) {
continue;
}
match objects_by_id.get(&object_id) {
Some(font_object) if is_font_object_body(&font_object.body) => {}
Some(_) => diagnostics.push(Diagnostic::warning(
"FONT_RESOURCE_NOT_FONT",
format!(
"font resource /{resource_name} points to non-font object {}",
object_id.number
),
)),
None => diagnostics.push(Diagnostic::warning(
"MISSING_FONT_RESOURCE",
format!(
"font resource /{resource_name} points to missing object {}",
object_id.number
),
)),
}
}
}
FontResourceSet { fonts, diagnostics }
}
#[must_use]
pub fn tounicode_maps_from_document(
document: &pdf_core::PdfDocument,
font_resources: &FontResourceSet,
) -> ToUnicodeMaps {
let objects_by_id = document
.objects
.iter()
.map(|object| (object.id, object))
.collect::<BTreeMap<_, _>>();
let mut maps = BTreeMap::new();
let mut diagnostics = Vec::new();
for (font_name, font_resource) in &font_resources.fonts {
if let Some(cmap_object_id) = font_resource.to_unicode {
let Some(cmap_stream) = objects_by_id
.get(&cmap_object_id)
.and_then(|object| object.stream.as_ref())
else {
continue;
};
let cmap = parse_tounicode_cmap(&cmap_stream.bytes);
diagnostics.extend(cmap.diagnostics.clone());
if !cmap.map.is_empty() {
maps.insert(font_name.clone(), cmap);
}
}
}
ToUnicodeMaps { maps, diagnostics }
}
pub fn apply_document_tounicode_maps(
program: &mut ContentProgram,
document: &pdf_core::PdfDocument,
font_resources: &FontResourceSet,
) -> ToUnicodeApplyResult {
let maps = tounicode_maps_from_document(document, font_resources);
apply_tounicode_maps(program, &maps)
}
pub fn apply_tounicode_maps(
program: &mut ContentProgram,
maps: &ToUnicodeMaps,
) -> ToUnicodeApplyResult {
if maps.maps.is_empty() {
return ToUnicodeApplyResult {
applied: false,
diagnostics: maps.diagnostics.clone(),
};
}
let mut current_font: Option<String> = None;
let mut applied = false;
for operation in &mut program.operations {
match operation {
ContentOp::SetFont { name, .. } => {
current_font = Some(name.clone());
}
ContentOp::ShowText {
text, raw_bytes, ..
}
| ContentOp::ShowAdjustedText {
text, raw_bytes, ..
} => {
let Some(font_name) = current_font.as_deref() else {
continue;
};
let Some(map) = maps.maps.get(font_name) else {
continue;
};
if let Some(decoded) = decode_with_tounicode(raw_bytes, &map.map) {
*text = decoded;
applied = true;
}
}
ContentOp::BeginText { .. }
| ContentOp::EndText { .. }
| ContentOp::MoveTextPosition { .. }
| ContentOp::MoveToNextLine { .. }
| ContentOp::SetTextLeading { .. }
| ContentOp::SetCharacterSpacing { .. }
| ContentOp::SetWordSpacing { .. }
| ContentOp::SetHorizontalScaling { .. }
| ContentOp::SetTextMatrix { .. }
| ContentOp::SaveGraphicsState { .. }
| ContentOp::RestoreGraphicsState { .. }
| ContentOp::ConcatMatrix { .. }
| ContentOp::AppendRectangle { .. }
| ContentOp::BeginMarkedContent { .. }
| ContentOp::EndMarkedContent { .. }
| ContentOp::RecognizedNonText { .. }
| ContentOp::Unknown { .. } => {}
}
}
ToUnicodeApplyResult {
applied,
diagnostics: maps.diagnostics.clone(),
}
}
#[must_use]
pub fn parse_tounicode_cmap(bytes: &[u8]) -> ToUnicodeCMap {
let text = String::from_utf8_lossy(bytes);
let mut map = BTreeMap::new();
let mut diagnostics = Vec::new();
let mut in_bfrange = false;
for line in text.lines() {
let trimmed = line.trim();
if trimmed.ends_with("beginbfrange") {
in_bfrange = true;
continue;
}
if trimmed == "endbfrange" {
in_bfrange = false;
continue;
}
if trimmed.ends_with("beginbfchar") || trimmed == "endbfchar" {
continue;
}
let hex_tokens = hex_tokens_in_line(line);
if in_bfrange && hex_tokens.len() >= 3 {
if let Err(message) = insert_bfrange(&mut map, &hex_tokens, trimmed.contains('[')) {
diagnostics.push(Diagnostic::warning("CMAP_UNSUPPORTED_RANGE", message));
}
} else if hex_tokens.len() == 2 {
if let Some(decoded) = unicode_hex_to_string(&hex_tokens[1]) {
map.insert(hex_tokens[0].clone(), decoded);
}
} else if trimmed.contains("begin")
|| trimmed.contains("end")
|| trimmed.is_empty()
|| trimmed.starts_with('%')
{
continue;
}
}
ToUnicodeCMap { map, diagnostics }
}
#[must_use]
pub fn decode_with_tounicode(raw_bytes: &[u8], map: &BTreeMap<Vec<u8>, String>) -> Option<String> {
let mut decoded = String::new();
let mut index = 0;
while index < raw_bytes.len() {
let mut matched = None;
for width in (1..=4).rev() {
let end = index + width;
if end <= raw_bytes.len() {
if let Some(value) = map.get(&raw_bytes[index..end]) {
matched = Some((width, value));
break;
}
}
}
let (width, value) = matched?;
decoded.push_str(value);
index += width;
}
Some(decoded)
}
#[derive(Debug, Clone)]
struct TextState {
x: f32,
y: f32,
font_name: Option<String>,
font_size: f32,
leading: f32,
character_spacing: f32,
word_spacing: f32,
horizontal_scale: f32,
}
impl Default for TextState {
fn default() -> Self {
Self {
x: 0.0,
y: 0.0,
font_name: None,
font_size: 12.0,
leading: 0.0,
character_spacing: 0.0,
word_spacing: 0.0,
horizontal_scale: 100.0,
}
}
}
#[must_use]
pub fn extract_text_runs(program: &ContentProgram, page_index: usize) -> TextExtraction {
extract_text_runs_with_optional_fonts(program, page_index, None)
}
#[must_use]
pub fn extract_text_runs_with_fonts(
program: &ContentProgram,
page_index: usize,
font_resources: &FontResourceSet,
) -> TextExtraction {
extract_text_runs_with_optional_fonts(program, page_index, Some(font_resources))
}
fn extract_text_runs_with_optional_fonts(
program: &ContentProgram,
page_index: usize,
font_resources: Option<&FontResourceSet>,
) -> TextExtraction {
let mut state = TextState::default();
let mut graphics_stack = Vec::new();
let mut runs = Vec::new();
let mut diagnostics = program.diagnostics.clone();
let mut emitted_missing_tounicode = false;
let mut marked_content_stack: Vec<MarkedContentRef> = Vec::new();
for operation in &program.operations {
match operation {
ContentOp::SetFont { name, size, .. } => {
state.font_name = Some(name.clone());
state.font_size = *size;
}
ContentOp::MoveTextPosition {
tx,
ty,
set_leading,
..
} => {
state.x += tx;
state.y += ty;
if let Some(leading) = set_leading {
state.leading = *leading;
}
}
ContentOp::MoveToNextLine { .. } => {
state.y -= state.leading;
}
ContentOp::SetTextLeading { leading, .. } => {
state.leading = *leading;
}
ContentOp::SetCharacterSpacing { spacing, .. } => {
state.character_spacing = *spacing;
}
ContentOp::SetWordSpacing { spacing, .. } => {
state.word_spacing = *spacing;
}
ContentOp::SetHorizontalScaling { scale, .. } => {
state.horizontal_scale = *scale;
}
ContentOp::SetTextMatrix { e, f, .. } => {
state.x = *e;
state.y = *f;
}
ContentOp::ShowText {
text,
raw_bytes,
source,
}
| ContentOp::ShowAdjustedText {
text,
raw_bytes,
source,
..
} => {
let run_page_index = source.page_index.unwrap_or(page_index);
let fallback_text = base14_latin_fallback_text(raw_bytes, &state, font_resources);
if fallback_text.is_none() && !emitted_missing_tounicode {
diagnostics.push(
Diagnostic::warning(
"MISSING_TOUNICODE",
"using literal-string fallback text because no ToUnicode map is available",
)
.with_page(run_page_index),
);
emitted_missing_tounicode = true;
}
emit_run(
&mut runs,
RunEmission {
page_index: run_page_index,
text: fallback_text.as_deref().unwrap_or(text),
raw_bytes,
source: source.clone(),
marked_content: marked_content_stack.last().cloned(),
font_resources,
},
&mut state,
);
}
ContentOp::SaveGraphicsState { .. } => {
graphics_stack.push(state.clone());
}
ContentOp::RestoreGraphicsState { .. } => {
if let Some(saved) = graphics_stack.pop() {
state = saved;
}
}
ContentOp::BeginMarkedContent { tag, mcid, .. } => {
marked_content_stack.push(MarkedContentRef {
tag: tag.clone(),
mcid: *mcid,
});
}
ContentOp::EndMarkedContent { .. } => {
marked_content_stack.pop();
}
ContentOp::BeginText { .. }
| ContentOp::EndText { .. }
| ContentOp::ConcatMatrix { .. }
| ContentOp::AppendRectangle { .. }
| ContentOp::RecognizedNonText { .. }
| ContentOp::Unknown { .. } => {}
}
}
TextExtraction { runs, diagnostics }
}
struct RunEmission<'a> {
page_index: usize,
text: &'a str,
raw_bytes: &'a [u8],
source: Provenance,
marked_content: Option<MarkedContentRef>,
font_resources: Option<&'a FontResourceSet>,
}
fn emit_run(runs: &mut Vec<TextRun>, emission: RunEmission<'_>, state: &mut TextState) {
let width = estimate_text_width(
emission.text,
emission.raw_bytes,
state,
emission.font_resources,
);
let bbox = Rect {
x0: state.x,
y0: state.y,
x1: state.x + width,
y1: state.y + state.font_size,
};
let glyph = GlyphToken {
id: format!("p{}.g{:04}", emission.page_index, runs.len()),
unicode: Some(emission.text.to_owned()),
raw_bytes: emission.raw_bytes.to_vec(),
page_index: emission.page_index,
bbox,
baseline: LineSegment {
start: Point {
x: state.x,
y: state.y,
},
end: Point {
x: state.x + width,
y: state.y,
},
},
source: emission.source.clone(),
};
runs.push(TextRun {
id: format!("p{}.r{:04}", emission.page_index, runs.len()),
text: emission.text.to_owned(),
normalized_text: normalize_text(emission.text),
glyphs: vec![glyph],
bbox,
source: emission.source,
marked_content: emission.marked_content,
});
state.x += width;
}
#[must_use]
pub fn normalize_text(text: &str) -> String {
text.split_whitespace().collect::<Vec<_>>().join(" ")
}
fn estimate_text_width(
text: &str,
raw_bytes: &[u8],
state: &TextState,
font_resources: Option<&FontResourceSet>,
) -> f32 {
let base_width = font_width_sum(raw_bytes, state, font_resources).unwrap_or_else(|| {
text.chars()
.map(|character| approximate_glyph_width(character) * state.font_size)
.sum::<f32>()
});
let spacing = text.chars().fold(0.0, |total, character| {
let word_spacing = if character == ' ' {
state.word_spacing
} else {
0.0
};
total + state.character_spacing + word_spacing
});
(base_width + spacing) * (state.horizontal_scale / 100.0)
}
fn font_width_sum(
raw_bytes: &[u8],
state: &TextState,
font_resources: Option<&FontResourceSet>,
) -> Option<f32> {
let font_name = state.font_name.as_deref()?;
let font = font_resources?.fonts.get(font_name)?;
let mut width = 0.0;
for byte in raw_bytes {
width += font.glyph_width_em_for_code(*byte)? * state.font_size;
}
Some(width)
}
fn base14_latin_fallback_text(
raw_bytes: &[u8],
state: &TextState,
font_resources: Option<&FontResourceSet>,
) -> Option<String> {
let font_name = state.font_name.as_deref()?;
let font = font_resources?.fonts.get(font_name)?;
if !font.supports_base14_latin_fallback() {
return None;
}
decode_base14_latin_bytes(raw_bytes)
}
fn decode_base14_latin_bytes(raw_bytes: &[u8]) -> Option<String> {
let mut text = String::with_capacity(raw_bytes.len());
for byte in raw_bytes {
match *byte {
b'\t' | b'\n' | b'\r' => text.push(char::from(*byte)),
0x20..=0x7e => text.push(char::from(*byte)),
_ => return None,
}
}
Some(text)
}
fn is_base14_latin_font_name(name: &str) -> bool {
let base_name = name.rsplit_once('+').map_or(name, |(_, suffix)| suffix);
matches!(
base_name,
"Helvetica"
| "Helvetica-Bold"
| "Helvetica-Oblique"
| "Helvetica-BoldOblique"
| "Times-Roman"
| "Times-Bold"
| "Times-Italic"
| "Times-BoldItalic"
| "Courier"
| "Courier-Bold"
| "Courier-Oblique"
| "Courier-BoldOblique"
)
}
fn approximate_glyph_width(character: char) -> f32 {
match character {
' ' => 0.25,
'i' | 'l' | 'I' | '!' | '.' | ',' | ':' | ';' | '|' => 0.28,
'm' | 'w' | 'M' | 'W' => 0.78,
character if character.is_ascii_digit() => 0.5,
character if character.is_ascii_punctuation() => 0.35,
character if character.len_utf8() > 1 => 0.6,
_ => 0.5,
}
}
fn font_resource_from_object(
resource_name: String,
object_id: ObjectId,
body: &str,
objects_by_id: &BTreeMap<ObjectId, &pdf_core::PdfObject>,
) -> FontResource {
let descendant_font_object_ids = named_references(body)
.into_iter()
.filter_map(|(name, object_id)| (name == "DescendantFonts").then_some(object_id))
.collect::<Vec<_>>();
let descendant_subtypes = descendant_font_object_ids
.iter()
.filter_map(|object_id| objects_by_id.get(object_id))
.filter_map(|object| name_after_key(&object.body, "Subtype"))
.collect::<Vec<_>>();
FontResource {
resource_name,
object_id,
subtype: name_after_key(body, "Subtype"),
base_font: name_after_key(body, "BaseFont"),
encoding: value_after_pdf_name(body, "Encoding"),
to_unicode: reference_after_key(body, "ToUnicode"),
first_char: number_after_key(body, "FirstChar"),
widths: number_array_after_key(body, "Widths"),
descendant_font_object_ids,
descendant_subtypes,
}
}
fn font_resource_references(body: &str) -> Vec<(String, ObjectId)> {
body.match_indices("/Font")
.flat_map(|(font_index, _)| {
let after_font = &body[font_index + "/Font".len()..];
let Some(dictionary_start) = after_font.find("<<") else {
return Vec::new();
};
let after_dictionary_start = &after_font[dictionary_start + "<<".len()..];
let Some(dictionary_end) = after_dictionary_start.find(">>") else {
return Vec::new();
};
named_references(&after_dictionary_start[..dictionary_end])
})
.collect()
}
fn is_font_object_body(body: &str) -> bool {
body.contains("/Type /Font")
|| body.contains("/Subtype /Type0")
|| body.contains("/Subtype /Type1")
|| body.contains("/Subtype /TrueType")
|| body.contains("/Subtype /CIDFontType")
}
fn named_references(body: &str) -> Vec<(String, ObjectId)> {
let tokens = body_tokens(body);
let mut references = Vec::new();
for index in 0..tokens.len().saturating_sub(3) {
let Some(name) = tokens[index].strip_prefix('/') else {
continue;
};
let Ok(number) = tokens[index + 1].parse::<u32>() else {
continue;
};
let Ok(generation) = tokens[index + 2].parse::<u16>() else {
continue;
};
if tokens[index + 3] == "R" {
references.push((name.to_owned(), ObjectId { number, generation }));
}
}
references
}
fn reference_after_key(body: &str, key: &str) -> Option<ObjectId> {
let start = body.find(&format!("/{key}"))? + key.len() + 1;
parse_reference_at(&body[start..])
}
fn parse_reference_at(body: &str) -> Option<ObjectId> {
let tokens = body_tokens(body);
let number = tokens.first()?.parse().ok()?;
let generation = tokens.get(1)?.parse().ok()?;
if tokens.get(2)? != "R" {
return None;
}
Some(ObjectId { number, generation })
}
fn name_after_key(body: &str, key: &str) -> Option<String> {
value_after_pdf_name(body, key).and_then(|value| value.strip_prefix('/').map(ToOwned::to_owned))
}
fn value_after_pdf_name(body: &str, key: &str) -> Option<String> {
let start = body.find(&format!("/{key}"))? + key.len() + 1;
let remaining = body[start..].trim_start();
if let Some(value) = remaining.strip_prefix('(') {
return value
.split_once(')')
.map(|(value, _)| value.chars().take(120).collect());
}
if let Some(value) = remaining.strip_prefix('/') {
return value
.split_whitespace()
.next()
.map(|value| format!("/{value}"));
}
Some(
remaining
.split_whitespace()
.take(3)
.collect::<Vec<_>>()
.join(" ")
.chars()
.take(120)
.collect::<String>(),
)
}
fn number_after_key<T>(body: &str, key: &str) -> Option<T>
where
T: std::str::FromStr,
{
let start = body.find(&format!("/{key}"))? + key.len() + 1;
body[start..].split_whitespace().next()?.parse().ok()
}
fn number_array_after_key(body: &str, key: &str) -> Vec<f32> {
let Some(start) = body.find(&format!("/{key}")) else {
return Vec::new();
};
let after_key = &body[start + key.len() + 1..];
let Some(array_start) = after_key.find('[') else {
return Vec::new();
};
let after_array_start = &after_key[array_start + 1..];
let Some(array_end) = after_array_start.find(']') else {
return Vec::new();
};
after_array_start[..array_end]
.split_whitespace()
.filter_map(|token| token.parse::<f32>().ok())
.collect()
}
fn body_tokens(body: &str) -> Vec<String> {
body.replace("<<", " ")
.replace(">>", " ")
.replace('/', " /")
.replace(['[', ']'], " ")
.split_whitespace()
.map(ToOwned::to_owned)
.collect()
}
trait DiagnosticExt {
fn with_page(self, page_index: usize) -> Self;
}
impl DiagnosticExt for Diagnostic {
fn with_page(mut self, page_index: usize) -> Self {
self.page_index = Some(page_index);
self
}
}
fn insert_bfrange(
map: &mut BTreeMap<Vec<u8>, String>,
hex_tokens: &[Vec<u8>],
array_mode: bool,
) -> Result<(), String> {
if hex_tokens.len() < 3 {
return Err("bfrange entry has fewer than three hex operands".into());
}
let start = bytes_to_u32(&hex_tokens[0]).ok_or("bfrange start is too wide")?;
let end = bytes_to_u32(&hex_tokens[1]).ok_or("bfrange end is too wide")?;
if end < start {
return Err("bfrange end is before start".into());
}
let count = usize::try_from(end - start + 1).map_err(|_| "bfrange is too large")?;
if !array_mode {
let destination_start =
bytes_to_u32(&hex_tokens[2]).ok_or("bfrange destination is too wide")?;
for offset in 0..count {
let source = int_to_be_bytes(start + offset as u32, hex_tokens[0].len());
let destination =
int_to_be_bytes(destination_start + offset as u32, hex_tokens[2].len());
let Some(decoded) = unicode_hex_to_string(&destination) else {
return Err("bfrange destination is not valid UTF-16BE".into());
};
map.insert(source, decoded);
}
} else {
if hex_tokens.len() - 2 < count {
return Err("bfrange destination array is shorter than source range".into());
}
for (offset, destination) in hex_tokens[2..2 + count].iter().enumerate() {
let source = int_to_be_bytes(start + offset as u32, hex_tokens[0].len());
let Some(decoded) = unicode_hex_to_string(destination) else {
return Err("bfrange array destination is not valid UTF-16BE".into());
};
map.insert(source, decoded);
}
}
Ok(())
}
fn bytes_to_u32(bytes: &[u8]) -> Option<u32> {
if bytes.len() > 4 {
return None;
}
let mut value = 0u32;
for byte in bytes {
value = (value << 8) | u32::from(*byte);
}
Some(value)
}
fn int_to_be_bytes(value: u32, width: usize) -> Vec<u8> {
let bytes = value.to_be_bytes();
bytes[bytes.len().saturating_sub(width)..].to_vec()
}
fn hex_token_bytes(token: &str) -> Option<Vec<u8>> {
let token = token.strip_prefix('<')?.strip_suffix('>')?;
if token.is_empty() || token.len() % 2 != 0 {
return None;
}
(0..token.len())
.step_by(2)
.map(|index| u8::from_str_radix(&token[index..index + 2], 16).ok())
.collect()
}
fn hex_tokens_in_line(line: &str) -> Vec<Vec<u8>> {
let bytes = line.as_bytes();
let mut tokens = Vec::new();
let mut index = 0;
while index < bytes.len() {
if bytes[index] != b'<' || bytes.get(index + 1) == Some(&b'<') {
index += 1;
continue;
}
let Some(relative_end) = bytes[index + 1..].iter().position(|byte| *byte == b'>') else {
break;
};
let end = index + 1 + relative_end;
if let Some(token) = line.get(index..=end).and_then(hex_token_bytes) {
tokens.push(token);
}
index = end + 1;
}
tokens
}
fn unicode_hex_to_string(bytes: &[u8]) -> Option<String> {
if bytes.len() % 2 != 0 {
return None;
}
let mut out = String::new();
for chunk in bytes.chunks_exact(2) {
let code_unit = u16::from_be_bytes([chunk[0], chunk[1]]);
let character = char::from_u32(u32::from(code_unit))?;
out.push(character);
}
Some(out)
}
#[cfg(test)]
mod tests {
use super::*;
use pdf_content::parse_content_stream;
use spdfdiff_types::DiagnosticSeverity;
#[test]
fn extracts_positioned_text_run() {
let program = parse_content_stream(b"BT /F1 12 Tf 72 720 Td (Hello) Tj ET");
let extraction = extract_text_runs(&program, 0);
assert_eq!(extraction.runs.len(), 1);
assert_eq!(extraction.runs[0].text, "Hello");
assert_eq!(extraction.runs[0].normalized_text, "Hello");
assert_eq!(extraction.runs[0].bbox.x0, 72.0);
assert_eq!(extraction.runs[0].bbox.y0, 720.0);
assert!(
extraction
.diagnostics
.iter()
.any(|diagnostic| diagnostic.code == "MISSING_TOUNICODE"
&& diagnostic.severity == DiagnosticSeverity::Warning)
);
}
#[test]
fn extracts_adjusted_text_array_as_one_run() {
let program = parse_content_stream(b"BT /F1 10 Tf 10 20 Td [(Hel) -120 (lo)] TJ ET");
let extraction = extract_text_runs(&program, 0);
assert_eq!(extraction.runs.len(), 1);
assert_eq!(extraction.runs[0].text, "Hello");
assert_eq!(extraction.runs[0].glyphs[0].raw_bytes, b"Hello");
assert!(extraction.runs[0].bbox.width() > 0.0);
}
#[test]
fn line_movement_changes_y_coordinate() {
let program =
parse_content_stream(b"BT /F1 12 Tf 72 720 Td 14 TL (First) Tj T* (Second) Tj ET");
let extraction = extract_text_runs(&program, 0);
assert_eq!(extraction.runs.len(), 2);
assert_eq!(extraction.runs[0].bbox.y0, 720.0);
assert_eq!(extraction.runs[1].bbox.y0, 706.0);
}
#[test]
fn spacing_state_affects_text_advance() {
let plain = extract_text_runs(
&parse_content_stream(b"BT /F1 10 Tf 0 0 Td (A A) Tj (B) Tj ET"),
0,
);
let spaced = extract_text_runs(
&parse_content_stream(b"BT /F1 10 Tf 0 0 Td 2 Tc 4 Tw (A A) Tj (B) Tj ET"),
0,
);
assert!(spaced.runs[1].bbox.x0 > plain.runs[1].bbox.x0);
}
#[test]
fn glyph_width_estimate_uses_character_shape_heuristics() {
let state = TextState {
font_size: 10.0,
..TextState::default()
};
assert!(
estimate_text_width("WWW", b"WWW", &state, None)
> estimate_text_width("iii", b"iii", &state, None)
);
assert!(
estimate_text_width("A A", b"A A", &state, None)
< estimate_text_width("AAA", b"AAA", &state, None)
);
}
#[test]
fn preserves_marked_content_mcid_on_text_runs() {
let program = parse_content_stream(b"/P << /MCID 3 >> BDC BT (Tagged) Tj ET EMC");
let extraction = extract_text_runs(&program, 0);
assert_eq!(extraction.runs.len(), 1);
assert_eq!(
extraction.runs[0].marked_content,
Some(MarkedContentRef {
tag: "P".to_owned(),
mcid: Some(3),
})
);
}
#[test]
fn builds_font_resource_model_from_page_resources() {
let document = pdf_core::PdfDocument::parse(font_resource_pdf().as_slice())
.expect("font resource fixture should parse");
let resources = font_resources_from_document(&document);
let font = resources.fonts.get("F1").expect("F1 should resolve");
assert!(resources.diagnostics.is_empty());
assert_eq!(font.resource_name, "F1");
assert_eq!(
font.object_id,
ObjectId {
number: 5,
generation: 0
}
);
assert_eq!(font.subtype.as_deref(), Some("Type0"));
assert_eq!(font.base_font.as_deref(), Some("CIDFont"));
assert_eq!(font.encoding.as_deref(), Some("/Identity-H"));
assert_eq!(font.first_char, None);
assert!(font.widths.is_empty());
assert_eq!(
font.to_unicode,
Some(ObjectId {
number: 7,
generation: 0
})
);
assert_eq!(
font.descendant_font_object_ids,
vec![ObjectId {
number: 6,
generation: 0
}]
);
assert_eq!(font.descendant_subtypes, vec!["CIDFontType2".to_owned()]);
assert!(font.is_cid_or_type0());
}
#[test]
fn font_resource_model_captures_simple_font_widths() {
let document = pdf_core::PdfDocument::parse(simple_font_widths_pdf().as_slice())
.expect("simple-font-width fixture should parse");
let resources = font_resources_from_document(&document);
let font = resources.fonts.get("F1").expect("F1 should resolve");
assert_eq!(font.first_char, Some(65));
assert_eq!(font.widths, vec![1000.0, 250.0]);
assert_eq!(font.glyph_width_em_for_code(b'A'), Some(1.0));
assert_eq!(font.glyph_width_em_for_code(b'B'), Some(0.25));
assert_eq!(font.glyph_width_em_for_code(b'C'), None);
}
#[test]
fn extraction_uses_simple_font_widths_when_available() {
let document = pdf_core::PdfDocument::parse(simple_font_widths_pdf().as_slice())
.expect("simple-font-width fixture should parse");
let resources = font_resources_from_document(&document);
let program = parse_content_stream(b"BT /F1 10 Tf 0 0 Td (AB) Tj ET");
let extraction = extract_text_runs_with_fonts(&program, 0, &resources);
assert_eq!(extraction.runs.len(), 1);
assert_eq!(extraction.runs[0].text, "AB");
assert_eq!(extraction.runs[0].bbox.width(), 12.5);
}
#[test]
fn base14_helvetica_ascii_extracts_without_missing_tounicode() {
let program = parse_content_stream(b"BT /F1 12 Tf 72 720 Td (Hello) Tj ET");
let extraction = extract_text_runs_with_fonts(&program, 0, &base14_font_resources(None));
assert_eq!(extraction.runs.len(), 1);
assert_eq!(extraction.runs[0].text, "Hello");
assert_no_diagnostic(&extraction.diagnostics, "MISSING_TOUNICODE");
}
#[test]
fn base14_fallback_handles_adjusted_text_arrays() {
let program = parse_content_stream(b"BT /F1 10 Tf 10 20 Td [(Hel) -120 (lo)] TJ ET");
let extraction = extract_text_runs_with_fonts(&program, 0, &base14_font_resources(None));
assert_eq!(extraction.runs.len(), 1);
assert_eq!(extraction.runs[0].text, "Hello");
assert_no_diagnostic(&extraction.diagnostics, "MISSING_TOUNICODE");
}
#[test]
fn base14_fallback_rejects_non_ascii_bytes() {
let program = parse_content_stream(b"BT /F1 12 Tf 72 720 Td <80> Tj ET");
let extraction = extract_text_runs_with_fonts(&program, 0, &base14_font_resources(None));
assert_has_diagnostic(&extraction.diagnostics, "MISSING_TOUNICODE");
}
#[test]
fn base14_fallback_rejects_custom_encoding() {
let program = parse_content_stream(b"BT /F1 12 Tf 72 720 Td (Hello) Tj ET");
let extraction = extract_text_runs_with_fonts(
&program,
0,
&base14_font_resources(Some("<< /Type /Encoding /Differences [65 /Alpha] >>")),
);
assert_has_diagnostic(&extraction.diagnostics, "MISSING_TOUNICODE");
}
#[test]
fn base14_fallback_rejects_symbol_and_cid_fonts() {
let program = parse_content_stream(b"BT /F1 12 Tf 72 720 Td (Hello) Tj ET");
let symbol = extract_text_runs_with_fonts(
&program,
0,
&font_resources(FontResource {
base_font: Some("Symbol".to_owned()),
..base14_font(None)
}),
);
let cid = extract_text_runs_with_fonts(
&program,
0,
&font_resources(FontResource {
subtype: Some("Type0".to_owned()),
base_font: Some("Helvetica".to_owned()),
..base14_font(None)
}),
);
assert_has_diagnostic(&symbol.diagnostics, "MISSING_TOUNICODE");
assert_has_diagnostic(&cid.diagnostics, "MISSING_TOUNICODE");
}
#[test]
fn tounicode_text_is_not_replaced_by_base14_fallback() {
let cmap = parse_tounicode_cmap(b"1 beginbfchar\n<26> <0043>\nendbfchar\n");
let mut maps = BTreeMap::new();
maps.insert("F1".to_owned(), cmap);
let mut program = parse_content_stream(b"BT /F1 12 Tf 72 720 Td <26> Tj ET");
let result = apply_tounicode_maps(
&mut program,
&ToUnicodeMaps {
maps,
diagnostics: Vec::new(),
},
);
let extraction = extract_text_runs_with_fonts(
&program,
0,
&font_resources(FontResource {
to_unicode: Some(ObjectId {
number: 7,
generation: 0,
}),
..base14_font(None)
}),
);
assert!(result.applied);
assert_eq!(extraction.runs[0].text, "C");
}
#[test]
fn builds_font_resource_model_from_indirect_resource_dictionary() {
let document = pdf_core::PdfDocument::parse(indirect_font_resource_pdf().as_slice())
.expect("indirect font resource fixture should parse");
let resources = font_resources_from_document(&document);
assert!(resources.diagnostics.is_empty());
assert_eq!(
resources
.fonts
.get("FIndirect")
.expect("FIndirect should resolve")
.to_unicode,
Some(ObjectId {
number: 7,
generation: 0
})
);
}
#[test]
fn parses_and_decodes_tounicode_cmap() {
let cmap =
parse_tounicode_cmap(b"2 beginbfchar\n<0026> <0043>\n<004f> <006c>\nendbfchar\n");
assert_eq!(
decode_with_tounicode(&[0x00, 0x26, 0x00, 0x4f], &cmap.map).as_deref(),
Some("Cl")
);
assert!(cmap.diagnostics.is_empty());
}
#[test]
fn parses_tounicode_bfrange_and_reports_unsupported_syntax() {
let cmap = parse_tounicode_cmap(
b"1 beginbfrange\n<0001> <0003> <0041>\n<0004> <0005> [<0058> <0059>]\n<0006> <0008> [<005a>]\nendbfrange\n",
);
assert_eq!(
decode_with_tounicode(&[0, 1, 0, 2, 0, 3], &cmap.map).as_deref(),
Some("ABC")
);
assert_eq!(
decode_with_tounicode(&[0, 4, 0, 5], &cmap.map).as_deref(),
Some("XY")
);
assert!(
cmap.diagnostics
.iter()
.any(|diagnostic| diagnostic.code == "CMAP_UNSUPPORTED_RANGE")
);
}
#[test]
fn applies_tounicode_maps_to_content_program_text() {
let cmap = parse_tounicode_cmap(b"1 beginbfchar\n<0026> <0043>\nendbfchar\n");
let mut maps = BTreeMap::new();
maps.insert("F1".to_owned(), cmap);
let mut program = parse_content_stream(b"BT /F1 12 Tf 72 720 Td <0026> Tj ET");
let result = apply_tounicode_maps(
&mut program,
&ToUnicodeMaps {
maps,
diagnostics: Vec::new(),
},
);
let extraction = extract_text_runs(&program, 0);
assert!(result.applied);
assert_eq!(extraction.runs[0].text, "C");
}
#[test]
fn font_resource_model_reports_missing_font_objects() {
let document = pdf_core::PdfDocument::parse(missing_font_resource_pdf().as_slice())
.expect("missing-font fixture should parse");
let resources = font_resources_from_document(&document);
assert!(resources.fonts.is_empty());
assert!(
resources
.diagnostics
.iter()
.any(|diagnostic| diagnostic.code == "MISSING_FONT_RESOURCE")
);
}
fn assert_no_diagnostic(diagnostics: &[Diagnostic], code: &str) {
assert!(
diagnostics.iter().all(|diagnostic| diagnostic.code != code),
"did not expect diagnostic {code} in {diagnostics:?}"
);
}
fn assert_has_diagnostic(diagnostics: &[Diagnostic], code: &str) {
assert!(
diagnostics.iter().any(|diagnostic| diagnostic.code == code),
"expected diagnostic {code} in {diagnostics:?}"
);
}
fn base14_font_resources(encoding: Option<&str>) -> FontResourceSet {
font_resources(base14_font(encoding))
}
fn font_resources(font: FontResource) -> FontResourceSet {
let mut fonts = BTreeMap::new();
fonts.insert("F1".to_owned(), font);
FontResourceSet {
fonts,
diagnostics: Vec::new(),
}
}
fn base14_font(encoding: Option<&str>) -> FontResource {
FontResource {
resource_name: "F1".to_owned(),
object_id: ObjectId {
number: 5,
generation: 0,
},
subtype: Some("Type1".to_owned()),
base_font: Some("Helvetica".to_owned()),
encoding: encoding.map(ToOwned::to_owned),
to_unicode: None,
first_char: None,
widths: Vec::new(),
descendant_font_object_ids: Vec::new(),
descendant_subtypes: Vec::new(),
}
}
fn font_resource_pdf() -> Vec<u8> {
"%PDF-1.7
1 0 obj
<< /Type /Catalog /Pages 2 0 R >>
endobj
2 0 obj
<< /Type /Pages /Kids [3 0 R] /Count 1 >>
endobj
3 0 obj
<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>
endobj
4 0 obj
<< /Length 38 >>
stream
BT /F1 12 Tf 72 720 Td (Hello) Tj ET
endstream
endobj
5 0 obj
<< /Type /Font /Subtype /Type0 /BaseFont /CIDFont /Encoding /Identity-H /DescendantFonts [6 0 R] /ToUnicode 7 0 R >>
endobj
6 0 obj
<< /Type /Font /Subtype /CIDFontType2 /BaseFont /CIDFont >>
endobj
7 0 obj
<< /Length 0 >>
stream
endstream
endobj
"
.as_bytes()
.to_vec()
}
fn indirect_font_resource_pdf() -> Vec<u8> {
"%PDF-1.7
1 0 obj
<< /Type /Catalog /Pages 2 0 R >>
endobj
2 0 obj
<< /Type /Pages /Kids [3 0 R] /Count 1 >>
endobj
3 0 obj
<< /Type /Page /Parent 2 0 R /Resources 8 0 R /Contents 4 0 R >>
endobj
4 0 obj
<< /Length 38 >>
stream
BT /FIndirect 12 Tf 72 720 Td (Hello) Tj ET
endstream
endobj
5 0 obj
<< /Type /Font /Subtype /Type0 /BaseFont /CIDFont /Encoding /Identity-H /DescendantFonts [6 0 R] /ToUnicode 7 0 R >>
endobj
6 0 obj
<< /Type /Font /Subtype /CIDFontType2 /BaseFont /CIDFont >>
endobj
7 0 obj
<< /Length 0 >>
stream
endstream
endobj
8 0 obj
<< /Font << /FIndirect 5 0 R >> >>
endobj
"
.as_bytes()
.to_vec()
}
fn simple_font_widths_pdf() -> Vec<u8> {
"%PDF-1.7
1 0 obj
<< /Type /Catalog /Pages 2 0 R >>
endobj
2 0 obj
<< /Type /Pages /Kids [3 0 R] /Count 1 >>
endobj
3 0 obj
<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>
endobj
4 0 obj
<< /Length 35 >>
stream
BT /F1 10 Tf 0 0 Td (AB) Tj ET
endstream
endobj
5 0 obj
<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /FirstChar 65 /LastChar 66 /Widths [1000 250] >>
endobj
"
.as_bytes()
.to_vec()
}
fn missing_font_resource_pdf() -> Vec<u8> {
"%PDF-1.7
1 0 obj
<< /Type /Catalog /Pages 2 0 R >>
endobj
2 0 obj
<< /Type /Pages /Kids [3 0 R] /Count 1 >>
endobj
3 0 obj
<< /Type /Page /Parent 2 0 R /Resources << /Font << /Missing 99 0 R >> >> /Contents 4 0 R >>
endobj
4 0 obj
<< /Length 38 >>
stream
BT /Missing 12 Tf 72 720 Td (Hello) Tj ET
endstream
endobj
"
.as_bytes()
.to_vec()
}
}