use pdf_oxide::PdfDocument;
use pdf_oxide::layout::TextChar;
use serde::Serialize;
pub mod markdown;
pub use markdown::{BareStyle, Descriptor, Escaping, Numbering, Section, Style};
mod date;
mod pdf_string;
mod actual_text;
use actual_text::{Frame, Unit, Unrepaired};
pub mod diagnostic;
pub use diagnostic::{Diagnostic, Diagnostics, Severity};
pub mod error;
pub use error::Error;
mod glyphs;
pub use glyphs::GlyphSpace;
mod document;
mod ligature;
mod matching;
use document::{
EncodingNotes, audit_annots, choose_text, has_quads, hex_color, is_interesting,
page_font_widths, raw_text, section_titles, subtype_name,
};
use glyphs::{debug_geometry, to_page_frame};
use matching::{Line, Matching, join_lines, quads_reach_segments, text_under_quads};
#[derive(Debug, Serialize)]
pub struct Record {
pub page: usize,
pub kind: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub covered_text: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub comment: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub author: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub modified: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub color: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub section: Option<String>,
pub rect: [f64; 4],
pub link: String,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub diagnostics: Vec<Diagnostic>,
}
#[derive(Clone, Debug)]
#[non_exhaustive]
pub struct Options {
pub min_overlap: f32,
pub space_gap: f32,
pub geometry: Option<GlyphSpace>,
pub keep_hyphens: bool,
pub keep_ligatures: bool,
pub split_quads: bool,
pub keep_empty: bool,
pub document_name: Option<String>,
pub page_counts: bool,
pub debug_geometry: bool,
pub debug_quads: bool,
}
impl Default for Options {
fn default() -> Self {
Options {
min_overlap: 0.5,
space_gap: 0.25,
geometry: None,
keep_hyphens: false,
keep_ligatures: false,
split_quads: false,
keep_empty: false,
document_name: None,
page_counts: false,
debug_geometry: false,
debug_quads: false,
}
}
}
#[derive(Debug)]
pub struct Report {
pub annotations: Vec<Record>,
pub diagnostics: Diagnostics,
}
pub fn extract(bytes: Vec<u8>, opts: &Options) -> Result<Report, Error> {
let mut diags = Diagnostics::default();
let doc = PdfDocument::from_bytes(bytes).map_err(Error::parse)?;
let file_name = opts.document_name.clone().unwrap_or_default();
let page_count = doc.page_count().map_err(Error::page_count)?;
let sections = section_titles(&doc, page_count);
let (space, inferred) = match opts.geometry {
Some(space) => (space, true),
None => {
let richest = (0..page_count.min(10))
.map(|page| (doc.extract_chars(page).map(|c| c.len()).unwrap_or(0), page))
.max_by_key(|&(len, _)| len)
.filter(|&(len, _)| len > 0);
let measured = richest.and_then(|(len, page)| {
let (mx0, my0, _, my1) = doc
.get_page_media_box(page)
.unwrap_or((0.0, 0.0, 612.0, 792.0));
let chars = to_page_frame(doc.extract_chars(page).unwrap_or_default(), mx0, my0);
if opts.debug_geometry {
eprintln!("geometry: inferring from page {} ({len} glyphs)", page + 1);
}
GlyphSpace::infer(&chars, my1 - my0, opts.debug_geometry)
});
match measured {
Some(space) => (space, true),
None => (GlyphSpace::FALLBACK, false),
}
}
};
if opts.debug_geometry {
eprintln!(
"geometry: using {space:?} ({})",
if inferred { "measured" } else { "assumed" }
);
}
let mut records = Vec::new();
let mut total_parsed = 0usize;
let mut total_dropped = 0usize;
let mut encoding_notes = EncodingNotes::default();
for page in 0..page_count {
let annots = match doc.get_annotations(page) {
Ok(annots) => annots,
Err(e) => {
diags.push(Diagnostic::AnnotsUnreadable {
page: page + 1,
error: e.to_string(),
});
continue;
}
};
let (raw_count, inline_count) = audit_annots(&doc, page);
total_parsed += annots.len();
total_dropped += raw_count.saturating_sub(annots.len());
if opts.page_counts && raw_count > 0 {
diags.push(Diagnostic::PageAnnotCounts {
page: page + 1,
in_annots: raw_count,
parsed: annots.len(),
inline: inline_count,
});
}
if annots.is_empty() {
continue;
}
let Ok((mx0, my0, _mx1, my1)) = doc.get_page_media_box(page) else {
diags.push(Diagnostic::NoMediaBox {
page: page + 1,
annotations: annots.len(),
});
continue;
};
let page_height = my1 - my0;
let rotation = doc.get_page_rotation(page).unwrap_or(0);
let mut units: Vec<Unit> = Vec::new();
let mut unrepaired: Vec<Unrepaired> = Vec::new();
let chars: Option<Vec<TextChar>> = if annots.iter().any(has_quads) {
match doc.extract_chars(page) {
Ok(mut cs) => {
let content = doc.get_page_content_data(page).unwrap_or_default();
if actual_text::present(&content) {
let fonts = page_font_widths(&doc, page);
let (decls, unsupported) = actual_text::scan(&content, &fonts, page + 1);
let (found, broken) = actual_text::repair(&mut cs, &decls, page + 1);
units = found;
for d in unsupported {
diags.push(d);
}
for u in &broken {
diags.push(u.diagnostic.clone());
}
unrepaired = broken;
}
Some(to_page_frame(cs, mx0, my0))
}
Err(e) => {
diags.push(Diagnostic::TextUnextractable {
page: page + 1,
error: e.to_string(),
});
None
}
}
} else {
None
};
let frame = Frame {
x0: mx0,
y0: my0,
height: page_height,
};
if opts.debug_geometry {
debug_geometry(page, my0, my1, chars.as_deref(), &annots);
}
for a in &annots {
if !is_interesting(&a.subtype_enum) {
continue;
}
let mut item_diags: Vec<Diagnostic> = Vec::new();
let covered_text = match (&a.quad_points, &chars) {
(Some(quads), Some(chars)) if !quads.is_empty() => {
let lines = text_under_quads(
quads,
chars,
&Matching {
space,
frame,
min_overlap: opts.min_overlap,
space_gap: opts.space_gap,
fold_ligatures: !opts.keep_ligatures,
debug: opts.debug_quads,
units: &units,
},
);
let empty = lines.iter().filter(|l| l.text.is_empty()).count();
if empty > 0 {
if !inferred {
let assumed = Diagnostic::GeometryAssumed { space };
if !diags.contains(&assumed) {
diags.push(assumed);
}
}
let unmatched = Diagnostic::UnmatchedQuads {
page: page + 1,
unmatched: empty,
total: quads.len(),
rotation,
};
item_diags.push(unmatched.clone());
diags.push(unmatched);
}
let nonempty: Vec<Line> =
lines.into_iter().filter(|l| !l.text.is_empty()).collect();
if nonempty.is_empty() {
None
} else if opts.split_quads {
Some(
nonempty
.iter()
.map(|l| l.text.clone())
.collect::<Vec<_>>()
.join("\n"),
)
} else {
Some(join_lines(&nonempty, opts.keep_hyphens, &units, frame))
}
}
_ => None,
};
if let Some(quads) = a.quad_points.as_ref() {
for u in &unrepaired {
if quads_reach_segments(quads, &u.segments) {
item_diags.push(u.diagnostic.clone());
}
}
}
let comment = choose_text(
raw_text(&doc, a, "Contents", &mut encoding_notes),
a.contents.as_deref(),
);
let author = choose_text(
raw_text(&doc, a, "T", &mut encoding_notes),
a.author.as_deref(),
);
if covered_text.is_none() && comment.is_none() && !opts.keep_empty {
continue;
}
records.push(Record {
page: page + 1,
kind: subtype_name(&a.subtype_enum).to_string(),
covered_text,
comment,
author,
modified: a.modification_date.clone(),
color: a.color.as_deref().and_then(hex_color),
section: sections.get(&page).cloned(),
rect: a.rect.unwrap_or([0.0; 4]),
link: format!("{file_name}#page={}", page + 1),
diagnostics: item_diags
.into_iter()
.filter(Diagnostic::is_annotation_scope)
.collect(),
});
}
}
if total_dropped > 0 {
diags.push(Diagnostic::UnparsedAnnots {
dropped: total_dropped,
total: total_parsed + total_dropped,
});
}
if encoding_notes.sniffed_utf8 > 0 {
diags.push(Diagnostic::EncodingSniffed {
count: encoding_notes.sniffed_utf8,
});
}
if encoding_notes.undefined_bytes > 0 {
diags.push(Diagnostic::UndefinedPdfDocBytes {
count: encoding_notes.undefined_bytes,
});
}
sort_records(&mut records);
Ok(Report {
annotations: records,
diagnostics: diags,
})
}
fn sort_records(records: &mut [Record]) {
records.sort_by(|a, b| {
a.page.cmp(&b.page).then_with(|| {
let ay = a.rect[1].max(a.rect[3]);
let by = b.rect[1].max(b.rect[3]);
by.partial_cmp(&ay)
.unwrap_or(std::cmp::Ordering::Equal)
.then(
a.rect[0]
.partial_cmp(&b.rect[0])
.unwrap_or(std::cmp::Ordering::Equal),
)
})
});
}
#[cfg(test)]
mod tests;