use std::collections::BTreeSet;
use std::fmt;
use std::io::{IsTerminal, Write};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use crate::result::ConversionStatus;
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum ProgressKind {
DocumentStarted,
PageCompleted {
page_no: usize,
success: bool,
completed_pages: usize,
total_pages: usize,
},
EnrichmentProgress {
step: EnrichmentStep,
label: &'static str,
completed_items: usize,
total_items: usize,
},
DocumentCompleted { status: ConversionStatus },
}
impl ProgressKind {
pub fn name(&self) -> &'static str {
match self {
Self::DocumentStarted => "document_started",
Self::PageCompleted { .. } => "page_completed",
Self::EnrichmentProgress { .. } => "enrichment_progress",
Self::DocumentCompleted { .. } => "document_completed",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[non_exhaustive]
pub enum EnrichmentStep {
CodeFormula,
PictureClassification,
PictureDescription,
ChartExtraction,
Other,
}
impl EnrichmentStep {
pub fn name(&self) -> &'static str {
match self {
Self::CodeFormula => "code_formula",
Self::PictureClassification => "picture_classification",
Self::PictureDescription => "picture_description",
Self::ChartExtraction => "chart_extraction",
Self::Other => "other",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ProgressEvent {
pub document_index: usize,
pub document_name: String,
pub kind: ProgressKind,
}
impl ProgressEvent {
pub fn to_json(&self) -> serde_json::Value {
let mut v = serde_json::json!({
"kind": self.kind.name(),
"document_index": self.document_index,
"document_name": self.document_name,
});
match &self.kind {
ProgressKind::DocumentStarted => {}
ProgressKind::PageCompleted {
page_no,
success,
completed_pages,
total_pages,
} => {
v["page_no"] = (*page_no).into();
v["success"] = (*success).into();
v["completed_pages"] = (*completed_pages).into();
v["total_pages"] = (*total_pages).into();
}
ProgressKind::EnrichmentProgress {
step,
label,
completed_items,
total_items,
} => {
v["step"] = step.name().into();
v["label"] = (*label).into();
v["completed_items"] = (*completed_items).into();
v["total_items"] = (*total_items).into();
}
ProgressKind::DocumentCompleted { status } => {
v["status"] = status.as_str().into();
}
}
v
}
}
pub type ProgressCallback = Arc<dyn Fn(&ProgressEvent) + Send + Sync>;
#[derive(Clone, Default)]
pub(crate) struct Callbacks(pub(crate) Vec<ProgressCallback>);
impl fmt::Debug for Callbacks {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{} callback(s)", self.0.len())
}
}
pub struct ProgressReporter {
callbacks: Vec<ProgressCallback>,
document_index: usize,
document_name: String,
finished: Mutex<BTreeSet<usize>>,
}
impl fmt::Debug for ProgressReporter {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ProgressReporter")
.field("document_index", &self.document_index)
.field("document_name", &self.document_name)
.field("callbacks", &self.callbacks.len())
.finish()
}
}
impl ProgressReporter {
pub fn new(
callbacks: Vec<ProgressCallback>,
document_index: usize,
document_name: impl Into<String>,
) -> Self {
Self {
callbacks,
document_index,
document_name: document_name.into(),
finished: Mutex::new(BTreeSet::new()),
}
}
pub fn enabled(&self) -> bool {
!self.callbacks.is_empty()
}
pub fn document_index(&self) -> usize {
self.document_index
}
pub fn document_name(&self) -> &str {
&self.document_name
}
fn emit(&self, kind: ProgressKind) {
if self.callbacks.is_empty() {
return;
}
let event = ProgressEvent {
document_index: self.document_index,
document_name: self.document_name.clone(),
kind,
};
for callback in &self.callbacks {
let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
callback(&event);
}));
if outcome.is_err() {
eprintln!(
"docling: warning: a progress callback panicked on {} of {}; ignored",
event.kind.name(),
event.document_name
);
}
}
}
pub fn document_started(&self) {
self.emit(ProgressKind::DocumentStarted);
}
pub fn page_completed(&self, page_no: usize, total_pages: usize, success: bool) {
if !self.enabled() || page_no < 1 {
return;
}
let completed_pages = {
let mut finished = self.finished.lock().unwrap_or_else(|p| p.into_inner());
if !finished.insert(page_no) {
return;
}
finished.len()
};
self.emit(ProgressKind::PageCompleted {
page_no,
success,
completed_pages,
total_pages,
});
}
pub fn fail_unfinished_pages(
&self,
pages: impl IntoIterator<Item = usize>,
total_pages: usize,
) {
if !self.enabled() {
return;
}
let mut pages: Vec<usize> = pages.into_iter().collect();
pages.sort_unstable();
for page_no in pages {
self.page_completed(page_no, total_pages, false);
}
}
pub fn enrichment_progress(
&self,
step: EnrichmentStep,
label: &'static str,
completed_items: usize,
total_items: usize,
) {
self.emit(ProgressKind::EnrichmentProgress {
step,
label,
completed_items,
total_items,
});
}
pub fn document_completed(&self, status: ConversionStatus) {
self.emit(ProgressKind::DocumentCompleted { status });
}
#[cfg(feature = "pdf")]
pub fn page_hook(self: &Arc<Self>) -> Option<docling_pdf::PageProgressHook> {
if !self.enabled() {
return None;
}
let reporter = Arc::clone(self);
Some(Arc::new(move |p: docling_pdf::PageProgress| {
reporter.page_completed(p.page_no, p.pages_selected, true);
}))
}
#[cfg(feature = "pdf")]
pub fn fail_pages_cut_off(&self, first_page: usize, pages_selected: usize) {
self.fail_unfinished_pages(first_page..first_page + pages_selected, pages_selected);
}
}
struct Bar {
document_index: usize,
label: &'static str,
done: usize,
total: usize,
drawn: bool,
last_draw: Instant,
}
pub struct ProgressPrinter {
total_documents: Option<usize>,
document_lines: bool,
terminal: bool,
color: bool,
bar: Mutex<Option<Bar>>,
}
impl fmt::Debug for ProgressPrinter {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ProgressPrinter")
.field("total_documents", &self.total_documents)
.field("terminal", &self.terminal)
.finish()
}
}
impl Default for ProgressPrinter {
fn default() -> Self {
Self::new()
}
}
const REDRAW_EVERY: Duration = Duration::from_millis(50);
const BAR_WIDTH: usize = 20;
impl ProgressPrinter {
pub fn new() -> Self {
Self {
total_documents: None,
document_lines: true,
terminal: Self::is_terminal(),
color: std::env::var_os("NO_COLOR").is_none_or(|v| v.is_empty()),
bar: Mutex::new(None),
}
}
pub fn with_total_documents(mut self, total: usize) -> Self {
self.total_documents = Some(total);
self
}
pub fn with_document_lines(mut self, enable: bool) -> Self {
self.document_lines = enable;
self
}
pub fn is_terminal() -> bool {
std::io::stderr().is_terminal()
}
pub fn callback(self: Arc<Self>) -> ProgressCallback {
Arc::new(move |event| self.handle(event))
}
pub fn handle(&self, event: &ProgressEvent) {
let mut bar = self.bar.lock().unwrap_or_else(|p| p.into_inner());
match &event.kind {
ProgressKind::DocumentStarted if !self.document_lines => {}
ProgressKind::DocumentStarted => {
let position = match self.total_documents {
Some(total) => format!("{}/{total}", event.document_index),
None => event.document_index.to_string(),
};
self.write_line(
&mut bar,
&format!("[{position}] Converting {}", event.document_name),
);
}
ProgressKind::PageCompleted {
completed_pages,
total_pages,
..
} => self.advance(
&mut bar,
event.document_index,
"pages",
*completed_pages,
*total_pages,
),
ProgressKind::EnrichmentProgress {
label,
completed_items,
total_items,
..
} => self.advance(
&mut bar,
event.document_index,
label,
*completed_items,
*total_items,
),
ProgressKind::DocumentCompleted { status } => {
self.close(&mut bar, Some(event.document_index));
if self.document_lines {
self.write_line(
&mut bar,
&format!("Finished {}: {}", event.document_name, status.as_str()),
);
}
}
}
}
fn write_line(&self, bar: &mut Option<Bar>, text: &str) {
let mut err = std::io::stderr().lock();
let redraw = bar.as_ref().is_some_and(|b| b.drawn);
if redraw {
let _ = write!(err, "\r\x1b[2K");
}
let _ = writeln!(err, "{text}");
if redraw {
if let Some(b) = bar.as_mut() {
let _ = err.write_all(self.render(b).as_bytes());
b.last_draw = Instant::now();
}
}
let _ = err.flush();
}
fn advance(
&self,
bar: &mut Option<Bar>,
document_index: usize,
label: &'static str,
done: usize,
total: usize,
) {
let same = bar
.as_ref()
.is_some_and(|b| b.document_index == document_index && b.label == label);
if !same {
self.close(bar, None);
*bar = Some(Bar {
document_index,
label,
done,
total,
drawn: false,
last_draw: Instant::now() - REDRAW_EVERY,
});
}
let Some(b) = bar.as_mut() else { return };
b.done = done;
b.total = total;
if !self.terminal {
return;
}
let finished = done >= total;
if !finished && b.drawn && b.last_draw.elapsed() < REDRAW_EVERY {
return;
}
let mut err = std::io::stderr().lock();
let _ = write!(err, "\r\x1b[2K");
let _ = err.write_all(self.render(b).as_bytes());
let _ = err.flush();
b.drawn = true;
b.last_draw = Instant::now();
}
fn close(&self, bar: &mut Option<Bar>, document_index: Option<usize>) {
let owned = bar
.as_ref()
.is_some_and(|b| document_index.is_none_or(|i| b.document_index == i));
if !owned {
return;
}
let Some(b) = bar.take() else { return };
let mut err = std::io::stderr().lock();
if self.terminal {
let _ = writeln!(err, "\r\x1b[2K{}", self.render(&b));
} else {
let _ = writeln!(
err,
" {}{} {}/{}",
self.prefix(b.document_index),
b.label,
b.done,
b.total
);
}
let _ = err.flush();
}
fn prefix(&self, document_index: usize) -> String {
match self.total_documents {
Some(total) if total > 1 => format!("[{document_index}] "),
_ => String::new(),
}
}
fn render(&self, b: &Bar) -> String {
let filled = (b.done.min(b.total) * BAR_WIDTH)
.checked_div(b.total)
.unwrap_or(BAR_WIDTH);
let (on, off) = if self.color {
("\x1b[38;5;208m", "\x1b[0m")
} else {
("", "")
};
format!(
" {}{} {}/{} [{on}{}{off}{}]",
self.prefix(b.document_index),
b.label,
b.done,
b.total,
"█".repeat(filled),
"░".repeat(BAR_WIDTH - filled),
)
}
}
#[cfg(test)]
mod tests {
use super::*;
fn collect() -> (ProgressCallback, Arc<Mutex<Vec<ProgressEvent>>>) {
let seen: Arc<Mutex<Vec<ProgressEvent>>> = Arc::new(Mutex::new(Vec::new()));
let sink = Arc::clone(&seen);
let cb: ProgressCallback = Arc::new(move |e| sink.lock().unwrap().push(e.clone()));
(cb, seen)
}
fn pages(events: &[ProgressEvent]) -> Vec<(usize, bool, usize, usize)> {
events
.iter()
.filter_map(|e| match e.kind {
ProgressKind::PageCompleted {
page_no,
success,
completed_pages,
total_pages,
} => Some((page_no, success, completed_pages, total_pages)),
_ => None,
})
.collect()
}
#[test]
fn pages_dedupe_and_cut_off_pages_fail_in_order() {
let (cb, seen) = collect();
let r = ProgressReporter::new(vec![cb], 2, "x.pdf");
r.document_started();
r.page_completed(3, 4, true);
r.page_completed(3, 4, true); r.page_completed(0, 4, true); r.page_completed(2, 4, true);
r.fail_unfinished_pages([5, 4], 4);
r.document_completed(ConversionStatus::PartialSuccess);
let events = seen.lock().unwrap().clone();
assert!(events
.iter()
.all(|e| e.document_index == 2 && e.document_name == "x.pdf"));
assert_eq!(
events.first().map(|e| e.kind.name()),
Some("document_started")
);
assert_eq!(
pages(&events),
vec![
(3, true, 1, 4),
(2, true, 2, 4),
(4, false, 3, 4),
(5, false, 4, 4)
]
);
assert_eq!(
events.last().map(|e| e.kind.clone()),
Some(ProgressKind::DocumentCompleted {
status: ConversionStatus::PartialSuccess
})
);
}
#[test]
fn silent_without_listeners_and_resilient_to_a_panic() {
let quiet = ProgressReporter::new(Vec::new(), 1, "a");
assert!(!quiet.enabled());
quiet.page_completed(1, 1, true);
assert!(quiet.finished.lock().unwrap().is_empty());
let (cb, seen) = collect();
let bad: ProgressCallback = Arc::new(|_| panic!("observer bug"));
let r = ProgressReporter::new(vec![bad, cb], 1, "a");
r.document_started();
assert_eq!(seen.lock().unwrap().len(), 1);
}
#[test]
fn json_shape_is_doclings() {
let e = ProgressEvent {
document_index: 1,
document_name: "a.pdf".into(),
kind: ProgressKind::PageCompleted {
page_no: 2,
success: true,
completed_pages: 1,
total_pages: 3,
},
};
assert_eq!(
e.to_json(),
serde_json::json!({
"kind": "page_completed", "document_index": 1, "document_name": "a.pdf",
"page_no": 2, "success": true, "completed_pages": 1, "total_pages": 3
})
);
let e = ProgressEvent {
document_index: 1,
document_name: "a.pdf".into(),
kind: ProgressKind::EnrichmentProgress {
step: EnrichmentStep::PictureDescription,
label: "picture description",
completed_items: 0,
total_items: 2,
},
};
assert_eq!(e.to_json()["step"], "picture_description");
let e = ProgressEvent {
document_index: 1,
document_name: "a.pdf".into(),
kind: ProgressKind::DocumentCompleted {
status: ConversionStatus::Failure,
},
};
assert_eq!(e.to_json()["status"], "failure");
}
#[test]
fn printer_renders_a_bar() {
let p = ProgressPrinter {
total_documents: Some(3),
document_lines: true,
terminal: false,
color: false,
bar: Mutex::new(None),
};
let b = Bar {
document_index: 2,
label: "pages",
done: 5,
total: 10,
drawn: false,
last_draw: Instant::now(),
};
assert_eq!(p.render(&b), " [2] pages 5/10 [██████████░░░░░░░░░░]");
let single = ProgressPrinter {
total_documents: None,
..p
};
assert_eq!(single.render(&b), " pages 5/10 [██████████░░░░░░░░░░]");
}
}