1use serde::de::DeserializeOwned;
16
17use crate::segments::Segment;
18
19use super::{ConvertError, Converter, Outcome, normalize_text};
20
21pub const PDF_SUBPROCESS_ID: &str = "pdf-subprocess";
23pub const PDF_SUBPROCESS_VERSION: &str = "1.0.0";
25
26pub const PDF_RECOVERED_ID: &str = "pdf-text-recovered";
31pub const PDF_RECOVERED_VERSION: &str = PDF_SUBPROCESS_VERSION;
42
43#[cfg_attr(not(unix), allow(dead_code))]
48pub(crate) fn pdf_converter_identity(recovered: bool) -> (&'static str, &'static str) {
49 if recovered {
50 (PDF_RECOVERED_ID, PDF_RECOVERED_VERSION)
51 } else {
52 (PDF_SUBPROCESS_ID, PDF_SUBPROCESS_VERSION)
53 }
54}
55
56pub const RECORDS_SUBPROCESS_ID: &str = "records-worker";
58pub const RECORDS_SUBPROCESS_VERSION: &str = "1.0.0";
60
61pub const OCR_ADAPTER_ID: &str = "ocr-adapter";
63pub const OCR_ADAPTER_VERSION: &str = "1.0.0";
65
66pub const ASR_ADAPTER_ID: &str = "asr-adapter";
68pub const ASR_ADAPTER_VERSION: &str = "1.0.0";
70
71pub const VIDEO_ADAPTER_ID: &str = "video-adapter";
73pub const VIDEO_ADAPTER_VERSION: &str = "1.0.0";
75
76pub const OCR_CONFIDENCE_WARNING: f64 = 0.5;
78
79pub fn format_timecode(seconds: f64) -> Result<String, String> {
82 if !seconds.is_finite() || seconds < 0.0 {
83 return Err(format!("timestamp {seconds} is not a non-negative time"));
84 }
85 let whole = seconds as u64;
86 Ok(format!(
87 "{:02}:{:02}:{:02}",
88 whole / 3600,
89 (whole % 3600) / 60,
90 whole % 60
91 ))
92}
93
94fn single_line(text: &str) -> String {
95 text.replace(['\n', '\r'], " ").trim().to_string()
96}
97
98#[cfg(unix)]
99mod imp {
100 use std::sync::OnceLock;
101
102 use super::*;
103 use crate::convert::RecordsLimits;
104 use crate::runner::protocol::bodies::{
105 AsrOk, AsrRequest, OcrInput, OcrOk, OcrRequest, PdfOk, PdfRequest, RecordsOk,
106 RecordsRequest, VideoOk, VideoRequest,
107 };
108 use crate::runner::{Runner, RunnerError};
109
110 fn static_code(code: &str) -> &'static str {
115 const KNOWN: &[&str] = &[
116 "pdf_no_text_layer",
117 "malformed",
118 "encrypted",
119 "resourceLimit",
120 "missingPart",
121 "unsupported",
122 "io",
123 "empty_output",
124 "invalid_utf8",
125 "converter_panic",
126 "unclaimed_format",
127 "resource_limit",
128 "record-limit-exceeded",
129 "records_read_error",
130 "not_sqlite",
131 ];
132 KNOWN
133 .iter()
134 .find(|known| **known == code)
135 .copied()
136 .unwrap_or("adapter_error")
137 }
138
139 fn runner_failure(error: RunnerError) -> ConvertError {
140 ConvertError {
141 code: error.code,
142 message: error.message,
143 }
144 }
145
146 fn call<Req: serde::Serialize, Ok: DeserializeOwned>(
148 runner: &Runner,
149 mode: &str,
150 request: &Req,
151 files: &[(&str, &[u8])],
152 ) -> Result<Ok, ConvertError> {
153 let payload = serde_json::to_value(request).map_err(|e| ConvertError {
154 code: "adapter_error",
155 message: format!("cannot encode the adapter request: {e}"),
156 })?;
157 let response = runner.run(mode, payload, files).map_err(runner_failure)?;
158 if let Some(error) = response.error {
159 return Err(ConvertError {
160 code: static_code(&error.code),
161 message: error.message,
162 });
163 }
164 let ok = response.ok.expect("validated response has a body");
165 serde_json::from_value(ok).map_err(|e| ConvertError {
166 code: "adapter_error",
167 message: format!("adapter success body did not parse: {e}"),
168 })
169 }
170
171 fn shared_runner() -> Result<&'static Runner, ConvertError> {
172 static RUNNER: OnceLock<Result<Runner, RunnerError>> = OnceLock::new();
173 match RUNNER.get_or_init(Runner::with_platform_defaults) {
174 Ok(runner) => Ok(runner),
175 Err(error) => Err(runner_failure(error.clone())),
176 }
177 }
178
179 pub struct PdfSubprocess;
185
186 impl Converter for PdfSubprocess {
187 fn id(&self) -> &'static str {
188 PDF_SUBPROCESS_ID
189 }
190
191 fn version(&self) -> &'static str {
192 PDF_SUBPROCESS_VERSION
193 }
194
195 fn convert(
196 &self,
197 source: &[u8],
198 detected_format: &str,
199 ) -> std::result::Result<Outcome, ConvertError> {
200 if detected_format != "pdf" {
201 return Err(ConvertError {
202 code: "unclaimed_format",
203 message: format!(
204 "the subprocess PDF adapter does not handle {detected_format}"
205 ),
206 });
207 }
208 let body: PdfOk = call(
209 shared_runner()?,
210 "pdf",
211 &PdfRequest {
212 input: "input.pdf".to_string(),
213 },
214 &[("input.pdf", source)],
215 )?;
216 let (converter_id, converter_version) = pdf_converter_identity(body.recovered);
217 Ok(Outcome {
218 converter_id: converter_id.to_string(),
219 converter_version: converter_version.to_string(),
220 detected_format: detected_format.to_string(),
221 text: body.text,
222 warnings: body.warnings,
223 segments: body.segments,
224 })
225 }
226 }
227
228 pub struct RecordsSubprocess {
239 limits: RecordsLimits,
240 }
241
242 impl RecordsSubprocess {
243 pub fn new(limits: RecordsLimits) -> RecordsSubprocess {
245 RecordsSubprocess { limits }
246 }
247 }
248
249 impl Converter for RecordsSubprocess {
250 fn id(&self) -> &'static str {
251 RECORDS_SUBPROCESS_ID
252 }
253
254 fn version(&self) -> &'static str {
255 RECORDS_SUBPROCESS_VERSION
256 }
257
258 fn convert(
259 &self,
260 source: &[u8],
261 detected_format: &str,
262 ) -> std::result::Result<Outcome, ConvertError> {
263 if !matches!(detected_format, "parquet" | "avro" | "sqlite") {
264 return Err(ConvertError {
265 code: "unclaimed_format",
266 message: format!("the records worker does not handle {detected_format}"),
267 });
268 }
269 let input = format!("input.{detected_format}");
270 let body: RecordsOk = call(
271 shared_runner()?,
272 "records",
273 &RecordsRequest {
274 input: input.clone(),
275 format: detected_format.to_string(),
276 max_records: self.limits.max_records,
277 max_tables: self.limits.max_tables,
278 max_output_bytes: self.limits.max_output_bytes,
279 },
280 &[(&input, source)],
281 )?;
282 Ok(Outcome {
283 converter_id: RECORDS_SUBPROCESS_ID.to_string(),
284 converter_version: RECORDS_SUBPROCESS_VERSION.to_string(),
285 detected_format: detected_format.to_string(),
286 text: body.text,
287 warnings: body.warnings,
288 segments: body.segments,
289 })
290 }
291 }
292
293 pub struct OcrAdapter {
296 runner: Runner,
297 }
298
299 impl OcrAdapter {
300 pub fn new(runner: Runner) -> OcrAdapter {
302 OcrAdapter { runner }
303 }
304 }
305
306 impl Converter for OcrAdapter {
307 fn id(&self) -> &'static str {
308 OCR_ADAPTER_ID
309 }
310
311 fn version(&self) -> &'static str {
312 OCR_ADAPTER_VERSION
313 }
314
315 fn convert(
316 &self,
317 source: &[u8],
318 detected_format: &str,
319 ) -> std::result::Result<Outcome, ConvertError> {
320 if !matches!(detected_format, "png" | "jpeg" | "tiff") {
321 return Err(ConvertError {
322 code: "unclaimed_format",
323 message: format!("the OCR adapter does not handle {detected_format}"),
324 });
325 }
326 let input = format!("input.{detected_format}");
327 let body: OcrOk = call(
328 &self.runner,
329 "ocr",
330 &OcrRequest {
331 input: input.clone(),
332 kind: OcrInput::Image,
333 },
334 &[(&input, source)],
335 )?;
336 let mut warnings = Vec::new();
337 let mut lines = Vec::new();
338 for (index, span) in body.spans.iter().enumerate() {
339 if !(0.0..=1.0).contains(&span.confidence) {
340 return Err(ConvertError {
341 code: "adapter_error",
342 message: format!(
343 "span {index} confidence {} is outside 0 to 1",
344 span.confidence
345 ),
346 });
347 }
348 if span.confidence < OCR_CONFIDENCE_WARNING {
349 warnings.push(format!(
350 "ocr_low_confidence: span {index} at {:.2}",
351 span.confidence
352 ));
353 }
354 lines.push(single_line(&span.text));
355 }
356 let text = normalize_text(&finish_lines(lines));
357 if !source.is_empty() && text.is_empty() {
358 return Err(empty_output(source.len()));
359 }
360 Ok(Outcome {
361 converter_id: OCR_ADAPTER_ID.to_string(),
362 converter_version: OCR_ADAPTER_VERSION.to_string(),
363 detected_format: detected_format.to_string(),
364 segments: vec![Segment::span(0, text.len(), "document")],
365 text,
366 warnings,
367 })
368 }
369 }
370
371 pub struct AsrAdapter {
373 runner: Runner,
374 }
375
376 impl AsrAdapter {
377 pub fn new(runner: Runner) -> AsrAdapter {
379 AsrAdapter { runner }
380 }
381 }
382
383 impl Converter for AsrAdapter {
384 fn id(&self) -> &'static str {
385 ASR_ADAPTER_ID
386 }
387
388 fn version(&self) -> &'static str {
389 ASR_ADAPTER_VERSION
390 }
391
392 fn convert(
393 &self,
394 source: &[u8],
395 detected_format: &str,
396 ) -> std::result::Result<Outcome, ConvertError> {
397 if !matches!(detected_format, "mp3" | "wav" | "m4a") {
398 return Err(ConvertError {
399 code: "unclaimed_format",
400 message: format!("the ASR adapter does not handle {detected_format}"),
401 });
402 }
403 let input = format!("input.{detected_format}");
404 let body: AsrOk = call(
405 &self.runner,
406 "asr",
407 &AsrRequest {
408 input: input.clone(),
409 language: None,
410 },
411 &[(&input, source)],
412 )?;
413 let text = normalize_text(&render_transcript(&body.segments).map_err(|detail| {
414 ConvertError {
415 code: "adapter_error",
416 message: detail,
417 }
418 })?);
419 if !source.is_empty() && text.is_empty() {
420 return Err(empty_output(source.len()));
421 }
422 Ok(Outcome {
423 converter_id: ASR_ADAPTER_ID.to_string(),
424 converter_version: ASR_ADAPTER_VERSION.to_string(),
425 detected_format: detected_format.to_string(),
426 segments: vec![Segment::span(0, text.len(), "document")],
427 text,
428 warnings: Vec::new(),
429 })
430 }
431 }
432
433 pub struct VideoAdapter {
435 runner: Runner,
436 }
437
438 impl VideoAdapter {
439 pub fn new(runner: Runner) -> VideoAdapter {
441 VideoAdapter { runner }
442 }
443 }
444
445 impl Converter for VideoAdapter {
446 fn id(&self) -> &'static str {
447 VIDEO_ADAPTER_ID
448 }
449
450 fn version(&self) -> &'static str {
451 VIDEO_ADAPTER_VERSION
452 }
453
454 fn convert(
455 &self,
456 source: &[u8],
457 detected_format: &str,
458 ) -> std::result::Result<Outcome, ConvertError> {
459 if !matches!(detected_format, "mp4" | "mov") {
460 return Err(ConvertError {
461 code: "unclaimed_format",
462 message: format!("the video adapter does not handle {detected_format}"),
463 });
464 }
465 let input = format!("input.{detected_format}");
466 let body: VideoOk = call(
467 &self.runner,
468 "video",
469 &VideoRequest {
470 input: input.clone(),
471 sample_interval_seconds: 1.0,
472 max_frames: 10_000,
473 },
474 &[(&input, source)],
475 )?;
476 let text = normalize_text(&render_screen_states(&body.states).map_err(|detail| {
477 ConvertError {
478 code: "adapter_error",
479 message: detail,
480 }
481 })?);
482 if !source.is_empty() && text.is_empty() {
483 return Err(empty_output(source.len()));
484 }
485 Ok(Outcome {
486 converter_id: VIDEO_ADAPTER_ID.to_string(),
487 converter_version: VIDEO_ADAPTER_VERSION.to_string(),
488 detected_format: detected_format.to_string(),
489 segments: vec![Segment::span(0, text.len(), "document")],
490 text,
491 warnings: Vec::new(),
492 })
493 }
494 }
495
496 fn empty_output(source_len: usize) -> ConvertError {
497 ConvertError {
498 code: "empty_output",
499 message: format!("source is {source_len} bytes but conversion produced no text"),
500 }
501 }
502}
503
504#[cfg(unix)]
505pub use imp::{AsrAdapter, OcrAdapter, PdfSubprocess, RecordsSubprocess, VideoAdapter};
506
507#[cfg(not(unix))]
508mod imp {
509 use super::*;
510 use crate::convert::RecordsLimits;
511
512 fn no_backend() -> ConvertError {
513 ConvertError {
514 code: "sandbox_unavailable",
515 message: "no jail backend exists for this platform, refusing to run adapters"
516 .to_string(),
517 }
518 }
519
520 pub struct PdfSubprocess;
523
524 impl Converter for PdfSubprocess {
525 fn id(&self) -> &'static str {
526 PDF_SUBPROCESS_ID
527 }
528
529 fn version(&self) -> &'static str {
530 PDF_SUBPROCESS_VERSION
531 }
532
533 fn convert(
534 &self,
535 _source: &[u8],
536 _detected_format: &str,
537 ) -> std::result::Result<Outcome, ConvertError> {
538 Err(no_backend())
539 }
540 }
541
542 pub struct RecordsSubprocess;
545
546 impl RecordsSubprocess {
547 pub fn new(_limits: RecordsLimits) -> RecordsSubprocess {
549 RecordsSubprocess
550 }
551 }
552
553 impl Converter for RecordsSubprocess {
554 fn id(&self) -> &'static str {
555 RECORDS_SUBPROCESS_ID
556 }
557
558 fn version(&self) -> &'static str {
559 RECORDS_SUBPROCESS_VERSION
560 }
561
562 fn convert(
563 &self,
564 _source: &[u8],
565 _detected_format: &str,
566 ) -> std::result::Result<Outcome, ConvertError> {
567 Err(no_backend())
568 }
569 }
570}
571
572#[cfg(not(unix))]
573pub use imp::{PdfSubprocess, RecordsSubprocess};
574
575pub fn render_transcript(segments: &[crate::runner_bodies::AsrSegment]) -> Result<String, String> {
579 let mut ordered: Vec<_> = segments.iter().collect();
580 ordered.sort_by(|a, b| {
581 a.start_seconds
582 .total_cmp(&b.start_seconds)
583 .then(a.end_seconds.total_cmp(&b.end_seconds))
584 .then(a.speaker.cmp(&b.speaker))
585 });
586 let mut lines = Vec::new();
587 for segment in ordered {
588 if segment.end_seconds < segment.start_seconds {
589 return Err(format!(
590 "segment ends at {} before it starts at {}",
591 segment.end_seconds, segment.start_seconds
592 ));
593 }
594 lines.push(format!(
595 "[{} -> {}] Speaker {}: {}",
596 format_timecode(segment.start_seconds)?,
597 format_timecode(segment.end_seconds)?,
598 segment.speaker,
599 single_line(&segment.text)
600 ));
601 }
602 Ok(finish_lines(lines))
603}
604
605pub fn render_screen_states(
609 states: &[crate::runner_bodies::ScreenState],
610) -> Result<String, String> {
611 let mut ordered: Vec<_> = states.iter().collect();
612 ordered.sort_by(|a, b| {
613 a.first_seen_seconds
614 .total_cmp(&b.first_seen_seconds)
615 .then(a.text.cmp(&b.text))
616 });
617 let mut lines = Vec::new();
618 let mut previous: Option<String> = None;
619 for state in ordered {
620 let text = single_line(&state.text);
621 if previous.as_deref() == Some(text.as_str()) {
622 continue;
623 }
624 lines.push(format!(
625 "[on-screen @ {}] {}",
626 format_timecode(state.first_seen_seconds)?,
627 text
628 ));
629 previous = Some(text);
630 }
631 Ok(finish_lines(lines))
632}
633
634fn finish_lines(lines: Vec<String>) -> String {
635 if lines.is_empty() {
636 String::new()
637 } else {
638 let mut text = lines.join("\n");
639 text.push('\n');
640 text
641 }
642}
643
644#[cfg(test)]
645mod tests {
646 use super::*;
647 use crate::runner_bodies::{AsrSegment, ScreenState};
648
649 #[test]
650 fn recovered_pdf_text_is_recorded_under_the_recovery_id() {
651 assert_eq!(
655 pdf_converter_identity(false),
656 (PDF_SUBPROCESS_ID, PDF_SUBPROCESS_VERSION)
657 );
658 assert_eq!(
659 pdf_converter_identity(true),
660 (PDF_RECOVERED_ID, PDF_RECOVERED_VERSION)
661 );
662 assert_ne!(PDF_RECOVERED_ID, PDF_SUBPROCESS_ID);
663 assert_eq!(PDF_RECOVERED_VERSION, PDF_SUBPROCESS_VERSION);
664 }
665
666 #[test]
667 fn timecodes_render_as_hours_minutes_seconds() {
668 assert_eq!(format_timecode(0.0).unwrap(), "00:00:00");
669 assert_eq!(format_timecode(12.9).unwrap(), "00:00:12");
670 assert_eq!(format_timecode(182.0).unwrap(), "00:03:02");
671 assert_eq!(format_timecode(3661.0).unwrap(), "01:01:01");
672 assert!(format_timecode(-1.0).is_err());
673 assert!(format_timecode(f64::NAN).is_err());
674 }
675
676 #[test]
677 fn transcripts_render_one_ordered_line_per_segment() {
678 let segments = vec![
679 AsrSegment {
680 start_seconds: 19.0,
681 end_seconds: 24.0,
682 speaker: 2,
683 text: "second\nline".to_string(),
684 },
685 AsrSegment {
686 start_seconds: 12.0,
687 end_seconds: 19.0,
688 speaker: 1,
689 text: "first remark".to_string(),
690 },
691 ];
692 assert_eq!(
693 render_transcript(&segments).unwrap(),
694 "[00:00:12 -> 00:00:19] Speaker 1: first remark\n\
695 [00:00:19 -> 00:00:24] Speaker 2: second line\n"
696 );
697 }
698
699 #[test]
700 fn transcripts_reject_a_segment_that_ends_before_it_starts() {
701 let segments = vec![AsrSegment {
702 start_seconds: 5.0,
703 end_seconds: 4.0,
704 speaker: 1,
705 text: "impossible".to_string(),
706 }];
707 assert!(render_transcript(&segments).is_err());
708 }
709
710 #[test]
711 fn screen_states_render_deduplicated_in_order() {
712 let states = vec![
713 ScreenState {
714 first_seen_seconds: 182.0,
715 text: "Quarterly revenue".to_string(),
716 },
717 ScreenState {
718 first_seen_seconds: 10.0,
719 text: "Agenda".to_string(),
720 },
721 ScreenState {
722 first_seen_seconds: 200.0,
723 text: "Quarterly revenue".to_string(),
724 },
725 ];
726 assert_eq!(
727 render_screen_states(&states).unwrap(),
728 "[on-screen @ 00:00:10] Agenda\n[on-screen @ 00:03:02] Quarterly revenue\n"
729 );
730 }
731}