1use super::builder_utils::{
8 build_optional_adapter, default_cpu_region_batch_size, resolve_device_batch_sizes,
9 resolve_model_path, resolve_model_source,
10};
11use oar_ocr_core::core::config::OrtSessionConfig;
12use oar_ocr_core::core::traits::OrtConfigurable;
13use oar_ocr_core::core::traits::adapter::{AdapterBuilder, ModelAdapter};
14use oar_ocr_core::core::{ModelSource, OCRError};
15use oar_ocr_core::domain::adapters::{
16 DocumentOrientationAdapter, DocumentOrientationAdapterBuilder, FormulaRecognitionAdapter,
17 LayoutDetectionAdapter, LayoutDetectionAdapterBuilder, PPFormulaNetAdapterBuilder,
18 SLANetWiredAdapterBuilder, SLANetWirelessAdapterBuilder, SealTextDetectionAdapter,
19 SealTextDetectionAdapterBuilder, TableCellDetectionAdapter, TableCellDetectionAdapterBuilder,
20 TableClassificationAdapter, TableClassificationAdapterBuilder,
21 TableStructureRecognitionAdapter, TextDetectionAdapter, TextDetectionAdapterBuilder,
22 TextLineOrientationAdapter, TextLineOrientationAdapterBuilder, TextRecognitionAdapter,
23 TextRecognitionAdapterBuilder, UVDocRectifierAdapter, UVDocRectifierAdapterBuilder,
24 UniMERNetAdapterBuilder,
25};
26use oar_ocr_core::domain::structure::{StructureResult, TableResult, TableType};
27use oar_ocr_core::domain::tasks::{
28 FormulaRecognitionConfig, LayoutDetectionConfig, TableCellDetectionConfig,
29 TableClassificationConfig, TableStructureRecognitionConfig, TextDetectionConfig,
30 TextRecognitionConfig,
31};
32use oar_ocr_core::predictors::FormulaModelKind;
33use std::path::PathBuf;
34use std::sync::Arc;
35use std::time::Instant;
36
37const LAYOUT_OVERLAP_IOU_THRESHOLD: f32 = 0.5;
39
40const CELL_OVERLAP_IOU_THRESHOLD: f32 = 0.5;
42
43const REGION_MEMBERSHIP_IOA_THRESHOLD: f32 = 0.1;
46
47const TEXT_BOX_SPLIT_IOA_THRESHOLD: f32 = 0.3;
50
51#[derive(Debug)]
53struct StructurePipeline {
54 document_orientation_adapter: Option<DocumentOrientationAdapter>,
56 rectification_adapter: Option<UVDocRectifierAdapter>,
57
58 layout_detection_adapter: LayoutDetectionAdapter,
60
61 region_detection_adapter: Option<LayoutDetectionAdapter>,
63
64 table_classification_adapter: Option<TableClassificationAdapter>,
66 table_orientation_adapter: Option<DocumentOrientationAdapter>, table_cell_detection_adapter: Option<TableCellDetectionAdapter>,
68 table_structure_recognition_adapter: Option<TableStructureRecognitionAdapter>,
69 wired_table_structure_adapter: Option<TableStructureRecognitionAdapter>,
71 wireless_table_structure_adapter: Option<TableStructureRecognitionAdapter>,
72 wired_table_cell_adapter: Option<TableCellDetectionAdapter>,
73 wireless_table_cell_adapter: Option<TableCellDetectionAdapter>,
74 use_e2e_wired_table_rec: bool,
76 use_e2e_wireless_table_rec: bool,
77 use_wired_table_cells_trans_to_html: bool,
79 use_wireless_table_cells_trans_to_html: bool,
80
81 formula_recognition_adapter: Option<FormulaRecognitionAdapter>,
82
83 seal_text_detection_adapter: Option<SealTextDetectionAdapter>,
84
85 text_detection_adapter: Option<TextDetectionAdapter>,
87 text_line_orientation_adapter: Option<TextLineOrientationAdapter>,
88 text_recognition_adapter: Option<TextRecognitionAdapter>,
89
90 image_batch_size: Option<usize>,
92 region_batch_size: Option<usize>,
93}
94
95#[derive(Debug, Clone)]
129pub struct OARStructureBuilder {
130 layout_detection_model: ModelSource,
132 layout_model_name: Option<String>,
133
134 document_orientation_model: Option<ModelSource>,
136 document_rectification_model: Option<ModelSource>,
137
138 region_detection_model: Option<ModelSource>,
140
141 table_classification_model: Option<ModelSource>,
143 table_orientation_model: Option<ModelSource>, table_cell_detection_model: Option<ModelSource>,
145 table_cell_detection_type: Option<Result<TableType, String>>,
146 table_structure_recognition_model: Option<ModelSource>,
147 table_structure_recognition_type: Option<Result<TableType, String>>,
148 table_structure_dict_path: Option<PathBuf>,
149
150 wired_table_structure_model: Option<ModelSource>,
151 wireless_table_structure_model: Option<ModelSource>,
152 wired_table_cell_model: Option<ModelSource>,
153 wireless_table_cell_model: Option<ModelSource>,
154 use_e2e_wired_table_rec: bool,
157 use_e2e_wireless_table_rec: bool,
158 use_wired_table_cells_trans_to_html: bool,
161 use_wireless_table_cells_trans_to_html: bool,
162
163 formula_recognition_model: Option<ModelSource>,
165 formula_recognition_type: Option<Result<FormulaModelKind, String>>,
166 formula_tokenizer_path: Option<PathBuf>,
167 formula_ort_session_config: Option<OrtSessionConfig>,
168
169 seal_text_detection_model: Option<ModelSource>,
171
172 text_detection_model: Option<ModelSource>,
174 text_line_orientation_model: Option<ModelSource>,
175 text_recognition_model: Option<ModelSource>,
176 character_dict_path: Option<PathBuf>,
177
178 region_model_name: Option<String>,
180 wired_table_structure_model_name: Option<String>,
181 wireless_table_structure_model_name: Option<String>,
182 wired_table_cell_model_name: Option<String>,
183 wireless_table_cell_model_name: Option<String>,
184 text_detection_model_name: Option<String>,
185 text_recognition_model_name: Option<String>,
186
187 ort_session_config: Option<OrtSessionConfig>,
189 layout_detection_config: Option<LayoutDetectionConfig>,
190 table_classification_config: Option<TableClassificationConfig>,
191 table_cell_detection_config: Option<TableCellDetectionConfig>,
192 table_structure_recognition_config: Option<TableStructureRecognitionConfig>,
193 formula_recognition_config: Option<FormulaRecognitionConfig>,
194 text_detection_config: Option<TextDetectionConfig>,
195 text_recognition_config: Option<TextRecognitionConfig>,
196
197 image_batch_size: Option<usize>,
199 region_batch_size: Option<usize>,
200}
201
202fn default_arena_shrinkage(config: &mut OrtSessionConfig) {
205 use oar_ocr_core::core::config::OrtExecutionProvider;
206 let uses_cuda = config
207 .execution_providers
208 .iter()
209 .flatten()
210 .any(|ep| matches!(ep, OrtExecutionProvider::CUDA { .. }));
211 if uses_cuda && config.arena_shrinkage.is_none() {
212 config.arena_shrinkage = Some(true);
213 }
214}
215
216impl OARStructureBuilder {
217 const MAX_BATCH_SIZE: usize = 4096;
218
219 pub fn new(layout_detection_model: impl Into<ModelSource>) -> Self {
225 Self {
226 layout_detection_model: layout_detection_model.into(),
227 layout_model_name: None,
228 document_orientation_model: None,
229 document_rectification_model: None,
230 region_detection_model: None,
231 table_classification_model: None,
232 table_orientation_model: None,
233 table_cell_detection_model: None,
234 table_cell_detection_type: None,
235 table_structure_recognition_model: None,
236 table_structure_recognition_type: None,
237 table_structure_dict_path: None,
238 wired_table_structure_model: None,
239 wireless_table_structure_model: None,
240 wired_table_cell_model: None,
241 wireless_table_cell_model: None,
242 use_e2e_wired_table_rec: false,
244 use_e2e_wireless_table_rec: true,
245 use_wired_table_cells_trans_to_html: false,
246 use_wireless_table_cells_trans_to_html: false,
247 formula_recognition_model: None,
248 formula_recognition_type: None,
249 formula_tokenizer_path: None,
250 formula_ort_session_config: None,
251 seal_text_detection_model: None,
252 text_detection_model: None,
253 text_line_orientation_model: None,
254 text_recognition_model: None,
255 character_dict_path: None,
256 region_model_name: None,
257 wired_table_structure_model_name: None,
258 wireless_table_structure_model_name: None,
259 wired_table_cell_model_name: None,
260 wireless_table_cell_model_name: None,
261 text_detection_model_name: None,
262 text_recognition_model_name: None,
263 ort_session_config: None,
264 layout_detection_config: None,
265 table_classification_config: None,
266 table_cell_detection_config: None,
267 table_structure_recognition_config: None,
268 formula_recognition_config: None,
269 text_detection_config: None,
270 text_recognition_config: None,
271 image_batch_size: None,
272 region_batch_size: None,
273 }
274 }
275
276 pub fn ort_session(mut self, config: OrtSessionConfig) -> Self {
280 self.ort_session_config = Some(config);
281 self
282 }
283
284 pub fn layout_detection_config(mut self, config: LayoutDetectionConfig) -> Self {
286 self.layout_detection_config = Some(config);
287 self
288 }
289
290 pub fn layout_model_name(mut self, name: impl Into<String>) -> Self {
306 self.layout_model_name = Some(name.into());
307 self
308 }
309
310 pub fn region_model_name(mut self, name: impl Into<String>) -> Self {
315 self.region_model_name = Some(name.into());
316 self
317 }
318
319 pub fn wired_table_structure_model_name(mut self, name: impl Into<String>) -> Self {
326 self.wired_table_structure_model_name = Some(name.into());
327 self
328 }
329
330 pub fn wireless_table_structure_model_name(mut self, name: impl Into<String>) -> Self {
334 self.wireless_table_structure_model_name = Some(name.into());
335 self
336 }
337
338 pub fn wired_table_cell_model_name(mut self, name: impl Into<String>) -> Self {
341 self.wired_table_cell_model_name = Some(name.into());
342 self
343 }
344
345 pub fn wireless_table_cell_model_name(mut self, name: impl Into<String>) -> Self {
348 self.wireless_table_cell_model_name = Some(name.into());
349 self
350 }
351
352 pub fn text_detection_model_name(mut self, name: impl Into<String>) -> Self {
356 self.text_detection_model_name = Some(name.into());
357 self
358 }
359
360 pub fn text_recognition_model_name(mut self, name: impl Into<String>) -> Self {
364 self.text_recognition_model_name = Some(name.into());
365 self
366 }
367
368 pub fn image_batch_size(mut self, size: usize) -> Self {
374 self.image_batch_size = Some(size);
375 self
376 }
377
378 pub fn region_batch_size(mut self, size: usize) -> Self {
385 self.region_batch_size = Some(size);
386 self
387 }
388
389 pub fn with_document_orientation(mut self, model_source: impl Into<ModelSource>) -> Self {
394 self.document_orientation_model = Some(model_source.into());
395 self
396 }
397
398 pub fn with_document_rectification(mut self, model_source: impl Into<ModelSource>) -> Self {
403 self.document_rectification_model = Some(model_source.into());
404 self
405 }
406
407 pub fn with_region_detection(mut self, model_source: impl Into<ModelSource>) -> Self {
420 self.region_detection_model = Some(model_source.into());
421 self
422 }
423
424 pub fn with_seal_text_detection(mut self, model_source: impl Into<ModelSource>) -> Self {
429 self.seal_text_detection_model = Some(model_source.into());
430 self
431 }
432
433 pub fn with_table_classification(mut self, model_source: impl Into<ModelSource>) -> Self {
437 self.table_classification_model = Some(model_source.into());
438 self
439 }
440
441 pub fn table_classification_config(mut self, config: TableClassificationConfig) -> Self {
443 self.table_classification_config = Some(config);
444 self
445 }
446
447 pub fn with_table_orientation(mut self, model_source: impl Into<ModelSource>) -> Self {
457 self.table_orientation_model = Some(model_source.into());
458 self
459 }
460
461 pub fn use_e2e_wired_table_rec(mut self, enabled: bool) -> Self {
469 self.use_e2e_wired_table_rec = enabled;
470 self
471 }
472
473 pub fn use_e2e_wireless_table_rec(mut self, enabled: bool) -> Self {
481 self.use_e2e_wireless_table_rec = enabled;
482 self
483 }
484
485 pub fn use_wired_table_cells_trans_to_html(mut self, enabled: bool) -> Self {
490 self.use_wired_table_cells_trans_to_html = enabled;
491 self
492 }
493
494 pub fn use_wireless_table_cells_trans_to_html(mut self, enabled: bool) -> Self {
499 self.use_wireless_table_cells_trans_to_html = enabled;
500 self
501 }
502
503 pub fn with_table_cell_detection(
510 mut self,
511 model_source: impl Into<ModelSource>,
512 cell_type: impl AsRef<str>,
513 ) -> Self {
514 self.table_cell_detection_model = Some(model_source.into());
515 self.table_cell_detection_type = Some(cell_type.as_ref().parse());
516 self
517 }
518
519 pub fn table_cell_detection_config(mut self, config: TableCellDetectionConfig) -> Self {
521 self.table_cell_detection_config = Some(config);
522 self
523 }
524
525 pub fn with_table_structure_recognition(
534 mut self,
535 model_source: impl Into<ModelSource>,
536 table_type: impl AsRef<str>,
537 ) -> Self {
538 self.table_structure_recognition_model = Some(model_source.into());
539 self.table_structure_recognition_type = Some(table_type.as_ref().parse());
540 self
541 }
542
543 pub fn table_structure_dict_path(mut self, path: impl Into<PathBuf>) -> Self {
550 self.table_structure_dict_path = Some(path.into());
551 self
552 }
553
554 pub fn table_structure_recognition_config(
556 mut self,
557 config: TableStructureRecognitionConfig,
558 ) -> Self {
559 self.table_structure_recognition_config = Some(config);
560 self
561 }
562
563 pub fn with_wired_table_structure(mut self, model_source: impl Into<ModelSource>) -> Self {
568 self.wired_table_structure_model = Some(model_source.into());
569 self
570 }
571
572 pub fn with_wireless_table_structure(mut self, model_source: impl Into<ModelSource>) -> Self {
577 self.wireless_table_structure_model = Some(model_source.into());
578 self
579 }
580
581 pub fn with_wired_table_cell_detection(mut self, model_source: impl Into<ModelSource>) -> Self {
586 self.wired_table_cell_model = Some(model_source.into());
587 self
588 }
589
590 pub fn with_wireless_table_cell_detection(
595 mut self,
596 model_source: impl Into<ModelSource>,
597 ) -> Self {
598 self.wireless_table_cell_model = Some(model_source.into());
599 self
600 }
601
602 pub fn with_formula_recognition(
613 mut self,
614 model_source: impl Into<ModelSource>,
615 tokenizer_path: impl Into<PathBuf>,
616 model_type: impl AsRef<str>,
617 ) -> Self {
618 self.formula_recognition_model = Some(model_source.into());
619 self.formula_tokenizer_path = Some(tokenizer_path.into());
620 self.formula_recognition_type = Some(model_type.as_ref().parse());
621 self
622 }
623
624 pub fn formula_recognition_config(mut self, config: FormulaRecognitionConfig) -> Self {
626 self.formula_recognition_config = Some(config);
627 self
628 }
629
630 pub fn formula_ort_session(mut self, config: OrtSessionConfig) -> Self {
632 self.formula_ort_session_config = Some(config);
633 self
634 }
635
636 pub fn with_ocr(
644 mut self,
645 text_detection_model: impl Into<ModelSource>,
646 text_recognition_model: impl Into<ModelSource>,
647 character_dict_path: impl Into<PathBuf>,
648 ) -> Self {
649 self.text_detection_model = Some(text_detection_model.into());
650 self.text_recognition_model = Some(text_recognition_model.into());
651 self.character_dict_path = Some(character_dict_path.into());
652 self
653 }
654
655 pub fn with_text_line_orientation(mut self, model_source: impl Into<ModelSource>) -> Self {
666 self.text_line_orientation_model = Some(model_source.into());
667 self
668 }
669
670 pub fn text_detection_config(mut self, config: TextDetectionConfig) -> Self {
672 self.text_detection_config = Some(config);
673 self
674 }
675
676 pub fn text_recognition_config(mut self, config: TextRecognitionConfig) -> Self {
678 self.text_recognition_config = Some(config);
679 self
680 }
681
682 pub fn build(mut self) -> Result<OARStructure, OCRError> {
686 for (component, selection) in [
687 ("table_cell_detection", &self.table_cell_detection_type),
688 (
689 "table_structure_recognition",
690 &self.table_structure_recognition_type,
691 ),
692 ] {
693 match selection {
694 Some(Err(message)) => {
695 return Err(OCRError::config_error_detailed(component, message.clone()));
696 }
697 Some(Ok(TableType::Unknown)) => {
698 return Err(OCRError::config_error_detailed(
699 component,
700 "'unknown' is not valid when selecting a table model".to_string(),
701 ));
702 }
703 _ => {}
704 }
705 }
706 if let Some(Err(message)) = self.formula_recognition_type.as_ref() {
707 return Err(OCRError::config_error_detailed(
708 "formula_recognition",
709 message.clone(),
710 ));
711 }
712
713 if let Some(size) = self.image_batch_size {
714 Self::validate_batch_size("image_batch_size", size)?;
715 }
716 if let Some(size) = self.region_batch_size {
717 Self::validate_batch_size("region_batch_size", size)?;
718 }
719
720 for config in [
727 self.ort_session_config.as_mut(),
728 self.formula_ort_session_config.as_mut(),
729 ]
730 .into_iter()
731 .flatten()
732 {
733 default_arena_shrinkage(config);
734 }
735
736 if self.formula_recognition_model.is_some() {
743 use oar_ocr_core::core::config::OrtExecutionProvider;
744 let uses_cuda = self
745 .formula_ort_session_config
746 .as_ref()
747 .or(self.ort_session_config.as_ref())
748 .and_then(|cfg| cfg.execution_providers.as_ref())
749 .is_some_and(|eps| {
750 eps.iter().any(|ep| {
751 matches!(
752 ep,
753 OrtExecutionProvider::CUDA { .. }
754 | OrtExecutionProvider::TensorRT { .. }
755 )
756 })
757 });
758 if uses_cuda {
759 oar_ocr_core::core::inference::ensure_cuda_launch_blocking();
760 }
761 }
762
763 self.layout_detection_model = resolve_model_source(&self.layout_detection_model)?;
767 fn resolve_opt_path(p: &mut Option<PathBuf>) -> Result<(), OCRError> {
768 if let Some(path) = p {
769 *path = resolve_model_path(path)?;
770 }
771 Ok(())
772 }
773 fn resolve_opt_source(s: &mut Option<ModelSource>) -> Result<(), OCRError> {
774 if let Some(source) = s {
775 *source = resolve_model_source(source)?;
776 }
777 Ok(())
778 }
779 resolve_opt_source(&mut self.document_orientation_model)?;
780 resolve_opt_source(&mut self.document_rectification_model)?;
781 resolve_opt_source(&mut self.region_detection_model)?;
782 resolve_opt_source(&mut self.table_classification_model)?;
783 resolve_opt_source(&mut self.table_orientation_model)?;
784 resolve_opt_source(&mut self.table_cell_detection_model)?;
785 resolve_opt_source(&mut self.table_structure_recognition_model)?;
786 resolve_opt_path(&mut self.table_structure_dict_path)?;
787 resolve_opt_source(&mut self.wired_table_structure_model)?;
788 resolve_opt_source(&mut self.wireless_table_structure_model)?;
789 resolve_opt_source(&mut self.wired_table_cell_model)?;
790 resolve_opt_source(&mut self.wireless_table_cell_model)?;
791 resolve_opt_source(&mut self.formula_recognition_model)?;
792 resolve_opt_path(&mut self.formula_tokenizer_path)?;
793 resolve_opt_source(&mut self.seal_text_detection_model)?;
794 resolve_opt_source(&mut self.text_detection_model)?;
795 resolve_opt_source(&mut self.text_line_orientation_model)?;
796 resolve_opt_source(&mut self.text_recognition_model)?;
797 resolve_opt_path(&mut self.character_dict_path)?;
798
799 let cpu_region_batch_size = default_cpu_region_batch_size(
800 self.text_recognition_model.as_ref(),
801 self.text_recognition_model_name.as_deref(),
802 );
803 (self.image_batch_size, self.region_batch_size) = resolve_device_batch_sizes(
804 self.ort_session_config.as_ref(),
805 self.image_batch_size,
806 self.region_batch_size,
807 1,
808 cpu_region_batch_size,
809 );
810
811 let char_dict = if let Some(ref dict_path) = self.character_dict_path {
813 Some(
814 std::fs::read_to_string(dict_path).map_err(|e| OCRError::InvalidInput {
815 message: format!(
816 "Failed to read character dictionary from '{}': {}",
817 dict_path.display(),
818 e
819 ),
820 })?,
821 )
822 } else {
823 None
824 };
825
826 let document_orientation_adapter = build_optional_adapter(
828 self.document_orientation_model.as_ref(),
829 self.ort_session_config.as_ref(),
830 DocumentOrientationAdapterBuilder::new,
831 )?;
832
833 let rectification_adapter = build_optional_adapter(
835 self.document_rectification_model.as_ref(),
836 self.ort_session_config.as_ref(),
837 UVDocRectifierAdapterBuilder::new,
838 )?;
839
840 let mut layout_builder = LayoutDetectionAdapterBuilder::new();
842
843 let layout_model_config = if let Some(name) = &self.layout_model_name {
845 use oar_ocr_core::domain::adapters::LayoutModelConfig;
846 match name.to_lowercase().replace('-', "_").as_str() {
851 "picodet_layout_1x" => LayoutModelConfig::picodet_layout_1x(),
852 "picodet_layout_1x_table" => LayoutModelConfig::picodet_layout_1x_table(),
853 "picodet_s_layout_3cls" => LayoutModelConfig::picodet_s_layout_3cls(),
854 "picodet_l_layout_3cls" => LayoutModelConfig::picodet_l_layout_3cls(),
855 "picodet_s_layout_17cls" => LayoutModelConfig::picodet_s_layout_17cls(),
856 "picodet_l_layout_17cls" => LayoutModelConfig::picodet_l_layout_17cls(),
857 "rt_detr_h_layout_3cls" => LayoutModelConfig::rtdetr_h_layout_3cls(),
858 "rt_detr_h_layout_17cls" => LayoutModelConfig::rtdetr_h_layout_17cls(),
859 "pp_docblocklayout" => LayoutModelConfig::pp_docblocklayout(),
860 "pp_doclayout_s" => LayoutModelConfig::pp_doclayout_s(),
861 "pp_doclayout_m" => LayoutModelConfig::pp_doclayout_m(),
862 "pp_doclayout_l" => LayoutModelConfig::pp_doclayout_l(),
863 "pp_doclayout_plus_l" => LayoutModelConfig::pp_doclayout_plus_l(),
864 _ => {
865 tracing::warn!(
866 requested = %name,
867 "Unknown --layout-model-name preset; falling back to PP-DocLayout_plus-L. \
868 This may apply the wrong class labels/preprocessing for your model."
869 );
870 LayoutModelConfig::pp_doclayout_plus_l()
871 }
872 }
873 } else {
874 crate::domain::adapters::LayoutModelConfig::pp_doclayout_plus_l()
876 };
877
878 layout_builder = layout_builder.model_config(layout_model_config);
879
880 let effective_layout_cfg = self
882 .layout_detection_config
883 .clone()
884 .unwrap_or_else(LayoutDetectionConfig::with_pp_structurev3_defaults);
885 layout_builder = layout_builder.with_config(effective_layout_cfg);
886
887 if let Some(ref ort_config) = self.ort_session_config {
888 layout_builder = layout_builder.with_ort_config(ort_config.clone());
889 }
890
891 let layout_detection_adapter = layout_builder.build(&self.layout_detection_model)?;
892
893 let region_detection_adapter = if let Some(ref model_path) = self.region_detection_model {
895 use oar_ocr_core::domain::adapters::LayoutModelConfig;
896 let mut region_builder = LayoutDetectionAdapterBuilder::new();
897
898 let region_model_config = if let Some(ref name) = self.region_model_name {
900 match name.to_lowercase().replace("-", "_").as_str() {
901 "pp_docblocklayout" => LayoutModelConfig::pp_docblocklayout(),
902 _ => LayoutModelConfig::pp_docblocklayout(),
903 }
904 } else {
905 LayoutModelConfig::pp_docblocklayout()
906 };
907 region_builder = region_builder.model_config(region_model_config);
908
909 let mut region_cfg = LayoutDetectionConfig::default();
911 let mut merge_modes = std::collections::HashMap::new();
912 merge_modes.insert(
913 "region".to_string(),
914 crate::domain::tasks::layout_detection::MergeBboxMode::Small,
915 );
916 region_cfg.class_merge_modes = Some(merge_modes);
917 region_builder = region_builder.with_config(region_cfg);
918
919 if let Some(ref ort_config) = self.ort_session_config {
920 region_builder = region_builder.with_ort_config(ort_config.clone());
921 }
922
923 Some(region_builder.build(model_path)?)
924 } else {
925 None
926 };
927
928 let table_classification_adapter =
930 if let Some(ref model_path) = self.table_classification_model {
931 let mut builder = TableClassificationAdapterBuilder::new();
932
933 if let Some(ref config) = self.table_classification_config {
934 builder = builder.with_config(config.clone());
935 }
936
937 if let Some(ref ort_config) = self.ort_session_config {
938 builder = builder.with_ort_config(ort_config.clone());
939 }
940
941 Some(builder.build(model_path)?)
942 } else {
943 None
944 };
945
946 let table_orientation_adapter = build_optional_adapter(
949 self.table_orientation_model.as_ref(),
950 self.ort_session_config.as_ref(),
951 DocumentOrientationAdapterBuilder::new,
952 )?;
953
954 let table_cell_detection_adapter = if let Some(ref model_path) =
956 self.table_cell_detection_model
957 {
958 let cell_type = match self.table_cell_detection_type.as_ref() {
959 Some(Ok(cell_type)) => *cell_type,
960 Some(Err(message)) => {
961 return Err(OCRError::config_error_detailed(
962 "table_cell_detection",
963 message.clone(),
964 ));
965 }
966 None => TableType::Wired,
967 };
968
969 use oar_ocr_core::domain::adapters::table_cell_detection_adapter::TableCellModelConfig;
970
971 let model_config = match cell_type {
972 TableType::Wired => TableCellModelConfig::rtdetr_l_wired_table_cell_det(),
973 TableType::Wireless => TableCellModelConfig::rtdetr_l_wireless_table_cell_det(),
974 TableType::Unknown => {
975 return Err(OCRError::config_error_detailed(
976 "table_cell_detection",
977 "TableType::Unknown is not valid for a table cell detector".to_string(),
978 ));
979 }
980 };
981
982 let mut builder = TableCellDetectionAdapterBuilder::new().model_config(model_config);
983
984 if let Some(ref config) = self.table_cell_detection_config {
985 builder = builder.with_config(config.clone());
986 }
987
988 if let Some(ref ort_config) = self.ort_session_config {
989 builder = builder.with_ort_config(ort_config.clone());
990 }
991
992 Some(builder.build(model_path)?)
993 } else {
994 None
995 };
996
997 let table_structure_recognition_adapter = if let Some(ref model_path) =
999 self.table_structure_recognition_model
1000 {
1001 let table_type = match self.table_structure_recognition_type.as_ref() {
1002 Some(Ok(table_type)) => *table_type,
1003 Some(Err(message)) => {
1004 return Err(OCRError::config_error_detailed(
1005 "table_structure_recognition",
1006 message.clone(),
1007 ));
1008 }
1009 None => TableType::Wired,
1010 };
1011 let dict_path = self
1012 .table_structure_dict_path
1013 .clone()
1014 .ok_or_else(|| {
1015 OCRError::config_error_detailed(
1016 "table_structure_recognition",
1017 "Dictionary path is required. Call table_structure_dict_path() when enabling table structure recognition.".to_string(),
1018 )
1019 })?;
1020
1021 let adapter: TableStructureRecognitionAdapter = match table_type {
1022 TableType::Wired => {
1023 let mut builder = SLANetWiredAdapterBuilder::new().dict_path(dict_path.clone());
1024
1025 if let Some(ref config) = self.table_structure_recognition_config {
1026 builder = builder.with_config(config.clone());
1027 }
1028
1029 if let Some(ref ort_config) = self.ort_session_config {
1030 builder = builder.with_ort_config(ort_config.clone());
1031 }
1032
1033 builder.build(model_path)?
1034 }
1035 TableType::Wireless => {
1036 let mut builder =
1037 SLANetWirelessAdapterBuilder::new().dict_path(dict_path.clone());
1038
1039 if let Some(ref config) = self.table_structure_recognition_config {
1040 builder = builder.with_config(config.clone());
1041 }
1042
1043 if let Some(ref ort_config) = self.ort_session_config {
1044 builder = builder.with_ort_config(ort_config.clone());
1045 }
1046
1047 builder.build(model_path)?
1048 }
1049 TableType::Unknown => {
1050 return Err(OCRError::config_error_detailed(
1051 "table_structure_recognition",
1052 "TableType::Unknown is not valid for table structure recognition"
1053 .to_string(),
1054 ));
1055 }
1056 };
1057
1058 Some(adapter)
1059 } else {
1060 None
1061 };
1062
1063 let wired_table_structure_adapter = if let Some(ref model_path) =
1065 self.wired_table_structure_model
1066 {
1067 let dict_path = self.table_structure_dict_path.clone().ok_or_else(|| {
1068 OCRError::config_error_detailed(
1069 "wired_table_structure",
1070 "Dictionary path is required. Call table_structure_dict_path() when enabling table structure recognition.".to_string(),
1071 )
1072 })?;
1073
1074 let mut builder = SLANetWiredAdapterBuilder::new().dict_path(dict_path);
1075
1076 if let Some(ref name) = self.wired_table_structure_model_name {
1080 builder = builder.model_name(name.clone());
1081 }
1082
1083 if let Some(ref config) = self.table_structure_recognition_config {
1084 builder = builder.with_config(config.clone());
1085 }
1086
1087 if let Some(ref ort_config) = self.ort_session_config {
1088 builder = builder.with_ort_config(ort_config.clone());
1089 }
1090
1091 Some(builder.build(model_path)?)
1092 } else {
1093 None
1094 };
1095
1096 let wireless_table_structure_adapter = if let Some(ref model_path) =
1097 self.wireless_table_structure_model
1098 {
1099 let dict_path = self.table_structure_dict_path.clone().ok_or_else(|| {
1100 OCRError::config_error_detailed(
1101 "wireless_table_structure",
1102 "Dictionary path is required. Call table_structure_dict_path() when enabling table structure recognition.".to_string(),
1103 )
1104 })?;
1105
1106 let mut builder = SLANetWirelessAdapterBuilder::new().dict_path(dict_path);
1107
1108 if let Some(ref name) = self.wireless_table_structure_model_name {
1109 builder = builder.model_name(name.clone());
1110 }
1111
1112 if let Some(ref config) = self.table_structure_recognition_config {
1113 builder = builder.with_config(config.clone());
1114 }
1115
1116 if let Some(ref ort_config) = self.ort_session_config {
1117 builder = builder.with_ort_config(ort_config.clone());
1118 }
1119
1120 Some(builder.build(model_path)?)
1121 } else {
1122 None
1123 };
1124
1125 let wired_table_cell_adapter = if let Some(ref model_path) = self.wired_table_cell_model {
1127 use oar_ocr_core::domain::adapters::table_cell_detection_adapter::TableCellModelConfig;
1128
1129 let mut model_config = TableCellModelConfig::rtdetr_l_wired_table_cell_det();
1130 if let Some(ref name) = self.wired_table_cell_model_name {
1132 model_config.model_name = name.clone();
1133 }
1134 let mut builder = TableCellDetectionAdapterBuilder::new().model_config(model_config);
1135
1136 if let Some(ref config) = self.table_cell_detection_config {
1137 builder = builder.with_config(config.clone());
1138 }
1139
1140 if let Some(ref ort_config) = self.ort_session_config {
1141 builder = builder.with_ort_config(ort_config.clone());
1142 }
1143
1144 Some(builder.build(model_path)?)
1145 } else {
1146 None
1147 };
1148
1149 let wireless_table_cell_adapter = if let Some(ref model_path) =
1150 self.wireless_table_cell_model
1151 {
1152 use oar_ocr_core::domain::adapters::table_cell_detection_adapter::TableCellModelConfig;
1153
1154 let mut model_config = TableCellModelConfig::rtdetr_l_wireless_table_cell_det();
1155 if let Some(ref name) = self.wireless_table_cell_model_name {
1156 model_config.model_name = name.clone();
1157 }
1158 let mut builder = TableCellDetectionAdapterBuilder::new().model_config(model_config);
1159
1160 if let Some(ref config) = self.table_cell_detection_config {
1161 builder = builder.with_config(config.clone());
1162 }
1163
1164 if let Some(ref ort_config) = self.ort_session_config {
1165 builder = builder.with_ort_config(ort_config.clone());
1166 }
1167
1168 Some(builder.build(model_path)?)
1169 } else {
1170 None
1171 };
1172
1173 let formula_recognition_adapter =
1175 if let Some(ref model_path) = self.formula_recognition_model {
1176 let tokenizer_path = self.formula_tokenizer_path.as_ref().ok_or_else(|| {
1177 OCRError::config_error_detailed(
1178 "formula_recognition",
1179 "Tokenizer path is required for formula recognition".to_string(),
1180 )
1181 })?;
1182
1183 let model_type = match self.formula_recognition_type.as_ref() {
1184 Some(Ok(model_type)) => *model_type,
1185 Some(Err(message)) => {
1186 return Err(OCRError::config_error_detailed(
1187 "formula_recognition",
1188 message.clone(),
1189 ));
1190 }
1191 None => {
1192 return Err(OCRError::config_error_detailed(
1193 "formula_recognition",
1194 "Formula model kind is required".to_string(),
1195 ));
1196 }
1197 };
1198
1199 let adapter: FormulaRecognitionAdapter = match model_type {
1200 FormulaModelKind::PPFormulaNet => {
1201 let mut builder = PPFormulaNetAdapterBuilder::new();
1202
1203 builder = builder.tokenizer_path(tokenizer_path);
1204
1205 if let Some(ref config) = self.formula_recognition_config {
1206 builder = builder.task_config(config.clone());
1207 }
1208
1209 if let Some(ort_config) = self
1210 .formula_ort_session_config
1211 .as_ref()
1212 .or(self.ort_session_config.as_ref())
1213 {
1214 builder = builder.with_ort_config(ort_config.clone());
1215 }
1216
1217 builder.build(model_path)?
1218 }
1219 FormulaModelKind::UniMERNet => {
1220 let mut builder = UniMERNetAdapterBuilder::new();
1221
1222 builder = builder.tokenizer_path(tokenizer_path);
1223
1224 if let Some(ref config) = self.formula_recognition_config {
1225 builder = builder.task_config(config.clone());
1226 }
1227
1228 if let Some(ort_config) = self
1229 .formula_ort_session_config
1230 .as_ref()
1231 .or(self.ort_session_config.as_ref())
1232 {
1233 builder = builder.with_ort_config(ort_config.clone());
1234 }
1235
1236 builder.build(model_path)?
1237 }
1238 };
1239
1240 Some(adapter)
1241 } else {
1242 None
1243 };
1244
1245 let seal_text_detection_adapter =
1247 if let Some(ref model_path) = self.seal_text_detection_model {
1248 let mut builder = SealTextDetectionAdapterBuilder::new();
1249
1250 if let Some(ref ort_config) = self.ort_session_config {
1251 builder = builder.with_ort_config(ort_config.clone());
1252 }
1253
1254 Some(builder.build(model_path)?)
1255 } else {
1256 None
1257 };
1258
1259 let text_detection_adapter = if let Some(ref model_path) = self.text_detection_model {
1268 let mut builder = TextDetectionAdapterBuilder::new();
1269
1270 let mut effective_cfg = self.text_detection_config.clone().unwrap_or_default();
1271
1272 let has_table_pipeline = self.table_classification_model.is_some()
1275 || self.table_structure_recognition_model.is_some()
1276 || self.wired_table_structure_model.is_some()
1277 || self.wireless_table_structure_model.is_some()
1278 || self.table_cell_detection_model.is_some()
1279 || self.wired_table_cell_model.is_some()
1280 || self.wireless_table_cell_model.is_some();
1281 if self.text_detection_config.is_none() && has_table_pipeline {
1282 effective_cfg.box_threshold = 0.4;
1283 }
1284
1285 if effective_cfg.limit_side_len.is_none() {
1286 effective_cfg.limit_side_len = Some(736);
1287 }
1288 if effective_cfg.limit_type.is_none() {
1289 effective_cfg.limit_type = Some(crate::processors::LimitType::Min);
1290 }
1291 if effective_cfg.max_side_len.is_none() {
1292 effective_cfg.max_side_len = Some(4000);
1293 }
1294 builder = builder.with_config(effective_cfg);
1295
1296 if let Some(ref name) = self.text_detection_model_name {
1300 builder = builder.model_name(name.clone());
1301 }
1302
1303 if let Some(ref ort_config) = self.ort_session_config {
1304 builder = builder.with_ort_config(ort_config.clone());
1305 }
1306
1307 Some(builder.build(model_path)?)
1308 } else {
1309 None
1310 };
1311
1312 let text_line_orientation_adapter =
1314 if let Some(ref model_path) = self.text_line_orientation_model {
1315 let mut builder = TextLineOrientationAdapterBuilder::new();
1316
1317 if let Some(ref ort_config) = self.ort_session_config {
1318 builder = builder.with_ort_config(ort_config.clone());
1319 }
1320
1321 Some(builder.build(model_path)?)
1322 } else {
1323 None
1324 };
1325
1326 let text_recognition_adapter = if let Some(ref model_path) = self.text_recognition_model {
1328 let dict = char_dict.ok_or_else(|| OCRError::InvalidInput {
1329 message: "Character dictionary is required for text recognition".to_string(),
1330 })?;
1331
1332 let char_vec: Vec<String> = dict.lines().map(|s| s.to_string()).collect();
1334
1335 let mut builder = TextRecognitionAdapterBuilder::new().character_dict(char_vec);
1336
1337 if let Some(ref config) = self.text_recognition_config {
1338 builder = builder.with_config(config.clone());
1339 }
1340
1341 if let Some(ref name) = self.text_recognition_model_name {
1345 builder = builder.model_name(name.clone());
1346 }
1347
1348 if let Some(ref ort_config) = self.ort_session_config {
1349 builder = builder.with_ort_config(ort_config.clone());
1350 }
1351
1352 Some(builder.build(model_path)?)
1353 } else {
1354 None
1355 };
1356
1357 let pipeline = StructurePipeline {
1358 document_orientation_adapter,
1359 rectification_adapter,
1360 layout_detection_adapter,
1361 region_detection_adapter,
1362 table_classification_adapter,
1363 table_orientation_adapter,
1364 table_cell_detection_adapter,
1365 table_structure_recognition_adapter,
1366 wired_table_structure_adapter,
1367 wireless_table_structure_adapter,
1368 wired_table_cell_adapter,
1369 wireless_table_cell_adapter,
1370 use_e2e_wired_table_rec: self.use_e2e_wired_table_rec,
1371 use_e2e_wireless_table_rec: self.use_e2e_wireless_table_rec,
1372 use_wired_table_cells_trans_to_html: self.use_wired_table_cells_trans_to_html,
1373 use_wireless_table_cells_trans_to_html: self.use_wireless_table_cells_trans_to_html,
1374 formula_recognition_adapter,
1375 seal_text_detection_adapter,
1376 text_detection_adapter,
1377 text_line_orientation_adapter,
1378 text_recognition_adapter,
1379 image_batch_size: self.image_batch_size,
1380 region_batch_size: self.region_batch_size,
1381 };
1382
1383 Ok(OARStructure { pipeline })
1384 }
1385
1386 fn validate_batch_size(field: &str, size: usize) -> Result<(), OCRError> {
1387 if size == 0 || size > Self::MAX_BATCH_SIZE {
1388 return Err(OCRError::validation_error(
1389 "OARStructureBuilder",
1390 field,
1391 &format!("1..={}", Self::MAX_BATCH_SIZE),
1392 &size.to_string(),
1393 ));
1394 }
1395
1396 Ok(())
1397 }
1398}
1399
1400#[derive(Debug)]
1404pub struct OARStructure {
1405 pipeline: StructurePipeline,
1406}
1407
1408struct PreparedPage {
1411 current_image: std::sync::Arc<image::RgbImage>,
1412 orientation_angle: Option<f32>,
1413 rectified_img: Option<std::sync::Arc<image::RgbImage>>,
1414 rotation: Option<crate::oarocr::preprocess::OrientationCorrection>,
1415 layout_elements: Vec<crate::domain::structure::LayoutElement>,
1416 detected_region_blocks: Option<Vec<crate::domain::structure::RegionBlock>>,
1417 precomputed_text_regions: Option<Vec<crate::oarocr::TextRegion>>,
1418}
1419
1420impl OARStructure {
1421 fn finish_layout_elements(layout_elements: &mut Vec<crate::domain::structure::LayoutElement>) {
1422 if layout_elements.len() > 1 {
1423 let removed = crate::domain::structure::remove_overlapping_layout_elements(
1424 layout_elements,
1425 LAYOUT_OVERLAP_IOU_THRESHOLD,
1426 );
1427 if removed > 0 {
1428 tracing::info!(
1429 "Removing {} overlapping layout elements (threshold={})",
1430 removed,
1431 LAYOUT_OVERLAP_IOU_THRESHOLD
1432 );
1433 }
1434 }
1435
1436 crate::domain::structure::apply_standardized_layout_label_fixes(layout_elements);
1437 }
1438
1439 fn layout_elements_from_detection(
1440 elements: &[oar_ocr_core::domain::tasks::LayoutDetectionElement],
1441 ) -> Vec<crate::domain::structure::LayoutElement> {
1442 use oar_ocr_core::domain::structure::LayoutElementType;
1443
1444 elements
1445 .iter()
1446 .map(|element| {
1447 let element_type_enum = LayoutElementType::from_label(&element.element_type);
1448 crate::domain::structure::LayoutElement::new(
1449 element.bbox.clone(),
1450 element_type_enum,
1451 element.score,
1452 )
1453 .with_label(element.element_type.clone())
1454 })
1455 .collect()
1456 }
1457
1458 fn refine_overall_ocr_with_layout(
1469 text_regions: &mut Vec<crate::oarocr::TextRegion>,
1470 layout_elements: &[crate::domain::structure::LayoutElement],
1471 region_blocks: Option<&[crate::domain::structure::RegionBlock]>,
1472 page_image: &image::RgbImage,
1473 text_recognition_adapter: &TextRecognitionAdapter,
1474 region_batch_size: usize,
1475 ) -> Result<(), OCRError> {
1476 use oar_ocr_core::core::traits::task::ImageTaskInput;
1477 use oar_ocr_core::domain::structure::LayoutElementType;
1478 use oar_ocr_core::processors::BoundingBox;
1479 use oar_ocr_core::utils::BBoxCrop;
1480
1481 if text_regions.is_empty() || layout_elements.is_empty() {
1482 return Ok(());
1483 }
1484
1485 fn aabb_intersection(b1: &BoundingBox, b2: &BoundingBox) -> Option<BoundingBox> {
1486 let x1 = b1.x_min().max(b2.x_min());
1487 let y1 = b1.y_min().max(b2.y_min());
1488 let x2 = b1.x_max().min(b2.x_max());
1489 let y2 = b1.y_max().min(b2.y_max());
1490 if x2 - x1 <= 1.0 || y2 - y1 <= 1.0 {
1491 None
1492 } else {
1493 Some(BoundingBox::from_coords(x1, y1, x2, y2))
1494 }
1495 }
1496
1497 let is_excluded_layout = |t: LayoutElementType| {
1499 matches!(
1500 t,
1501 LayoutElementType::Formula
1502 | LayoutElementType::FormulaNumber
1503 | LayoutElementType::Table
1504 | LayoutElementType::Seal
1505 )
1506 };
1507
1508 let min_pixels = 3.0;
1512 let mut matched_ocr: Vec<Vec<usize>> = vec![Vec::new(); text_regions.len()];
1513 for (ocr_idx, region) in text_regions.iter().enumerate() {
1514 for (layout_idx, elem) in layout_elements.iter().enumerate() {
1515 if is_excluded_layout(elem.element_type) {
1516 continue;
1517 }
1518 let inter_x_min = region.bounding_box.x_min().max(elem.bbox.x_min());
1519 let inter_y_min = region.bounding_box.y_min().max(elem.bbox.y_min());
1520 let inter_x_max = region.bounding_box.x_max().min(elem.bbox.x_max());
1521 let inter_y_max = region.bounding_box.y_max().min(elem.bbox.y_max());
1522 if inter_x_max - inter_x_min > min_pixels && inter_y_max - inter_y_min > min_pixels
1523 {
1524 matched_ocr[ocr_idx].push(layout_idx);
1525 }
1526 }
1527 }
1528
1529 let mut appended_regions: Vec<crate::oarocr::TextRegion> = Vec::new();
1531 let original_ocr_len = text_regions.len();
1532 let mut multi_layout_ocr_count = 0usize;
1533 let mut multi_layout_crop_count = 0usize;
1534
1535 for ocr_idx in 0..original_ocr_len {
1536 let layout_ids = matched_ocr[ocr_idx].clone();
1537 if layout_ids.len() <= 1 {
1538 continue;
1539 }
1540 multi_layout_ocr_count += 1;
1541
1542 let ocr_box = text_regions[ocr_idx].bounding_box.clone();
1543
1544 let mut crops: Vec<image::RgbImage> = Vec::new();
1545 let mut crop_boxes: Vec<(BoundingBox, bool)> = Vec::new(); for (j, layout_idx) in layout_ids.iter().enumerate() {
1548 let layout_box = &layout_elements[*layout_idx].bbox;
1549 let Some(crop_box) = aabb_intersection(&ocr_box, layout_box) else {
1550 continue;
1551 };
1552
1553 for (other_idx, other_region) in text_regions.iter_mut().enumerate() {
1555 if other_idx == ocr_idx {
1556 continue;
1557 }
1558 if other_region.bounding_box.iou(&crop_box) > 0.8 {
1559 other_region.text = None;
1560 }
1561 }
1562
1563 if let Ok(crop_img) = BBoxCrop::crop_bounding_box(page_image, &crop_box) {
1564 crops.push(crop_img);
1565 crop_boxes.push((crop_box, j == 0));
1566 }
1567 }
1568 multi_layout_crop_count += crop_boxes.len();
1569
1570 if crops.is_empty() {
1571 continue;
1572 }
1573
1574 let mut rec_texts: Vec<String> = Vec::with_capacity(crops.len());
1576 let mut rec_scores: Vec<f32> = Vec::with_capacity(crops.len());
1577
1578 for batch_start in (0..crops.len()).step_by(region_batch_size.max(1)) {
1579 let batch_end = (batch_start + region_batch_size).min(crops.len());
1580 let batch: Vec<_> = crops[batch_start..batch_end].to_vec();
1581 let rec_input = ImageTaskInput::new(batch);
1582 let rec_result = text_recognition_adapter.execute(rec_input, None)?;
1583 rec_texts.extend(rec_result.texts);
1584 rec_scores.extend(rec_result.scores);
1585 }
1586
1587 for ((crop_box, is_first), (text, score)) in crop_boxes
1588 .into_iter()
1589 .zip(rec_texts.into_iter().zip(rec_scores))
1590 {
1591 if text.is_empty() {
1592 continue;
1593 }
1594 if is_first {
1595 text_regions[ocr_idx].bounding_box = crop_box.clone();
1596 text_regions[ocr_idx].dt_poly = Some(crop_box.clone());
1597 text_regions[ocr_idx].rec_poly = Some(crop_box.clone());
1598 text_regions[ocr_idx].text = Some(Arc::from(text));
1599 text_regions[ocr_idx].confidence = Some(score);
1600 } else {
1601 appended_regions.push(crate::oarocr::TextRegion {
1602 bounding_box: crop_box.clone(),
1603 dt_poly: Some(crop_box.clone()),
1604 rec_poly: Some(crop_box),
1605 text: Some(Arc::from(text)),
1606 confidence: Some(score),
1607 orientation_angle: None,
1608 word_boxes: None,
1609 label: None,
1610 });
1611 }
1612 }
1613 }
1614
1615 if !appended_regions.is_empty() {
1616 text_regions.extend(appended_regions);
1617 }
1618
1619 let mut fallback_blocks = 0usize;
1622 for elem in layout_elements.iter() {
1623 if is_excluded_layout(elem.element_type) {
1624 continue;
1625 }
1626 if matches!(
1627 elem.element_type,
1628 LayoutElementType::Image | LayoutElementType::Chart
1629 ) {
1630 continue;
1631 }
1632
1633 let mut has_text = false;
1634 for region in text_regions.iter() {
1635 if !region.text.as_ref().map(|t| !t.is_empty()).unwrap_or(false) {
1636 continue;
1637 }
1638 let inter_x_min = region.bounding_box.x_min().max(elem.bbox.x_min());
1639 let inter_y_min = region.bounding_box.y_min().max(elem.bbox.y_min());
1640 let inter_x_max = region.bounding_box.x_max().min(elem.bbox.x_max());
1641 let inter_y_max = region.bounding_box.y_max().min(elem.bbox.y_max());
1642 if inter_x_max - inter_x_min > min_pixels && inter_y_max - inter_y_min > min_pixels
1643 {
1644 has_text = true;
1645 break;
1646 }
1647 }
1648
1649 if has_text {
1650 continue;
1651 }
1652 fallback_blocks += 1;
1653
1654 if let Ok(crop_img) = BBoxCrop::crop_bounding_box(page_image, &elem.bbox) {
1656 let rec_input = ImageTaskInput::new(vec![crop_img]);
1657 let rec_result = text_recognition_adapter.execute(rec_input, None)?;
1658 if let (Some(text), Some(score)) =
1659 (rec_result.texts.first(), rec_result.scores.first())
1660 && !text.is_empty()
1661 {
1662 let crop_box = elem.bbox.clone();
1663 text_regions.push(crate::oarocr::TextRegion {
1664 bounding_box: crop_box.clone(),
1665 dt_poly: Some(crop_box.clone()),
1666 rec_poly: Some(crop_box),
1667 text: Some(Arc::from(text.as_str())),
1668 confidence: Some(*score),
1669 orientation_angle: None,
1670 word_boxes: None,
1671 label: None,
1672 });
1673 }
1674 }
1675 }
1676
1677 tracing::info!(
1678 "overall OCR refine: multi-layout OCR boxes={}, crops={}, fallback layout blocks={}",
1679 multi_layout_ocr_count,
1680 multi_layout_crop_count,
1681 fallback_blocks
1682 );
1683
1684 let _ = region_blocks;
1687
1688 Ok(())
1689 }
1690
1691 fn split_ocr_bboxes_by_table_cells(
1699 tables: &[TableResult],
1700 text_regions: &mut Vec<crate::oarocr::TextRegion>,
1701 page_image: &image::RgbImage,
1702 text_recognition_adapter: &TextRecognitionAdapter,
1703 ) -> Result<(), OCRError> {
1704 use oar_ocr_core::core::traits::task::ImageTaskInput;
1705 use oar_ocr_core::processors::BoundingBox;
1706
1707 let mut cell_boxes: Vec<[f32; 4]> = Vec::new();
1709 for table in tables {
1710 for cell in &table.cells {
1711 let x1 = cell.bbox.x_min();
1712 let y1 = cell.bbox.y_min();
1713 let x2 = cell.bbox.x_max();
1714 let y2 = cell.bbox.y_max();
1715 if x2 > x1 && y2 > y1 {
1716 cell_boxes.push([x1, y1, x2, y2]);
1717 }
1718 }
1719 }
1720
1721 if cell_boxes.is_empty() || text_regions.is_empty() {
1722 return Ok(());
1723 }
1724
1725 fn overlap_ratio_box_over_cell(box1: &[f32; 4], box2: &[f32; 4]) -> f32 {
1727 let x_left = box1[0].max(box2[0]);
1728 let y_top = box1[1].max(box2[1]);
1729 let x_right = box1[2].min(box2[2]);
1730 let y_bottom = box1[3].min(box2[3]);
1731
1732 if x_right <= x_left || y_bottom <= y_top {
1733 return 0.0;
1734 }
1735
1736 let inter_area = (x_right - x_left) * (y_bottom - y_top);
1737 let cell_area = (box2[2] - box2[0]) * (box2[3] - box2[1]);
1738 if cell_area <= 0.0 {
1739 0.0
1740 } else {
1741 inter_area / cell_area
1742 }
1743 }
1744
1745 fn get_overlapping_cells(
1747 ocr_box: &[f32; 4],
1748 cells: &[[f32; 4]],
1749 threshold: f32,
1750 ) -> Vec<usize> {
1751 let mut overlapping = Vec::new();
1752 for (idx, cell) in cells.iter().enumerate() {
1753 if overlap_ratio_box_over_cell(ocr_box, cell) > threshold {
1754 overlapping.push(idx);
1755 }
1756 }
1757 overlapping.sort_by(|&i, &j| {
1759 cells[i][0]
1760 .partial_cmp(&cells[j][0])
1761 .unwrap_or(std::cmp::Ordering::Equal)
1762 });
1763 overlapping
1764 }
1765
1766 fn split_box_by_cells(
1768 ocr_box: &[f32; 4],
1769 cell_indices: &[usize],
1770 cells: &[[f32; 4]],
1771 ) -> Vec<[f32; 4]> {
1772 if cell_indices.is_empty() {
1773 return vec![*ocr_box];
1774 }
1775
1776 let mut split_boxes: Vec<[f32; 4]> = Vec::new();
1777 let cells_to_split: Vec<[f32; 4]> = cell_indices.iter().map(|&i| cells[i]).collect();
1778
1779 if ocr_box[0] < cells_to_split[0][0] {
1781 split_boxes.push([ocr_box[0], ocr_box[1], cells_to_split[0][0], ocr_box[3]]);
1782 }
1783
1784 for (i, current_cell) in cells_to_split.iter().enumerate() {
1786 split_boxes.push([
1788 ocr_box[0].max(current_cell[0]),
1789 ocr_box[1],
1790 ocr_box[2].min(current_cell[2]),
1791 ocr_box[3],
1792 ]);
1793
1794 if i + 1 < cells_to_split.len() {
1796 let next_cell = cells_to_split[i + 1];
1797 if current_cell[2] < next_cell[0] {
1798 split_boxes.push([current_cell[2], ocr_box[1], next_cell[0], ocr_box[3]]);
1799 }
1800 }
1801 }
1802
1803 let last_cell = cells_to_split[cells_to_split.len() - 1];
1805 if last_cell[2] < ocr_box[2] {
1806 split_boxes.push([last_cell[2], ocr_box[1], ocr_box[2], ocr_box[3]]);
1807 }
1808
1809 let mut unique = Vec::new();
1811 let mut seen = std::collections::HashSet::new();
1812 for b in split_boxes {
1813 let key = (
1814 b[0].to_bits(),
1815 b[1].to_bits(),
1816 b[2].to_bits(),
1817 b[3].to_bits(),
1818 );
1819 if seen.insert(key) {
1820 unique.push(b);
1821 }
1822 }
1823 unique
1824 }
1825
1826 let k_min_cells = 2usize;
1827 let overlap_threshold = CELL_OVERLAP_IOU_THRESHOLD;
1828
1829 let mut new_regions: Vec<crate::oarocr::TextRegion> =
1830 Vec::with_capacity(text_regions.len());
1831
1832 for region in text_regions.iter() {
1833 let ocr_box = [
1834 region.bounding_box.x_min(),
1835 region.bounding_box.y_min(),
1836 region.bounding_box.x_max(),
1837 region.bounding_box.y_max(),
1838 ];
1839
1840 let overlapping_cells = get_overlapping_cells(&ocr_box, &cell_boxes, overlap_threshold);
1841
1842 if overlapping_cells.len() < k_min_cells {
1844 new_regions.push(region.clone());
1845 continue;
1846 }
1847
1848 let split_boxes = split_box_by_cells(&ocr_box, &overlapping_cells, &cell_boxes);
1849
1850 for box_coords in split_boxes {
1851 let img_w = page_image.width() as i32;
1853 let img_h = page_image.height() as i32;
1854
1855 let mut x1 = box_coords[0].floor() as i32;
1856 let mut y1 = box_coords[1].floor() as i32;
1857 let mut x2 = box_coords[2].ceil() as i32;
1858 let mut y2 = box_coords[3].ceil() as i32;
1859
1860 x1 = x1.clamp(0, img_w.saturating_sub(1));
1861 y1 = y1.clamp(0, img_h.saturating_sub(1));
1862 x2 = x2.clamp(0, img_w);
1863 y2 = y2.clamp(0, img_h);
1864
1865 if x2 - x1 <= 1 || y2 - y1 <= 1 {
1866 continue;
1867 }
1868
1869 let crop_w = (x2 - x1) as u32;
1870 let crop_h = (y2 - y1) as u32;
1871 if crop_w <= 1 || crop_h <= 1 {
1872 continue;
1873 }
1874
1875 let x1u = x1 as u32;
1876 let y1u = y1 as u32;
1877 if x1u >= page_image.width() || y1u >= page_image.height() {
1878 continue;
1879 }
1880 let crop_w = crop_w.min(page_image.width() - x1u);
1881 let crop_h = crop_h.min(page_image.height() - y1u);
1882 if crop_w <= 1 || crop_h <= 1 {
1883 continue;
1884 }
1885
1886 let crop =
1887 image::imageops::crop_imm(page_image, x1u, y1u, crop_w, crop_h).to_image();
1888
1889 let rec_input = ImageTaskInput::new(vec![crop]);
1890 let rec_result = text_recognition_adapter.execute(rec_input, None)?;
1891 if let (Some(text), Some(score)) =
1892 (rec_result.texts.first(), rec_result.scores.first())
1893 && !text.is_empty()
1894 {
1895 let bbox = BoundingBox::from_coords(
1896 box_coords[0],
1897 box_coords[1],
1898 box_coords[2],
1899 box_coords[3],
1900 );
1901 new_regions.push(crate::oarocr::TextRegion {
1902 bounding_box: bbox.clone(),
1903 dt_poly: Some(bbox.clone()),
1904 rec_poly: Some(bbox),
1905 text: Some(Arc::from(text.as_str())),
1906 confidence: Some(*score),
1907 orientation_angle: None,
1908 word_boxes: None,
1909 label: None,
1910 });
1911 }
1912 }
1913 }
1914
1915 *text_regions = new_regions;
1916 Ok(())
1917 }
1918
1919 fn detect_layout_and_regions(
1920 &self,
1921 page_image: &image::RgbImage,
1922 ) -> Result<
1923 (
1924 Vec<crate::domain::structure::LayoutElement>,
1925 Option<Vec<crate::domain::structure::RegionBlock>>,
1926 ),
1927 OCRError,
1928 > {
1929 use oar_ocr_core::core::traits::task::ImageTaskInput;
1930 use oar_ocr_core::domain::structure::RegionBlock;
1931
1932 let input = ImageTaskInput::new(vec![page_image.clone()]);
1933 let t_layout = Instant::now();
1934 let layout_result = self
1935 .pipeline
1936 .layout_detection_adapter
1937 .execute(input, None)?;
1938 let layout_dur = t_layout.elapsed();
1939
1940 let mut layout_elements = layout_result
1941 .elements
1942 .first()
1943 .map(|elements| Self::layout_elements_from_detection(elements))
1944 .unwrap_or_default();
1945
1946 let mut detected_region_blocks: Option<Vec<RegionBlock>> = None;
1947 if let Some(ref region_adapter) = self.pipeline.region_detection_adapter {
1948 let region_input = ImageTaskInput::new(vec![page_image.clone()]);
1949 let t_region = Instant::now();
1950 if let Ok(region_result) = region_adapter.execute(region_input, None)
1951 && let Some(region_elements) = region_result.elements.first()
1952 && !region_elements.is_empty()
1953 {
1954 let blocks: Vec<RegionBlock> = region_elements
1955 .iter()
1956 .map(|e| RegionBlock {
1957 bbox: e.bbox.clone(),
1958 confidence: e.score,
1959 order_index: None,
1960 element_indices: Vec::new(),
1961 })
1962 .collect();
1963 detected_region_blocks = Some(blocks);
1964 }
1965 tracing::debug!(
1966 "structure stage: region detection {:.1} ms, blocks={}",
1967 t_region.elapsed().as_secs_f64() * 1000.0,
1968 detected_region_blocks.as_ref().map_or(0, Vec::len)
1969 );
1970 }
1971
1972 Self::finish_layout_elements(&mut layout_elements);
1973 tracing::debug!(
1974 "structure stage: layout detection {:.1} ms, elements={}",
1975 layout_dur.as_secs_f64() * 1000.0,
1976 layout_elements.len()
1977 );
1978
1979 Ok((layout_elements, detected_region_blocks))
1980 }
1981
1982 fn recognize_formulas(
1983 &self,
1984 page_image: &image::RgbImage,
1985 layout_elements: &[crate::domain::structure::LayoutElement],
1986 ) -> Result<Vec<crate::domain::structure::FormulaResult>, OCRError> {
1987 use oar_ocr_core::core::traits::task::ImageTaskInput;
1988 use oar_ocr_core::domain::structure::FormulaResult;
1989 use oar_ocr_core::utils::BBoxCrop;
1990
1991 let Some(ref formula_adapter) = self.pipeline.formula_recognition_adapter else {
1992 return Ok(Vec::new());
1993 };
1994
1995 let formula_elements: Vec<_> = layout_elements
1996 .iter()
1997 .filter(|e| e.element_type.is_formula())
1998 .collect();
1999
2000 if formula_elements.is_empty() {
2001 tracing::debug!(
2002 "Formula recognition skipped: no formula regions from layout detection"
2003 );
2004 return Ok(Vec::new());
2005 }
2006
2007 let mut crops = Vec::new();
2008 let mut bboxes = Vec::new();
2009
2010 for elem in &formula_elements {
2011 match BBoxCrop::crop_bounding_box(page_image, &elem.bbox) {
2012 Ok(crop) => {
2013 crops.push(crop);
2014 bboxes.push(elem.bbox.clone());
2015 }
2016 Err(err) => {
2017 tracing::warn!("Formula region crop failed: {}", err);
2018 }
2019 }
2020 }
2021
2022 if crops.is_empty() {
2023 tracing::debug!(
2024 "Formula recognition skipped: all formula crops failed for {} regions",
2025 formula_elements.len()
2026 );
2027 return Ok(Vec::new());
2028 }
2029
2030 let t_formula = Instant::now();
2031 let batch_size = formula_adapter.recommended_batch_size().max(1);
2032 let crop_count = bboxes.len();
2033 let mut formula_results = Vec::with_capacity(crop_count);
2034 let mut score_results = Vec::with_capacity(crop_count);
2035 let mut remaining_crops = crops.into_iter();
2036 loop {
2037 let chunk_vec: Vec<_> = remaining_crops.by_ref().take(batch_size).collect();
2038 if chunk_vec.is_empty() {
2039 break;
2040 }
2041
2042 let output = formula_adapter.execute(ImageTaskInput::new(chunk_vec), None)?;
2043 formula_results.extend(output.formulas);
2044 score_results.extend(output.scores);
2045 }
2046 tracing::debug!(
2047 "structure stage: formula recognition {:.1} ms, crops={}, batches={}, batch_size={}",
2048 t_formula.elapsed().as_secs_f64() * 1000.0,
2049 crop_count,
2050 crop_count.div_ceil(batch_size),
2051 batch_size
2052 );
2053
2054 let mut formulas = Vec::new();
2055 for ((bbox, formula), score) in bboxes.into_iter().zip(formula_results).zip(score_results) {
2056 let width = bbox.x_max() - bbox.x_min();
2057 let height = bbox.y_max() - bbox.y_min();
2058 if width <= 0.0 || height <= 0.0 {
2059 tracing::warn!(
2060 "Skipping formula with non-positive bbox dimensions: w={:.2}, h={:.2}",
2061 width,
2062 height
2063 );
2064 continue;
2065 }
2066
2067 formulas.push(FormulaResult {
2068 bbox,
2069 latex: formula,
2070 confidence: score.unwrap_or(0.0),
2071 });
2072 }
2073
2074 Ok(formulas)
2075 }
2076
2077 fn detect_seal_text(
2078 &self,
2079 page_image: &image::RgbImage,
2080 layout_elements: &mut Vec<crate::domain::structure::LayoutElement>,
2081 ) -> Result<(), OCRError> {
2082 use oar_ocr_core::core::traits::task::ImageTaskInput;
2083 use oar_ocr_core::domain::structure::{LayoutElement, LayoutElementType};
2084 use oar_ocr_core::processors::Point;
2085 use oar_ocr_core::utils::BBoxCrop;
2086
2087 let Some(ref seal_adapter) = self.pipeline.seal_text_detection_adapter else {
2088 return Ok(());
2089 };
2090
2091 let seal_regions: Vec<_> = layout_elements
2092 .iter()
2093 .filter(|e| e.element_type == LayoutElementType::Seal)
2094 .map(|e| e.bbox.clone())
2095 .collect();
2096
2097 if seal_regions.is_empty() {
2098 tracing::debug!("Seal detection skipped: no seal regions from layout detection");
2099 return Ok(());
2100 }
2101
2102 let mut seal_crops = Vec::new();
2103 let mut crop_offsets = Vec::new();
2104
2105 for region_bbox in &seal_regions {
2106 match BBoxCrop::crop_bounding_box(page_image, region_bbox) {
2107 Ok(crop) => {
2108 seal_crops.push(crop);
2109 crop_offsets.push((region_bbox.x_min(), region_bbox.y_min()));
2110 }
2111 Err(err) => {
2112 tracing::warn!("Seal region crop failed: {}", err);
2113 }
2114 }
2115 }
2116
2117 if seal_crops.is_empty() {
2118 return Ok(());
2119 }
2120
2121 let input = ImageTaskInput::new(seal_crops);
2122 let seal_result = seal_adapter.execute(input, None)?;
2123
2124 for ((dx, dy), detections) in crop_offsets.iter().zip(seal_result.detections) {
2125 for detection in detections {
2126 let translated_bbox = crate::processors::BoundingBox::new(
2127 detection
2128 .bbox
2129 .points
2130 .iter()
2131 .map(|p| Point::new(p.x + dx, p.y + dy))
2132 .collect(),
2133 );
2134
2135 layout_elements.push(
2136 LayoutElement::new(translated_bbox, LayoutElementType::Seal, detection.score)
2137 .with_label("seal".to_string()),
2138 );
2139 }
2140 }
2141
2142 Ok(())
2143 }
2144
2145 fn sort_layout_elements_enhanced(
2146 layout_elements: &mut Vec<crate::domain::structure::LayoutElement>,
2147 page_width: f32,
2148 page_height: f32,
2149 ) {
2150 use oar_ocr_core::processors::layout_sorting::{SortableElement, sort_layout_enhanced};
2151
2152 if layout_elements.is_empty() {
2153 return;
2154 }
2155
2156 let sortable_elements: Vec<_> = layout_elements
2157 .iter()
2158 .map(|e| SortableElement {
2159 bbox: e.bbox.clone(),
2160 element_type: e.element_type,
2161 num_lines: e.num_lines,
2162 })
2163 .collect();
2164
2165 let sorted_indices = sort_layout_enhanced(&sortable_elements, page_width, page_height);
2166 if sorted_indices.len() != layout_elements.len() {
2167 return;
2168 }
2169
2170 let sorted_elements: Vec<_> = sorted_indices
2171 .into_iter()
2172 .map(|idx| layout_elements[idx].clone())
2173 .collect();
2174 *layout_elements = sorted_elements;
2175 }
2176
2177 fn assign_region_block_membership(
2178 region_blocks: &mut [crate::domain::structure::RegionBlock],
2179 layout_elements: &[crate::domain::structure::LayoutElement],
2180 ) {
2181 use std::cmp::Ordering;
2182
2183 if region_blocks.is_empty() {
2184 return;
2185 }
2186
2187 region_blocks.sort_by(|a, b| {
2188 a.bbox
2189 .y_min()
2190 .partial_cmp(&b.bbox.y_min())
2191 .unwrap_or(Ordering::Equal)
2192 .then_with(|| {
2193 a.bbox
2194 .x_min()
2195 .partial_cmp(&b.bbox.x_min())
2196 .unwrap_or(Ordering::Equal)
2197 })
2198 });
2199
2200 for (i, region) in region_blocks.iter_mut().enumerate() {
2201 region.order_index = Some((i + 1) as u32);
2202 region.element_indices.clear();
2203 }
2204
2205 if layout_elements.is_empty() {
2206 return;
2207 }
2208
2209 for (elem_idx, elem) in layout_elements.iter().enumerate() {
2210 let elem_area = elem.bbox.area();
2211 if elem_area <= 0.0 {
2212 continue;
2213 }
2214
2215 let mut best_region: Option<usize> = None;
2216 let mut best_ioa = 0.0f32;
2217
2218 for (region_idx, region) in region_blocks.iter().enumerate() {
2219 let intersection = elem.bbox.intersection_area(®ion.bbox);
2220 if intersection <= 0.0 {
2221 continue;
2222 }
2223 let ioa = intersection / elem_area;
2224 if ioa > best_ioa {
2225 best_ioa = ioa;
2226 best_region = Some(region_idx);
2227 }
2228 }
2229
2230 if let Some(region_idx) = best_region
2231 && best_ioa >= REGION_MEMBERSHIP_IOA_THRESHOLD
2232 {
2233 region_blocks[region_idx].element_indices.push(elem_idx);
2234 }
2235 }
2236 }
2237
2238 fn run_overall_ocr(
2239 &self,
2240 page_image: &image::RgbImage,
2241 layout_elements: &[crate::domain::structure::LayoutElement],
2242 region_blocks: Option<&[crate::domain::structure::RegionBlock]>,
2243 ) -> Result<Vec<crate::oarocr::TextRegion>, OCRError> {
2244 use crate::oarocr::TextRegion;
2245 use oar_ocr_core::core::traits::task::ImageTaskInput;
2246 use std::sync::Arc;
2247
2248 let Some(ref text_detection_adapter) = self.pipeline.text_detection_adapter else {
2249 return Ok(Vec::new());
2250 };
2251 let Some(ref text_recognition_adapter) = self.pipeline.text_recognition_adapter else {
2252 return Ok(Vec::new());
2253 };
2254
2255 let mut text_regions = Vec::new();
2256
2257 let mut ocr_image = page_image.clone();
2261 if self.pipeline.formula_recognition_adapter.is_some() {
2262 let mask_bboxes: Vec<crate::processors::BoundingBox> = layout_elements
2263 .iter()
2264 .filter(|e| e.element_type.is_formula())
2265 .map(|e| e.bbox.clone())
2266 .collect();
2267
2268 if !mask_bboxes.is_empty() {
2269 crate::utils::mask_regions(&mut ocr_image, &mask_bboxes, [255, 255, 255]);
2270 }
2271 }
2272
2273 let input = ImageTaskInput::new(vec![ocr_image.clone()]);
2275 let t_text_det = Instant::now();
2276 let det_result = text_detection_adapter.execute(input, None)?;
2277 let text_det_dur = t_text_det.elapsed();
2278
2279 let mut detection_boxes = if let Some(detections) = det_result.detections.first() {
2280 detections
2281 .iter()
2282 .map(|d| d.bbox.clone())
2283 .collect::<Vec<_>>()
2284 } else {
2285 Vec::new()
2286 };
2287
2288 let raw_detection_boxes = detection_boxes.clone();
2290 if tracing::enabled!(tracing::Level::DEBUG) && !raw_detection_boxes.is_empty() {
2291 let raw_rects: Vec<[f32; 4]> = raw_detection_boxes
2292 .iter()
2293 .map(|b| [b.x_min(), b.y_min(), b.x_max(), b.y_max()])
2294 .collect();
2295 tracing::debug!("overall OCR text det boxes (raw): {:?}", raw_rects);
2296 }
2297
2298 if !detection_boxes.is_empty() {
2300 let mut split_boxes = Vec::new();
2301 let mut split_count = 0usize;
2302
2303 let container_boxes: Vec<crate::processors::BoundingBox> =
2304 if let Some(regions) = region_blocks {
2305 regions.iter().map(|r| r.bbox.clone()).collect()
2306 } else {
2307 layout_elements
2308 .iter()
2309 .filter(|e| {
2310 matches!(
2311 e.element_type,
2312 crate::domain::structure::LayoutElementType::DocTitle
2313 | crate::domain::structure::LayoutElementType::ParagraphTitle
2314 | crate::domain::structure::LayoutElementType::Text
2315 | crate::domain::structure::LayoutElementType::Content
2316 | crate::domain::structure::LayoutElementType::Abstract
2317 | crate::domain::structure::LayoutElementType::Header
2318 | crate::domain::structure::LayoutElementType::Footer
2319 | crate::domain::structure::LayoutElementType::Footnote
2320 | crate::domain::structure::LayoutElementType::Number
2321 | crate::domain::structure::LayoutElementType::Reference
2322 | crate::domain::structure::LayoutElementType::ReferenceContent
2323 | crate::domain::structure::LayoutElementType::Algorithm
2324 | crate::domain::structure::LayoutElementType::AsideText
2325 | crate::domain::structure::LayoutElementType::List
2326 | crate::domain::structure::LayoutElementType::FigureTitle
2327 | crate::domain::structure::LayoutElementType::TableTitle
2328 | crate::domain::structure::LayoutElementType::ChartTitle
2329 | crate::domain::structure::LayoutElementType::FigureTableChartTitle
2330 )
2331 })
2332 .map(|e| e.bbox.clone())
2333 .collect()
2334 };
2335
2336 if !container_boxes.is_empty() {
2337 for bbox in detection_boxes.into_iter() {
2338 let mut intersections: Vec<crate::processors::BoundingBox> = Vec::new();
2339 let self_area = bbox.area();
2340 if self_area <= 0.0 {
2341 split_boxes.push(bbox);
2342 continue;
2343 }
2344
2345 for container in &container_boxes {
2346 let inter_x_min = bbox.x_min().max(container.x_min());
2347 let inter_y_min = bbox.y_min().max(container.y_min());
2348 let inter_x_max = bbox.x_max().min(container.x_max());
2349 let inter_y_max = bbox.y_max().min(container.y_max());
2350
2351 if inter_x_max - inter_x_min <= 2.0 || inter_y_max - inter_y_min <= 2.0 {
2352 continue;
2353 }
2354
2355 let inter_bbox = crate::processors::BoundingBox::from_coords(
2356 inter_x_min,
2357 inter_y_min,
2358 inter_x_max,
2359 inter_y_max,
2360 );
2361 let inter_area = inter_bbox.area();
2362 if inter_area <= 0.0 {
2363 continue;
2364 }
2365
2366 let ioa = inter_area / self_area;
2367 if ioa >= TEXT_BOX_SPLIT_IOA_THRESHOLD {
2368 intersections.push(inter_bbox);
2369 }
2370 }
2371
2372 if intersections.len() >= 2 {
2373 split_count += intersections.len();
2374 split_boxes.extend(intersections);
2375 } else {
2376 split_boxes.push(bbox);
2377 }
2378 }
2379
2380 if split_count > 0 {
2381 tracing::debug!(
2382 "Cross-layout re-recognition: split {} text boxes into {} sub-boxes",
2383 split_count,
2384 split_boxes.len()
2385 );
2386 }
2387
2388 detection_boxes = split_boxes;
2389 }
2390 }
2391
2392 if !detection_boxes.is_empty() {
2394 detection_boxes = oar_ocr_core::processors::sort_quad_boxes(&detection_boxes);
2395 }
2396
2397 if tracing::enabled!(tracing::Level::DEBUG) && !detection_boxes.is_empty() {
2399 let pre_rec_rects: Vec<[f32; 4]> = detection_boxes
2400 .iter()
2401 .map(|b| [b.x_min(), b.y_min(), b.x_max(), b.y_max()])
2402 .collect();
2403 tracing::debug!(
2404 "overall OCR boxes pre-recognition (after splitting): {:?}",
2405 pre_rec_rects
2406 );
2407 }
2408
2409 if !detection_boxes.is_empty() {
2410 use crate::oarocr::processors::{EdgeProcessor, TextCroppingProcessor};
2411
2412 let processor = TextCroppingProcessor::new(true);
2413 let cropped =
2414 processor.process((Arc::new(page_image.clone()), detection_boxes.clone()))?;
2415
2416 let mut cropped_images: Vec<image::RgbImage> = Vec::new();
2417 let mut valid_indices: Vec<usize> = Vec::new();
2418
2419 for (idx, crop_result) in cropped.into_iter().enumerate() {
2420 if let Some(img) = crop_result {
2421 cropped_images.push((*img).clone());
2422 valid_indices.push(idx);
2423 }
2424 }
2425
2426 if !cropped_images.is_empty() {
2427 if let Some(ref tlo_adapter) = self.pipeline.text_line_orientation_adapter {
2429 let tlo_input = ImageTaskInput::new(cropped_images.clone());
2430 match tlo_adapter.execute(tlo_input, None) {
2431 Ok(tlo_result) => {
2432 for (i, classifications) in
2433 tlo_result.classifications.iter().enumerate()
2434 {
2435 if i >= cropped_images.len() {
2436 break;
2437 }
2438 if let Some(top_cls) = classifications.first()
2439 && top_cls.class_id == 1
2440 {
2441 cropped_images[i] =
2442 image::imageops::rotate180(&cropped_images[i]);
2443 }
2444 }
2445 }
2446 Err(err) => {
2447 tracing::warn!(
2448 "Text-line orientation failed; proceeding without rotation: {}",
2449 err
2450 );
2451 }
2452 }
2453 }
2454
2455 let mut items: Vec<(usize, f32, image::RgbImage)> = valid_indices
2456 .into_iter()
2457 .zip(cropped_images)
2458 .map(|(det_idx, img)| {
2459 let wh_ratio = img.width() as f32 / img.height().max(1) as f32;
2460 (det_idx, wh_ratio, img)
2461 })
2462 .collect();
2463
2464 items.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal));
2465
2466 let batch_size = self
2467 .pipeline
2468 .region_batch_size
2469 .unwrap_or_else(|| text_recognition_adapter.recommended_batch_size())
2470 .max(1);
2471 let mut recognized_by_det_idx: Vec<Option<(String, f32)>> =
2472 vec![None; detection_boxes.len()];
2473 let mut rec_batches = 0usize;
2474 let t_text_rec = Instant::now();
2475
2476 let mut remaining_items = items.into_iter();
2477 loop {
2478 let batch_items: Vec<_> = remaining_items.by_ref().take(batch_size).collect();
2479 if batch_items.is_empty() {
2480 break;
2481 }
2482
2483 let mut det_indices: Vec<usize> = Vec::with_capacity(batch_items.len());
2484 let mut rec_imgs: Vec<image::RgbImage> = Vec::with_capacity(batch_items.len());
2485 for (det_idx, _ratio, img) in batch_items {
2486 det_indices.push(det_idx);
2487 rec_imgs.push(img);
2488 }
2489
2490 let rec_input = ImageTaskInput::new(rec_imgs);
2491 rec_batches += 1;
2492 match text_recognition_adapter.execute(rec_input, None) {
2493 Ok(rec_result) => {
2494 for ((det_idx, text), score) in det_indices
2495 .into_iter()
2496 .zip(rec_result.texts)
2497 .zip(rec_result.scores)
2498 {
2499 if text.is_empty() {
2500 continue;
2501 }
2502 if let Some(slot) = recognized_by_det_idx.get_mut(det_idx) {
2503 *slot = Some((text, score));
2504 }
2505 }
2506 }
2507 Err(err) => {
2511 tracing::warn!(
2512 "Text recognition batch failed for {} crops and will be skipped: {}",
2513 det_indices.len(),
2514 err
2515 );
2516 }
2517 }
2518 }
2519 tracing::debug!(
2520 "structure stage: text recognition {:.1} ms, crops={}, batches={}, batch_size={}",
2521 t_text_rec.elapsed().as_secs_f64() * 1000.0,
2522 detection_boxes.len(),
2523 rec_batches,
2524 batch_size
2525 );
2526
2527 for (det_idx, rec) in recognized_by_det_idx.into_iter().enumerate() {
2529 let Some((text, score)) = rec else {
2530 continue;
2531 };
2532 let bbox = detection_boxes[det_idx].clone();
2533 text_regions.push(TextRegion {
2534 bounding_box: bbox.clone(),
2535 dt_poly: Some(bbox.clone()),
2536 rec_poly: Some(bbox),
2537 text: Some(Arc::from(text)),
2538 confidence: Some(score),
2539 orientation_angle: None,
2540 word_boxes: None,
2541 label: None,
2542 });
2543 }
2544 }
2545 }
2546
2547 let batch_size = self
2548 .pipeline
2549 .region_batch_size
2550 .unwrap_or_else(|| text_recognition_adapter.recommended_batch_size())
2551 .max(1);
2552 Self::refine_overall_ocr_with_layout(
2553 &mut text_regions,
2554 layout_elements,
2555 region_blocks,
2556 page_image,
2557 text_recognition_adapter,
2558 batch_size,
2559 )?;
2560 tracing::debug!(
2561 "structure stage: text detection {:.1} ms, boxes={}, recognized_regions={}",
2562 text_det_dur.as_secs_f64() * 1000.0,
2563 detection_boxes.len(),
2564 text_regions.len()
2565 );
2566
2567 Ok(text_regions)
2568 }
2569
2570 pub fn predict(&self, image_path: impl Into<PathBuf>) -> Result<StructureResult, OCRError> {
2580 let image_path = image_path.into();
2581
2582 let image = image::open(&image_path).map_err(|e| OCRError::InvalidInput {
2584 message: format!(
2585 "failed to load image from '{}': {}",
2586 image_path.display(),
2587 e
2588 ),
2589 })?;
2590
2591 let mut result = self.predict_image(image.to_rgb8())?;
2592 result.input_path = std::sync::Arc::from(image_path.to_string_lossy().as_ref());
2593 Ok(result)
2594 }
2595
2596 fn preprocess_page(&self, image: image::RgbImage) -> Result<PreparedPage, OCRError> {
2599 use crate::oarocr::preprocess::DocumentPreprocessor;
2600 use std::sync::Arc;
2601
2602 let preprocessor = DocumentPreprocessor::new(
2603 self.pipeline.document_orientation_adapter.as_ref(),
2604 self.pipeline.rectification_adapter.as_ref(),
2605 );
2606 let preprocess = preprocessor.preprocess(Arc::new(image))?;
2607 let current_image = preprocess.image;
2608 let orientation_angle = preprocess.orientation_angle;
2609 let rectified_img = preprocess.rectified_img;
2610 let rotation = preprocess.rotation;
2611
2612 Ok(PreparedPage {
2613 current_image,
2614 orientation_angle,
2615 rectified_img,
2616 rotation,
2617 layout_elements: Vec::new(),
2618 detected_region_blocks: None,
2619 precomputed_text_regions: None,
2620 })
2621 }
2622
2623 fn prepare_page(&self, image: image::RgbImage) -> Result<PreparedPage, OCRError> {
2626 let mut prepared = self.preprocess_page(image)?;
2627 let (layout_elements, detected_region_blocks) =
2628 self.detect_layout_and_regions(&prepared.current_image)?;
2629 prepared.layout_elements = layout_elements;
2630 prepared.detected_region_blocks = detected_region_blocks;
2631 Ok(prepared)
2632 }
2633
2634 fn complete_page(
2637 &self,
2638 prepared: PreparedPage,
2639 mut formulas: Vec<crate::domain::structure::FormulaResult>,
2640 ) -> Result<StructureResult, OCRError> {
2641 use std::sync::Arc;
2642
2643 let PreparedPage {
2644 current_image,
2645 orientation_angle,
2646 rectified_img,
2647 rotation,
2648 mut layout_elements,
2649 mut detected_region_blocks,
2650 precomputed_text_regions,
2651 } = prepared;
2652
2653 let mut tables = Vec::new();
2654
2655 self.detect_seal_text(¤t_image, &mut layout_elements)?;
2656
2657 if !layout_elements.is_empty() {
2660 let (width, height) = if let Some(img) = &rectified_img {
2661 (img.width() as f32, img.height() as f32)
2662 } else {
2663 (current_image.width() as f32, current_image.height() as f32)
2664 };
2665 Self::sort_layout_elements_enhanced(&mut layout_elements, width, height);
2666 }
2667
2668 if let Some(ref mut regions) = detected_region_blocks {
2669 Self::assign_region_block_membership(regions, &layout_elements);
2670 }
2671
2672 let t_ocr = Instant::now();
2673 let mut text_regions = if let Some(text_regions) = precomputed_text_regions {
2674 text_regions
2675 } else {
2676 self.run_overall_ocr(
2677 ¤t_image,
2678 &layout_elements,
2679 detected_region_blocks.as_deref(),
2680 )?
2681 };
2682 let ocr_dur = t_ocr.elapsed();
2683
2684 {
2685 let t_tables = Instant::now();
2686 let analyzer = crate::oarocr::table_analyzer::TableAnalyzer::new(
2687 crate::oarocr::table_analyzer::TableAnalyzerConfig {
2688 table_classification_adapter: self
2689 .pipeline
2690 .table_classification_adapter
2691 .as_ref(),
2692 table_orientation_adapter: self.pipeline.table_orientation_adapter.as_ref(),
2693 table_structure_recognition_adapter: self
2694 .pipeline
2695 .table_structure_recognition_adapter
2696 .as_ref(),
2697 wired_table_structure_adapter: self
2698 .pipeline
2699 .wired_table_structure_adapter
2700 .as_ref(),
2701 wireless_table_structure_adapter: self
2702 .pipeline
2703 .wireless_table_structure_adapter
2704 .as_ref(),
2705 table_cell_detection_adapter: self
2706 .pipeline
2707 .table_cell_detection_adapter
2708 .as_ref(),
2709 wired_table_cell_adapter: self.pipeline.wired_table_cell_adapter.as_ref(),
2710 wireless_table_cell_adapter: self.pipeline.wireless_table_cell_adapter.as_ref(),
2711 use_e2e_wired_table_rec: self.pipeline.use_e2e_wired_table_rec,
2712 use_e2e_wireless_table_rec: self.pipeline.use_e2e_wireless_table_rec,
2713 use_wired_table_cells_trans_to_html: self
2714 .pipeline
2715 .use_wired_table_cells_trans_to_html,
2716 use_wireless_table_cells_trans_to_html: self
2717 .pipeline
2718 .use_wireless_table_cells_trans_to_html,
2719 },
2720 );
2721 tables.extend(analyzer.analyze_tables(¤t_image, &layout_elements)?);
2722 tracing::debug!(
2723 "structure stage: table analysis {:.1} ms, tables={}",
2724 t_tables.elapsed().as_secs_f64() * 1000.0,
2725 tables.len()
2726 );
2727 }
2728 tracing::debug!(
2729 "structure stage: overall OCR total {:.1} ms, regions={}",
2730 ocr_dur.as_secs_f64() * 1000.0,
2731 text_regions.len()
2732 );
2733
2734 let has_detection_backed_table_cells = tables.iter().any(|table| !table.is_e2e);
2742 if has_detection_backed_table_cells
2743 && !text_regions.is_empty()
2744 && let Some(ref text_rec_adapter) = self.pipeline.text_recognition_adapter
2745 {
2746 Self::split_ocr_bboxes_by_table_cells(
2747 &tables,
2748 &mut text_regions,
2749 ¤t_image,
2750 text_rec_adapter,
2751 )?;
2752 }
2753
2754 if let Some(rot) = rotation {
2757 let rotated_width = rot.rotated_width;
2758 let rotated_height = rot.rotated_height;
2759 let angle = rot.angle;
2760
2761 for element in &mut layout_elements {
2763 element.bbox =
2764 element
2765 .bbox
2766 .rotate_back_to_original(angle, rotated_width, rotated_height);
2767 }
2768
2769 for table in &mut tables {
2771 table.bbox =
2772 table
2773 .bbox
2774 .rotate_back_to_original(angle, rotated_width, rotated_height);
2775
2776 for cell in &mut table.cells {
2778 cell.bbox =
2779 cell.bbox
2780 .rotate_back_to_original(angle, rotated_width, rotated_height);
2781 }
2782 }
2783
2784 for formula in &mut formulas {
2786 formula.bbox =
2787 formula
2788 .bbox
2789 .rotate_back_to_original(angle, rotated_width, rotated_height);
2790 }
2791
2792 for region in &mut text_regions {
2794 region.dt_poly = region
2795 .dt_poly
2796 .take()
2797 .map(|poly| poly.rotate_back_to_original(angle, rotated_width, rotated_height));
2798 region.rec_poly = region
2799 .rec_poly
2800 .take()
2801 .map(|poly| poly.rotate_back_to_original(angle, rotated_width, rotated_height));
2802 region.bounding_box = region.bounding_box.rotate_back_to_original(
2803 angle,
2804 rotated_width,
2805 rotated_height,
2806 );
2807
2808 if let Some(ref word_boxes) = region.word_boxes {
2809 let transformed_word_boxes: Vec<_> = word_boxes
2810 .iter()
2811 .map(|wb| wb.rotate_back_to_original(angle, rotated_width, rotated_height))
2812 .collect();
2813 region.word_boxes = Some(transformed_word_boxes);
2814 }
2815 }
2816
2817 if let Some(ref mut regions) = detected_region_blocks {
2819 for region in regions.iter_mut() {
2820 region.bbox =
2821 region
2822 .bbox
2823 .rotate_back_to_original(angle, rotated_width, rotated_height);
2824 }
2825 }
2826 }
2827
2828 for formula in &formulas {
2834 let w = formula.bbox.x_max() - formula.bbox.x_min();
2835 let h = formula.bbox.y_max() - formula.bbox.y_min();
2836 if w > 1.0 && h > 1.0 {
2837 let mut region = crate::oarocr::TextRegion::new(formula.bbox.clone());
2838 region.text = Some(formula.latex.clone().into());
2839 region.confidence = Some(1.0);
2840 region.label = Some("formula".into()); text_regions.push(region);
2842 }
2843 }
2844
2845 let final_image = rectified_img.unwrap_or_else(|| Arc::new((*current_image).clone()));
2849 let mut result = StructureResult {
2850 input_path: Arc::from("memory"),
2851 index: 0,
2852 layout_elements,
2853 tables,
2854 formulas,
2855 text_regions: if text_regions.is_empty() {
2856 None
2857 } else {
2858 Some(text_regions)
2859 },
2860 orientation_angle,
2861 region_blocks: detected_region_blocks,
2862 rectified_img: Some(final_image),
2863 page_continuation_flags: None,
2864 };
2865
2866 use crate::oarocr::stitching::{ResultStitcher, StitchConfig};
2869 let stitch_cfg = StitchConfig::default();
2870 ResultStitcher::stitch_with_config(&mut result, &stitch_cfg);
2871
2872 Ok(result)
2873 }
2874
2875 pub fn predict_image(&self, image: image::RgbImage) -> Result<StructureResult, OCRError> {
2877 let t_total = Instant::now();
2878 let prepared = self.prepare_page(image)?;
2879 let formulas =
2880 self.recognize_formulas(&prepared.current_image, &prepared.layout_elements)?;
2881 let result = self.complete_page(prepared, formulas)?;
2882 tracing::debug!(
2883 "structure stage: total predict_image {:.1} ms",
2884 t_total.elapsed().as_secs_f64() * 1000.0
2885 );
2886 Ok(result)
2887 }
2888
2889 fn precompute_overall_ocr_across_pages(
2890 &self,
2891 prepared_pages: &mut [Result<PreparedPage, OCRError>],
2892 ) {
2893 use crate::oarocr::TextRegion;
2894 use crate::oarocr::processors::{EdgeProcessor, TextCroppingProcessor};
2895 use oar_ocr_core::core::traits::task::ImageTaskInput;
2896 use std::sync::Arc;
2897
2898 let Some(ref text_detection_adapter) = self.pipeline.text_detection_adapter else {
2899 return;
2900 };
2901 let Some(ref text_recognition_adapter) = self.pipeline.text_recognition_adapter else {
2902 return;
2903 };
2904
2905 if self.pipeline.seal_text_detection_adapter.is_some() {
2908 return;
2909 }
2910
2911 let image_batch_size = self
2912 .pipeline
2913 .image_batch_size
2914 .unwrap_or_else(|| text_detection_adapter.recommended_batch_size())
2915 .max(1);
2916
2917 let t_total = Instant::now();
2918
2919 #[derive(Default)]
2920 struct PageOcrState {
2921 detection_boxes: Vec<crate::processors::BoundingBox>,
2922 recognized: Vec<Option<(String, f32)>>,
2923 }
2924
2925 struct RecItem {
2926 page_idx: usize,
2927 det_idx: usize,
2928 wh_ratio: f32,
2929 image: Arc<image::RgbImage>,
2930 }
2931
2932 let mut page_states: Vec<Option<PageOcrState>> =
2933 (0..prepared_pages.len()).map(|_| None).collect();
2934 let mut rec_items: Vec<RecItem> = Vec::new();
2935 let cropper = TextCroppingProcessor::new(true);
2936 let mut batched_detection_boxes: Vec<Option<Vec<crate::processors::BoundingBox>>> =
2937 (0..prepared_pages.len()).map(|_| None).collect();
2938
2939 let t_detection = Instant::now();
2940 let mut det_page_indices = Vec::new();
2941 let mut det_images = Vec::new();
2942 for (page_idx, prepared) in prepared_pages.iter().enumerate() {
2943 let Ok(prepared) = prepared else {
2944 continue;
2945 };
2946
2947 let mut ocr_image = (*prepared.current_image).clone();
2948 if self.pipeline.formula_recognition_adapter.is_some() {
2949 let mask_bboxes: Vec<crate::processors::BoundingBox> = prepared
2950 .layout_elements
2951 .iter()
2952 .filter(|e| e.element_type.is_formula())
2953 .map(|e| e.bbox.clone())
2954 .collect();
2955 if !mask_bboxes.is_empty() {
2956 crate::utils::mask_regions(&mut ocr_image, &mask_bboxes, [255, 255, 255]);
2957 }
2958 }
2959
2960 det_page_indices.push(page_idx);
2961 det_images.push(ocr_image);
2962 }
2963
2964 let mut remaining_pages = det_page_indices.into_iter().zip(det_images);
2965 loop {
2966 let mut batch_page_indices = Vec::with_capacity(image_batch_size);
2967 let mut batch_images = Vec::with_capacity(image_batch_size);
2968 for (page_idx, image) in remaining_pages.by_ref().take(image_batch_size) {
2969 batch_page_indices.push(page_idx);
2970 batch_images.push(image);
2971 }
2972 if batch_page_indices.is_empty() {
2973 break;
2974 }
2975 match text_detection_adapter.execute(ImageTaskInput::new(batch_images), None) {
2976 Ok(det_result) => {
2977 for (offset, detections) in det_result.detections.into_iter().enumerate() {
2978 let page_idx = batch_page_indices[offset];
2979 batched_detection_boxes[page_idx] =
2980 Some(detections.into_iter().map(|d| d.bbox).collect());
2981 }
2982 }
2983 Err(err) => {
2984 tracing::warn!(
2985 "Batch structure OCR text detection failed; falling back to per-page detection: {}",
2986 err
2987 );
2988 }
2989 }
2990 }
2991 let detection_ms = t_detection.elapsed().as_secs_f64() * 1000.0;
2992
2993 let t_crop = Instant::now();
2994 for page_idx in 0..prepared_pages.len() {
2995 let prepared = match &prepared_pages[page_idx] {
2996 Ok(prepared) => prepared,
2997 Err(_) => continue,
2998 };
2999
3000 let mut detection_boxes = if let Some(boxes) = batched_detection_boxes[page_idx].take()
3001 {
3002 boxes
3003 } else {
3004 let mut ocr_image = (*prepared.current_image).clone();
3005 if self.pipeline.formula_recognition_adapter.is_some() {
3006 let mask_bboxes: Vec<crate::processors::BoundingBox> = prepared
3007 .layout_elements
3008 .iter()
3009 .filter(|e| e.element_type.is_formula())
3010 .map(|e| e.bbox.clone())
3011 .collect();
3012 if !mask_bboxes.is_empty() {
3013 crate::utils::mask_regions(&mut ocr_image, &mask_bboxes, [255, 255, 255]);
3014 }
3015 }
3016
3017 let det_result = match text_detection_adapter
3018 .execute(ImageTaskInput::new(vec![ocr_image]), None)
3019 {
3020 Ok(result) => result,
3021 Err(err) => {
3022 prepared_pages[page_idx] = Err(err);
3023 continue;
3024 }
3025 };
3026
3027 det_result
3028 .detections
3029 .first()
3030 .map(|detections| {
3031 detections
3032 .iter()
3033 .map(|d| d.bbox.clone())
3034 .collect::<Vec<_>>()
3035 })
3036 .unwrap_or_default()
3037 };
3038
3039 if !detection_boxes.is_empty() {
3040 let mut split_boxes = Vec::new();
3041 let container_boxes: Vec<crate::processors::BoundingBox> = prepared
3042 .detected_region_blocks
3043 .as_ref()
3044 .map(|regions| regions.iter().map(|r| r.bbox.clone()).collect())
3045 .unwrap_or_else(|| {
3046 prepared
3047 .layout_elements
3048 .iter()
3049 .filter(|e| {
3050 matches!(
3051 e.element_type,
3052 crate::domain::structure::LayoutElementType::DocTitle
3053 | crate::domain::structure::LayoutElementType::ParagraphTitle
3054 | crate::domain::structure::LayoutElementType::Text
3055 | crate::domain::structure::LayoutElementType::Content
3056 | crate::domain::structure::LayoutElementType::Abstract
3057 | crate::domain::structure::LayoutElementType::Header
3058 | crate::domain::structure::LayoutElementType::Footer
3059 | crate::domain::structure::LayoutElementType::Footnote
3060 | crate::domain::structure::LayoutElementType::Number
3061 | crate::domain::structure::LayoutElementType::Reference
3062 | crate::domain::structure::LayoutElementType::ReferenceContent
3063 | crate::domain::structure::LayoutElementType::Algorithm
3064 | crate::domain::structure::LayoutElementType::AsideText
3065 | crate::domain::structure::LayoutElementType::List
3066 | crate::domain::structure::LayoutElementType::FigureTitle
3067 | crate::domain::structure::LayoutElementType::TableTitle
3068 | crate::domain::structure::LayoutElementType::ChartTitle
3069 | crate::domain::structure::LayoutElementType::FigureTableChartTitle
3070 )
3071 })
3072 .map(|e| e.bbox.clone())
3073 .collect()
3074 });
3075
3076 if !container_boxes.is_empty() {
3077 for bbox in detection_boxes.into_iter() {
3078 let mut intersections: Vec<crate::processors::BoundingBox> = Vec::new();
3079 let self_area = bbox.area();
3080 if self_area <= 0.0 {
3081 split_boxes.push(bbox);
3082 continue;
3083 }
3084
3085 for container in &container_boxes {
3086 let inter_x_min = bbox.x_min().max(container.x_min());
3087 let inter_y_min = bbox.y_min().max(container.y_min());
3088 let inter_x_max = bbox.x_max().min(container.x_max());
3089 let inter_y_max = bbox.y_max().min(container.y_max());
3090
3091 if inter_x_max - inter_x_min <= 2.0 || inter_y_max - inter_y_min <= 2.0
3092 {
3093 continue;
3094 }
3095
3096 let inter_bbox = crate::processors::BoundingBox::from_coords(
3097 inter_x_min,
3098 inter_y_min,
3099 inter_x_max,
3100 inter_y_max,
3101 );
3102 let inter_area = inter_bbox.area();
3103 if inter_area <= 0.0 {
3104 continue;
3105 }
3106
3107 if inter_area / self_area >= TEXT_BOX_SPLIT_IOA_THRESHOLD {
3108 intersections.push(inter_bbox);
3109 }
3110 }
3111
3112 if intersections.len() >= 2 {
3113 split_boxes.extend(intersections);
3114 } else {
3115 split_boxes.push(bbox);
3116 }
3117 }
3118 detection_boxes = split_boxes;
3119 }
3120 }
3121
3122 if !detection_boxes.is_empty() {
3123 detection_boxes = oar_ocr_core::processors::sort_quad_boxes(&detection_boxes);
3124 }
3125
3126 let state = PageOcrState {
3127 recognized: vec![None; detection_boxes.len()],
3128 detection_boxes,
3129 };
3130
3131 if !state.detection_boxes.is_empty() {
3132 match cropper.process((
3133 Arc::clone(&prepared.current_image),
3134 state.detection_boxes.clone(),
3135 )) {
3136 Ok(cropped) => {
3137 for (det_idx, crop_result) in cropped.into_iter().enumerate() {
3138 let Some(img) = crop_result else {
3139 continue;
3140 };
3141 let wh_ratio = img.width() as f32 / img.height().max(1) as f32;
3142 rec_items.push(RecItem {
3143 page_idx,
3144 det_idx,
3145 wh_ratio,
3146 image: img,
3147 });
3148 }
3149 }
3150 Err(err) => {
3151 prepared_pages[page_idx] = Err(err);
3152 continue;
3153 }
3154 }
3155 }
3156
3157 page_states[page_idx] = Some(state);
3158 }
3159 let crop_ms = t_crop.elapsed().as_secs_f64() * 1000.0;
3160
3161 let mut tlo_ms = 0.0;
3162 let mut recognition_ms = 0.0;
3163 if !rec_items.is_empty() {
3164 if let Some(ref tlo_adapter) = self.pipeline.text_line_orientation_adapter {
3165 let t_tlo = Instant::now();
3166 let input = ImageTaskInput::from_arc_images(
3167 rec_items
3168 .iter()
3169 .map(|item| Arc::clone(&item.image))
3170 .collect(),
3171 );
3172 match tlo_adapter.execute(input, None) {
3173 Ok(tlo_result) => {
3174 for (item, classifications) in
3175 rec_items.iter_mut().zip(tlo_result.classifications)
3176 {
3177 if let Some(top_cls) = classifications.first()
3178 && top_cls.class_id == 1
3179 {
3180 item.image = Arc::new(image::imageops::rotate180(&*item.image));
3181 }
3182 }
3183 }
3184 Err(err) => {
3185 tracing::warn!(
3186 "Text-line orientation failed; proceeding without rotation: {}",
3187 err
3188 );
3189 }
3190 }
3191 tlo_ms = t_tlo.elapsed().as_secs_f64() * 1000.0;
3192 }
3193
3194 rec_items.sort_by(|a, b| {
3195 a.wh_ratio
3196 .partial_cmp(&b.wh_ratio)
3197 .unwrap_or(std::cmp::Ordering::Equal)
3198 });
3199
3200 let batch_size = self
3201 .pipeline
3202 .region_batch_size
3203 .unwrap_or_else(|| text_recognition_adapter.recommended_batch_size())
3204 .max(1);
3205
3206 let t_recognition = Instant::now();
3207 let mut start = 0usize;
3208 while start < rec_items.len() {
3209 let end = (start + batch_size).min(rec_items.len());
3210 let chunk = &rec_items[start..end];
3211 let rec_input = ImageTaskInput::from_arc_images(
3212 chunk.iter().map(|item| Arc::clone(&item.image)).collect(),
3213 );
3214 match text_recognition_adapter.execute(rec_input, None) {
3215 Ok(rec_result) => {
3216 for (i, item) in chunk.iter().enumerate() {
3217 let text = rec_result.texts.get(i).cloned().unwrap_or_default();
3218 if text.is_empty() {
3219 continue;
3220 }
3221 let score = rec_result.scores.get(i).copied().unwrap_or(0.0);
3222 if let Some(Some(state)) = page_states.get_mut(item.page_idx)
3223 && let Some(slot) = state.recognized.get_mut(item.det_idx)
3224 {
3225 *slot = Some((text, score));
3226 }
3227 }
3228 }
3229 Err(err) => {
3230 tracing::warn!(
3231 "Text recognition batch failed for {} crops and will be skipped: {}",
3232 end - start,
3233 err
3234 );
3235 }
3236 }
3237 start = end;
3238 }
3239 recognition_ms = t_recognition.elapsed().as_secs_f64() * 1000.0;
3240 }
3241
3242 let batch_size = self
3243 .pipeline
3244 .region_batch_size
3245 .unwrap_or_else(|| text_recognition_adapter.recommended_batch_size())
3246 .max(1);
3247
3248 let t_refine = Instant::now();
3249 let mut precomputed_pages = 0usize;
3250 let mut text_region_count = 0usize;
3251 for page_idx in 0..prepared_pages.len() {
3252 let Some(state) = page_states[page_idx].take() else {
3253 continue;
3254 };
3255 let Ok(prepared) = &mut prepared_pages[page_idx] else {
3256 continue;
3257 };
3258
3259 let mut text_regions = Vec::new();
3260 for (det_idx, rec) in state.recognized.into_iter().enumerate() {
3261 let Some((text, score)) = rec else {
3262 continue;
3263 };
3264 let bbox = state.detection_boxes[det_idx].clone();
3265 text_regions.push(TextRegion {
3266 bounding_box: bbox.clone(),
3267 dt_poly: Some(bbox.clone()),
3268 rec_poly: Some(bbox),
3269 text: Some(Arc::from(text)),
3270 confidence: Some(score),
3271 orientation_angle: None,
3272 word_boxes: None,
3273 label: None,
3274 });
3275 }
3276
3277 if let Err(err) = Self::refine_overall_ocr_with_layout(
3278 &mut text_regions,
3279 &prepared.layout_elements,
3280 prepared.detected_region_blocks.as_deref(),
3281 &prepared.current_image,
3282 text_recognition_adapter,
3283 batch_size,
3284 ) {
3285 prepared_pages[page_idx] = Err(err);
3286 continue;
3287 }
3288
3289 text_region_count += text_regions.len();
3290 prepared.precomputed_text_regions = Some(text_regions);
3291 precomputed_pages += 1;
3292 }
3293 let refine_ms = t_refine.elapsed().as_secs_f64() * 1000.0;
3294
3295 tracing::debug!(
3296 "structure batch OCR: pages={}, regions={}, detection={:.1} ms, crop/split={:.1} ms, tlo={:.1} ms, recognition={:.1} ms, refine={:.1} ms, total={:.1} ms",
3297 precomputed_pages,
3298 text_region_count,
3299 detection_ms,
3300 crop_ms,
3301 tlo_ms,
3302 recognition_ms,
3303 refine_ms,
3304 t_total.elapsed().as_secs_f64() * 1000.0
3305 );
3306 }
3307
3308 pub fn predict_images(
3318 &self,
3319 images: Vec<image::RgbImage>,
3320 ) -> Vec<Result<StructureResult, OCRError>> {
3321 use oar_ocr_core::core::traits::task::ImageTaskInput;
3322 use oar_ocr_core::domain::structure::FormulaResult;
3323 use oar_ocr_core::utils::BBoxCrop;
3324
3325 let image_batch_size = self
3326 .pipeline
3327 .image_batch_size
3328 .unwrap_or_else(|| {
3329 self.pipeline
3330 .layout_detection_adapter
3331 .recommended_batch_size()
3332 })
3333 .max(1);
3334
3335 if images.is_empty() {
3336 return Vec::new();
3337 }
3338
3339 let t_total = Instant::now();
3340
3341 let t_preprocess = Instant::now();
3346 let mut prepared_pages: Vec<Result<PreparedPage, OCRError>> = images
3347 .into_iter()
3348 .map(|image| self.preprocess_page(image))
3349 .collect();
3350 let preprocess_ms = t_preprocess.elapsed().as_secs_f64() * 1000.0;
3351
3352 let batch_pages: Vec<(usize, std::sync::Arc<image::RgbImage>)> = prepared_pages
3353 .iter()
3354 .enumerate()
3355 .filter_map(|(page_idx, prepared)| {
3356 prepared
3357 .as_ref()
3358 .ok()
3359 .map(|page| (page_idx, std::sync::Arc::clone(&page.current_image)))
3360 })
3361 .collect();
3362
3363 let t_layout = Instant::now();
3364 if !batch_pages.is_empty() {
3365 for page_chunk in batch_pages.chunks(image_batch_size) {
3366 let layout_input = ImageTaskInput::from_arc_images(
3367 page_chunk
3368 .iter()
3369 .map(|(_, img)| std::sync::Arc::clone(img))
3370 .collect(),
3371 );
3372 match self
3373 .pipeline
3374 .layout_detection_adapter
3375 .execute(layout_input, None)
3376 {
3377 Ok(layout_result) => {
3378 for (batch_idx, (page_idx, _)) in page_chunk.iter().enumerate() {
3379 if let Ok(prepared) = &mut prepared_pages[*page_idx] {
3380 let mut layout_elements = layout_result
3381 .elements
3382 .get(batch_idx)
3383 .map(|elements| Self::layout_elements_from_detection(elements))
3384 .unwrap_or_default();
3385 Self::finish_layout_elements(&mut layout_elements);
3386 prepared.layout_elements = layout_elements;
3387 }
3388 }
3389
3390 if let Some(ref region_adapter) = self.pipeline.region_detection_adapter {
3391 let region_input = ImageTaskInput::from_arc_images(
3392 page_chunk
3393 .iter()
3394 .map(|(_, img)| std::sync::Arc::clone(img))
3395 .collect(),
3396 );
3397 match region_adapter.execute(region_input, None) {
3398 Ok(region_result) => {
3399 for (batch_idx, (page_idx, _)) in page_chunk.iter().enumerate()
3400 {
3401 let Some(region_elements) =
3402 region_result.elements.get(batch_idx)
3403 else {
3404 continue;
3405 };
3406 if region_elements.is_empty() {
3407 continue;
3408 }
3409 if let Ok(prepared) = &mut prepared_pages[*page_idx] {
3410 prepared.detected_region_blocks = Some(
3411 region_elements
3412 .iter()
3413 .map(|e| {
3414 crate::domain::structure::RegionBlock {
3415 bbox: e.bbox.clone(),
3416 confidence: e.score,
3417 order_index: None,
3418 element_indices: Vec::new(),
3419 }
3420 })
3421 .collect(),
3422 );
3423 }
3424 }
3425 }
3426 Err(err) => {
3427 tracing::warn!("Batch region detection failed: {}", err);
3428 }
3429 }
3430 }
3431 }
3432 Err(err) => {
3433 tracing::warn!(
3434 "Batch layout detection failed; falling back to per-page layout: {}",
3435 err
3436 );
3437 for (page_idx, _) in page_chunk {
3438 if let Ok(prepared) = &mut prepared_pages[*page_idx] {
3439 match self.detect_layout_and_regions(&prepared.current_image) {
3440 Ok((layout_elements, region_blocks)) => {
3441 prepared.layout_elements = layout_elements;
3442 prepared.detected_region_blocks = region_blocks;
3443 }
3444 Err(err) => {
3445 prepared_pages[*page_idx] = Err(err);
3446 }
3447 }
3448 }
3449 }
3450 }
3451 }
3452 }
3453 }
3454 let layout_ms = t_layout.elapsed().as_secs_f64() * 1000.0;
3455
3456 let t_formula = Instant::now();
3458 let num_pages = prepared_pages.len();
3459 let mut per_page_formulas: Vec<Vec<FormulaResult>> =
3460 (0..num_pages).map(|_| Vec::new()).collect();
3461
3462 if let Some(ref formula_adapter) = self.pipeline.formula_recognition_adapter {
3463 let mut all_crops: Vec<image::RgbImage> = Vec::new();
3464 let mut crop_meta: Vec<(usize, oar_ocr_core::processors::BoundingBox)> = Vec::new();
3465
3466 for (page_idx, prepared) in prepared_pages.iter().enumerate() {
3467 let prepared = match prepared {
3468 Ok(p) => p,
3469 Err(_) => continue,
3470 };
3471 for elem in prepared
3472 .layout_elements
3473 .iter()
3474 .filter(|e| e.element_type.is_formula())
3475 {
3476 match BBoxCrop::crop_bounding_box(&prepared.current_image, &elem.bbox) {
3477 Ok(crop) => {
3478 all_crops.push(crop);
3479 crop_meta.push((page_idx, elem.bbox.clone()));
3480 }
3481 Err(err) => {
3482 tracing::warn!("Formula region crop failed (batch): {}", err);
3483 }
3484 }
3485 }
3486 }
3487
3488 if !all_crops.is_empty() {
3489 let batch_size = formula_adapter.recommended_batch_size().max(1);
3490 let mut remaining_crops = all_crops.into_iter();
3491 let mut meta_offset = 0;
3492
3493 loop {
3494 let chunk_vec: Vec<_> = remaining_crops.by_ref().take(batch_size).collect();
3495 let chunk_len = chunk_vec.len();
3496 if chunk_len == 0 {
3497 break;
3498 }
3499
3500 let chunk_meta = &crop_meta[meta_offset..meta_offset + chunk_len];
3501 match formula_adapter.execute(ImageTaskInput::new(chunk_vec), None) {
3502 Ok(formula_output) => {
3503 for ((page_idx, bbox), (formula_text, score)) in
3504 chunk_meta.iter().cloned().zip(
3505 formula_output
3506 .formulas
3507 .into_iter()
3508 .zip(formula_output.scores),
3509 )
3510 {
3511 let width = bbox.x_max() - bbox.x_min();
3512 let height = bbox.y_max() - bbox.y_min();
3513 if width > 0.0 && height > 0.0 {
3514 per_page_formulas[page_idx].push(FormulaResult {
3515 bbox,
3516 latex: formula_text,
3517 confidence: score.unwrap_or(0.0),
3518 });
3519 }
3520 }
3521 }
3522 Err(err) => {
3523 tracing::warn!("Batch formula recognition failed: {}", err);
3524 }
3525 }
3526 meta_offset += chunk_len;
3527 }
3528 }
3529 }
3530 let formula_ms = t_formula.elapsed().as_secs_f64() * 1000.0;
3531
3532 let t_ocr = Instant::now();
3533 self.precompute_overall_ocr_across_pages(&mut prepared_pages);
3534 let ocr_ms = t_ocr.elapsed().as_secs_f64() * 1000.0;
3535
3536 let t_complete = Instant::now();
3538 let results: Vec<_> = prepared_pages
3539 .into_iter()
3540 .zip(per_page_formulas)
3541 .map(|(prepared, formulas)| self.complete_page(prepared?, formulas))
3542 .collect();
3543 tracing::debug!(
3544 "structure batch: pages={}, preprocess={:.1} ms, layout/region={:.1} ms, formula={:.1} ms, ocr={:.1} ms, complete={:.1} ms, total={:.1} ms",
3545 num_pages,
3546 preprocess_ms,
3547 layout_ms,
3548 formula_ms,
3549 ocr_ms,
3550 t_complete.elapsed().as_secs_f64() * 1000.0,
3551 t_total.elapsed().as_secs_f64() * 1000.0
3552 );
3553 results
3554 }
3555}
3556
3557#[cfg(test)]
3558mod tests {
3559 use super::*;
3560
3561 #[test]
3562 fn arena_shrinkage_defaults_on_for_cuda_only() {
3563 use oar_ocr_core::core::config::OrtExecutionProvider;
3564 let cuda = || OrtExecutionProvider::CUDA {
3565 device_id: None,
3566 gpu_mem_limit: None,
3567 arena_extend_strategy: None,
3568 cudnn_conv_algo_search: None,
3569 cudnn_conv_use_max_workspace: None,
3570 };
3571
3572 let mut config = OrtSessionConfig::new().with_execution_providers(vec![cuda()]);
3573 default_arena_shrinkage(&mut config);
3574 assert_eq!(config.arena_shrinkage, Some(true));
3575
3576 let mut explicit = OrtSessionConfig::new()
3577 .with_execution_providers(vec![cuda()])
3578 .with_arena_shrinkage(false);
3579 default_arena_shrinkage(&mut explicit);
3580 assert_eq!(explicit.arena_shrinkage, Some(false));
3581
3582 let mut cpu =
3583 OrtSessionConfig::new().with_execution_providers(vec![OrtExecutionProvider::CPU]);
3584 default_arena_shrinkage(&mut cpu);
3585 assert_eq!(cpu.arena_shrinkage, None);
3586 }
3587
3588 #[test]
3589 fn test_structure_builder_new() {
3590 let builder = OARStructureBuilder::new("layout.onnx");
3591 assert_eq!(
3592 builder.layout_detection_model.as_path(),
3593 Some(std::path::Path::new("layout.onnx"))
3594 );
3595 assert!(builder.table_classification_model.is_none());
3596 assert!(builder.formula_recognition_model.is_none());
3597 }
3598
3599 #[test]
3600 fn test_structure_builder_with_table_components() {
3601 let builder = OARStructureBuilder::new("layout.onnx")
3602 .with_table_classification("table_cls.onnx")
3603 .with_table_cell_detection("table_cell.onnx", TableType::Wired)
3604 .with_table_structure_recognition("table_struct.onnx", TableType::Wired)
3605 .table_structure_dict_path("table_structure_dict.txt");
3606
3607 assert!(builder.table_classification_model.is_some());
3608 assert!(builder.table_cell_detection_model.is_some());
3609 assert!(builder.table_structure_recognition_model.is_some());
3610 assert_eq!(
3611 builder.table_cell_detection_type,
3612 Some(Ok(TableType::Wired))
3613 );
3614 assert_eq!(
3615 builder.table_structure_recognition_type,
3616 Some(Ok(TableType::Wired))
3617 );
3618 assert_eq!(
3619 builder.table_structure_dict_path,
3620 Some(PathBuf::from("table_structure_dict.txt"))
3621 );
3622 }
3623
3624 #[test]
3625 fn test_structure_builder_with_formula() {
3626 let builder = OARStructureBuilder::new("layout.onnx").with_formula_recognition(
3627 "formula.onnx",
3628 "tokenizer.json",
3629 FormulaModelKind::PPFormulaNet,
3630 );
3631
3632 assert!(builder.formula_recognition_model.is_some());
3633 assert!(builder.formula_tokenizer_path.is_some());
3634 assert_eq!(
3635 builder.formula_recognition_type,
3636 Some(Ok(FormulaModelKind::PPFormulaNet))
3637 );
3638 }
3639
3640 #[test]
3641 fn test_structure_builder_accepts_strict_string_kinds() {
3642 let builder = OARStructureBuilder::new("layout.onnx")
3643 .with_table_cell_detection("table_cell.onnx", "wireless")
3644 .with_table_structure_recognition("table_struct.onnx", "wired")
3645 .with_formula_recognition("formula.onnx", "tokenizer.json", "pp_formulanet");
3646
3647 assert_eq!(
3648 builder.table_cell_detection_type,
3649 Some(Ok(TableType::Wireless))
3650 );
3651 assert_eq!(
3652 builder.table_structure_recognition_type,
3653 Some(Ok(TableType::Wired))
3654 );
3655 assert_eq!(
3656 builder.formula_recognition_type,
3657 Some(Ok(FormulaModelKind::PPFormulaNet))
3658 );
3659 }
3660
3661 #[test]
3662 fn test_structure_builder_preserves_invalid_string_kinds() {
3663 let builder = OARStructureBuilder::new("layout.onnx")
3664 .with_table_cell_detection("table_cell.onnx", "border-ish")
3665 .with_formula_recognition("formula.onnx", "tokenizer.json", "mystery");
3666
3667 assert!(matches!(
3668 builder.table_cell_detection_type,
3669 Some(Err(ref message)) if message.contains("border-ish")
3670 ));
3671 assert!(matches!(
3672 builder.formula_recognition_type,
3673 Some(Err(ref message)) if message.contains("mystery")
3674 ));
3675 }
3676
3677 #[test]
3678 fn test_structure_builder_reports_invalid_string_before_loading_models() {
3679 let result = OARStructureBuilder::new("missing-layout.onnx")
3680 .with_table_cell_detection("missing-table.onnx", "border-ish")
3681 .build();
3682 let error = match result {
3683 Ok(_) => panic!("invalid selector must fail"),
3684 Err(error) => error,
3685 };
3686 let message = error.to_string();
3687 assert!(message.contains("table_cell_detection"));
3688 assert!(message.contains("border-ish"));
3689 }
3690
3691 #[test]
3692 fn test_structure_builder_with_ocr() {
3693 let builder =
3694 OARStructureBuilder::new("layout.onnx").with_ocr("det.onnx", "rec.onnx", "dict.txt");
3695
3696 assert!(builder.text_detection_model.is_some());
3697 assert!(builder.text_recognition_model.is_some());
3698 assert!(builder.character_dict_path.is_some());
3699 }
3700
3701 #[test]
3702 fn test_structure_builder_with_configuration() {
3703 let layout_config = LayoutDetectionConfig {
3704 score_threshold: 0.5,
3705 max_elements: 100,
3706 ..Default::default()
3707 };
3708
3709 let builder = OARStructureBuilder::new("layout.onnx")
3710 .layout_detection_config(layout_config.clone())
3711 .image_batch_size(4)
3712 .region_batch_size(64);
3713
3714 assert!(builder.layout_detection_config.is_some());
3715 assert_eq!(builder.image_batch_size, Some(4));
3716 assert_eq!(builder.region_batch_size, Some(64));
3717 }
3718
3719 #[test]
3720 fn test_structure_batch_size_validation() {
3721 assert!(OARStructureBuilder::validate_batch_size("image_batch_size", 1).is_ok());
3722 assert!(
3723 OARStructureBuilder::validate_batch_size(
3724 "region_batch_size",
3725 OARStructureBuilder::MAX_BATCH_SIZE,
3726 )
3727 .is_ok()
3728 );
3729
3730 let err = OARStructureBuilder::validate_batch_size("image_batch_size", 0).unwrap_err();
3731 let msg = err.to_string();
3732 assert!(msg.contains("image_batch_size"));
3733 assert!(msg.contains(&format!("1..={}", OARStructureBuilder::MAX_BATCH_SIZE)));
3734 }
3735}