1use crate::Error;
18use crate::{ConvertOptions, Flip, ImageProcessor, ImageProcessorTrait, Result, Rotation};
19use edgefirst_tensor::{CpuAccess, DType, PixelFormat, Region, TensorDyn, TensorMemory};
20
21pub use edgefirst_decoder_abi::TilePlacement;
22
23#[derive(Debug, Clone, Copy, PartialEq, Eq)]
25pub struct TileSpec {
26 pub source: Region,
28 pub index: usize,
30 pub row: usize,
32 pub col: usize,
34}
35
36#[derive(Debug, Clone, Copy, PartialEq)]
38pub struct TilingConfig {
39 pub tile_w: usize,
41 pub tile_h: usize,
43 pub overlap_ratio: f32,
47 options: ConvertOptions,
48}
49
50impl TilingConfig {
51 pub fn new(tile_w: usize, tile_h: usize) -> Self {
55 Self {
56 tile_w,
57 tile_h,
58 overlap_ratio: 0.2,
59 options: ConvertOptions::new(),
60 }
61 }
62
63 pub fn with_overlap(mut self, overlap_ratio: f32) -> Self {
65 self.overlap_ratio = overlap_ratio;
66 self
67 }
68
69 pub fn with_options(mut self, options: ConvertOptions) -> Self {
75 self.options = options;
76 self
77 }
78
79 pub fn options(&self) -> ConvertOptions {
81 self.options
82 }
83
84 pub fn validate(&self) -> Result<()> {
93 if !(self.overlap_ratio >= 0.0 && self.overlap_ratio < 1.0) {
94 return Err(Error::CropInvalid(format!(
95 "tiling overlap_ratio must be in [0.0, 1.0), got {}",
96 self.overlap_ratio
97 )));
98 }
99 if self.tile_w == 0 || self.tile_h == 0 {
100 return Err(Error::CropInvalid(
101 "tiling tile size must be non-zero".into(),
102 ));
103 }
104 if self.options.get_rotation() != Rotation::None || self.options.get_flip() != Flip::None {
105 return Err(Error::InvalidOptions(
106 "tiling options cannot rotate or flip".into(),
107 ));
108 }
109 Ok(())
110 }
111}
112
113fn axis_origins(frame: usize, tile: usize, overlap: f64) -> Vec<usize> {
119 if frame <= tile {
120 return vec![0];
121 }
122 let last = frame - tile;
123 let max_step = ((1.0 - overlap) * tile as f64).floor().max(1.0) as usize;
129 let total = last.div_ceil(max_step);
130 (0..=total)
131 .map(|i| (i as f32 * last as f32 / total as f32).round() as usize)
132 .collect()
133}
134
135pub fn tile_grid(
140 frame_h: usize,
141 frame_w: usize,
142 tile_h: usize,
143 tile_w: usize,
144 overlap_ratio: f32,
145) -> Vec<TileSpec> {
146 let cw = tile_w.min(frame_w);
147 let ch = tile_h.min(frame_h);
148 let xs = axis_origins(frame_w, tile_w, overlap_ratio as f64);
149 let ys = axis_origins(frame_h, tile_h, overlap_ratio as f64);
150 let mut tiles = Vec::with_capacity(xs.len() * ys.len());
151 let mut index = 0;
152 for (row, &oy) in ys.iter().enumerate() {
153 for (col, &ox) in xs.iter().enumerate() {
154 debug_assert!(
155 ox + cw <= frame_w && oy + ch <= frame_h,
156 "tile out of bounds"
157 );
158 tiles.push(TileSpec {
159 source: Region::new(ox, oy, cw, ch),
160 index,
161 row,
162 col,
163 });
164 index += 1;
165 }
166 }
167 tiles
168}
169
170impl ImageProcessor {
171 pub fn alloc_tile_batch(
187 &self,
188 n: usize,
189 cfg: &TilingConfig,
190 format: PixelFormat,
191 dtype: DType,
192 memory: Option<TensorMemory>,
193 access: CpuAccess,
194 ) -> Result<TensorDyn> {
195 cfg.validate()?;
196 self.create_image_with(
197 cfg.tile_w,
198 n.saturating_mul(cfg.tile_h),
199 format,
200 dtype,
201 memory,
202 access,
203 true,
204 )
205 }
206
207 pub fn plan_tiles(
223 &self,
224 src_w: usize,
225 src_h: usize,
226 cfg: &TilingConfig,
227 ) -> Result<Vec<TilePlacement>> {
228 let span = tracing::trace_span!(
229 "image.plan_tiles",
230 tiles = tracing::field::Empty,
231 overlap = cfg.overlap_ratio,
232 );
233 let _s = span.enter();
234 cfg.validate()?;
235 let grid = tile_grid(src_h, src_w, cfg.tile_h, cfg.tile_w, cfg.overlap_ratio);
236 let count = grid.len();
237 span.record("tiles", count);
238 grid.iter()
239 .map(|t| {
240 let region = cfg
241 .options
242 .source(Some(t.source))
243 .content_region(src_w, src_h, cfg.tile_w, cfg.tile_h)?;
244 Ok(TilePlacement {
245 index: t.index,
246 count,
247 origin: (t.source.x as f32, t.source.y as f32),
248 crop_size: (t.source.width as f32, t.source.height as f32),
249 content_region: region.letterboxed(),
250 frame_dims: (src_w as f32, src_h as f32),
251 })
252 })
253 .collect()
254 }
255
256 pub fn tile_into(
300 &mut self,
301 src: &TensorDyn,
302 dst_batched: &mut TensorDyn,
303 cfg: &TilingConfig,
304 ) -> Result<Vec<TilePlacement>> {
305 let span = tracing::trace_span!("image.tile_into", tiles = tracing::field::Empty);
306 let _s = span.enter();
307 cfg.validate()?;
308 let (src_w, src_h) = (
309 src.width().ok_or(Error::NotAnImage)?,
310 src.height().ok_or(Error::NotAnImage)?,
311 );
312 let placements = self.plan_tiles(src_w, src_h, cfg)?;
313 let count = placements.len();
314 span.record("tiles", count);
315
316 let dst_h = dst_batched.height().ok_or(Error::NotAnImage)?;
317 let required_h = count.saturating_mul(cfg.tile_h);
318 if dst_h < required_h {
319 return Err(Error::InvalidShape(format!(
320 "tile_into dst height {dst_h} < count*tile_h {required_h}"
321 )));
322 }
323
324 for p in &placements {
325 self.render_tile(src, dst_batched, p, cfg)?;
326 }
327 self.flush()?;
328 Ok(placements)
329 }
330
331 pub fn tile_one(
343 &mut self,
344 src: &TensorDyn,
345 dst_slot: &mut TensorDyn,
346 placement: &TilePlacement,
347 cfg: &TilingConfig,
348 ) -> Result<()> {
349 let _s = tracing::trace_span!(
350 "image.tile_one",
351 index = placement.index,
352 count = placement.count,
353 )
354 .entered();
355 cfg.validate()?;
356 self.render_tile(src, dst_slot, placement, cfg)
357 }
358
359 fn render_tile(
362 &mut self,
363 src: &TensorDyn,
364 dst: &mut TensorDyn,
365 placement: &TilePlacement,
366 cfg: &TilingConfig,
367 ) -> Result<()> {
368 let options = cfg
369 .options
370 .source(Some(placement_to_source_region(placement)?));
371
372 let dst_h = dst.height().ok_or(Error::NotAnImage)?;
377 let batched = dst_h >= placement.count.saturating_mul(cfg.tile_h);
378 if batched {
379 let mut band = dst.view(Region::new(
380 0,
381 placement.index * cfg.tile_h,
382 cfg.tile_w,
383 cfg.tile_h,
384 ))?;
385 self.convert_deferred(src, &mut band, Some(options))?;
386 } else {
387 self.convert_deferred(src, dst, Some(options))?;
388 }
389 Ok(())
390 }
391}
392
393fn placement_to_source_region(placement: &TilePlacement) -> Result<Region> {
399 let (ox, oy) = placement.origin;
400 let (cw, ch) = placement.crop_size;
401 for (name, v) in [
402 ("origin.x", ox),
403 ("origin.y", oy),
404 ("crop.w", cw),
405 ("crop.h", ch),
406 ] {
407 if !v.is_finite() || v < 0.0 {
408 return Err(Error::CropInvalid(format!(
409 "tile placement {name} must be finite and non-negative, got {v}"
410 )));
411 }
412 }
413 let (ox_i, oy_i, cw_i, ch_i) = (ox as usize, oy as usize, cw as usize, ch as usize);
414 for (name, f, i) in [
416 ("origin.x", ox, ox_i),
417 ("origin.y", oy, oy_i),
418 ("crop.w", cw, cw_i),
419 ("crop.h", ch, ch_i),
420 ] {
421 if (f - i as f32).abs() > f32::EPSILON {
422 return Err(Error::CropInvalid(format!(
423 "tile placement {name} must be an integral pixel value, got {f}"
424 )));
425 }
426 }
427 if cw_i == 0 || ch_i == 0 {
428 return Err(Error::CropInvalid(
429 "tile placement crop size must be non-zero".into(),
430 ));
431 }
432 Ok(Region::new(ox_i, oy_i, cw_i, ch_i))
433}
434
435#[cfg(test)]
436mod tests {
437 use super::*;
438
439 #[test]
442 fn axis_origins_matches_adis_worked_example() {
443 assert_eq!(axis_origins(1920, 640, 0.1), vec![0, 427, 853, 1280]);
445 }
446
447 #[test]
448 fn axis_origins_4k_axes() {
449 let xs = axis_origins(3840, 640, 0.2);
451 assert_eq!(xs, vec![0, 457, 914, 1371, 1829, 2286, 2743, 3200]);
452 let ys = axis_origins(2160, 640, 0.2);
454 assert_eq!(ys, vec![0, 507, 1013, 1520]);
455 assert_eq!(*xs.last().unwrap(), 3840 - 640);
456 assert_eq!(*ys.last().unwrap(), 2160 - 640);
457 assert_eq!(xs[0], 0);
458 }
459
460 #[test]
461 fn axis_origins_frame_le_tile_single() {
462 assert_eq!(axis_origins(640, 640, 0.2), vec![0]); assert_eq!(axis_origins(400, 640, 0.2), vec![0]); }
465
466 #[test]
467 fn axis_origins_frame_tile_plus_one() {
468 assert_eq!(axis_origins(641, 640, 0.2), vec![0, 1]);
470 }
471
472 #[test]
473 fn axis_origins_overlap_zero_is_exact_tiling() {
474 assert_eq!(axis_origins(1920, 640, 0.0), vec![0, 640, 1280]);
476 }
477
478 #[test]
479 fn axis_origins_overlap_near_one_no_panic() {
480 let xs = axis_origins(1920, 640, 0.99);
482 assert_eq!(xs[0], 0);
483 assert_eq!(*xs.last().unwrap(), 1280);
484 assert!(xs.windows(2).all(|w| w[0] <= w[1]));
486 assert!(xs.len() > 8); }
488
489 #[test]
490 fn axis_origins_8k_no_overflow() {
491 let xs = axis_origins(7680, 640, 0.2);
493 assert_eq!(xs[0], 0);
494 assert_eq!(*xs.last().unwrap(), 7680 - 640);
495 assert!(xs.windows(2).all(|w| w[0] <= w[1]));
496 }
497
498 #[test]
499 fn tile_grid_4k_all_full_size_and_in_bounds() {
500 let grid = tile_grid(2160, 3840, 640, 640, 0.2);
501 assert_eq!(grid.len(), 8 * 4); for t in &grid {
503 assert_eq!(t.source.width, 640);
504 assert_eq!(t.source.height, 640);
505 assert!(t.source.x + 640 <= 3840);
506 assert!(t.source.y + 640 <= 2160);
507 }
508 assert_eq!(grid[0].source, Region::new(0, 0, 640, 640));
510 assert_eq!(grid[8].source, Region::new(0, 507, 640, 640)); }
512
513 #[test]
514 fn tile_grid_realized_overlap_at_least_requested() {
515 let grid = tile_grid(2160, 3840, 640, 640, 0.2);
516 let step = grid[1].source.x - grid[0].source.x;
518 let realized_overlap = 1.0 - (step as f64 / 640.0);
519 assert!(
520 realized_overlap >= 0.2 - 1e-9,
521 "realized {realized_overlap} < 0.2"
522 );
523 }
524
525 #[test]
526 fn tile_grid_frame_smaller_than_tile_single_whole_frame() {
527 let grid = tile_grid(400, 500, 640, 640, 0.2);
528 assert_eq!(grid.len(), 1);
529 assert_eq!(grid[0].source, Region::new(0, 0, 500, 400));
530 }
531
532 #[test]
533 fn rounding_uses_f32_path() {
534 let xs = axis_origins(4000, 640, 0.333);
538 assert_eq!(xs[0], 0);
539 assert_eq!(*xs.last().unwrap(), 4000 - 640);
540 assert!(xs.windows(2).all(|w| w[0] <= w[1]));
541 let max_step = xs.windows(2).map(|w| w[1] - w[0]).max().unwrap();
542 let realized = 1.0 - (max_step as f64 / 640.0);
543 assert!(
544 realized >= 0.333 - 1e-9,
545 "realized overlap {realized} < 0.333 (max_step={max_step})"
546 );
547 }
548
549 #[test]
550 fn tiling_options_refuse_rotation_and_flip() {
551 let cfg = TilingConfig::new(640, 640);
552 assert!(cfg.validate().is_ok());
553 let rotated = cfg.with_options(ConvertOptions::new().rotation(Rotation::Clockwise90));
554 assert!(matches!(rotated.validate(), Err(Error::InvalidOptions(_))));
555 let flipped = cfg.with_options(ConvertOptions::new().flip(Flip::Horizontal));
556 assert!(matches!(flipped.validate(), Err(Error::InvalidOptions(_))));
557 }
558
559 fn cpu() -> ImageProcessor {
560 ImageProcessor::with_config(crate::ImageProcessorConfig {
561 backend: crate::ComputeBackend::Cpu,
562 ..Default::default()
563 })
564 .unwrap()
565 }
566
567 #[test]
570 fn plan_tiles_content_region_on_every_tile() {
571 let options = ConvertOptions::new()
572 .letterbox([114, 114, 114, 255])
573 .placement(crate::LetterboxPlacement::TopLeft);
574 let cfg = TilingConfig::new(640, 640).with_options(options);
575 let plan = cpu().plan_tiles(1500, 400, &cfg).unwrap();
576 assert!(plan.len() >= 3);
577 for p in &plan {
578 assert_eq!(
579 p.content_region,
580 Some([0.0, 0.0, 1.0, 0.625]),
581 "tile {}",
582 p.index
583 );
584 }
585 let centred = cfg.with_options(options.placement(crate::LetterboxPlacement::Center));
586 let plan = cpu().plan_tiles(1500, 400, ¢red).unwrap();
587 assert!(plan
588 .iter()
589 .all(|p| p.content_region == Some([0.0, 0.1875, 1.0, 0.8125])));
590 }
591
592 #[test]
595 fn top_left_tile_boxes_lift_exactly() {
596 use edgefirst_tensor::{BoundingBox, DetectBox};
597 let cfg = TilingConfig::new(640, 640).with_options(
598 ConvertOptions::new()
599 .letterbox([0, 0, 0, 255])
600 .placement(crate::LetterboxPlacement::TopLeft),
601 );
602 for p in cpu().plan_tiles(1500, 400, &cfg).unwrap() {
603 let [x0, y0, x1, y1] = p.content_region.unwrap();
604 let b = DetectBox {
605 bbox: BoundingBox {
606 xmin: x0,
607 ymin: y0,
608 xmax: x1,
609 ymax: y1,
610 },
611 score: 1.0,
612 label: 0,
613 };
614 let lifted = edgefirst_decoder::tiling::lift_tile_boxes(vec![b], &p);
615 let l = lifted[0].bbox;
616 let want = [p.origin.0, 0.0, p.origin.0 + p.crop_size.0, 400.0];
617 for (got, want) in [l.xmin, l.ymin, l.xmax, l.ymax].iter().zip(want) {
618 assert!(
619 (got - want).abs() < 1e-3,
620 "tile {}: {l:?} vs {want:?}",
621 p.index
622 );
623 }
624 }
625 }
626
627 #[test]
629 fn tile_render_applies_options() {
630 use edgefirst_tensor::{TensorMapTrait, TensorTrait};
631 let mut proc = cpu();
632 let src = TensorDyn::image(
633 1500,
634 400,
635 PixelFormat::Rgb,
636 DType::U8,
637 Some(TensorMemory::Mem),
638 CpuAccess::ReadWrite,
639 )
640 .unwrap();
641 {
642 let t = src.as_u8().unwrap();
643 let mut m = t.map().unwrap();
644 for (i, v) in m.as_mut_slice().iter_mut().enumerate() {
645 *v = (i * 7 % 251) as u8;
646 }
647 }
648 let options = ConvertOptions::new()
649 .letterbox([0, 0, 0, 255])
650 .placement(crate::LetterboxPlacement::TopLeft)
651 .pad_output([0.0; 4])
652 .normalization(crate::Normalization::imagenet());
653 let cfg = TilingConfig::new(320, 320).with_options(options);
654 let plan = proc.plan_tiles(1500, 400, &cfg).unwrap();
655 let p = plan[plan.len() / 2];
656 let image = || {
657 TensorDyn::image(
658 320,
659 320,
660 PixelFormat::PlanarRgb,
661 DType::F32,
662 Some(TensorMemory::Mem),
663 CpuAccess::ReadWrite,
664 )
665 .unwrap()
666 };
667 let mut tile = image();
668 proc.tile_one(&src, &mut tile, &p, &cfg).unwrap();
669 proc.flush().unwrap();
670 let mut direct = image();
671 let source = Region::new(
672 p.origin.0 as usize,
673 p.origin.1 as usize,
674 p.crop_size.0 as usize,
675 p.crop_size.1 as usize,
676 );
677 proc.convert(&src, &mut direct, Some(options.source(Some(source))))
678 .unwrap();
679 let a = tile
680 .as_f32()
681 .unwrap()
682 .map_read()
683 .unwrap()
684 .as_slice()
685 .to_vec();
686 let b = direct
687 .as_f32()
688 .unwrap()
689 .map_read()
690 .unwrap()
691 .as_slice()
692 .to_vec();
693 assert_eq!(a, b);
694 assert!(a.iter().any(|v| *v < 0.0), "imagenet normalization applied");
695 }
696
697 #[test]
700 fn tile_render_applies_interpolation() {
701 use edgefirst_tensor::{TensorMapTrait, TensorTrait};
702 let mut proc = cpu();
703 let src = TensorDyn::image(
704 200,
705 100,
706 PixelFormat::Rgb,
707 DType::U8,
708 Some(TensorMemory::Mem),
709 CpuAccess::ReadWrite,
710 )
711 .unwrap();
712 {
713 let t = src.as_u8().unwrap();
714 let mut m = t.map().unwrap();
715 for (i, v) in m.as_mut_slice().iter_mut().enumerate() {
716 *v = (i * 7 % 251) as u8;
717 }
718 }
719 let base = ConvertOptions::new().letterbox([0, 0, 0, 255]);
720 let options = base.interpolation(crate::Interpolation::Nearest);
721 let cfg = TilingConfig::new(320, 320).with_options(options);
722 let plan = proc.plan_tiles(200, 100, &cfg).unwrap();
723 let read = |t: &TensorDyn| t.as_u8().unwrap().map().unwrap().as_slice().to_vec();
724 let image = || {
725 TensorDyn::image(
726 320,
727 320,
728 PixelFormat::Rgb,
729 DType::U8,
730 Some(TensorMemory::Mem),
731 CpuAccess::ReadWrite,
732 )
733 .unwrap()
734 };
735 let mut tile = image();
736 proc.tile_one(&src, &mut tile, &plan[0], &cfg).unwrap();
737 proc.flush().unwrap();
738 let mut nearest = image();
739 proc.convert(&src, &mut nearest, Some(options)).unwrap();
740 let mut bilinear = image();
741 proc.convert(&src, &mut bilinear, Some(base)).unwrap();
742 assert_ne!(read(&nearest), read(&bilinear));
743 assert_eq!(read(&tile), read(&nearest));
744 }
745
746 #[test]
747 fn placement_to_source_region_rejects_invalid() {
748 let good = TilePlacement {
749 index: 0,
750 count: 1,
751 origin: (10.0, 20.0),
752 crop_size: (640.0, 640.0),
753 content_region: None,
754 frame_dims: (3840.0, 2160.0),
755 };
756 assert!(placement_to_source_region(&good).is_ok());
757
758 let mut bad = good;
759 bad.origin.0 = -1.0;
760 assert!(placement_to_source_region(&bad).is_err());
761 bad = good;
762 bad.origin.0 = f32::NAN;
763 assert!(placement_to_source_region(&bad).is_err());
764 bad = good;
765 bad.origin.0 = 1.5;
766 assert!(placement_to_source_region(&bad).is_err());
767 bad = good;
768 bad.crop_size.0 = 0.0;
769 assert!(placement_to_source_region(&bad).is_err());
770 }
771
772 #[test]
773 fn tiling_config_validate_rejects_bad_overlap() {
774 assert!(TilingConfig::new(640, 640)
775 .with_overlap(1.0)
776 .validate()
777 .is_err());
778 assert!(TilingConfig::new(640, 640)
779 .with_overlap(-0.1)
780 .validate()
781 .is_err());
782 assert!(TilingConfig::new(640, 640)
783 .with_overlap(0.2)
784 .validate()
785 .is_ok());
786 }
787
788 #[test]
789 fn tiling_config_validate_rejects_zero_tile_size() {
790 assert!(TilingConfig::new(0, 640).validate().is_err());
791 assert!(TilingConfig::new(640, 0).validate().is_err());
792 }
793
794 struct SpanNameCapture {
801 names: std::sync::Arc<std::sync::Mutex<Vec<String>>>,
802 next_id: std::sync::atomic::AtomicU64,
803 }
804
805 impl tracing::Subscriber for SpanNameCapture {
806 fn enabled(&self, _metadata: &tracing::Metadata<'_>) -> bool {
807 true
808 }
809
810 fn new_span(&self, span: &tracing::span::Attributes<'_>) -> tracing::span::Id {
811 self.names
812 .lock()
813 .unwrap()
814 .push(span.metadata().name().to_string());
815 let id = self
816 .next_id
817 .fetch_add(1, std::sync::atomic::Ordering::Relaxed)
818 + 1;
819 tracing::span::Id::from_u64(id)
820 }
821
822 fn record(&self, _span: &tracing::span::Id, _values: &tracing::span::Record<'_>) {}
823 fn record_follows_from(&self, _span: &tracing::span::Id, _follows: &tracing::span::Id) {}
824 fn event(&self, _event: &tracing::Event<'_>) {}
825 fn enter(&self, _span: &tracing::span::Id) {}
826 fn exit(&self, _span: &tracing::span::Id) {}
827 }
828
829 #[test]
830 fn tiling_emits_spans() {
831 let names = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
832 let subscriber = SpanNameCapture {
833 names: names.clone(),
834 next_id: std::sync::atomic::AtomicU64::new(0),
835 };
836
837 let _other = tracing::Dispatch::new(tracing::subscriber::NoSubscriber::default());
839 tracing::subscriber::with_default(subscriber, || {
840 let mut proc = ImageProcessor::new().expect("processor");
841 let cfg = TilingConfig::new(320, 240);
842 let placements = proc.plan_tiles(640, 480, &cfg).expect("plan_tiles");
843
844 let src = TensorDyn::image(
845 640,
846 480,
847 PixelFormat::Nv12,
848 DType::U8,
849 Some(TensorMemory::Mem),
850 CpuAccess::ReadWrite,
851 )
852 .expect("src");
853 let mut dst = TensorDyn::image(
854 320,
855 240,
856 PixelFormat::Rgb,
857 DType::U8,
858 Some(TensorMemory::Mem),
859 CpuAccess::ReadWrite,
860 )
861 .expect("dst");
862
863 proc.tile_one(&src, &mut dst, &placements[0], &cfg)
864 .expect("tile_one");
865 let _ = proc.flush();
866 });
867
868 let names = names.lock().unwrap();
869 assert!(
870 names.iter().any(|n| n == "image.plan_tiles"),
871 "expected image.plan_tiles in {names:?}"
872 );
873 assert!(
874 names.iter().any(|n| n == "image.tile_one"),
875 "expected image.tile_one in {names:?}"
876 );
877 }
878}