1#![cfg_attr(coverage_nightly, feature(coverage_attribute))]
68
69use ndarray::{Array, Array2, ArrayView, ArrayView1, ArrayView3, Dimension};
70use num_traits::{AsPrimitive, Float, PrimInt};
71
72pub mod byte;
73pub mod error;
74pub mod float;
75pub mod infer;
76pub mod modelpack;
77pub mod per_scale;
78pub mod schema;
79pub mod tiling;
80pub mod yolo;
81
82pub use infer::{
83 infer_ultralytics_schema, InferError, InferredSchema, ModelSignals, ModelSource, TensorInfo,
84};
85
86mod decoder;
87pub use decoder::*;
88#[cfg(test)]
89mod test_support;
90
91pub use edgefirst_tensor::{BoundingBox, DetectBox, ProtoData, ProtoLayout, Segmentation};
95
96pub use edgefirst_decoder_abi::TilePlacement;
101
102pub use configs::{DecoderVersion, Nms};
103pub use error::{DecoderError, DecoderResult};
104pub use per_scale::DecodeDtype;
105
106use crate::{
107 decoder::configs::QuantTuple, modelpack::modelpack_segmentation_to_mask,
108 yolo::yolo_segmentation_to_mask,
109};
110
111fn mask_to_tensor(mask: ndarray::ArrayView3<'_, u8>) -> DecoderResult<edgefirst_tensor::TensorDyn> {
120 edgefirst_tensor::Tensor::from_arrayview3(mask)
121 .map(edgefirst_tensor::TensorDyn::from)
122 .map_err(|e| DecoderError::Internal(format!("mask -> TensorDyn: {e}")))
123}
124
125#[cfg(test)]
133pub(crate) trait DetectBoxTestExt {
134 fn equal_within_delta(&self, rhs: &DetectBox, eps: f32) -> bool;
135}
136
137#[cfg(test)]
138impl DetectBoxTestExt for DetectBox {
139 fn equal_within_delta(&self, rhs: &DetectBox, eps: f32) -> bool {
140 let eq_delta = |a: f32, b: f32| (a - b).abs() <= eps;
141 self.label == rhs.label
142 && eq_delta(self.score, rhs.score)
143 && eq_delta(self.bbox.xmin, rhs.bbox.xmin)
144 && eq_delta(self.bbox.ymin, rhs.bbox.ymin)
145 && eq_delta(self.bbox.xmax, rhs.bbox.xmax)
146 && eq_delta(self.bbox.ymax, rhs.bbox.ymax)
147 }
148}
149
150pub trait BBoxTypeTrait {
152 fn to_xyxy_float<A: Float + 'static, B: AsPrimitive<A>>(input: &[B; 4]) -> [A; 4];
154
155 fn to_xyxy_dequant<A: Float + 'static, B: AsPrimitive<A>>(
157 input: &[B; 4],
158 quant: Quantization,
159 ) -> [A; 4]
160 where
161 f32: AsPrimitive<A>,
162 i32: AsPrimitive<A>;
163
164 #[inline(always)]
175 fn ndarray_to_xyxy_float<A: Float + 'static, B: AsPrimitive<A>>(
176 input: ArrayView1<B>,
177 ) -> [A; 4] {
178 Self::to_xyxy_float(&[input[0], input[1], input[2], input[3]])
179 }
180
181 #[inline(always)]
182 fn ndarray_to_xyxy_dequant<A: Float + 'static, B: AsPrimitive<A>>(
184 input: ArrayView1<B>,
185 quant: Quantization,
186 ) -> [A; 4]
187 where
188 f32: AsPrimitive<A>,
189 i32: AsPrimitive<A>,
190 {
191 Self::to_xyxy_dequant(&[input[0], input[1], input[2], input[3]], quant)
192 }
193}
194
195#[derive(Debug, Clone, Copy, PartialEq, Eq)]
197pub struct XYXY {}
198
199impl BBoxTypeTrait for XYXY {
200 fn to_xyxy_float<A: Float + 'static, B: AsPrimitive<A>>(input: &[B; 4]) -> [A; 4] {
201 input.map(|b| b.as_())
202 }
203
204 fn to_xyxy_dequant<A: Float + 'static, B: AsPrimitive<A>>(
205 input: &[B; 4],
206 quant: Quantization,
207 ) -> [A; 4]
208 where
209 f32: AsPrimitive<A>,
210 i32: AsPrimitive<A>,
211 {
212 let scale = quant.scale.as_();
213 let zp = quant.zero_point.as_();
214 input.map(|b| (b.as_() - zp) * scale)
215 }
216
217 #[inline(always)]
218 fn ndarray_to_xyxy_float<A: Float + 'static, B: AsPrimitive<A>>(
219 input: ArrayView1<B>,
220 ) -> [A; 4] {
221 [
222 input[0].as_(),
223 input[1].as_(),
224 input[2].as_(),
225 input[3].as_(),
226 ]
227 }
228}
229
230#[derive(Debug, Clone, Copy, PartialEq, Eq)]
233pub struct XYWH {}
234
235impl BBoxTypeTrait for XYWH {
236 #[inline(always)]
237 fn to_xyxy_float<A: Float + 'static, B: AsPrimitive<A>>(input: &[B; 4]) -> [A; 4] {
238 let half = A::one() / (A::one() + A::one());
239 [
240 (input[0].as_()) - (input[2].as_() * half),
241 (input[1].as_()) - (input[3].as_() * half),
242 (input[0].as_()) + (input[2].as_() * half),
243 (input[1].as_()) + (input[3].as_() * half),
244 ]
245 }
246
247 #[inline(always)]
248 fn to_xyxy_dequant<A: Float + 'static, B: AsPrimitive<A>>(
249 input: &[B; 4],
250 quant: Quantization,
251 ) -> [A; 4]
252 where
253 f32: AsPrimitive<A>,
254 i32: AsPrimitive<A>,
255 {
256 let scale = quant.scale.as_();
257 let half_scale = (quant.scale * 0.5).as_();
258 let zp = quant.zero_point.as_();
259 let [x, y, w, h] = [
260 (input[0].as_() - zp) * scale,
261 (input[1].as_() - zp) * scale,
262 (input[2].as_() - zp) * half_scale,
263 (input[3].as_() - zp) * half_scale,
264 ];
265
266 [x - w, y - h, x + w, y + h]
267 }
268
269 #[inline(always)]
270 fn ndarray_to_xyxy_float<A: Float + 'static, B: AsPrimitive<A>>(
271 input: ArrayView1<B>,
272 ) -> [A; 4] {
273 let half = A::one() / (A::one() + A::one());
274 [
275 (input[0].as_()) - (input[2].as_() * half),
276 (input[1].as_()) - (input[3].as_() * half),
277 (input[0].as_()) + (input[2].as_() * half),
278 (input[1].as_()) + (input[3].as_() * half),
279 ]
280 }
281}
282
283#[derive(Debug, Clone, Copy, PartialEq)]
285pub struct Quantization {
286 pub scale: f32,
287 pub zero_point: i32,
288}
289
290impl Quantization {
291 pub fn new(scale: f32, zero_point: i32) -> Self {
300 Self { scale, zero_point }
301 }
302
303 pub fn identity() -> Self {
315 Self {
316 scale: 1.0,
317 zero_point: 0,
318 }
319 }
320}
321
322impl From<QuantTuple> for Quantization {
323 fn from(quant_tuple: QuantTuple) -> Quantization {
334 Quantization {
335 scale: quant_tuple.0,
336 zero_point: quant_tuple.1,
337 }
338 }
339}
340
341impl<S, Z> From<(S, Z)> for Quantization
342where
343 S: AsPrimitive<f32>,
344 Z: AsPrimitive<i32>,
345{
346 fn from((scale, zp): (S, Z)) -> Quantization {
355 Self {
356 scale: scale.as_(),
357 zero_point: zp.as_(),
358 }
359 }
360}
361
362impl Default for Quantization {
363 fn default() -> Self {
372 Self {
373 scale: 1.0,
374 zero_point: 0,
375 }
376 }
377}
378
379pub fn dequant_detect_box<SCORE: PrimInt + AsPrimitive<f32>>(
397 detect: &DetectBoxQuantized<SCORE>,
398 quant_scores: Quantization,
399) -> DetectBox {
400 let scaled_zp = -quant_scores.scale * quant_scores.zero_point as f32;
401 DetectBox {
402 bbox: detect.bbox,
403 score: quant_scores.scale * detect.score.as_() + scaled_zp,
404 label: detect.label,
405 }
406}
407#[derive(Debug, Clone, Copy, PartialEq)]
409pub struct DetectBoxQuantized<
410 SCORE: PrimInt + AsPrimitive<f32>,
412> {
413 pub bbox: BoundingBox,
415 pub score: SCORE,
418 pub label: usize,
420}
421
422pub fn dequantize_ndarray<T: AsPrimitive<F>, D: Dimension, F: Float + 'static>(
435 input: ArrayView<T, D>,
436 quant: Quantization,
437) -> Array<F, D>
438where
439 i32: num_traits::AsPrimitive<F>,
440 f32: num_traits::AsPrimitive<F>,
441{
442 let zero_point = quant.zero_point.as_();
443 let scale = quant.scale.as_();
444 if zero_point != F::zero() {
445 let scaled_zero = -zero_point * scale;
446 input.mapv(|d| d.as_() * scale + scaled_zero)
447 } else {
448 input.mapv(|d| d.as_() * scale)
449 }
450}
451
452pub fn dequantize_cpu<T: AsPrimitive<F>, F: Float + 'static>(
465 input: &[T],
466 quant: Quantization,
467 output: &mut [F],
468) where
469 f32: num_traits::AsPrimitive<F>,
470 i32: num_traits::AsPrimitive<F>,
471{
472 assert!(input.len() == output.len());
473 let zero_point = quant.zero_point.as_();
474 let scale = quant.scale.as_();
475 if zero_point != F::zero() {
476 let scaled_zero = -zero_point * scale; input
478 .iter()
479 .zip(output)
480 .for_each(|(d, deq)| *deq = d.as_() * scale + scaled_zero);
481 } else {
482 input
483 .iter()
484 .zip(output)
485 .for_each(|(d, deq)| *deq = d.as_() * scale);
486 }
487}
488
489pub fn dequantize_cpu_chunked<T: AsPrimitive<F>, F: Float + 'static>(
503 input: &[T],
504 quant: Quantization,
505 output: &mut [F],
506) where
507 f32: num_traits::AsPrimitive<F>,
508 i32: num_traits::AsPrimitive<F>,
509{
510 assert!(input.len() == output.len());
511 let zero_point = quant.zero_point.as_();
512 let scale = quant.scale.as_();
513
514 let input = input.as_chunks::<4>();
515 let output = output.as_chunks_mut::<4>();
516
517 if zero_point != F::zero() {
518 let scaled_zero = -zero_point * scale; input
521 .0
522 .iter()
523 .zip(output.0)
524 .for_each(|(d, deq)| *deq = d.map(|d| d.as_() * scale + scaled_zero));
525 input
526 .1
527 .iter()
528 .zip(output.1)
529 .for_each(|(d, deq)| *deq = d.as_() * scale + scaled_zero);
530 } else {
531 input
532 .0
533 .iter()
534 .zip(output.0)
535 .for_each(|(d, deq)| *deq = d.map(|d| d.as_() * scale));
536 input
537 .1
538 .iter()
539 .zip(output.1)
540 .for_each(|(d, deq)| *deq = d.as_() * scale);
541 }
542}
543
544pub fn segmentation_to_mask(segmentation: ArrayView3<u8>) -> Result<Array2<u8>, DecoderError> {
562 if segmentation.shape()[2] == 0 {
563 return Err(DecoderError::InvalidShape(
564 "Segmentation tensor must have non-zero depth".to_string(),
565 ));
566 }
567 if segmentation.shape()[2] == 1 {
568 yolo_segmentation_to_mask(segmentation, 128)
569 } else {
570 Ok(modelpack_segmentation_to_mask(segmentation))
571 }
572}
573
574fn arg_max<T: PartialOrd + Copy>(score: ArrayView1<T>) -> (T, usize) {
576 score
577 .iter()
578 .enumerate()
579 .fold((score[0], 0), |(max, arg_max), (ind, s)| {
580 if max > *s {
581 (max, arg_max)
582 } else {
583 (*s, ind)
584 }
585 })
586}
587
588#[cfg(target_arch = "aarch64")]
594pub(crate) fn arg_max_i8(scores: &[i8]) -> (i8, usize) {
595 use std::arch::aarch64::*; let n = scores.len();
598 if n < 16 {
599 let mut max = scores[0];
601 let mut idx = 0;
602 for (i, &s) in scores.iter().enumerate().skip(1) {
603 if s >= max {
604 max = s;
605 idx = i;
606 }
607 }
608 return (max, idx);
609 }
610
611 unsafe {
612 let chunks = n / 16;
614 let mut vmax = vld1q_s8(scores.as_ptr());
615 for i in 1..chunks {
616 let v = vld1q_s8(scores.as_ptr().add(i * 16));
617 vmax = vmaxq_s8(vmax, v);
618 }
619 let global_max = vmaxvq_s8(vmax);
620
621 let remainder_start = chunks * 16;
623 let mut final_max = global_max;
624 for &s in &scores[remainder_start..] {
625 if s > final_max {
626 final_max = s;
627 }
628 }
629
630 let mut idx = 0;
633 for i in (0..n).rev() {
634 if scores[i] == final_max {
635 idx = i;
636 break;
637 }
638 }
639 (final_max, idx)
640 }
641}
642#[cfg(test)]
643#[cfg_attr(coverage_nightly, coverage(off))]
644mod decoder_tests {
645 #![allow(clippy::excessive_precision)]
646 use crate::DetectBoxTestExt;
647 use crate::{
648 configs::{DecoderType, DimName, Protos},
649 modelpack::{decode_modelpack_det, decode_modelpack_split_quant},
650 yolo::{
651 decode_yolo_det, decode_yolo_det_float, decode_yolo_segdet_float,
652 decode_yolo_segdet_quant,
653 },
654 *,
655 };
656 use edgefirst_tensor::{Tensor, TensorMapTrait, TensorTrait};
657 use ndarray::Dimension;
658 use ndarray::{array, s, Array2, Array3, Array4, Axis};
659 use num_traits::{AsPrimitive, PrimInt};
660
661 fn compare_outputs(
662 boxes: (&[DetectBox], &[DetectBox]),
663 masks: (&[Segmentation], &[Segmentation]),
664 ) {
665 let (boxes0, boxes1) = boxes;
666 let (masks0, masks1) = masks;
667
668 assert_eq!(boxes0.len(), boxes1.len());
669 assert_eq!(masks0.len(), masks1.len());
670
671 for (b_i8, b_f32) in boxes0.iter().zip(boxes1) {
672 assert!(
673 b_i8.equal_within_delta(b_f32, 1e-6),
674 "{b_i8:?} is not equal to {b_f32:?}"
675 );
676 }
677
678 for (m_i8, m_f32) in masks0.iter().zip(masks1) {
679 assert_eq!(
680 [m_i8.xmin, m_i8.ymin, m_i8.xmax, m_i8.ymax],
681 [m_f32.xmin, m_f32.ymin, m_f32.xmax, m_f32.ymax],
682 );
683 let shape = m_i8.segmentation.shape().to_vec();
684 assert_eq!(shape, m_f32.segmentation.shape());
685 let (t_i8, t_f32) = (
688 m_i8.segmentation.as_u8().expect("mask must be U8"),
689 m_f32.segmentation.as_u8().expect("mask must be U8"),
690 );
691 let (map_i8, map_f32) = (
692 t_i8.map_read().expect("map i8 mask"),
693 t_f32.map_read().expect("map f32 mask"),
694 );
695 let (a, b) = (map_i8.as_slice(), map_f32.as_slice());
696 let (w, c) = (shape[1], shape[2]);
697 let mut sq_err = 0f64;
698 for (i, (&va, &vb)) in a.iter().zip(b.iter()).enumerate() {
699 let val = va as i32 - vb as i32;
700 assert!(
701 val.abs() <= 1,
702 "Difference between mask0 and mask1 is greater than 1 at ({}, {}, {}): {}",
703 i / (w * c),
704 (i / c) % w,
705 i % c,
706 val
707 );
708 sq_err += (val as f64) * (val as f64);
709 }
710 let mean_sq_err = sq_err / a.len() as f64;
711 assert!(
712 mean_sq_err < 1e-2,
713 "Mean Square Error between masks was greater than 1%: {:.2}%",
714 mean_sq_err * 100.0
715 );
716 }
717 }
718
719 fn seg_view(s: &Segmentation) -> ndarray::Array3<u8> {
722 let t = s.segmentation.as_u8().expect("mask must be U8");
723 let m = t.map_read().expect("map mask");
724 let sh = s.segmentation.shape();
725 ndarray::Array3::from_shape_vec((sh[0], sh[1], sh[2]), m.as_slice().to_vec())
726 .expect("mask shape")
727 }
728
729 fn load_yolov8_boxes() -> Array3<i8> {
732 let raw = edgefirst_bench::testdata::read("yolov8_boxes_116x8400.bin");
733 let raw = unsafe { std::slice::from_raw_parts(raw.as_ptr() as *const i8, raw.len()) };
734 Array3::from_shape_vec((1, 116, 8400), raw.to_vec()).unwrap()
735 }
736
737 fn load_yolov8_protos() -> Array4<i8> {
738 let raw = edgefirst_bench::testdata::read("yolov8_protos_160x160x32.bin");
739 let raw = unsafe { std::slice::from_raw_parts(raw.as_ptr() as *const i8, raw.len()) };
740 Array4::from_shape_vec((1, 160, 160, 32), raw.to_vec()).unwrap()
741 }
742
743 fn load_yolov8s_det() -> Array3<i8> {
744 let raw = edgefirst_bench::testdata::read("yolov8s_80_classes.bin");
745 let raw = unsafe { std::slice::from_raw_parts(raw.as_ptr() as *const i8, raw.len()) };
746 Array3::from_shape_vec((1, 84, 8400), raw.to_vec()).unwrap()
747 }
748
749 #[test]
750 fn test_decoder_modelpack() {
751 let score_threshold = 0.45;
752 let iou_threshold = 0.45;
753 let boxes = edgefirst_bench::testdata::read("modelpack_boxes_1935x1x4.bin");
754 let boxes = ndarray::Array4::from_shape_vec((1, 1935, 1, 4), boxes.to_vec()).unwrap();
755
756 let scores = edgefirst_bench::testdata::read("modelpack_scores_1935x1.bin");
757 let scores = ndarray::Array3::from_shape_vec((1, 1935, 1), scores.to_vec()).unwrap();
758
759 let quant_boxes = (0.004656755365431309, 21).into();
760 let quant_scores = (0.0019603664986789227, 0).into();
761
762 let decoder = DecoderBuilder::default()
763 .with_config_modelpack_det(
764 configs::Boxes {
765 decoder: DecoderType::ModelPack,
766 quantization: Some(quant_boxes),
767 shape: vec![1, 1935, 1, 4],
768 dshape: vec![
769 (DimName::Batch, 1),
770 (DimName::NumBoxes, 1935),
771 (DimName::Padding, 1),
772 (DimName::BoxCoords, 4),
773 ],
774 normalized: Some(true),
775 },
776 configs::Scores {
777 decoder: DecoderType::ModelPack,
778 quantization: Some(quant_scores),
779 shape: vec![1, 1935, 1],
780 dshape: vec![
781 (DimName::Batch, 1),
782 (DimName::NumBoxes, 1935),
783 (DimName::NumClasses, 1),
784 ],
785 },
786 )
787 .with_score_threshold(score_threshold)
788 .with_iou_threshold(iou_threshold)
789 .build()
790 .unwrap();
791
792 let quant_boxes = quant_boxes.into();
793 let quant_scores = quant_scores.into();
794
795 let mut output_boxes: Vec<_> = Vec::with_capacity(50);
796 decode_modelpack_det(
797 (boxes.slice(s![0, .., 0, ..]), quant_boxes),
798 (scores.slice(s![0, .., ..]), quant_scores),
799 score_threshold,
800 iou_threshold,
801 300,
802 &mut output_boxes,
803 );
804 assert!(output_boxes[0].equal_within_delta(
805 &DetectBox {
806 bbox: BoundingBox {
807 xmin: 0.40513772,
808 ymin: 0.6379755,
809 xmax: 0.5122431,
810 ymax: 0.7730214,
811 },
812 score: 0.4861709,
813 label: 0
814 },
815 1e-6
816 ));
817
818 let mut output_boxes1 = Vec::with_capacity(50);
819 let mut output_masks1 = Vec::with_capacity(50);
820
821 decoder
822 .decode_quantized(
823 &[boxes.view().into(), scores.view().into()],
824 &mut output_boxes1,
825 &mut output_masks1,
826 )
827 .unwrap();
828
829 let mut output_boxes_float = Vec::with_capacity(50);
830 let mut output_masks_float = Vec::with_capacity(50);
831
832 let boxes = dequantize_ndarray(boxes.view(), quant_boxes);
833 let scores = dequantize_ndarray(scores.view(), quant_scores);
834
835 decoder
836 .decode_float::<f32>(
837 &[boxes.view().into_dyn(), scores.view().into_dyn()],
838 &mut output_boxes_float,
839 &mut output_masks_float,
840 )
841 .unwrap();
842
843 compare_outputs((&output_boxes, &output_boxes1), (&[], &output_masks1));
844 compare_outputs(
845 (&output_boxes, &output_boxes_float),
846 (&[], &output_masks_float),
847 );
848 }
849
850 #[test]
851 fn test_decoder_modelpack_split_u8() {
852 let score_threshold = 0.45;
853 let iou_threshold = 0.45;
854 let detect0 = edgefirst_bench::testdata::read("modelpack_split_9x15x18.bin");
855 let detect0 = ndarray::Array4::from_shape_vec((1, 9, 15, 18), detect0.to_vec()).unwrap();
856
857 let detect1 = edgefirst_bench::testdata::read("modelpack_split_17x30x18.bin");
858 let detect1 = ndarray::Array4::from_shape_vec((1, 17, 30, 18), detect1.to_vec()).unwrap();
859
860 let quant0 = (0.08547406643629074, 174).into();
861 let quant1 = (0.09929127991199493, 183).into();
862 let anchors0 = vec![
863 [0.36666667461395264, 0.31481480598449707],
864 [0.38749998807907104, 0.4740740656852722],
865 [0.5333333611488342, 0.644444465637207],
866 ];
867 let anchors1 = vec![
868 [0.13750000298023224, 0.2074074000120163],
869 [0.2541666626930237, 0.21481481194496155],
870 [0.23125000298023224, 0.35185185074806213],
871 ];
872
873 let detect_config0 = configs::Detection {
874 decoder: DecoderType::ModelPack,
875 shape: vec![1, 9, 15, 18],
876 anchors: Some(anchors0.clone()),
877 quantization: Some(quant0),
878 dshape: vec![
879 (DimName::Batch, 1),
880 (DimName::Height, 9),
881 (DimName::Width, 15),
882 (DimName::NumAnchorsXFeatures, 18),
883 ],
884 normalized: Some(true),
885 };
886
887 let detect_config1 = configs::Detection {
888 decoder: DecoderType::ModelPack,
889 shape: vec![1, 17, 30, 18],
890 anchors: Some(anchors1.clone()),
891 quantization: Some(quant1),
892 dshape: vec![
893 (DimName::Batch, 1),
894 (DimName::Height, 17),
895 (DimName::Width, 30),
896 (DimName::NumAnchorsXFeatures, 18),
897 ],
898 normalized: Some(true),
899 };
900
901 let config0 = (&detect_config0).try_into().unwrap();
902 let config1 = (&detect_config1).try_into().unwrap();
903
904 let decoder = DecoderBuilder::default()
905 .with_config_modelpack_det_split(vec![detect_config1, detect_config0])
906 .with_score_threshold(score_threshold)
907 .with_iou_threshold(iou_threshold)
908 .build()
909 .unwrap();
910
911 let quant0 = quant0.into();
912 let quant1 = quant1.into();
913
914 let mut output_boxes: Vec<_> = Vec::with_capacity(10);
915 decode_modelpack_split_quant(
916 &[
917 detect0.slice(s![0, .., .., ..]),
918 detect1.slice(s![0, .., .., ..]),
919 ],
920 &[config0, config1],
921 score_threshold,
922 iou_threshold,
923 300,
924 &mut output_boxes,
925 );
926 assert!(output_boxes[0].equal_within_delta(
927 &DetectBox {
928 bbox: BoundingBox {
929 xmin: 0.43171933,
930 ymin: 0.68243736,
931 xmax: 0.5626645,
932 ymax: 0.808863,
933 },
934 score: 0.99240804,
935 label: 0
936 },
937 1e-6
938 ));
939
940 let mut output_boxes1: Vec<_> = Vec::with_capacity(10);
941 let mut output_masks1: Vec<_> = Vec::with_capacity(10);
942 decoder
943 .decode_quantized(
944 &[detect0.view().into(), detect1.view().into()],
945 &mut output_boxes1,
946 &mut output_masks1,
947 )
948 .unwrap();
949
950 let mut output_boxes1_f32: Vec<_> = Vec::with_capacity(10);
951 let mut output_masks1_f32: Vec<_> = Vec::with_capacity(10);
952
953 let detect0 = dequantize_ndarray(detect0.view(), quant0);
954 let detect1 = dequantize_ndarray(detect1.view(), quant1);
955 decoder
956 .decode_float::<f32>(
957 &[detect0.view().into_dyn(), detect1.view().into_dyn()],
958 &mut output_boxes1_f32,
959 &mut output_masks1_f32,
960 )
961 .unwrap();
962
963 compare_outputs((&output_boxes, &output_boxes1), (&[], &output_masks1));
964 compare_outputs(
965 (&output_boxes, &output_boxes1_f32),
966 (&[], &output_masks1_f32),
967 );
968 }
969
970 #[test]
971 fn test_decoder_parse_config_modelpack_split_u8() {
972 let score_threshold = 0.45;
973 let iou_threshold = 0.45;
974 let detect0 = edgefirst_bench::testdata::read("modelpack_split_9x15x18.bin");
975 let detect0 = ndarray::Array4::from_shape_vec((1, 9, 15, 18), detect0.to_vec()).unwrap();
976
977 let detect1 = edgefirst_bench::testdata::read("modelpack_split_17x30x18.bin");
978 let detect1 = ndarray::Array4::from_shape_vec((1, 17, 30, 18), detect1.to_vec()).unwrap();
979
980 let decoder = DecoderBuilder::default()
981 .with_config_yaml_str(
982 edgefirst_bench::testdata::read_to_string("modelpack_split.yaml").to_string(),
983 )
984 .with_score_threshold(score_threshold)
985 .with_iou_threshold(iou_threshold)
986 .build()
987 .unwrap();
988
989 let mut output_boxes: Vec<_> = Vec::with_capacity(10);
990 let mut output_masks: Vec<_> = Vec::with_capacity(10);
991 decoder
992 .decode_quantized(
993 &[
994 ArrayViewDQuantized::from(detect1.view()),
995 ArrayViewDQuantized::from(detect0.view()),
996 ],
997 &mut output_boxes,
998 &mut output_masks,
999 )
1000 .unwrap();
1001 assert!(output_boxes[0].equal_within_delta(
1002 &DetectBox {
1003 bbox: BoundingBox {
1004 xmin: 0.43171933,
1005 ymin: 0.68243736,
1006 xmax: 0.5626645,
1007 ymax: 0.808863,
1008 },
1009 score: 0.99240804,
1010 label: 0
1011 },
1012 1e-6
1013 ));
1014 }
1015
1016 #[test]
1017 fn test_modelpack_seg() {
1018 let out = edgefirst_bench::testdata::read("modelpack_seg_2x160x160.bin");
1019 let out = ndarray::Array4::from_shape_vec((1, 2, 160, 160), out.to_vec()).unwrap();
1020 let quant = (1.0 / 255.0, 0).into();
1021
1022 let decoder = DecoderBuilder::default()
1023 .with_config_modelpack_seg(configs::Segmentation {
1024 decoder: DecoderType::ModelPack,
1025 quantization: Some(quant),
1026 shape: vec![1, 2, 160, 160],
1027 dshape: vec![
1028 (DimName::Batch, 1),
1029 (DimName::NumClasses, 2),
1030 (DimName::Height, 160),
1031 (DimName::Width, 160),
1032 ],
1033 })
1034 .build()
1035 .unwrap();
1036 let mut output_boxes: Vec<_> = Vec::with_capacity(10);
1037 let mut output_masks: Vec<_> = Vec::with_capacity(10);
1038 decoder
1039 .decode_quantized(&[out.view().into()], &mut output_boxes, &mut output_masks)
1040 .unwrap();
1041
1042 let mut mask = out.slice(s![0, .., .., ..]);
1043 mask.swap_axes(0, 1);
1044 mask.swap_axes(1, 2);
1045 let mask = [Segmentation {
1046 xmin: 0.0,
1047 ymin: 0.0,
1048 xmax: 1.0,
1049 ymax: 1.0,
1050 segmentation: crate::mask_to_tensor(mask.view()).unwrap(),
1051 }];
1052 compare_outputs((&[], &output_boxes), (&mask, &output_masks));
1053
1054 decoder
1055 .decode_float::<f32>(
1056 &[dequantize_ndarray(out.view(), quant.into())
1057 .view()
1058 .into_dyn()],
1059 &mut output_boxes,
1060 &mut output_masks,
1061 )
1062 .unwrap();
1063
1064 compare_outputs((&[], &output_boxes), (&[], &[]));
1070 let mask0 = segmentation_to_mask(seg_view(&mask[0]).view()).unwrap();
1071 let mask1 = segmentation_to_mask(seg_view(&output_masks[0]).view()).unwrap();
1072
1073 assert_eq!(mask0, mask1);
1074 }
1075 #[test]
1076 fn test_modelpack_seg_quant() {
1077 let out = edgefirst_bench::testdata::read("modelpack_seg_2x160x160.bin");
1078 let out_u8 = ndarray::Array4::from_shape_vec((1, 2, 160, 160), out.to_vec()).unwrap();
1079 let out_i8 = out_u8.mapv(|x| (x as i16 - 128) as i8);
1080 let out_u16 = out_u8.mapv(|x| (x as u16) << 8);
1081 let out_i16 = out_u8.mapv(|x| (((x as i32) << 8) - 32768) as i16);
1082 let out_u32 = out_u8.mapv(|x| (x as u32) << 24);
1083 let out_i32 = out_u8.mapv(|x| (((x as i64) << 24) - 2147483648) as i32);
1084
1085 let quant = (1.0 / 255.0, 0).into();
1086
1087 let decoder = DecoderBuilder::default()
1088 .with_config_modelpack_seg(configs::Segmentation {
1089 decoder: DecoderType::ModelPack,
1090 quantization: Some(quant),
1091 shape: vec![1, 2, 160, 160],
1092 dshape: vec![
1093 (DimName::Batch, 1),
1094 (DimName::NumClasses, 2),
1095 (DimName::Height, 160),
1096 (DimName::Width, 160),
1097 ],
1098 })
1099 .build()
1100 .unwrap();
1101 let mut output_boxes: Vec<_> = Vec::with_capacity(10);
1102 let mut output_masks_u8: Vec<_> = Vec::with_capacity(10);
1103 decoder
1104 .decode_quantized(
1105 &[out_u8.view().into()],
1106 &mut output_boxes,
1107 &mut output_masks_u8,
1108 )
1109 .unwrap();
1110
1111 let mut output_masks_i8: Vec<_> = Vec::with_capacity(10);
1112 decoder
1113 .decode_quantized(
1114 &[out_i8.view().into()],
1115 &mut output_boxes,
1116 &mut output_masks_i8,
1117 )
1118 .unwrap();
1119
1120 let mut output_masks_u16: Vec<_> = Vec::with_capacity(10);
1121 decoder
1122 .decode_quantized(
1123 &[out_u16.view().into()],
1124 &mut output_boxes,
1125 &mut output_masks_u16,
1126 )
1127 .unwrap();
1128
1129 let mut output_masks_i16: Vec<_> = Vec::with_capacity(10);
1130 decoder
1131 .decode_quantized(
1132 &[out_i16.view().into()],
1133 &mut output_boxes,
1134 &mut output_masks_i16,
1135 )
1136 .unwrap();
1137
1138 let mut output_masks_u32: Vec<_> = Vec::with_capacity(10);
1139 decoder
1140 .decode_quantized(
1141 &[out_u32.view().into()],
1142 &mut output_boxes,
1143 &mut output_masks_u32,
1144 )
1145 .unwrap();
1146
1147 let mut output_masks_i32: Vec<_> = Vec::with_capacity(10);
1148 decoder
1149 .decode_quantized(
1150 &[out_i32.view().into()],
1151 &mut output_boxes,
1152 &mut output_masks_i32,
1153 )
1154 .unwrap();
1155
1156 compare_outputs((&[], &output_boxes), (&[], &[]));
1157 let mask_u8 = segmentation_to_mask(seg_view(&output_masks_u8[0]).view()).unwrap();
1158 let mask_i8 = segmentation_to_mask(seg_view(&output_masks_i8[0]).view()).unwrap();
1159 let mask_u16 = segmentation_to_mask(seg_view(&output_masks_u16[0]).view()).unwrap();
1160 let mask_i16 = segmentation_to_mask(seg_view(&output_masks_i16[0]).view()).unwrap();
1161 let mask_u32 = segmentation_to_mask(seg_view(&output_masks_u32[0]).view()).unwrap();
1162 let mask_i32 = segmentation_to_mask(seg_view(&output_masks_i32[0]).view()).unwrap();
1163 assert_eq!(mask_u8, mask_i8);
1164 assert_eq!(mask_u8, mask_u16);
1165 assert_eq!(mask_u8, mask_i16);
1166 assert_eq!(mask_u8, mask_u32);
1167 assert_eq!(mask_u8, mask_i32);
1168 }
1169
1170 #[test]
1171 fn test_modelpack_segdet() {
1172 let score_threshold = 0.45;
1173 let iou_threshold = 0.45;
1174
1175 let boxes = edgefirst_bench::testdata::read("modelpack_boxes_1935x1x4.bin");
1176 let boxes = Array4::from_shape_vec((1, 1935, 1, 4), boxes.to_vec()).unwrap();
1177
1178 let scores = edgefirst_bench::testdata::read("modelpack_scores_1935x1.bin");
1179 let scores = Array3::from_shape_vec((1, 1935, 1), scores.to_vec()).unwrap();
1180
1181 let seg = edgefirst_bench::testdata::read("modelpack_seg_2x160x160.bin");
1182 let seg = Array4::from_shape_vec((1, 2, 160, 160), seg.to_vec()).unwrap();
1183
1184 let quant_boxes = (0.004656755365431309, 21).into();
1185 let quant_scores = (0.0019603664986789227, 0).into();
1186 let quant_seg = (1.0 / 255.0, 0).into();
1187
1188 let decoder = DecoderBuilder::default()
1189 .with_config_modelpack_segdet(
1190 configs::Boxes {
1191 decoder: DecoderType::ModelPack,
1192 quantization: Some(quant_boxes),
1193 shape: vec![1, 1935, 1, 4],
1194 dshape: vec![
1195 (DimName::Batch, 1),
1196 (DimName::NumBoxes, 1935),
1197 (DimName::Padding, 1),
1198 (DimName::BoxCoords, 4),
1199 ],
1200 normalized: Some(true),
1201 },
1202 configs::Scores {
1203 decoder: DecoderType::ModelPack,
1204 quantization: Some(quant_scores),
1205 shape: vec![1, 1935, 1],
1206 dshape: vec![
1207 (DimName::Batch, 1),
1208 (DimName::NumBoxes, 1935),
1209 (DimName::NumClasses, 1),
1210 ],
1211 },
1212 configs::Segmentation {
1213 decoder: DecoderType::ModelPack,
1214 quantization: Some(quant_seg),
1215 shape: vec![1, 2, 160, 160],
1216 dshape: vec![
1217 (DimName::Batch, 1),
1218 (DimName::NumClasses, 2),
1219 (DimName::Height, 160),
1220 (DimName::Width, 160),
1221 ],
1222 },
1223 )
1224 .with_iou_threshold(iou_threshold)
1225 .with_score_threshold(score_threshold)
1226 .build()
1227 .unwrap();
1228 let mut output_boxes: Vec<_> = Vec::with_capacity(10);
1229 let mut output_masks: Vec<_> = Vec::with_capacity(10);
1230 decoder
1231 .decode_quantized(
1232 &[scores.view().into(), boxes.view().into(), seg.view().into()],
1233 &mut output_boxes,
1234 &mut output_masks,
1235 )
1236 .unwrap();
1237
1238 let mut mask = seg.slice(s![0, .., .., ..]);
1239 mask.swap_axes(0, 1);
1240 mask.swap_axes(1, 2);
1241 let mask = [Segmentation {
1242 xmin: 0.0,
1243 ymin: 0.0,
1244 xmax: 1.0,
1245 ymax: 1.0,
1246 segmentation: crate::mask_to_tensor(mask.view()).unwrap(),
1247 }];
1248 let correct_boxes = [DetectBox {
1249 bbox: BoundingBox {
1250 xmin: 0.40513772,
1251 ymin: 0.6379755,
1252 xmax: 0.5122431,
1253 ymax: 0.7730214,
1254 },
1255 score: 0.4861709,
1256 label: 0,
1257 }];
1258 compare_outputs((&correct_boxes, &output_boxes), (&mask, &output_masks));
1259
1260 let scores = dequantize_ndarray(scores.view(), quant_scores.into());
1261 let boxes = dequantize_ndarray(boxes.view(), quant_boxes.into());
1262 let seg = dequantize_ndarray(seg.view(), quant_seg.into());
1263 decoder
1264 .decode_float::<f32>(
1265 &[
1266 scores.view().into_dyn(),
1267 boxes.view().into_dyn(),
1268 seg.view().into_dyn(),
1269 ],
1270 &mut output_boxes,
1271 &mut output_masks,
1272 )
1273 .unwrap();
1274
1275 compare_outputs((&correct_boxes, &output_boxes), (&[], &[]));
1281 let mask0 = segmentation_to_mask(seg_view(&mask[0]).view()).unwrap();
1282 let mask1 = segmentation_to_mask(seg_view(&output_masks[0]).view()).unwrap();
1283
1284 assert_eq!(mask0, mask1);
1285 }
1286
1287 #[test]
1288 fn test_modelpack_segdet_split() {
1289 let score_threshold = 0.8;
1290 let iou_threshold = 0.5;
1291
1292 let seg = edgefirst_bench::testdata::read("modelpack_seg_2x160x160.bin");
1293 let seg = ndarray::Array4::from_shape_vec((1, 2, 160, 160), seg.to_vec()).unwrap();
1294
1295 let detect0 = edgefirst_bench::testdata::read("modelpack_split_9x15x18.bin");
1296 let detect0 = ndarray::Array4::from_shape_vec((1, 9, 15, 18), detect0.to_vec()).unwrap();
1297
1298 let detect1 = edgefirst_bench::testdata::read("modelpack_split_17x30x18.bin");
1299 let detect1 = ndarray::Array4::from_shape_vec((1, 17, 30, 18), detect1.to_vec()).unwrap();
1300
1301 let quant0 = (0.08547406643629074, 174).into();
1302 let quant1 = (0.09929127991199493, 183).into();
1303 let quant_seg = (1.0 / 255.0, 0).into();
1304
1305 let anchors0 = vec![
1306 [0.36666667461395264, 0.31481480598449707],
1307 [0.38749998807907104, 0.4740740656852722],
1308 [0.5333333611488342, 0.644444465637207],
1309 ];
1310 let anchors1 = vec![
1311 [0.13750000298023224, 0.2074074000120163],
1312 [0.2541666626930237, 0.21481481194496155],
1313 [0.23125000298023224, 0.35185185074806213],
1314 ];
1315
1316 let decoder = DecoderBuilder::default()
1317 .with_config_modelpack_segdet_split(
1318 vec![
1319 configs::Detection {
1320 decoder: DecoderType::ModelPack,
1321 shape: vec![1, 17, 30, 18],
1322 anchors: Some(anchors1),
1323 quantization: Some(quant1),
1324 dshape: vec![
1325 (DimName::Batch, 1),
1326 (DimName::Height, 17),
1327 (DimName::Width, 30),
1328 (DimName::NumAnchorsXFeatures, 18),
1329 ],
1330 normalized: Some(true),
1331 },
1332 configs::Detection {
1333 decoder: DecoderType::ModelPack,
1334 shape: vec![1, 9, 15, 18],
1335 anchors: Some(anchors0),
1336 quantization: Some(quant0),
1337 dshape: vec![
1338 (DimName::Batch, 1),
1339 (DimName::Height, 9),
1340 (DimName::Width, 15),
1341 (DimName::NumAnchorsXFeatures, 18),
1342 ],
1343 normalized: Some(true),
1344 },
1345 ],
1346 configs::Segmentation {
1347 decoder: DecoderType::ModelPack,
1348 quantization: Some(quant_seg),
1349 shape: vec![1, 2, 160, 160],
1350 dshape: vec![
1351 (DimName::Batch, 1),
1352 (DimName::NumClasses, 2),
1353 (DimName::Height, 160),
1354 (DimName::Width, 160),
1355 ],
1356 },
1357 )
1358 .with_score_threshold(score_threshold)
1359 .with_iou_threshold(iou_threshold)
1360 .build()
1361 .unwrap();
1362 let mut output_boxes: Vec<_> = Vec::with_capacity(10);
1363 let mut output_masks: Vec<_> = Vec::with_capacity(10);
1364 decoder
1365 .decode_quantized(
1366 &[
1367 detect0.view().into(),
1368 detect1.view().into(),
1369 seg.view().into(),
1370 ],
1371 &mut output_boxes,
1372 &mut output_masks,
1373 )
1374 .unwrap();
1375
1376 let mut mask = seg.slice(s![0, .., .., ..]);
1377 mask.swap_axes(0, 1);
1378 mask.swap_axes(1, 2);
1379 let mask = [Segmentation {
1380 xmin: 0.0,
1381 ymin: 0.0,
1382 xmax: 1.0,
1383 ymax: 1.0,
1384 segmentation: crate::mask_to_tensor(mask.view()).unwrap(),
1385 }];
1386 let correct_boxes = [DetectBox {
1387 bbox: BoundingBox {
1388 xmin: 0.43171933,
1389 ymin: 0.68243736,
1390 xmax: 0.5626645,
1391 ymax: 0.808863,
1392 },
1393 score: 0.99240804,
1394 label: 0,
1395 }];
1396 println!("Output Boxes: {:?}", output_boxes);
1397 compare_outputs((&correct_boxes, &output_boxes), (&mask, &output_masks));
1398
1399 let detect0 = dequantize_ndarray(detect0.view(), quant0.into());
1400 let detect1 = dequantize_ndarray(detect1.view(), quant1.into());
1401 let seg = dequantize_ndarray(seg.view(), quant_seg.into());
1402 decoder
1403 .decode_float::<f32>(
1404 &[
1405 detect0.view().into_dyn(),
1406 detect1.view().into_dyn(),
1407 seg.view().into_dyn(),
1408 ],
1409 &mut output_boxes,
1410 &mut output_masks,
1411 )
1412 .unwrap();
1413
1414 compare_outputs((&correct_boxes, &output_boxes), (&[], &[]));
1420 let mask0 = segmentation_to_mask(seg_view(&mask[0]).view()).unwrap();
1421 let mask1 = segmentation_to_mask(seg_view(&output_masks[0]).view()).unwrap();
1422
1423 assert_eq!(mask0, mask1);
1424 }
1425
1426 #[test]
1427 fn test_dequant_chunked() {
1428 let mut out = load_yolov8s_det().into_raw_vec_and_offset().0;
1429 out.push(123); let mut out_dequant = vec![0.0; 84 * 8400 + 1];
1432 let mut out_dequant_simd = vec![0.0; 84 * 8400 + 1];
1433 let quant = Quantization::new(0.0040811873, -123);
1434 dequantize_cpu(&out, quant, &mut out_dequant);
1435
1436 dequantize_cpu_chunked(&out, quant, &mut out_dequant_simd);
1437 assert_eq!(out_dequant, out_dequant_simd);
1438
1439 let quant = Quantization::new(0.0040811873, 0);
1440 dequantize_cpu(&out, quant, &mut out_dequant);
1441
1442 dequantize_cpu_chunked(&out, quant, &mut out_dequant_simd);
1443 assert_eq!(out_dequant, out_dequant_simd);
1444 }
1445
1446 #[test]
1447 fn test_dequant_ground_truth() {
1448 let quant = Quantization::new(0.1, -128);
1453 let input: Vec<i8> = vec![0, 127, -128, 64];
1454 let mut output = vec![0.0f32; 4];
1455 let mut output_chunked = vec![0.0f32; 4];
1456 dequantize_cpu(&input, quant, &mut output);
1457 dequantize_cpu_chunked(&input, quant, &mut output_chunked);
1458 let expected: Vec<f32> = vec![12.8, 25.5, 0.0, 19.2];
1463 for (i, (&out, &exp)) in output.iter().zip(expected.iter()).enumerate() {
1464 assert!((out - exp).abs() < 1e-5, "cpu[{i}]: {out} != {exp}");
1465 }
1466 for (i, (&out, &exp)) in output_chunked.iter().zip(expected.iter()).enumerate() {
1467 assert!((out - exp).abs() < 1e-5, "chunked[{i}]: {out} != {exp}");
1468 }
1469
1470 let quant = Quantization::new(1.0, 0);
1472 dequantize_cpu(&input, quant, &mut output);
1473 dequantize_cpu_chunked(&input, quant, &mut output_chunked);
1474 let expected: Vec<f32> = vec![0.0, 127.0, -128.0, 64.0];
1475 assert_eq!(output, expected);
1476 assert_eq!(output_chunked, expected);
1477
1478 let quant = Quantization::new(0.5, 0);
1480 dequantize_cpu(&input, quant, &mut output);
1481 dequantize_cpu_chunked(&input, quant, &mut output_chunked);
1482 let expected: Vec<f32> = vec![0.0, 63.5, -64.0, 32.0];
1483 assert_eq!(output, expected);
1484 assert_eq!(output_chunked, expected);
1485
1486 let quant = Quantization::new(0.021287762, 31);
1488 let input: Vec<i8> = vec![-128, -1, 0, 1, 31, 127];
1489 let mut output = vec![0.0f32; 6];
1490 let mut output_chunked = vec![0.0f32; 6];
1491 dequantize_cpu(&input, quant, &mut output);
1492 dequantize_cpu_chunked(&input, quant, &mut output_chunked);
1493 for i in 0..6 {
1494 let expected = (input[i] as f32 - 31.0) * 0.021287762;
1495 assert!(
1496 (output[i] - expected).abs() < 1e-5,
1497 "cpu[{i}]: {} != {expected}",
1498 output[i]
1499 );
1500 assert!(
1501 (output_chunked[i] - expected).abs() < 1e-5,
1502 "chunked[{i}]: {} != {expected}",
1503 output_chunked[i]
1504 );
1505 }
1506 }
1507
1508 #[test]
1509 fn test_decoder_yolo_det() {
1510 let score_threshold = 0.25;
1511 let iou_threshold = 0.7;
1512 let out = load_yolov8s_det();
1513 let quant = (0.0040811873, -123).into();
1514
1515 let decoder = DecoderBuilder::default()
1516 .with_config_yolo_det(
1517 configs::Detection {
1518 decoder: DecoderType::Ultralytics,
1519 shape: vec![1, 84, 8400],
1520 anchors: None,
1521 quantization: Some(quant),
1522 dshape: vec![
1523 (DimName::Batch, 1),
1524 (DimName::NumFeatures, 84),
1525 (DimName::NumBoxes, 8400),
1526 ],
1527 normalized: Some(true),
1528 },
1529 Some(DecoderVersion::Yolo11),
1530 )
1531 .with_score_threshold(score_threshold)
1532 .with_iou_threshold(iou_threshold)
1533 .build()
1534 .unwrap();
1535
1536 let mut output_boxes: Vec<_> = Vec::with_capacity(50);
1537 decode_yolo_det(
1538 (out.slice(s![0, .., ..]), quant.into()),
1539 score_threshold,
1540 iou_threshold,
1541 Some(configs::Nms::ClassAgnostic),
1542 &mut output_boxes,
1543 );
1544 assert!(output_boxes[0].equal_within_delta(
1545 &DetectBox {
1546 bbox: BoundingBox {
1547 xmin: 0.5285137,
1548 ymin: 0.05305544,
1549 xmax: 0.87541467,
1550 ymax: 0.9998909,
1551 },
1552 score: 0.5591227,
1553 label: 0
1554 },
1555 1e-6
1556 ));
1557
1558 assert!(output_boxes[1].equal_within_delta(
1559 &DetectBox {
1560 bbox: BoundingBox {
1561 xmin: 0.130598,
1562 ymin: 0.43260583,
1563 xmax: 0.35098213,
1564 ymax: 0.9958097,
1565 },
1566 score: 0.33057618,
1567 label: 75
1568 },
1569 1e-6
1570 ));
1571
1572 let mut output_boxes1: Vec<_> = Vec::with_capacity(50);
1573 let mut output_masks1: Vec<_> = Vec::with_capacity(50);
1574 decoder
1575 .decode_quantized(&[out.view().into()], &mut output_boxes1, &mut output_masks1)
1576 .unwrap();
1577
1578 let out = dequantize_ndarray(out.view(), quant.into());
1579 let mut output_boxes_f32: Vec<_> = Vec::with_capacity(50);
1580 let mut output_masks_f32: Vec<_> = Vec::with_capacity(50);
1581 decoder
1582 .decode_float::<f32>(
1583 &[out.view().into_dyn()],
1584 &mut output_boxes_f32,
1585 &mut output_masks_f32,
1586 )
1587 .unwrap();
1588
1589 compare_outputs((&output_boxes, &output_boxes1), (&[], &output_masks1));
1590 compare_outputs((&output_boxes, &output_boxes_f32), (&[], &output_masks_f32));
1591 }
1592
1593 #[test]
1594 fn test_decoder_masks() {
1595 let score_threshold = 0.45;
1596 let iou_threshold = 0.45;
1597 let boxes = load_yolov8_boxes();
1598 let quant_boxes = Quantization::new(0.021287761628627777, 31);
1599
1600 let protos = load_yolov8_protos();
1601 let quant_protos = Quantization::new(0.02491161972284317, -117);
1602 let protos = dequantize_ndarray::<_, _, f32>(protos.view(), quant_protos);
1603 let seg = dequantize_ndarray::<_, _, f32>(boxes.view(), quant_boxes);
1604 let mut output_boxes: Vec<_> = Vec::with_capacity(10);
1605 let mut output_masks: Vec<_> = Vec::with_capacity(10);
1606 decode_yolo_segdet_float(
1607 seg.slice(s![0, .., ..]),
1608 protos.slice(s![0, .., .., ..]),
1609 score_threshold,
1610 iou_threshold,
1611 Some(configs::Nms::ClassAgnostic),
1612 &mut output_boxes,
1613 &mut output_masks,
1614 )
1615 .unwrap();
1616 assert_eq!(output_boxes.len(), 2);
1617 assert_eq!(output_boxes.len(), output_masks.len());
1618
1619 for (b, m) in output_boxes.iter().zip(&output_masks) {
1620 assert!(b.bbox.xmin >= m.xmin);
1623 assert!(b.bbox.ymin >= m.ymin);
1624 assert!(b.bbox.xmax <= m.xmax);
1625 assert!(b.bbox.ymax <= m.ymax);
1626 }
1627 assert!(output_boxes[0].equal_within_delta(
1628 &DetectBox {
1629 bbox: BoundingBox {
1630 xmin: 0.08515105,
1631 ymin: 0.7131401,
1632 xmax: 0.29802868,
1633 ymax: 0.8195788,
1634 },
1635 score: 0.91537374,
1636 label: 23
1637 },
1638 1.0 / 160.0, ));
1640
1641 assert!(output_boxes[1].equal_within_delta(
1642 &DetectBox {
1643 bbox: BoundingBox {
1644 xmin: 0.59605736,
1645 ymin: 0.25545314,
1646 xmax: 0.93666154,
1647 ymax: 0.72378385,
1648 },
1649 score: 0.91537374,
1650 label: 23
1651 },
1652 1.0 / 160.0, ));
1654
1655 let full_mask = edgefirst_bench::testdata::read("yolov8_mask_results.bin");
1656 let full_mask = ndarray::Array2::from_shape_vec((160, 160), full_mask.to_vec()).unwrap();
1657
1658 let cropped_mask = full_mask.slice(ndarray::s![
1659 (output_masks[1].ymin * 160.0) as usize..(output_masks[1].ymax * 160.0) as usize,
1660 (output_masks[1].xmin * 160.0) as usize..(output_masks[1].xmax * 160.0) as usize,
1661 ]);
1662
1663 assert_eq!(
1664 cropped_mask,
1665 segmentation_to_mask(seg_view(&output_masks[1]).view()).unwrap()
1666 );
1667 }
1668
1669 #[test]
1679 fn test_decoder_masks_nchw_protos() {
1680 let score_threshold = 0.45;
1681 let iou_threshold = 0.45;
1682
1683 let boxes_2d = load_yolov8_boxes().slice_move(s![0, .., ..]);
1685 let quant_boxes = Quantization::new(0.021287761628627777, 31);
1686
1687 let protos_hwc = load_yolov8_protos().slice_move(s![0, .., .., ..]);
1689 let quant_protos = Quantization::new(0.02491161972284317, -117);
1690 let protos_f32_hwc = dequantize_ndarray::<_, _, f32>(protos_hwc.view(), quant_protos);
1691
1692 let seg = dequantize_ndarray::<_, _, f32>(boxes_2d.view(), quant_boxes);
1694 let mut ref_boxes: Vec<_> = Vec::with_capacity(10);
1695 let mut ref_masks: Vec<_> = Vec::with_capacity(10);
1696 decode_yolo_segdet_float(
1697 seg.view(),
1698 protos_f32_hwc.view(),
1699 score_threshold,
1700 iou_threshold,
1701 Some(configs::Nms::ClassAgnostic),
1702 &mut ref_boxes,
1703 &mut ref_masks,
1704 )
1705 .unwrap();
1706 assert_eq!(ref_boxes.len(), 2);
1707
1708 let protos_f32_chw_view = protos_f32_hwc.view().permuted_axes([2, 0, 1]); let protos_f32_chw = protos_f32_chw_view.to_owned();
1715 let protos_nchw = protos_f32_chw.insert_axis(ndarray::Axis(0)); let seg_3d = seg.insert_axis(ndarray::Axis(0)); let decoder = DecoderBuilder::default()
1724 .with_config_yolo_segdet(
1725 configs::Detection {
1726 decoder: configs::DecoderType::Ultralytics,
1727 quantization: None,
1728 shape: vec![1, 116, 8400],
1729 dshape: vec![
1730 (configs::DimName::Batch, 1),
1731 (configs::DimName::NumFeatures, 116),
1732 (configs::DimName::NumBoxes, 8400),
1733 ],
1734 normalized: Some(true),
1735 anchors: None,
1736 },
1737 configs::Protos {
1738 decoder: configs::DecoderType::Ultralytics,
1739 quantization: None,
1740 shape: vec![1, 32, 160, 160],
1741 dshape: vec![
1742 (configs::DimName::Batch, 1),
1743 (configs::DimName::NumProtos, 32),
1744 (configs::DimName::Height, 160),
1745 (configs::DimName::Width, 160),
1746 ],
1747 },
1748 None, )
1750 .with_score_threshold(score_threshold)
1751 .with_iou_threshold(iou_threshold)
1752 .build()
1753 .unwrap();
1754
1755 let mut cfg_boxes: Vec<_> = Vec::with_capacity(10);
1756 let mut cfg_masks: Vec<_> = Vec::with_capacity(10);
1757 decoder
1758 .decode_float(
1759 &[seg_3d.view().into_dyn(), protos_nchw.view().into_dyn()],
1760 &mut cfg_boxes,
1761 &mut cfg_masks,
1762 )
1763 .unwrap();
1764
1765 assert_eq!(
1767 cfg_boxes.len(),
1768 ref_boxes.len(),
1769 "config path produced {} boxes, reference produced {}",
1770 cfg_boxes.len(),
1771 ref_boxes.len()
1772 );
1773
1774 for (i, (cb, rb)) in cfg_boxes.iter().zip(&ref_boxes).enumerate() {
1776 assert!(
1777 cb.equal_within_delta(rb, 0.01),
1778 "box {i} mismatch: config={cb:?}, reference={rb:?}"
1779 );
1780 }
1781
1782 for (i, (cm, rm)) in cfg_masks.iter().zip(&ref_masks).enumerate() {
1784 let cm_arr = segmentation_to_mask(seg_view(cm).view()).unwrap();
1785 let rm_arr = segmentation_to_mask(seg_view(rm).view()).unwrap();
1786 assert_eq!(
1787 cm_arr, rm_arr,
1788 "mask {i} pixel mismatch between config-driven and reference paths"
1789 );
1790 }
1791 }
1792
1793 #[test]
1794 fn test_decoder_masks_i8() {
1795 let score_threshold = 0.45;
1796 let iou_threshold = 0.45;
1797 let boxes = load_yolov8_boxes();
1798 let quant_boxes = (0.021287761628627777, 31).into();
1799
1800 let protos = load_yolov8_protos();
1801 let quant_protos = (0.02491161972284317, -117).into();
1802 let mut output_boxes: Vec<_> = Vec::with_capacity(500);
1803 let mut output_masks: Vec<_> = Vec::with_capacity(500);
1804
1805 let decoder = DecoderBuilder::default()
1806 .with_config_yolo_segdet(
1807 configs::Detection {
1808 decoder: configs::DecoderType::Ultralytics,
1809 quantization: Some(quant_boxes),
1810 shape: vec![1, 116, 8400],
1811 anchors: None,
1812 dshape: vec![
1813 (DimName::Batch, 1),
1814 (DimName::NumFeatures, 116),
1815 (DimName::NumBoxes, 8400),
1816 ],
1817 normalized: Some(true),
1818 },
1819 Protos {
1820 decoder: configs::DecoderType::Ultralytics,
1821 quantization: Some(quant_protos),
1822 shape: vec![1, 160, 160, 32],
1823 dshape: vec![
1824 (DimName::Batch, 1),
1825 (DimName::Height, 160),
1826 (DimName::Width, 160),
1827 (DimName::NumProtos, 32),
1828 ],
1829 },
1830 Some(DecoderVersion::Yolo11),
1831 )
1832 .with_score_threshold(score_threshold)
1833 .with_iou_threshold(iou_threshold)
1834 .build()
1835 .unwrap();
1836
1837 let quant_boxes = quant_boxes.into();
1838 let quant_protos = quant_protos.into();
1839
1840 decode_yolo_segdet_quant(
1841 (boxes.slice(s![0, .., ..]), quant_boxes),
1842 (protos.slice(s![0, .., .., ..]), quant_protos),
1843 score_threshold,
1844 iou_threshold,
1845 Some(configs::Nms::ClassAgnostic),
1846 &mut output_boxes,
1847 &mut output_masks,
1848 )
1849 .unwrap();
1850
1851 let mut output_boxes1: Vec<_> = Vec::with_capacity(500);
1852 let mut output_masks1: Vec<_> = Vec::with_capacity(500);
1853
1854 decoder
1855 .decode_quantized(
1856 &[boxes.view().into(), protos.view().into()],
1857 &mut output_boxes1,
1858 &mut output_masks1,
1859 )
1860 .unwrap();
1861
1862 let protos = dequantize_ndarray::<_, _, f32>(protos.view(), quant_protos);
1863 let seg = dequantize_ndarray::<_, _, f32>(boxes.view(), quant_boxes);
1864
1865 let mut output_boxes_f32: Vec<_> = Vec::with_capacity(500);
1866 let mut output_masks_f32: Vec<_> = Vec::with_capacity(500);
1867 decode_yolo_segdet_float(
1868 seg.slice(s![0, .., ..]),
1869 protos.slice(s![0, .., .., ..]),
1870 score_threshold,
1871 iou_threshold,
1872 Some(configs::Nms::ClassAgnostic),
1873 &mut output_boxes_f32,
1874 &mut output_masks_f32,
1875 )
1876 .unwrap();
1877
1878 let mut output_boxes1_f32: Vec<_> = Vec::with_capacity(500);
1879 let mut output_masks1_f32: Vec<_> = Vec::with_capacity(500);
1880
1881 decoder
1882 .decode_float(
1883 &[seg.view().into_dyn(), protos.view().into_dyn()],
1884 &mut output_boxes1_f32,
1885 &mut output_masks1_f32,
1886 )
1887 .unwrap();
1888
1889 compare_outputs(
1890 (&output_boxes, &output_boxes1),
1891 (&output_masks, &output_masks1),
1892 );
1893
1894 compare_outputs(
1895 (&output_boxes, &output_boxes_f32),
1896 (&output_masks, &output_masks_f32),
1897 );
1898
1899 compare_outputs(
1900 (&output_boxes_f32, &output_boxes1_f32),
1901 (&output_masks_f32, &output_masks1_f32),
1902 );
1903 }
1904
1905 #[test]
1906 fn test_decoder_yolo_split() {
1907 let score_threshold = 0.45;
1908 let iou_threshold = 0.45;
1909 let boxes = load_yolov8_boxes();
1910 let boxes: Vec<_> = boxes.iter().map(|x| *x as i16 * 256).collect();
1911 let boxes = ndarray::Array3::from_shape_vec((1, 116, 8400), boxes).unwrap();
1912
1913 let quant_boxes = Quantization::new(0.021287761628627777 / 256.0, 31 * 256);
1914
1915 let decoder = DecoderBuilder::default()
1916 .with_config_yolo_split_det(
1917 configs::Boxes {
1918 decoder: configs::DecoderType::Ultralytics,
1919 quantization: Some(QuantTuple(quant_boxes.scale, quant_boxes.zero_point)),
1920 shape: vec![1, 4, 8400],
1921 dshape: vec![
1922 (DimName::Batch, 1),
1923 (DimName::BoxCoords, 4),
1924 (DimName::NumBoxes, 8400),
1925 ],
1926 normalized: Some(true),
1927 },
1928 configs::Scores {
1929 decoder: configs::DecoderType::Ultralytics,
1930 quantization: Some(QuantTuple(quant_boxes.scale, quant_boxes.zero_point)),
1931 shape: vec![1, 80, 8400],
1932 dshape: vec![
1933 (DimName::Batch, 1),
1934 (DimName::NumClasses, 80),
1935 (DimName::NumBoxes, 8400),
1936 ],
1937 },
1938 )
1939 .with_score_threshold(score_threshold)
1940 .with_iou_threshold(iou_threshold)
1941 .build()
1942 .unwrap();
1943
1944 let mut output_boxes: Vec<_> = Vec::with_capacity(500);
1945 let mut output_masks: Vec<_> = Vec::with_capacity(500);
1946
1947 decoder
1948 .decode_quantized(
1949 &[
1950 boxes.slice(s![.., ..4, ..]).into(),
1951 boxes.slice(s![.., 4..84, ..]).into(),
1952 ],
1953 &mut output_boxes,
1954 &mut output_masks,
1955 )
1956 .unwrap();
1957
1958 let seg = dequantize_ndarray::<_, _, f32>(boxes.view(), quant_boxes);
1959 let mut output_boxes_f32: Vec<_> = Vec::with_capacity(500);
1960 decode_yolo_det_float(
1961 seg.slice(s![0, ..84, ..]),
1962 score_threshold,
1963 iou_threshold,
1964 Some(configs::Nms::ClassAgnostic),
1965 &mut output_boxes_f32,
1966 );
1967
1968 let mut output_boxes1: Vec<_> = Vec::with_capacity(500);
1969 let mut output_masks1: Vec<_> = Vec::with_capacity(500);
1970
1971 decoder
1972 .decode_float(
1973 &[
1974 seg.slice(s![.., ..4, ..]).into_dyn(),
1975 seg.slice(s![.., 4..84, ..]).into_dyn(),
1976 ],
1977 &mut output_boxes1,
1978 &mut output_masks1,
1979 )
1980 .unwrap();
1981 compare_outputs((&output_boxes, &output_boxes_f32), (&output_masks, &[]));
1982 compare_outputs((&output_boxes_f32, &output_boxes1), (&[], &output_masks1));
1983 }
1984
1985 #[test]
1986 fn test_decoder_masks_config_mixed() {
1987 let score_threshold = 0.45;
1988 let iou_threshold = 0.45;
1989 let boxes_raw = load_yolov8_boxes();
1990 let boxes: Vec<_> = boxes_raw.iter().map(|x| *x as i16 * 256).collect();
1991 let boxes = ndarray::Array3::from_shape_vec((1, 116, 8400), boxes).unwrap();
1992
1993 let quant_boxes = (0.021287761628627777 / 256.0, 31 * 256);
1994
1995 let protos = load_yolov8_protos();
1996 let quant_protos = (0.02491161972284317, -117);
1997
1998 let decoder = build_yolo_split_segdet_decoder(
1999 score_threshold,
2000 iou_threshold,
2001 quant_boxes,
2002 quant_protos,
2003 );
2004 let mut output_boxes: Vec<_> = Vec::with_capacity(500);
2005 let mut output_masks: Vec<_> = Vec::with_capacity(500);
2006
2007 decoder
2008 .decode_quantized(
2009 &[
2010 boxes.slice(s![.., ..4, ..]).into(),
2011 boxes.slice(s![.., 4..84, ..]).into(),
2012 boxes.slice(s![.., 84.., ..]).into(),
2013 protos.view().into(),
2014 ],
2015 &mut output_boxes,
2016 &mut output_masks,
2017 )
2018 .unwrap();
2019
2020 let protos = dequantize_ndarray::<_, _, f32>(protos.view(), quant_protos.into());
2021 let seg = dequantize_ndarray::<_, _, f32>(boxes.view(), quant_boxes.into());
2022 let mut output_boxes_f32: Vec<_> = Vec::with_capacity(500);
2023 let mut output_masks_f32: Vec<_> = Vec::with_capacity(500);
2024 decode_yolo_segdet_float(
2025 seg.slice(s![0, .., ..]),
2026 protos.slice(s![0, .., .., ..]),
2027 score_threshold,
2028 iou_threshold,
2029 Some(configs::Nms::ClassAgnostic),
2030 &mut output_boxes_f32,
2031 &mut output_masks_f32,
2032 )
2033 .unwrap();
2034
2035 let mut output_boxes1: Vec<_> = Vec::with_capacity(500);
2036 let mut output_masks1: Vec<_> = Vec::with_capacity(500);
2037
2038 decoder
2039 .decode_float(
2040 &[
2041 seg.slice(s![.., ..4, ..]).into_dyn(),
2042 seg.slice(s![.., 4..84, ..]).into_dyn(),
2043 seg.slice(s![.., 84.., ..]).into_dyn(),
2044 protos.view().into_dyn(),
2045 ],
2046 &mut output_boxes1,
2047 &mut output_masks1,
2048 )
2049 .unwrap();
2050 compare_outputs(
2051 (&output_boxes, &output_boxes_f32),
2052 (&output_masks, &output_masks_f32),
2053 );
2054 compare_outputs(
2055 (&output_boxes_f32, &output_boxes1),
2056 (&output_masks_f32, &output_masks1),
2057 );
2058 }
2059
2060 fn build_yolo_split_segdet_decoder(
2061 score_threshold: f32,
2062 iou_threshold: f32,
2063 quant_boxes: (f32, i32),
2064 quant_protos: (f32, i32),
2065 ) -> crate::Decoder {
2066 DecoderBuilder::default()
2067 .with_config_yolo_split_segdet(
2068 configs::Boxes {
2069 decoder: configs::DecoderType::Ultralytics,
2070 quantization: Some(quant_boxes.into()),
2071 shape: vec![1, 4, 8400],
2072 dshape: vec![
2073 (DimName::Batch, 1),
2074 (DimName::BoxCoords, 4),
2075 (DimName::NumBoxes, 8400),
2076 ],
2077 normalized: Some(true),
2078 },
2079 configs::Scores {
2080 decoder: configs::DecoderType::Ultralytics,
2081 quantization: Some(quant_boxes.into()),
2082 shape: vec![1, 80, 8400],
2083 dshape: vec![
2084 (DimName::Batch, 1),
2085 (DimName::NumClasses, 80),
2086 (DimName::NumBoxes, 8400),
2087 ],
2088 },
2089 configs::MaskCoefficients {
2090 decoder: configs::DecoderType::Ultralytics,
2091 quantization: Some(quant_boxes.into()),
2092 shape: vec![1, 32, 8400],
2093 dshape: vec![
2094 (DimName::Batch, 1),
2095 (DimName::NumProtos, 32),
2096 (DimName::NumBoxes, 8400),
2097 ],
2098 },
2099 configs::Protos {
2100 decoder: configs::DecoderType::Ultralytics,
2101 quantization: Some(quant_protos.into()),
2102 shape: vec![1, 160, 160, 32],
2103 dshape: vec![
2104 (DimName::Batch, 1),
2105 (DimName::Height, 160),
2106 (DimName::Width, 160),
2107 (DimName::NumProtos, 32),
2108 ],
2109 },
2110 )
2111 .with_score_threshold(score_threshold)
2112 .with_iou_threshold(iou_threshold)
2113 .build()
2114 .unwrap()
2115 }
2116
2117 fn build_yolov8_seg_decoder(score_threshold: f32, iou_threshold: f32) -> crate::Decoder {
2118 let config_yaml = edgefirst_bench::testdata::read_to_string("yolov8_seg.yaml");
2119 DecoderBuilder::default()
2120 .with_config_yaml_str(config_yaml.to_string())
2121 .with_score_threshold(score_threshold)
2122 .with_iou_threshold(iou_threshold)
2123 .build()
2124 .unwrap()
2125 }
2126 #[test]
2127 fn test_decoder_masks_config_i32() {
2128 let score_threshold = 0.45;
2129 let iou_threshold = 0.45;
2130 let boxes_raw = load_yolov8_boxes();
2131 let scale = 1 << 23;
2132 let boxes: Vec<_> = boxes_raw.iter().map(|x| *x as i32 * scale).collect();
2133 let boxes = ndarray::Array3::from_shape_vec((1, 116, 8400), boxes).unwrap();
2134
2135 let quant_boxes = (0.021287761628627777 / scale as f32, 31 * scale);
2136
2137 let protos_raw = load_yolov8_protos();
2138 let protos: Vec<_> = protos_raw.iter().map(|x| *x as i32 * scale).collect();
2139 let protos = ndarray::Array4::from_shape_vec((1, 160, 160, 32), protos).unwrap();
2140 let quant_protos = (0.02491161972284317 / scale as f32, -117 * scale);
2141
2142 let decoder = build_yolo_split_segdet_decoder(
2143 score_threshold,
2144 iou_threshold,
2145 quant_boxes,
2146 quant_protos,
2147 );
2148
2149 let mut output_boxes: Vec<_> = Vec::with_capacity(500);
2150 let mut output_masks: Vec<_> = Vec::with_capacity(500);
2151
2152 decoder
2153 .decode_quantized(
2154 &[
2155 boxes.slice(s![.., ..4, ..]).into(),
2156 boxes.slice(s![.., 4..84, ..]).into(),
2157 boxes.slice(s![.., 84.., ..]).into(),
2158 protos.view().into(),
2159 ],
2160 &mut output_boxes,
2161 &mut output_masks,
2162 )
2163 .unwrap();
2164
2165 let protos = dequantize_ndarray::<_, _, f32>(protos.view(), quant_protos.into());
2166 let seg = dequantize_ndarray::<_, _, f32>(boxes.view(), quant_boxes.into());
2167 let mut output_boxes_f32: Vec<_> = Vec::with_capacity(500);
2168 let mut output_masks_f32: Vec<Segmentation> = Vec::with_capacity(500);
2169 decode_yolo_segdet_float(
2170 seg.slice(s![0, .., ..]),
2171 protos.slice(s![0, .., .., ..]),
2172 score_threshold,
2173 iou_threshold,
2174 Some(configs::Nms::ClassAgnostic),
2175 &mut output_boxes_f32,
2176 &mut output_masks_f32,
2177 )
2178 .unwrap();
2179
2180 assert_eq!(output_boxes.len(), output_boxes_f32.len());
2181 assert_eq!(output_masks.len(), output_masks_f32.len());
2182
2183 compare_outputs(
2184 (&output_boxes, &output_boxes_f32),
2185 (&output_masks, &output_masks_f32),
2186 );
2187 }
2188
2189 #[test]
2191 fn test_context_switch() {
2192 let yolo_det = || {
2193 let score_threshold = 0.25;
2194 let iou_threshold = 0.7;
2195 let out = load_yolov8s_det();
2196 let quant = (0.0040811873, -123).into();
2197
2198 let decoder = DecoderBuilder::default()
2199 .with_config_yolo_det(
2200 configs::Detection {
2201 decoder: DecoderType::Ultralytics,
2202 shape: vec![1, 84, 8400],
2203 anchors: None,
2204 quantization: Some(quant),
2205 dshape: vec![
2206 (DimName::Batch, 1),
2207 (DimName::NumFeatures, 84),
2208 (DimName::NumBoxes, 8400),
2209 ],
2210 normalized: None,
2211 },
2212 None,
2213 )
2214 .with_score_threshold(score_threshold)
2215 .with_iou_threshold(iou_threshold)
2216 .build()
2217 .unwrap();
2218
2219 let mut output_boxes: Vec<_> = Vec::with_capacity(50);
2220 let mut output_masks: Vec<_> = Vec::with_capacity(50);
2221
2222 for _ in 0..100 {
2223 decoder
2224 .decode_quantized(&[out.view().into()], &mut output_boxes, &mut output_masks)
2225 .unwrap();
2226
2227 assert!(output_boxes[0].equal_within_delta(
2228 &DetectBox {
2229 bbox: BoundingBox {
2230 xmin: 0.5285137,
2231 ymin: 0.05305544,
2232 xmax: 0.87541467,
2233 ymax: 0.9998909,
2234 },
2235 score: 0.5591227,
2236 label: 0
2237 },
2238 1e-6
2239 ));
2240
2241 assert!(output_boxes[1].equal_within_delta(
2242 &DetectBox {
2243 bbox: BoundingBox {
2244 xmin: 0.130598,
2245 ymin: 0.43260583,
2246 xmax: 0.35098213,
2247 ymax: 0.9958097,
2248 },
2249 score: 0.33057618,
2250 label: 75
2251 },
2252 1e-6
2253 ));
2254 assert!(output_masks.is_empty());
2255 }
2256 };
2257
2258 let modelpack_det_split = || {
2259 let score_threshold = 0.8;
2260 let iou_threshold = 0.5;
2261
2262 let seg = edgefirst_bench::testdata::read("modelpack_seg_2x160x160.bin");
2263 let seg = ndarray::Array4::from_shape_vec((1, 2, 160, 160), seg.to_vec()).unwrap();
2264
2265 let detect0 = edgefirst_bench::testdata::read("modelpack_split_9x15x18.bin");
2266 let detect0 =
2267 ndarray::Array4::from_shape_vec((1, 9, 15, 18), detect0.to_vec()).unwrap();
2268
2269 let detect1 = edgefirst_bench::testdata::read("modelpack_split_17x30x18.bin");
2270 let detect1 =
2271 ndarray::Array4::from_shape_vec((1, 17, 30, 18), detect1.to_vec()).unwrap();
2272
2273 let mut mask = seg.slice(s![0, .., .., ..]);
2274 mask.swap_axes(0, 1);
2275 mask.swap_axes(1, 2);
2276 let mask = [Segmentation {
2277 xmin: 0.0,
2278 ymin: 0.0,
2279 xmax: 1.0,
2280 ymax: 1.0,
2281 segmentation: crate::mask_to_tensor(mask.view()).unwrap(),
2282 }];
2283 let correct_boxes = [DetectBox {
2284 bbox: BoundingBox {
2285 xmin: 0.43171933,
2286 ymin: 0.68243736,
2287 xmax: 0.5626645,
2288 ymax: 0.808863,
2289 },
2290 score: 0.99240804,
2291 label: 0,
2292 }];
2293
2294 let quant0 = (0.08547406643629074, 174).into();
2295 let quant1 = (0.09929127991199493, 183).into();
2296 let quant_seg = (1.0 / 255.0, 0).into();
2297
2298 let anchors0 = vec![
2299 [0.36666667461395264, 0.31481480598449707],
2300 [0.38749998807907104, 0.4740740656852722],
2301 [0.5333333611488342, 0.644444465637207],
2302 ];
2303 let anchors1 = vec![
2304 [0.13750000298023224, 0.2074074000120163],
2305 [0.2541666626930237, 0.21481481194496155],
2306 [0.23125000298023224, 0.35185185074806213],
2307 ];
2308
2309 let decoder = DecoderBuilder::default()
2310 .with_config_modelpack_segdet_split(
2311 vec![
2312 configs::Detection {
2313 decoder: DecoderType::ModelPack,
2314 shape: vec![1, 17, 30, 18],
2315 anchors: Some(anchors1),
2316 quantization: Some(quant1),
2317 dshape: vec![
2318 (DimName::Batch, 1),
2319 (DimName::Height, 17),
2320 (DimName::Width, 30),
2321 (DimName::NumAnchorsXFeatures, 18),
2322 ],
2323 normalized: None,
2324 },
2325 configs::Detection {
2326 decoder: DecoderType::ModelPack,
2327 shape: vec![1, 9, 15, 18],
2328 anchors: Some(anchors0),
2329 quantization: Some(quant0),
2330 dshape: vec![
2331 (DimName::Batch, 1),
2332 (DimName::Height, 9),
2333 (DimName::Width, 15),
2334 (DimName::NumAnchorsXFeatures, 18),
2335 ],
2336 normalized: None,
2337 },
2338 ],
2339 configs::Segmentation {
2340 decoder: DecoderType::ModelPack,
2341 quantization: Some(quant_seg),
2342 shape: vec![1, 2, 160, 160],
2343 dshape: vec![
2344 (DimName::Batch, 1),
2345 (DimName::NumClasses, 2),
2346 (DimName::Height, 160),
2347 (DimName::Width, 160),
2348 ],
2349 },
2350 )
2351 .with_score_threshold(score_threshold)
2352 .with_iou_threshold(iou_threshold)
2353 .build()
2354 .unwrap();
2355 let mut output_boxes: Vec<_> = Vec::with_capacity(10);
2356 let mut output_masks: Vec<_> = Vec::with_capacity(10);
2357
2358 for _ in 0..100 {
2359 decoder
2360 .decode_quantized(
2361 &[
2362 detect0.view().into(),
2363 detect1.view().into(),
2364 seg.view().into(),
2365 ],
2366 &mut output_boxes,
2367 &mut output_masks,
2368 )
2369 .unwrap();
2370
2371 compare_outputs((&correct_boxes, &output_boxes), (&mask, &output_masks));
2372 }
2373 };
2374
2375 let handles = vec![
2376 std::thread::spawn(yolo_det),
2377 std::thread::spawn(modelpack_det_split),
2378 std::thread::spawn(yolo_det),
2379 std::thread::spawn(modelpack_det_split),
2380 std::thread::spawn(yolo_det),
2381 std::thread::spawn(modelpack_det_split),
2382 std::thread::spawn(yolo_det),
2383 std::thread::spawn(modelpack_det_split),
2384 ];
2385 for handle in handles {
2386 handle.join().unwrap();
2387 }
2388 }
2389
2390 #[test]
2391 fn test_ndarray_to_xyxy_float() {
2392 let arr = array![10.0_f32, 20.0, 20.0, 20.0];
2393 let xyxy: [f32; 4] = XYWH::ndarray_to_xyxy_float(arr.view());
2394 assert_eq!(xyxy, [0.0_f32, 10.0, 20.0, 30.0]);
2395
2396 let arr = array![10.0_f32, 20.0, 20.0, 20.0];
2397 let xyxy: [f32; 4] = XYXY::ndarray_to_xyxy_float(arr.view());
2398 assert_eq!(xyxy, [10.0_f32, 20.0, 20.0, 20.0]);
2399 }
2400
2401 #[test]
2402 fn test_class_aware_nms_float() {
2403 use crate::float::nms_class_aware_float;
2404
2405 let boxes = vec![
2407 DetectBox {
2408 bbox: BoundingBox {
2409 xmin: 0.0,
2410 ymin: 0.0,
2411 xmax: 0.5,
2412 ymax: 0.5,
2413 },
2414 score: 0.9,
2415 label: 0, },
2417 DetectBox {
2418 bbox: BoundingBox {
2419 xmin: 0.1,
2420 ymin: 0.1,
2421 xmax: 0.6,
2422 ymax: 0.6,
2423 },
2424 score: 0.8,
2425 label: 1, },
2427 ];
2428
2429 let result = nms_class_aware_float(0.3, None, boxes.clone());
2432 assert_eq!(
2433 result.len(),
2434 2,
2435 "Class-aware NMS should keep both boxes with different classes"
2436 );
2437
2438 let same_class_boxes = vec![
2440 DetectBox {
2441 bbox: BoundingBox {
2442 xmin: 0.0,
2443 ymin: 0.0,
2444 xmax: 0.5,
2445 ymax: 0.5,
2446 },
2447 score: 0.9,
2448 label: 0,
2449 },
2450 DetectBox {
2451 bbox: BoundingBox {
2452 xmin: 0.1,
2453 ymin: 0.1,
2454 xmax: 0.6,
2455 ymax: 0.6,
2456 },
2457 score: 0.8,
2458 label: 0, },
2460 ];
2461
2462 let result = nms_class_aware_float(0.3, None, same_class_boxes);
2463 assert_eq!(
2464 result.len(),
2465 1,
2466 "Class-aware NMS should suppress overlapping box with same class"
2467 );
2468 assert_eq!(result[0].label, 0);
2469 assert!((result[0].score - 0.9).abs() < 1e-6);
2470 }
2471
2472 #[test]
2473 fn test_class_agnostic_vs_aware_nms() {
2474 use crate::float::{nms_class_aware_float, nms_float};
2475
2476 let boxes = vec![
2478 DetectBox {
2479 bbox: BoundingBox {
2480 xmin: 0.0,
2481 ymin: 0.0,
2482 xmax: 0.5,
2483 ymax: 0.5,
2484 },
2485 score: 0.9,
2486 label: 0,
2487 },
2488 DetectBox {
2489 bbox: BoundingBox {
2490 xmin: 0.1,
2491 ymin: 0.1,
2492 xmax: 0.6,
2493 ymax: 0.6,
2494 },
2495 score: 0.8,
2496 label: 1,
2497 },
2498 ];
2499
2500 let agnostic_result = nms_float(0.3, None, boxes.clone());
2502 assert_eq!(
2503 agnostic_result.len(),
2504 1,
2505 "Class-agnostic NMS should suppress overlapping boxes"
2506 );
2507
2508 let aware_result = nms_class_aware_float(0.3, None, boxes);
2510 assert_eq!(
2511 aware_result.len(),
2512 2,
2513 "Class-aware NMS should keep boxes with different classes"
2514 );
2515 }
2516
2517 #[test]
2518 fn test_class_aware_nms_int() {
2519 use crate::byte::nms_class_aware_int;
2520
2521 let boxes = vec![
2523 DetectBoxQuantized {
2524 bbox: BoundingBox {
2525 xmin: 0.0,
2526 ymin: 0.0,
2527 xmax: 0.5,
2528 ymax: 0.5,
2529 },
2530 score: 200_u8,
2531 label: 0,
2532 },
2533 DetectBoxQuantized {
2534 bbox: BoundingBox {
2535 xmin: 0.1,
2536 ymin: 0.1,
2537 xmax: 0.6,
2538 ymax: 0.6,
2539 },
2540 score: 180_u8,
2541 label: 1, },
2543 ];
2544
2545 let result = nms_class_aware_int(0.5, None, boxes);
2547 assert_eq!(
2548 result.len(),
2549 2,
2550 "Class-aware NMS (int) should keep boxes with different classes"
2551 );
2552 }
2553
2554 #[test]
2555 fn test_nms_enum_default() {
2556 let default_nms: configs::Nms = Default::default();
2558 assert_eq!(default_nms, configs::Nms::ClassAgnostic);
2559 }
2560
2561 #[test]
2562 fn test_decoder_nms_mode() {
2563 let decoder = DecoderBuilder::default()
2565 .with_config_yolo_det(
2566 configs::Detection {
2567 anchors: None,
2568 decoder: DecoderType::Ultralytics,
2569 quantization: None,
2570 shape: vec![1, 84, 8400],
2571 dshape: Vec::new(),
2572 normalized: Some(true),
2573 },
2574 None,
2575 )
2576 .with_nms(Some(configs::Nms::ClassAware))
2577 .build()
2578 .unwrap();
2579
2580 assert_eq!(decoder.nms, Some(configs::Nms::ClassAware));
2581 }
2582
2583 #[test]
2584 fn test_decoder_nms_bypass() {
2585 let decoder = DecoderBuilder::default()
2587 .with_config_yolo_det(
2588 configs::Detection {
2589 anchors: None,
2590 decoder: DecoderType::Ultralytics,
2591 quantization: None,
2592 shape: vec![1, 84, 8400],
2593 dshape: Vec::new(),
2594 normalized: Some(true),
2595 },
2596 None,
2597 )
2598 .with_nms(None)
2599 .build()
2600 .unwrap();
2601
2602 assert_eq!(decoder.nms, None);
2603 }
2604
2605 #[test]
2606 fn test_decoder_normalized_boxes_true() {
2607 let decoder = DecoderBuilder::default()
2609 .with_config_yolo_det(
2610 configs::Detection {
2611 anchors: None,
2612 decoder: DecoderType::Ultralytics,
2613 quantization: None,
2614 shape: vec![1, 84, 8400],
2615 dshape: Vec::new(),
2616 normalized: Some(true),
2617 },
2618 None,
2619 )
2620 .build()
2621 .unwrap();
2622
2623 assert_eq!(decoder.normalized_boxes(), Some(true));
2624 }
2625
2626 #[test]
2627 fn test_decoder_normalized_boxes_false() {
2628 let decoder = DecoderBuilder::default()
2631 .with_config_yolo_det(
2632 configs::Detection {
2633 anchors: None,
2634 decoder: DecoderType::Ultralytics,
2635 quantization: None,
2636 shape: vec![1, 84, 8400],
2637 dshape: Vec::new(),
2638 normalized: Some(false),
2639 },
2640 None,
2641 )
2642 .build()
2643 .unwrap();
2644
2645 assert_eq!(decoder.normalized_boxes(), Some(false));
2646 }
2647
2648 #[test]
2649 fn test_decoder_normalized_boxes_unknown() {
2650 let decoder = DecoderBuilder::default()
2652 .with_config_yolo_det(
2653 configs::Detection {
2654 anchors: None,
2655 decoder: DecoderType::Ultralytics,
2656 quantization: None,
2657 shape: vec![1, 84, 8400],
2658 dshape: Vec::new(),
2659 normalized: None,
2660 },
2661 Some(DecoderVersion::Yolo11),
2662 )
2663 .build()
2664 .unwrap();
2665
2666 assert_eq!(decoder.normalized_boxes(), None);
2667 }
2668
2669 pub fn quantize_ndarray<T: PrimInt + 'static, D: Dimension, F: Float + AsPrimitive<T>>(
2670 input: ArrayView<F, D>,
2671 quant: Quantization,
2672 ) -> Array<T, D>
2673 where
2674 i32: num_traits::AsPrimitive<F>,
2675 f32: num_traits::AsPrimitive<F>,
2676 {
2677 let zero_point = quant.zero_point.as_();
2678 let div_scale = F::one() / quant.scale.as_();
2679 if zero_point != F::zero() {
2680 input.mapv(|d| (d * div_scale + zero_point).round().as_())
2681 } else {
2682 input.mapv(|d| (d * div_scale).round().as_())
2683 }
2684 }
2685
2686 fn real_data_expected_boxes() -> [DetectBox; 2] {
2687 [
2688 DetectBox {
2689 bbox: BoundingBox {
2690 xmin: 0.08515105,
2691 ymin: 0.7131401,
2692 xmax: 0.29802868,
2693 ymax: 0.8195788,
2694 },
2695 score: 0.91537374,
2696 label: 23,
2697 },
2698 DetectBox {
2699 bbox: BoundingBox {
2700 xmin: 0.59605736,
2701 ymin: 0.25545314,
2702 xmax: 0.93666154,
2703 ymax: 0.72378385,
2704 },
2705 score: 0.91537374,
2706 label: 23,
2707 },
2708 ]
2709 }
2710
2711 fn e2e_expected_boxes_quant() -> [DetectBox; 1] {
2712 [DetectBox {
2713 bbox: BoundingBox {
2714 xmin: 0.12549022,
2715 ymin: 0.12549022,
2716 xmax: 0.23529413,
2717 ymax: 0.23529413,
2718 },
2719 score: 0.98823535,
2720 label: 2,
2721 }]
2722 }
2723
2724 fn e2e_expected_boxes_float() -> [DetectBox; 1] {
2725 [DetectBox {
2726 bbox: BoundingBox {
2727 xmin: 0.1234,
2728 ymin: 0.1234,
2729 xmax: 0.2345,
2730 ymax: 0.2345,
2731 },
2732 score: 0.9876,
2733 label: 2,
2734 }]
2735 }
2736
2737 macro_rules! real_data_proto_test {
2738 ($name:ident, quantized, $layout:ident) => {
2739 #[test]
2740 fn $name() {
2741 let is_split = matches!(stringify!($layout), "split");
2742
2743 let score_threshold = 0.45;
2744 let iou_threshold = 0.45;
2745 let quant_boxes = (0.021287762_f32, 31_i32);
2746 let quant_protos = (0.02491162_f32, -117_i32);
2747
2748 let raw_boxes = edgefirst_bench::testdata::read("yolov8_boxes_116x8400.bin");
2749 let raw_boxes = unsafe {
2750 std::slice::from_raw_parts(raw_boxes.as_ptr() as *const i8, raw_boxes.len())
2751 };
2752 let boxes_i8 =
2753 ndarray::Array3::from_shape_vec((1, 116, 8400), raw_boxes.to_vec()).unwrap();
2754
2755 let raw_protos = edgefirst_bench::testdata::read("yolov8_protos_160x160x32.bin");
2756 let raw_protos = unsafe {
2757 std::slice::from_raw_parts(raw_protos.as_ptr() as *const i8, raw_protos.len())
2758 };
2759 let protos_i8 =
2760 ndarray::Array4::from_shape_vec((1, 160, 160, 32), raw_protos.to_vec())
2761 .unwrap();
2762
2763 let mask_split = boxes_i8.slice(s![.., 84.., ..]).to_owned();
2765 let scores_split = boxes_i8.slice(s![.., 4..84, ..]).to_owned();
2766 let boxes_split = boxes_i8.slice(s![.., ..4, ..]).to_owned();
2767 let boxes_combined = boxes_i8;
2768
2769 let decoder = if is_split {
2770 build_yolo_split_segdet_decoder(
2771 score_threshold,
2772 iou_threshold,
2773 quant_boxes,
2774 quant_protos,
2775 )
2776 } else {
2777 build_yolov8_seg_decoder(score_threshold, iou_threshold)
2778 };
2779
2780 let expected = real_data_expected_boxes();
2781 let mut output_boxes = Vec::with_capacity(50);
2782
2783 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> = if is_split {
2784 vec![
2785 boxes_split.view().into(),
2786 scores_split.view().into(),
2787 mask_split.view().into(),
2788 protos_i8.view().into(),
2789 ]
2790 } else {
2791 vec![boxes_combined.view().into(), protos_i8.view().into()]
2792 };
2793 decoder
2794 .decode_quantized_proto(&inputs, &mut output_boxes)
2795 .unwrap();
2796
2797 assert_eq!(output_boxes.len(), 2);
2798 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
2799 assert!(output_boxes[1].equal_within_delta(&expected[1], 1.0 / 160.0));
2800 }
2801 };
2802 ($name:ident, float, $layout:ident) => {
2803 #[test]
2804 fn $name() {
2805 let is_split = matches!(stringify!($layout), "split");
2806
2807 let score_threshold = 0.45;
2808 let iou_threshold = 0.45;
2809 let quant_boxes = (0.021287762_f32, 31_i32);
2810 let quant_protos = (0.02491162_f32, -117_i32);
2811
2812 let raw_boxes = edgefirst_bench::testdata::read("yolov8_boxes_116x8400.bin");
2813 let raw_boxes = unsafe {
2814 std::slice::from_raw_parts(raw_boxes.as_ptr() as *const i8, raw_boxes.len())
2815 };
2816 let boxes_i8 =
2817 ndarray::Array3::from_shape_vec((1, 116, 8400), raw_boxes.to_vec()).unwrap();
2818 let boxes_f32: Array3<f32> =
2819 dequantize_ndarray(boxes_i8.view(), quant_boxes.into());
2820
2821 let raw_protos = edgefirst_bench::testdata::read("yolov8_protos_160x160x32.bin");
2822 let raw_protos = unsafe {
2823 std::slice::from_raw_parts(raw_protos.as_ptr() as *const i8, raw_protos.len())
2824 };
2825 let protos_i8 =
2826 ndarray::Array4::from_shape_vec((1, 160, 160, 32), raw_protos.to_vec())
2827 .unwrap();
2828 let protos_f32: Array4<f32> =
2829 dequantize_ndarray(protos_i8.view(), quant_protos.into());
2830
2831 let mask_split = boxes_f32.slice(s![.., 84.., ..]).to_owned();
2833 let scores_split = boxes_f32.slice(s![.., 4..84, ..]).to_owned();
2834 let boxes_split = boxes_f32.slice(s![.., ..4, ..]).to_owned();
2835 let boxes_combined = boxes_f32;
2836
2837 let decoder = if is_split {
2838 build_yolo_split_segdet_decoder(
2839 score_threshold,
2840 iou_threshold,
2841 quant_boxes,
2842 quant_protos,
2843 )
2844 } else {
2845 build_yolov8_seg_decoder(score_threshold, iou_threshold)
2846 };
2847
2848 let expected = real_data_expected_boxes();
2849 let mut output_boxes = Vec::with_capacity(50);
2850
2851 let inputs = if is_split {
2852 vec![
2853 boxes_split.view().into_dyn(),
2854 scores_split.view().into_dyn(),
2855 mask_split.view().into_dyn(),
2856 protos_f32.view().into_dyn(),
2857 ]
2858 } else {
2859 vec![
2860 boxes_combined.view().into_dyn(),
2861 protos_f32.view().into_dyn(),
2862 ]
2863 };
2864 decoder
2865 .decode_float_proto(&inputs, &mut output_boxes)
2866 .unwrap();
2867
2868 assert_eq!(output_boxes.len(), 2);
2869 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
2870 assert!(output_boxes[1].equal_within_delta(&expected[1], 1.0 / 160.0));
2871 }
2872 };
2873 }
2874
2875 real_data_proto_test!(test_decoder_segdet_proto, quantized, combined);
2876 real_data_proto_test!(test_decoder_segdet_proto_float, float, combined);
2877 real_data_proto_test!(test_decoder_segdet_split_proto, quantized, split);
2878 real_data_proto_test!(test_decoder_segdet_split_proto_float, float, split);
2879
2880 const E2E_COMBINED_DET_CONFIG: &str = "
2881decoder_version: yolo26
2882outputs:
2883 - type: detection
2884 decoder: ultralytics
2885 quantization: [0.00784313725490196, 0]
2886 shape: [1, 10, 6]
2887 dshape:
2888 - [batch, 1]
2889 - [num_boxes, 10]
2890 - [num_features, 6]
2891 normalized: true
2892";
2893
2894 const E2E_COMBINED_SEGDET_CONFIG: &str = "
2895decoder_version: yolo26
2896outputs:
2897 - type: detection
2898 decoder: ultralytics
2899 quantization: [0.00784313725490196, 0]
2900 shape: [1, 10, 38]
2901 dshape:
2902 - [batch, 1]
2903 - [num_boxes, 10]
2904 - [num_features, 38]
2905 normalized: true
2906 - type: protos
2907 decoder: ultralytics
2908 quantization: [0.0039215686274509803921568627451, 128]
2909 shape: [1, 160, 160, 32]
2910 dshape:
2911 - [batch, 1]
2912 - [height, 160]
2913 - [width, 160]
2914 - [num_protos, 32]
2915";
2916
2917 const E2E_SPLIT_DET_CONFIG: &str = "
2918decoder_version: yolo26
2919outputs:
2920 - type: boxes
2921 decoder: ultralytics
2922 quantization: [0.00784313725490196, 0]
2923 shape: [1, 10, 4]
2924 dshape:
2925 - [batch, 1]
2926 - [num_boxes, 10]
2927 - [box_coords, 4]
2928 normalized: true
2929 - type: scores
2930 decoder: ultralytics
2931 quantization: [0.00784313725490196, 0]
2932 shape: [1, 10, 1]
2933 dshape:
2934 - [batch, 1]
2935 - [num_boxes, 10]
2936 - [num_classes, 1]
2937 - type: classes
2938 decoder: ultralytics
2939 quantization: [0.00784313725490196, 0]
2940 shape: [1, 10, 1]
2941 dshape:
2942 - [batch, 1]
2943 - [num_boxes, 10]
2944 - [num_classes, 1]
2945";
2946
2947 const E2E_SPLIT_SEGDET_CONFIG: &str = "
2948decoder_version: yolo26
2949outputs:
2950 - type: boxes
2951 decoder: ultralytics
2952 quantization: [0.00784313725490196, 0]
2953 shape: [1, 10, 4]
2954 dshape:
2955 - [batch, 1]
2956 - [num_boxes, 10]
2957 - [box_coords, 4]
2958 normalized: true
2959 - type: scores
2960 decoder: ultralytics
2961 quantization: [0.00784313725490196, 0]
2962 shape: [1, 10, 1]
2963 dshape:
2964 - [batch, 1]
2965 - [num_boxes, 10]
2966 - [num_classes, 1]
2967 - type: classes
2968 decoder: ultralytics
2969 quantization: [0.00784313725490196, 0]
2970 shape: [1, 10, 1]
2971 dshape:
2972 - [batch, 1]
2973 - [num_boxes, 10]
2974 - [num_classes, 1]
2975 - type: mask_coefficients
2976 decoder: ultralytics
2977 quantization: [0.00784313725490196, 0]
2978 shape: [1, 10, 32]
2979 dshape:
2980 - [batch, 1]
2981 - [num_boxes, 10]
2982 - [num_protos, 32]
2983 - type: protos
2984 decoder: ultralytics
2985 quantization: [0.0039215686274509803921568627451, 128]
2986 shape: [1, 160, 160, 32]
2987 dshape:
2988 - [batch, 1]
2989 - [height, 160]
2990 - [width, 160]
2991 - [num_protos, 32]
2992";
2993
2994 macro_rules! e2e_segdet_test {
2995 ($name:ident, quantized, $layout:ident, $output:ident) => {
2996 #[test]
2997 fn $name() {
2998 let is_split = matches!(stringify!($layout), "split");
2999 let is_proto = matches!(stringify!($output), "proto");
3000
3001 let score_threshold = 0.45;
3002 let iou_threshold = 0.45;
3003
3004 let mut boxes = Array2::zeros((10, 4));
3005 let mut scores = Array2::zeros((10, 1));
3006 let mut classes = Array2::zeros((10, 1));
3007 let mask = Array2::zeros((10, 32));
3008 let protos = Array3::<f64>::zeros((160, 160, 32));
3009 let protos = protos.insert_axis(Axis(0));
3010 let protos_quant = (1.0 / 255.0, 0.0);
3011 let protos: Array4<u8> = quantize_ndarray(protos.view(), protos_quant.into());
3012
3013 boxes
3014 .slice_mut(s![0, ..])
3015 .assign(&array![0.1234, 0.1234, 0.2345, 0.2345]);
3016 scores.slice_mut(s![0, ..]).assign(&array![0.9876]);
3017 classes.slice_mut(s![0, ..]).assign(&array![2.0]);
3018
3019 let detect_quant = (2.0 / 255.0, 0.0);
3020
3021 let decoder = if is_split {
3022 DecoderBuilder::default()
3023 .with_config_yaml_str(E2E_SPLIT_SEGDET_CONFIG.to_string())
3024 .with_score_threshold(score_threshold)
3025 .with_iou_threshold(iou_threshold)
3026 .build()
3027 .unwrap()
3028 } else {
3029 DecoderBuilder::default()
3030 .with_config_yaml_str(E2E_COMBINED_SEGDET_CONFIG.to_string())
3031 .with_score_threshold(score_threshold)
3032 .with_iou_threshold(iou_threshold)
3033 .build()
3034 .unwrap()
3035 };
3036
3037 let expected = e2e_expected_boxes_quant();
3038 let mut output_boxes = Vec::with_capacity(50);
3039
3040 if is_split {
3041 let boxes = boxes.insert_axis(Axis(0));
3042 let scores = scores.insert_axis(Axis(0));
3043 let classes = classes.insert_axis(Axis(0));
3044 let mask = mask.insert_axis(Axis(0));
3045
3046 let boxes: Array3<u8> = quantize_ndarray(boxes.view(), detect_quant.into());
3047 let scores: Array3<u8> = quantize_ndarray(scores.view(), detect_quant.into());
3048 let classes: Array3<u8> = quantize_ndarray(classes.view(), detect_quant.into());
3049 let mask: Array3<u8> = quantize_ndarray(mask.view(), detect_quant.into());
3050
3051 if is_proto {
3052 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> = vec![
3053 boxes.view().into(),
3054 scores.view().into(),
3055 classes.view().into(),
3056 mask.view().into(),
3057 protos.view().into(),
3058 ];
3059 decoder
3060 .decode_quantized_proto(&inputs, &mut output_boxes)
3061 .unwrap();
3062
3063 assert_eq!(output_boxes.len(), 1);
3064 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
3065 } else {
3066 let mut output_masks = Vec::with_capacity(50);
3067 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> = vec![
3068 boxes.view().into(),
3069 scores.view().into(),
3070 classes.view().into(),
3071 mask.view().into(),
3072 protos.view().into(),
3073 ];
3074 decoder
3075 .decode_quantized(&inputs, &mut output_boxes, &mut output_masks)
3076 .unwrap();
3077
3078 assert_eq!(output_boxes.len(), 1);
3079 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
3080 }
3081 } else {
3082 let detect = ndarray::concatenate![
3084 Axis(1),
3085 boxes.view(),
3086 scores.view(),
3087 classes.view(),
3088 mask.view()
3089 ];
3090 let detect = detect.insert_axis(Axis(0));
3091 assert_eq!(detect.shape(), &[1, 10, 38]);
3092 let detect: Array3<u8> = quantize_ndarray(detect.view(), detect_quant.into());
3093
3094 if is_proto {
3095 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> =
3096 vec![detect.view().into(), protos.view().into()];
3097 decoder
3098 .decode_quantized_proto(&inputs, &mut output_boxes)
3099 .unwrap();
3100
3101 assert_eq!(output_boxes.len(), 1);
3102 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
3103 } else {
3104 let mut output_masks = Vec::with_capacity(50);
3105 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> =
3106 vec![detect.view().into(), protos.view().into()];
3107 decoder
3108 .decode_quantized(&inputs, &mut output_boxes, &mut output_masks)
3109 .unwrap();
3110
3111 assert_eq!(output_boxes.len(), 1);
3112 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
3113 }
3114 }
3115 }
3116 };
3117 ($name:ident, float, $layout:ident, $output:ident) => {
3118 #[test]
3119 fn $name() {
3120 let is_split = matches!(stringify!($layout), "split");
3121 let is_proto = matches!(stringify!($output), "proto");
3122
3123 let score_threshold = 0.45;
3124 let iou_threshold = 0.45;
3125
3126 let mut boxes = Array2::zeros((10, 4));
3127 let mut scores = Array2::zeros((10, 1));
3128 let mut classes = Array2::zeros((10, 1));
3129 let mask: Array2<f64> = Array2::zeros((10, 32));
3130 let protos = Array3::<f64>::zeros((160, 160, 32));
3131 let protos = protos.insert_axis(Axis(0));
3132
3133 boxes
3134 .slice_mut(s![0, ..])
3135 .assign(&array![0.1234, 0.1234, 0.2345, 0.2345]);
3136 scores.slice_mut(s![0, ..]).assign(&array![0.9876]);
3137 classes.slice_mut(s![0, ..]).assign(&array![2.0]);
3138
3139 let decoder = if is_split {
3140 DecoderBuilder::default()
3141 .with_config_yaml_str(E2E_SPLIT_SEGDET_CONFIG.to_string())
3142 .with_score_threshold(score_threshold)
3143 .with_iou_threshold(iou_threshold)
3144 .build()
3145 .unwrap()
3146 } else {
3147 DecoderBuilder::default()
3148 .with_config_yaml_str(E2E_COMBINED_SEGDET_CONFIG.to_string())
3149 .with_score_threshold(score_threshold)
3150 .with_iou_threshold(iou_threshold)
3151 .build()
3152 .unwrap()
3153 };
3154
3155 let expected = e2e_expected_boxes_float();
3156 let mut output_boxes = Vec::with_capacity(50);
3157
3158 if is_split {
3159 let boxes = boxes.insert_axis(Axis(0));
3160 let scores = scores.insert_axis(Axis(0));
3161 let classes = classes.insert_axis(Axis(0));
3162 let mask = mask.insert_axis(Axis(0));
3163
3164 if is_proto {
3165 let inputs = vec![
3166 boxes.view().into_dyn(),
3167 scores.view().into_dyn(),
3168 classes.view().into_dyn(),
3169 mask.view().into_dyn(),
3170 protos.view().into_dyn(),
3171 ];
3172 decoder
3173 .decode_float_proto(&inputs, &mut output_boxes)
3174 .unwrap();
3175
3176 assert_eq!(output_boxes.len(), 1);
3177 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
3178 } else {
3179 let mut output_masks = Vec::with_capacity(50);
3180 let inputs = vec![
3181 boxes.view().into_dyn(),
3182 scores.view().into_dyn(),
3183 classes.view().into_dyn(),
3184 mask.view().into_dyn(),
3185 protos.view().into_dyn(),
3186 ];
3187 decoder
3188 .decode_float(&inputs, &mut output_boxes, &mut output_masks)
3189 .unwrap();
3190
3191 assert_eq!(output_boxes.len(), 1);
3192 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
3193 }
3194 } else {
3195 let detect = ndarray::concatenate![
3197 Axis(1),
3198 boxes.view(),
3199 scores.view(),
3200 classes.view(),
3201 mask.view()
3202 ];
3203 let detect = detect.insert_axis(Axis(0));
3204 assert_eq!(detect.shape(), &[1, 10, 38]);
3205
3206 if is_proto {
3207 let inputs = vec![detect.view().into_dyn(), protos.view().into_dyn()];
3208 decoder
3209 .decode_float_proto(&inputs, &mut output_boxes)
3210 .unwrap();
3211
3212 assert_eq!(output_boxes.len(), 1);
3213 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
3214 } else {
3215 let mut output_masks = Vec::with_capacity(50);
3216 let inputs = vec![detect.view().into_dyn(), protos.view().into_dyn()];
3217 decoder
3218 .decode_float(&inputs, &mut output_boxes, &mut output_masks)
3219 .unwrap();
3220
3221 assert_eq!(output_boxes.len(), 1);
3222 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
3223 }
3224 }
3225 }
3226 };
3227 }
3228
3229 e2e_segdet_test!(test_decoder_end_to_end_segdet, quantized, combined, masks);
3230 e2e_segdet_test!(test_decoder_end_to_end_segdet_float, float, combined, masks);
3231 e2e_segdet_test!(
3232 test_decoder_end_to_end_segdet_proto,
3233 quantized,
3234 combined,
3235 proto
3236 );
3237 e2e_segdet_test!(
3238 test_decoder_end_to_end_segdet_proto_float,
3239 float,
3240 combined,
3241 proto
3242 );
3243 e2e_segdet_test!(
3244 test_decoder_end_to_end_segdet_split,
3245 quantized,
3246 split,
3247 masks
3248 );
3249 e2e_segdet_test!(
3250 test_decoder_end_to_end_segdet_split_float,
3251 float,
3252 split,
3253 masks
3254 );
3255 e2e_segdet_test!(
3256 test_decoder_end_to_end_segdet_split_proto,
3257 quantized,
3258 split,
3259 proto
3260 );
3261 e2e_segdet_test!(
3262 test_decoder_end_to_end_segdet_split_proto_float,
3263 float,
3264 split,
3265 proto
3266 );
3267
3268 macro_rules! e2e_det_test {
3269 ($name:ident, quantized, $layout:ident) => {
3270 #[test]
3271 fn $name() {
3272 let is_split = matches!(stringify!($layout), "split");
3273
3274 let score_threshold = 0.45;
3275 let iou_threshold = 0.45;
3276
3277 let mut boxes = Array3::zeros((1, 10, 4));
3278 let mut scores = Array3::zeros((1, 10, 1));
3279 let mut classes = Array3::zeros((1, 10, 1));
3280
3281 boxes
3282 .slice_mut(s![0, 0, ..])
3283 .assign(&array![0.1234, 0.1234, 0.2345, 0.2345]);
3284 scores.slice_mut(s![0, 0, ..]).assign(&array![0.9876]);
3285 classes.slice_mut(s![0, 0, ..]).assign(&array![2.0]);
3286
3287 let detect_quant = (2.0 / 255.0, 0_i32);
3288
3289 let decoder = if is_split {
3290 DecoderBuilder::default()
3291 .with_config_yaml_str(E2E_SPLIT_DET_CONFIG.to_string())
3292 .with_score_threshold(score_threshold)
3293 .with_iou_threshold(iou_threshold)
3294 .build()
3295 .unwrap()
3296 } else {
3297 DecoderBuilder::default()
3298 .with_config_yaml_str(E2E_COMBINED_DET_CONFIG.to_string())
3299 .with_score_threshold(score_threshold)
3300 .with_iou_threshold(iou_threshold)
3301 .build()
3302 .unwrap()
3303 };
3304
3305 let expected = e2e_expected_boxes_quant();
3306 let mut output_boxes = Vec::with_capacity(50);
3307
3308 if is_split {
3309 let boxes: Array<u8, _> = quantize_ndarray(boxes.view(), detect_quant.into());
3310 let scores: Array<u8, _> = quantize_ndarray(scores.view(), detect_quant.into());
3311 let classes: Array<u8, _> =
3312 quantize_ndarray(classes.view(), detect_quant.into());
3313 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> = vec![
3314 boxes.view().into(),
3315 scores.view().into(),
3316 classes.view().into(),
3317 ];
3318 decoder
3319 .decode_quantized(&inputs, &mut output_boxes, &mut Vec::new())
3320 .unwrap();
3321 } else {
3322 let detect =
3323 ndarray::concatenate![Axis(2), boxes.view(), scores.view(), classes.view()];
3324 assert_eq!(detect.shape(), &[1, 10, 6]);
3325 let detect: Array3<u8> = quantize_ndarray(detect.view(), detect_quant.into());
3326 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> =
3327 vec![detect.view().into()];
3328 decoder
3329 .decode_quantized(&inputs, &mut output_boxes, &mut Vec::new())
3330 .unwrap();
3331 }
3332
3333 assert_eq!(output_boxes.len(), 1);
3334 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
3335 }
3336 };
3337 ($name:ident, float, $layout:ident) => {
3338 #[test]
3339 fn $name() {
3340 let is_split = matches!(stringify!($layout), "split");
3341
3342 let score_threshold = 0.45;
3343 let iou_threshold = 0.45;
3344
3345 let mut boxes = Array3::zeros((1, 10, 4));
3346 let mut scores = Array3::zeros((1, 10, 1));
3347 let mut classes = Array3::zeros((1, 10, 1));
3348
3349 boxes
3350 .slice_mut(s![0, 0, ..])
3351 .assign(&array![0.1234, 0.1234, 0.2345, 0.2345]);
3352 scores.slice_mut(s![0, 0, ..]).assign(&array![0.9876]);
3353 classes.slice_mut(s![0, 0, ..]).assign(&array![2.0]);
3354
3355 let decoder = if is_split {
3356 DecoderBuilder::default()
3357 .with_config_yaml_str(E2E_SPLIT_DET_CONFIG.to_string())
3358 .with_score_threshold(score_threshold)
3359 .with_iou_threshold(iou_threshold)
3360 .build()
3361 .unwrap()
3362 } else {
3363 DecoderBuilder::default()
3364 .with_config_yaml_str(E2E_COMBINED_DET_CONFIG.to_string())
3365 .with_score_threshold(score_threshold)
3366 .with_iou_threshold(iou_threshold)
3367 .build()
3368 .unwrap()
3369 };
3370
3371 let expected = e2e_expected_boxes_float();
3372 let mut output_boxes = Vec::with_capacity(50);
3373
3374 if is_split {
3375 let inputs = vec![
3376 boxes.view().into_dyn(),
3377 scores.view().into_dyn(),
3378 classes.view().into_dyn(),
3379 ];
3380 decoder
3381 .decode_float(&inputs, &mut output_boxes, &mut Vec::new())
3382 .unwrap();
3383 } else {
3384 let detect =
3385 ndarray::concatenate![Axis(2), boxes.view(), scores.view(), classes.view()];
3386 assert_eq!(detect.shape(), &[1, 10, 6]);
3387 let inputs = vec![detect.view().into_dyn()];
3388 decoder
3389 .decode_float(&inputs, &mut output_boxes, &mut Vec::new())
3390 .unwrap();
3391 }
3392
3393 assert_eq!(output_boxes.len(), 1);
3394 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
3395 }
3396 };
3397 }
3398
3399 e2e_det_test!(test_decoder_end_to_end_combined_det, quantized, combined);
3400 e2e_det_test!(test_decoder_end_to_end_combined_det_float, float, combined);
3401 e2e_det_test!(test_decoder_end_to_end_split_det, quantized, split);
3402 e2e_det_test!(test_decoder_end_to_end_split_det_float, float, split);
3403
3404 #[test]
3405 fn test_decode_tensor() {
3406 let score_threshold = 0.45;
3407 let iou_threshold = 0.45;
3408
3409 let raw_boxes = edgefirst_bench::testdata::read("yolov8_boxes_116x8400.bin");
3410 let raw_boxes =
3411 unsafe { std::slice::from_raw_parts(raw_boxes.as_ptr() as *const i8, raw_boxes.len()) };
3412 let boxes_i8: Tensor<i8> = Tensor::new(&[1, 116, 8400], None, None).unwrap();
3413 boxes_i8
3414 .map()
3415 .unwrap()
3416 .as_mut_slice()
3417 .copy_from_slice(raw_boxes);
3418 let boxes_i8 = boxes_i8.into();
3419
3420 let raw_protos = edgefirst_bench::testdata::read("yolov8_protos_160x160x32.bin");
3421 let raw_protos = unsafe {
3422 std::slice::from_raw_parts(raw_protos.as_ptr() as *const i8, raw_protos.len())
3423 };
3424 let protos_i8: Tensor<i8> = Tensor::new(&[1, 160, 160, 32], None, None).unwrap();
3425 protos_i8
3426 .map()
3427 .unwrap()
3428 .as_mut_slice()
3429 .copy_from_slice(raw_protos);
3430 let protos_i8 = protos_i8.into();
3431
3432 let decoder = build_yolov8_seg_decoder(score_threshold, iou_threshold);
3433 let expected = real_data_expected_boxes();
3434 let mut output_boxes = Vec::with_capacity(50);
3435
3436 decoder
3437 .decode(&[&boxes_i8, &protos_i8], &mut output_boxes, &mut Vec::new())
3438 .unwrap();
3439
3440 assert_eq!(output_boxes.len(), 2);
3441 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
3442 assert!(output_boxes[1].equal_within_delta(&expected[1], 1.0 / 160.0));
3443 }
3444
3445 #[test]
3446 fn test_decode_tensor_f32() {
3447 let score_threshold = 0.45;
3448 let iou_threshold = 0.45;
3449
3450 let quant_boxes = (0.021287762_f32, 31_i32);
3451 let quant_protos = (0.02491162_f32, -117_i32);
3452 let raw_boxes = edgefirst_bench::testdata::read("yolov8_boxes_116x8400.bin");
3453 let raw_boxes =
3454 unsafe { std::slice::from_raw_parts(raw_boxes.as_ptr() as *const i8, raw_boxes.len()) };
3455 let mut raw_boxes_f32 = vec![0f32; raw_boxes.len()];
3456 dequantize_cpu(raw_boxes, quant_boxes.into(), &mut raw_boxes_f32);
3457 let boxes_f32: Tensor<f32> = Tensor::new(&[1, 116, 8400], None, None).unwrap();
3458 boxes_f32
3459 .map()
3460 .unwrap()
3461 .as_mut_slice()
3462 .copy_from_slice(&raw_boxes_f32);
3463 let boxes_f32 = boxes_f32.into();
3464
3465 let raw_protos = edgefirst_bench::testdata::read("yolov8_protos_160x160x32.bin");
3466 let raw_protos = unsafe {
3467 std::slice::from_raw_parts(raw_protos.as_ptr() as *const i8, raw_protos.len())
3468 };
3469 let mut raw_protos_f32 = vec![0f32; raw_protos.len()];
3470 dequantize_cpu(raw_protos, quant_protos.into(), &mut raw_protos_f32);
3471 let protos_f32: Tensor<f32> = Tensor::new(&[1, 160, 160, 32], None, None).unwrap();
3472 protos_f32
3473 .map()
3474 .unwrap()
3475 .as_mut_slice()
3476 .copy_from_slice(&raw_protos_f32);
3477 let protos_f32 = protos_f32.into();
3478
3479 let decoder = build_yolov8_seg_decoder(score_threshold, iou_threshold);
3480
3481 let expected = real_data_expected_boxes();
3482 let mut output_boxes = Vec::with_capacity(50);
3483
3484 decoder
3485 .decode(
3486 &[&boxes_f32, &protos_f32],
3487 &mut output_boxes,
3488 &mut Vec::new(),
3489 )
3490 .unwrap();
3491
3492 assert_eq!(output_boxes.len(), 2);
3493 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
3494 assert!(output_boxes[1].equal_within_delta(&expected[1], 1.0 / 160.0));
3495 }
3496
3497 #[test]
3498 fn test_decode_tensor_f64() {
3499 let score_threshold = 0.45;
3500 let iou_threshold = 0.45;
3501
3502 let quant_boxes = (0.021287762_f32, 31_i32);
3503 let quant_protos = (0.02491162_f32, -117_i32);
3504 let raw_boxes = edgefirst_bench::testdata::read("yolov8_boxes_116x8400.bin");
3505 let raw_boxes =
3506 unsafe { std::slice::from_raw_parts(raw_boxes.as_ptr() as *const i8, raw_boxes.len()) };
3507 let mut raw_boxes_f64 = vec![0f64; raw_boxes.len()];
3508 dequantize_cpu(raw_boxes, quant_boxes.into(), &mut raw_boxes_f64);
3509 let boxes_f64: Tensor<f64> = Tensor::new(&[1, 116, 8400], None, None).unwrap();
3510 boxes_f64
3511 .map()
3512 .unwrap()
3513 .as_mut_slice()
3514 .copy_from_slice(&raw_boxes_f64);
3515 let boxes_f64 = boxes_f64.into();
3516
3517 let raw_protos = edgefirst_bench::testdata::read("yolov8_protos_160x160x32.bin");
3518 let raw_protos = unsafe {
3519 std::slice::from_raw_parts(raw_protos.as_ptr() as *const i8, raw_protos.len())
3520 };
3521 let mut raw_protos_f64 = vec![0f64; raw_protos.len()];
3522 dequantize_cpu(raw_protos, quant_protos.into(), &mut raw_protos_f64);
3523 let protos_f64: Tensor<f64> = Tensor::new(&[1, 160, 160, 32], None, None).unwrap();
3524 protos_f64
3525 .map()
3526 .unwrap()
3527 .as_mut_slice()
3528 .copy_from_slice(&raw_protos_f64);
3529 let protos_f64 = protos_f64.into();
3530
3531 let decoder = build_yolov8_seg_decoder(score_threshold, iou_threshold);
3532
3533 let expected = real_data_expected_boxes();
3534 let mut output_boxes = Vec::with_capacity(50);
3535
3536 decoder
3537 .decode(
3538 &[&boxes_f64, &protos_f64],
3539 &mut output_boxes,
3540 &mut Vec::new(),
3541 )
3542 .unwrap();
3543
3544 assert_eq!(output_boxes.len(), 2);
3545 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
3546 assert!(output_boxes[1].equal_within_delta(&expected[1], 1.0 / 160.0));
3547 }
3548
3549 #[test]
3550 fn test_decode_tensor_proto() {
3551 let score_threshold = 0.45;
3552 let iou_threshold = 0.45;
3553
3554 let raw_boxes = edgefirst_bench::testdata::read("yolov8_boxes_116x8400.bin");
3555 let raw_boxes =
3556 unsafe { std::slice::from_raw_parts(raw_boxes.as_ptr() as *const i8, raw_boxes.len()) };
3557 let boxes_i8: Tensor<i8> = Tensor::new(&[1, 116, 8400], None, None).unwrap();
3558 boxes_i8
3559 .map()
3560 .unwrap()
3561 .as_mut_slice()
3562 .copy_from_slice(raw_boxes);
3563 let boxes_i8 = boxes_i8.into();
3564
3565 let raw_protos = edgefirst_bench::testdata::read("yolov8_protos_160x160x32.bin");
3566 let raw_protos = unsafe {
3567 std::slice::from_raw_parts(raw_protos.as_ptr() as *const i8, raw_protos.len())
3568 };
3569 let protos_i8: Tensor<i8> = Tensor::new(&[1, 160, 160, 32], None, None).unwrap();
3570 protos_i8
3571 .map()
3572 .unwrap()
3573 .as_mut_slice()
3574 .copy_from_slice(raw_protos);
3575 let protos_i8 = protos_i8.into();
3576
3577 let decoder = build_yolov8_seg_decoder(score_threshold, iou_threshold);
3578
3579 let expected = real_data_expected_boxes();
3580 let mut output_boxes = Vec::with_capacity(50);
3581
3582 let proto_data = decoder
3583 .decode_proto(&[&boxes_i8, &protos_i8], &mut output_boxes)
3584 .unwrap();
3585
3586 assert_eq!(output_boxes.len(), 2);
3587 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
3588 assert!(output_boxes[1].equal_within_delta(&expected[1], 1.0 / 160.0));
3589
3590 let proto_data = proto_data.expect("segmentation model should return ProtoData");
3591 let coeffs_shape = proto_data.mask_coefficients.shape();
3592 assert_eq!(
3593 coeffs_shape[0],
3594 output_boxes.len(),
3595 "mask_coefficients count must match detection count"
3596 );
3597 assert_eq!(
3598 coeffs_shape[1], 32,
3599 "each detection should have 32 mask coefficients"
3600 );
3601 }
3602
3603 #[test]
3621 fn test_physical_order_tflite_nhwc_protos() {
3622 let score_threshold = 0.45;
3623 let iou_threshold = 0.45;
3624
3625 let protos_hwc = load_yolov8_protos().slice_move(s![0, .., .., ..]);
3627 let quant_protos = Quantization::new(0.02491161972284317, -117);
3628 let protos_f32_hwc = dequantize_ndarray::<_, _, f32>(protos_hwc.view(), quant_protos);
3629
3630 let boxes_2d = load_yolov8_boxes().slice_move(s![0, .., ..]);
3631 let quant_boxes = Quantization::new(0.021287761628627777, 31);
3632 let seg = dequantize_ndarray::<_, _, f32>(boxes_2d.view(), quant_boxes);
3633
3634 let mut ref_boxes: Vec<_> = Vec::with_capacity(10);
3636 let mut ref_masks: Vec<_> = Vec::with_capacity(10);
3637 decode_yolo_segdet_float(
3638 seg.view(),
3639 protos_f32_hwc.view(),
3640 score_threshold,
3641 iou_threshold,
3642 Some(configs::Nms::ClassAgnostic),
3643 &mut ref_boxes,
3644 &mut ref_masks,
3645 )
3646 .unwrap();
3647
3648 let protos_nhwc = protos_f32_hwc.clone().insert_axis(Axis(0)); let seg_3d = seg.insert_axis(Axis(0)); let decoder = DecoderBuilder::default()
3654 .with_config_yolo_segdet(
3655 configs::Detection {
3656 decoder: configs::DecoderType::Ultralytics,
3657 quantization: None,
3658 shape: vec![1, 116, 8400],
3659 dshape: vec![
3660 (DimName::Batch, 1),
3661 (DimName::NumFeatures, 116),
3662 (DimName::NumBoxes, 8400),
3663 ],
3664 normalized: Some(true),
3665 anchors: None,
3666 },
3667 configs::Protos {
3668 decoder: configs::DecoderType::Ultralytics,
3669 quantization: None,
3670 shape: vec![1, 160, 160, 32],
3671 dshape: vec![
3673 (DimName::Batch, 1),
3674 (DimName::Height, 160),
3675 (DimName::Width, 160),
3676 (DimName::NumProtos, 32),
3677 ],
3678 },
3679 None,
3680 )
3681 .with_score_threshold(score_threshold)
3682 .with_iou_threshold(iou_threshold)
3683 .build()
3684 .expect("config with NHWC protos dshape must build");
3685
3686 let mut cfg_boxes = Vec::with_capacity(10);
3687 let mut cfg_masks = Vec::with_capacity(10);
3688 decoder
3689 .decode_float(
3690 &[seg_3d.view().into_dyn(), protos_nhwc.view().into_dyn()],
3691 &mut cfg_boxes,
3692 &mut cfg_masks,
3693 )
3694 .unwrap();
3695
3696 assert_eq!(cfg_boxes.len(), ref_boxes.len(), "box count mismatch");
3697 for (c, r) in cfg_boxes.iter().zip(&ref_boxes) {
3698 assert!(
3699 c.equal_within_delta(r, 0.01),
3700 "NHWC-declared box does not match reference: {c:?} vs {r:?}"
3701 );
3702 }
3703 for (cm, rm) in cfg_masks.iter().zip(&ref_masks) {
3704 let cm_arr = segmentation_to_mask(seg_view(cm).view()).unwrap();
3705 let rm_arr = segmentation_to_mask(seg_view(rm).view()).unwrap();
3706 assert_eq!(
3707 cm_arr, rm_arr,
3708 "NHWC-declared mask must match reference pixel-for-pixel"
3709 );
3710 }
3711 }
3712
3713 #[test]
3722 fn test_physical_order_ara2_anchor_first_split_boxes() {
3723 use configs::{Boxes, Scores};
3724
3725 const N: usize = 8400;
3728 let mut boxes_canonical = Array3::<f32>::zeros((1, 4, N));
3729 let target_anchor = 42usize;
3730 boxes_canonical[[0, 0, target_anchor]] = 0.4; boxes_canonical[[0, 1, target_anchor]] = 0.5; boxes_canonical[[0, 2, target_anchor]] = 0.2; boxes_canonical[[0, 3, target_anchor]] = 0.2; let mut scores_canonical = Array3::<f32>::zeros((1, 80, N));
3737 scores_canonical[[0, 0, target_anchor]] = 0.9;
3738
3739 let ref_decoder = DecoderBuilder::default()
3741 .with_config_yolo_split_det(
3742 Boxes {
3743 decoder: configs::DecoderType::Ultralytics,
3744 quantization: None,
3745 shape: vec![1, 4, N],
3746 dshape: vec![
3747 (DimName::Batch, 1),
3748 (DimName::BoxCoords, 4),
3749 (DimName::NumBoxes, N),
3750 ],
3751 normalized: Some(true),
3752 },
3753 Scores {
3754 decoder: configs::DecoderType::Ultralytics,
3755 quantization: None,
3756 shape: vec![1, 80, N],
3757 dshape: vec![
3758 (DimName::Batch, 1),
3759 (DimName::NumClasses, 80),
3760 (DimName::NumBoxes, N),
3761 ],
3762 },
3763 )
3764 .with_score_threshold(0.5)
3765 .with_iou_threshold(0.5)
3766 .with_nms(Some(configs::Nms::ClassAgnostic))
3767 .build()
3768 .expect("reference canonical split decoder must build");
3769
3770 let mut ref_boxes = Vec::with_capacity(4);
3771 let mut ref_masks = Vec::with_capacity(0);
3772 ref_decoder
3773 .decode_float(
3774 &[
3775 boxes_canonical.view().into_dyn(),
3776 scores_canonical.view().into_dyn(),
3777 ],
3778 &mut ref_boxes,
3779 &mut ref_masks,
3780 )
3781 .unwrap();
3782 assert_eq!(ref_boxes.len(), 1, "reference should produce one box");
3783
3784 let boxes_ara2 = boxes_canonical.view().permuted_axes([0, 2, 1]).to_owned(); let scores_ara2 = scores_canonical.view().permuted_axes([0, 2, 1]).to_owned(); let ara2_decoder = DecoderBuilder::default()
3792 .with_config_yolo_split_det(
3793 Boxes {
3794 decoder: configs::DecoderType::Ultralytics,
3795 quantization: None,
3796 shape: vec![1, N, 4],
3797 dshape: vec![
3798 (DimName::Batch, 1),
3799 (DimName::NumBoxes, N),
3800 (DimName::BoxCoords, 4),
3801 ],
3802 normalized: Some(true),
3803 },
3804 Scores {
3805 decoder: configs::DecoderType::Ultralytics,
3806 quantization: None,
3807 shape: vec![1, N, 80],
3808 dshape: vec![
3809 (DimName::Batch, 1),
3810 (DimName::NumBoxes, N),
3811 (DimName::NumClasses, 80),
3812 ],
3813 },
3814 )
3815 .with_score_threshold(0.5)
3816 .with_iou_threshold(0.5)
3817 .with_nms(Some(configs::Nms::ClassAgnostic))
3818 .build()
3819 .expect("Ara-2 anchor-first decoder must build");
3820
3821 let mut ara2_boxes = Vec::with_capacity(4);
3822 let mut ara2_masks = Vec::with_capacity(0);
3823 ara2_decoder
3824 .decode_float(
3825 &[boxes_ara2.view().into_dyn(), scores_ara2.view().into_dyn()],
3826 &mut ara2_boxes,
3827 &mut ara2_masks,
3828 )
3829 .unwrap();
3830
3831 assert_eq!(
3832 ara2_boxes.len(),
3833 ref_boxes.len(),
3834 "Ara-2 anchor-first declaration must produce the same number \
3835 of boxes as the canonical features-first reference"
3836 );
3837 for (a, r) in ara2_boxes.iter().zip(&ref_boxes) {
3838 assert!(
3839 a.equal_within_delta(r, 1e-4),
3840 "Ara-2 box differs from reference: {a:?} vs {r:?}"
3841 );
3842 }
3843 }
3844
3845 #[test]
3849 fn test_physical_order_rejects_shape_dshape_mismatch() {
3850 let result = DecoderBuilder::default()
3851 .with_config_yolo_segdet(
3852 configs::Detection {
3853 decoder: configs::DecoderType::Ultralytics,
3854 quantization: None,
3855 shape: vec![1, 116, 8400],
3856 dshape: vec![
3857 (DimName::Batch, 1),
3858 (DimName::NumFeatures, 116),
3859 (DimName::NumBoxes, 8400),
3860 ],
3861 normalized: Some(true),
3862 anchors: None,
3863 },
3864 configs::Protos {
3865 decoder: configs::DecoderType::Ultralytics,
3866 quantization: None,
3867 shape: vec![1, 32, 160, 160],
3869 dshape: vec![
3872 (DimName::Batch, 1),
3873 (DimName::Height, 160),
3874 (DimName::Width, 160),
3875 (DimName::NumProtos, 32),
3876 ],
3877 },
3878 None,
3879 )
3880 .build();
3881
3882 match result {
3883 Err(DecoderError::InvalidConfig(msg)) => {
3884 assert!(
3885 msg.contains("does not match shape"),
3886 "expected shape/dshape size mismatch error, got: {msg}"
3887 );
3888 }
3889 other => panic!("expected InvalidConfig, got {other:?}"),
3890 }
3891 }
3892
3893 #[test]
3896 fn test_physical_order_rejects_duplicate_dshape_axis() {
3897 let result = DecoderBuilder::default()
3898 .with_config_yolo_split_det(
3899 configs::Boxes {
3900 decoder: configs::DecoderType::Ultralytics,
3901 quantization: None,
3902 shape: vec![1, 4, 8400],
3903 dshape: vec![
3904 (DimName::Batch, 1),
3905 (DimName::BoxCoords, 4),
3906 (DimName::BoxCoords, 4), ],
3908 normalized: Some(true),
3909 },
3910 configs::Scores {
3911 decoder: configs::DecoderType::Ultralytics,
3912 quantization: None,
3913 shape: vec![1, 80, 8400],
3914 dshape: vec![
3915 (DimName::Batch, 1),
3916 (DimName::NumClasses, 80),
3917 (DimName::NumBoxes, 8400),
3918 ],
3919 },
3920 )
3921 .build();
3922
3923 match result {
3928 Err(DecoderError::InvalidConfig(msg)) => {
3929 assert!(
3930 msg.contains("appears at both index") || msg.contains("does not match shape"),
3931 "expected positional or duplicate-axis error, got: {msg}"
3932 );
3933 }
3934 other => panic!("expected InvalidConfig, got {other:?}"),
3935 }
3936
3937 let result = DecoderBuilder::default()
3942 .with_config_yolo_split_det(
3943 configs::Boxes {
3944 decoder: configs::DecoderType::Ultralytics,
3945 quantization: None,
3946 shape: vec![1, 1, 4, 8400],
3947 dshape: vec![
3948 (DimName::Batch, 1),
3949 (DimName::Batch, 1), (DimName::BoxCoords, 4),
3951 (DimName::NumBoxes, 8400),
3952 ],
3953 normalized: Some(true),
3954 },
3955 configs::Scores {
3956 decoder: configs::DecoderType::Ultralytics,
3957 quantization: None,
3958 shape: vec![1, 80, 8400],
3959 dshape: vec![
3960 (DimName::Batch, 1),
3961 (DimName::NumClasses, 80),
3962 (DimName::NumBoxes, 8400),
3963 ],
3964 },
3965 )
3966 .build();
3967 match result {
3968 Err(DecoderError::InvalidConfig(msg)) => {
3969 assert!(
3970 msg.contains("appears at both index"),
3971 "expected duplicate-axis error, got: {msg}"
3972 );
3973 }
3974 other => panic!("expected InvalidConfig, got {other:?}"),
3975 }
3976 }
3977
3978 #[test]
3984 fn test_physical_order_dshape_omitted_decodes_numerically() {
3985 let score_threshold = 0.45;
3986 let iou_threshold = 0.45;
3987
3988 let protos_hwc = load_yolov8_protos().slice_move(s![0, .., .., ..]);
3989 let quant_protos = Quantization::new(0.02491161972284317, -117);
3990 let protos_f32_hwc = dequantize_ndarray::<_, _, f32>(protos_hwc.view(), quant_protos);
3991
3992 let boxes_2d = load_yolov8_boxes().slice_move(s![0, .., ..]);
3993 let quant_boxes = Quantization::new(0.021287761628627777, 31);
3994 let seg = dequantize_ndarray::<_, _, f32>(boxes_2d.view(), quant_boxes);
3995
3996 let protos_nhwc = protos_f32_hwc.clone().insert_axis(Axis(0));
3997 let seg_3d = seg.insert_axis(Axis(0));
3998
3999 let build_decoder = |det_dshape: Vec<(DimName, usize)>,
4000 proto_dshape: Vec<(DimName, usize)>| {
4001 DecoderBuilder::default()
4002 .with_config_yolo_segdet(
4003 configs::Detection {
4004 decoder: configs::DecoderType::Ultralytics,
4005 quantization: None,
4006 shape: vec![1, 116, 8400],
4007 dshape: det_dshape,
4008 normalized: Some(true),
4009 anchors: None,
4010 },
4011 configs::Protos {
4012 decoder: configs::DecoderType::Ultralytics,
4013 quantization: None,
4014 shape: vec![1, 160, 160, 32],
4015 dshape: proto_dshape,
4016 },
4017 None,
4018 )
4019 .with_score_threshold(score_threshold)
4020 .with_iou_threshold(iou_threshold)
4021 .build()
4022 .unwrap()
4023 };
4024
4025 let dshaped = build_decoder(
4027 vec![
4028 (DimName::Batch, 1),
4029 (DimName::NumFeatures, 116),
4030 (DimName::NumBoxes, 8400),
4031 ],
4032 vec![
4033 (DimName::Batch, 1),
4034 (DimName::Height, 160),
4035 (DimName::Width, 160),
4036 (DimName::NumProtos, 32),
4037 ],
4038 );
4039 let mut dshaped_boxes = Vec::new();
4040 let mut dshaped_masks = Vec::new();
4041 dshaped
4042 .decode_float(
4043 &[seg_3d.view().into_dyn(), protos_nhwc.view().into_dyn()],
4044 &mut dshaped_boxes,
4045 &mut dshaped_masks,
4046 )
4047 .unwrap();
4048
4049 let bare = build_decoder(vec![], vec![]);
4052 let mut bare_boxes = Vec::new();
4053 let mut bare_masks = Vec::new();
4054 bare.decode_float(
4055 &[seg_3d.view().into_dyn(), protos_nhwc.view().into_dyn()],
4056 &mut bare_boxes,
4057 &mut bare_masks,
4058 )
4059 .unwrap();
4060
4061 assert_eq!(bare_boxes.len(), dshaped_boxes.len());
4062 for (b, d) in bare_boxes.iter().zip(&dshaped_boxes) {
4063 assert!(
4064 b.equal_within_delta(d, 1e-4),
4065 "dshape-omitted box {b:?} differs from dshape-populated {d:?}"
4066 );
4067 }
4068 for (bm, dm) in bare_masks.iter().zip(&dshaped_masks) {
4069 let bm_arr = segmentation_to_mask(seg_view(bm).view()).unwrap();
4070 let dm_arr = segmentation_to_mask(seg_view(dm).view()).unwrap();
4071 assert_eq!(
4072 bm_arr, dm_arr,
4073 "dshape-omitted mask must match dshape-populated pixel-for-pixel"
4074 );
4075 }
4076 }
4077
4078 #[test]
4088 fn test_physical_order_ara2_4d_anchor_first_with_padding() {
4089 const N: usize = 8400;
4094 let mut boxes = Array3::<f32>::zeros((1, N, 4));
4095 let target = 42usize;
4096 boxes[[0, target, 0]] = 0.4;
4097 boxes[[0, target, 1]] = 0.5;
4098 boxes[[0, target, 2]] = 0.2;
4099 boxes[[0, target, 3]] = 0.2;
4100 let mut scores = Array3::<f32>::zeros((1, N, 80));
4101 scores[[0, target, 0]] = 0.9;
4102
4103 let json = r#"{
4109 "schema_version": 2,
4110 "decoder_version": "yolov8",
4111 "nms": "class_agnostic",
4112 "outputs": [
4113 {"name": "boxes", "type": "boxes",
4114 "shape": [1, 8400, 1, 4],
4115 "dshape": [{"batch":1},{"num_boxes":8400},{"padding":1},{"box_coords":4}],
4116 "encoding": "direct",
4117 "decoder": "ultralytics",
4118 "normalized": true},
4119 {"name": "scores", "type": "scores",
4120 "shape": [1, 8400, 1, 80],
4121 "dshape": [{"batch":1},{"num_boxes":8400},{"padding":1},{"num_classes":80}],
4122 "decoder": "ultralytics",
4123 "score_format": "per_class"}
4124 ]
4125 }"#;
4126 let decoder = DecoderBuilder::default()
4127 .with_config_json_str(json.to_string())
4128 .with_score_threshold(0.5)
4129 .with_iou_threshold(0.5)
4130 .build()
4131 .expect("4D anchor-first schema should build via squeeze_padding_dims");
4132
4133 let mut out_boxes = Vec::with_capacity(4);
4134 let mut out_masks = Vec::with_capacity(0);
4135 decoder
4136 .decode_float(
4137 &[boxes.view().into_dyn(), scores.view().into_dyn()],
4138 &mut out_boxes,
4139 &mut out_masks,
4140 )
4141 .unwrap();
4142
4143 assert_eq!(
4144 out_boxes.len(),
4145 1,
4146 "4D anchor-first with padding should decode exactly one box from the seeded anchor"
4147 );
4148 let b = &out_boxes[0];
4149 assert!((b.bbox.xmin - 0.3).abs() < 1e-3, "xmin wrong: {b:?}");
4151 assert!((b.bbox.ymin - 0.4).abs() < 1e-3, "ymin wrong: {b:?}");
4152 assert!((b.bbox.xmax - 0.5).abs() < 1e-3, "xmax wrong: {b:?}");
4153 assert!((b.bbox.ymax - 0.6).abs() < 1e-3, "ymax wrong: {b:?}");
4154 assert_eq!(b.label, 0);
4155 assert!(b.score > 0.85, "score {}: {b:?}", b.score);
4156 }
4157}
4158
4159#[cfg(feature = "tracker")]
4160#[cfg(test)]
4161#[cfg_attr(coverage_nightly, coverage(off))]
4162mod decoder_tracked_tests {
4163
4164 use crate::DetectBoxTestExt;
4165 use edgefirst_tracker::{ByteTrackBuilder, Tracker};
4166 use ndarray::{array, s, Array, Array2, Array3, Array4, ArrayView, Axis, Dimension};
4167 use num_traits::{AsPrimitive, Float, PrimInt};
4168 use rand::{RngExt, SeedableRng};
4169 use rand_distr::StandardNormal;
4170
4171 use crate::{
4172 configs::{self, DimName},
4173 dequantize_ndarray, BoundingBox, DecoderBuilder, DetectBox, Quantization,
4174 };
4175
4176 pub fn quantize_ndarray<T: PrimInt + 'static, D: Dimension, F: Float + AsPrimitive<T>>(
4177 input: ArrayView<F, D>,
4178 quant: Quantization,
4179 ) -> Array<T, D>
4180 where
4181 i32: num_traits::AsPrimitive<F>,
4182 f32: num_traits::AsPrimitive<F>,
4183 {
4184 let zero_point = quant.zero_point.as_();
4185 let div_scale = F::one() / quant.scale.as_();
4186 if zero_point != F::zero() {
4187 input.mapv(|d| (d * div_scale + zero_point).round().as_())
4188 } else {
4189 input.mapv(|d| (d * div_scale).round().as_())
4190 }
4191 }
4192
4193 #[test]
4194 fn test_decoder_tracked_random_jitter() {
4195 use crate::configs::{DecoderType, Nms};
4196 use crate::DecoderBuilder;
4197
4198 let score_threshold = 0.25;
4199 let iou_threshold = 0.1;
4200 let out = edgefirst_bench::testdata::read("yolov8s_80_classes.bin");
4201 let out = unsafe { std::slice::from_raw_parts(out.as_ptr() as *const i8, out.len()) };
4202 let out = Array3::from_shape_vec((1, 84, 8400), out.to_vec()).unwrap();
4203 let quant = (0.0040811873, -123).into();
4204
4205 let decoder = DecoderBuilder::default()
4206 .with_config_yolo_det(
4207 crate::configs::Detection {
4208 decoder: DecoderType::Ultralytics,
4209 shape: vec![1, 84, 8400],
4210 anchors: None,
4211 quantization: Some(quant),
4212 dshape: vec![
4213 (crate::configs::DimName::Batch, 1),
4214 (crate::configs::DimName::NumFeatures, 84),
4215 (crate::configs::DimName::NumBoxes, 8400),
4216 ],
4217 normalized: Some(true),
4218 },
4219 None,
4220 )
4221 .with_score_threshold(score_threshold)
4222 .with_iou_threshold(iou_threshold)
4223 .with_nms(Some(Nms::ClassAgnostic))
4224 .build()
4225 .unwrap();
4226 let mut rng = rand::rngs::StdRng::seed_from_u64(0xAB_BEEF); let expected_boxes = [
4229 crate::DetectBox {
4230 bbox: crate::BoundingBox {
4231 xmin: 0.5285137,
4232 ymin: 0.05305544,
4233 xmax: 0.87541467,
4234 ymax: 0.9998909,
4235 },
4236 score: 0.5591227,
4237 label: 0,
4238 },
4239 crate::DetectBox {
4240 bbox: crate::BoundingBox {
4241 xmin: 0.130598,
4242 ymin: 0.43260583,
4243 xmax: 0.35098213,
4244 ymax: 0.9958097,
4245 },
4246 score: 0.33057618,
4247 label: 75,
4248 },
4249 ];
4250
4251 let mut tracker = ByteTrackBuilder::new()
4252 .track_update(0.1)
4253 .track_high_conf(0.3)
4254 .build();
4255
4256 let mut output_boxes = Vec::with_capacity(50);
4257 let mut output_masks = Vec::with_capacity(50);
4258 let mut output_tracks = Vec::with_capacity(50);
4259
4260 decoder
4261 .decode_tracked_quantized(
4262 &mut tracker,
4263 0,
4264 &[out.view().into()],
4265 &mut output_boxes,
4266 &mut output_masks,
4267 &mut output_tracks,
4268 )
4269 .unwrap();
4270
4271 assert_eq!(output_boxes.len(), 2);
4272 assert!(output_boxes[0].equal_within_delta(&expected_boxes[0], 1e-6));
4273 assert!(output_boxes[1].equal_within_delta(&expected_boxes[1], 1e-6));
4274
4275 let mut last_boxes = output_boxes.clone();
4276
4277 for i in 1..=100 {
4278 let mut out = out.clone();
4279 let mut x_values = out.slice_mut(s![0, 0, ..]);
4281 for x in x_values.iter_mut() {
4282 let r: f32 = rng.sample(StandardNormal);
4283 let r = r.clamp(-2.0, 2.0) / 2.0;
4284 *x = x.saturating_add((r * 1e-2 / quant.0) as i8);
4285 }
4286
4287 let mut y_values = out.slice_mut(s![0, 1, ..]);
4288 for y in y_values.iter_mut() {
4289 let r: f32 = rng.sample(StandardNormal);
4290 let r = r.clamp(-2.0, 2.0) / 2.0;
4291 *y = y.saturating_add((r * 1e-2 / quant.0) as i8);
4292 }
4293
4294 decoder
4295 .decode_tracked_quantized(
4296 &mut tracker,
4297 100_000_000 * i / 3, &[out.view().into()],
4299 &mut output_boxes,
4300 &mut output_masks,
4301 &mut output_tracks,
4302 )
4303 .unwrap();
4304
4305 assert_eq!(output_boxes.len(), 2);
4306 assert!(output_boxes[0].equal_within_delta(&expected_boxes[0], 5e-3));
4307 assert!(output_boxes[1].equal_within_delta(&expected_boxes[1], 5e-3));
4308
4309 assert!(output_boxes[0].equal_within_delta(&last_boxes[0], 1e-3));
4310 assert!(output_boxes[1].equal_within_delta(&last_boxes[1], 1e-3));
4311 last_boxes = output_boxes.clone();
4312 }
4313 }
4314
4315 fn real_data_expected_boxes() -> [DetectBox; 2] {
4318 [
4319 DetectBox {
4320 bbox: BoundingBox {
4321 xmin: 0.08515105,
4322 ymin: 0.7131401,
4323 xmax: 0.29802868,
4324 ymax: 0.8195788,
4325 },
4326 score: 0.91537374,
4327 label: 23,
4328 },
4329 DetectBox {
4330 bbox: BoundingBox {
4331 xmin: 0.59605736,
4332 ymin: 0.25545314,
4333 xmax: 0.93666154,
4334 ymax: 0.72378385,
4335 },
4336 score: 0.91537374,
4337 label: 23,
4338 },
4339 ]
4340 }
4341
4342 fn e2e_expected_boxes_quant() -> [DetectBox; 1] {
4343 [DetectBox {
4344 bbox: BoundingBox {
4345 xmin: 0.12549022,
4346 ymin: 0.12549022,
4347 xmax: 0.23529413,
4348 ymax: 0.23529413,
4349 },
4350 score: 0.98823535,
4351 label: 2,
4352 }]
4353 }
4354
4355 fn e2e_expected_boxes_float() -> [DetectBox; 1] {
4356 [DetectBox {
4357 bbox: BoundingBox {
4358 xmin: 0.1234,
4359 ymin: 0.1234,
4360 xmax: 0.2345,
4361 ymax: 0.2345,
4362 },
4363 score: 0.9876,
4364 label: 2,
4365 }]
4366 }
4367
4368 fn build_yolo_split_segdet_decoder(
4369 score_threshold: f32,
4370 iou_threshold: f32,
4371 quant_boxes: (f32, i32),
4372 quant_protos: (f32, i32),
4373 ) -> crate::Decoder {
4374 DecoderBuilder::default()
4375 .with_config_yolo_split_segdet(
4376 configs::Boxes {
4377 decoder: configs::DecoderType::Ultralytics,
4378 quantization: Some(quant_boxes.into()),
4379 shape: vec![1, 4, 8400],
4380 dshape: vec![
4381 (DimName::Batch, 1),
4382 (DimName::BoxCoords, 4),
4383 (DimName::NumBoxes, 8400),
4384 ],
4385 normalized: Some(true),
4386 },
4387 configs::Scores {
4388 decoder: configs::DecoderType::Ultralytics,
4389 quantization: Some(quant_boxes.into()),
4390 shape: vec![1, 80, 8400],
4391 dshape: vec![
4392 (DimName::Batch, 1),
4393 (DimName::NumClasses, 80),
4394 (DimName::NumBoxes, 8400),
4395 ],
4396 },
4397 configs::MaskCoefficients {
4398 decoder: configs::DecoderType::Ultralytics,
4399 quantization: Some(quant_boxes.into()),
4400 shape: vec![1, 32, 8400],
4401 dshape: vec![
4402 (DimName::Batch, 1),
4403 (DimName::NumProtos, 32),
4404 (DimName::NumBoxes, 8400),
4405 ],
4406 },
4407 configs::Protos {
4408 decoder: configs::DecoderType::Ultralytics,
4409 quantization: Some(quant_protos.into()),
4410 shape: vec![1, 160, 160, 32],
4411 dshape: vec![
4412 (DimName::Batch, 1),
4413 (DimName::Height, 160),
4414 (DimName::Width, 160),
4415 (DimName::NumProtos, 32),
4416 ],
4417 },
4418 )
4419 .with_score_threshold(score_threshold)
4420 .with_iou_threshold(iou_threshold)
4421 .build()
4422 .unwrap()
4423 }
4424
4425 fn build_yolov8_seg_decoder(score_threshold: f32, iou_threshold: f32) -> crate::Decoder {
4426 let config_yaml = edgefirst_bench::testdata::read_to_string("yolov8_seg.yaml");
4427 DecoderBuilder::default()
4428 .with_config_yaml_str(config_yaml.to_string())
4429 .with_score_threshold(score_threshold)
4430 .with_iou_threshold(iou_threshold)
4431 .build()
4432 .unwrap()
4433 }
4434
4435 macro_rules! real_data_tracked_test {
4442 ($name:ident, quantized, $layout:ident, $output:ident) => {
4443 #[test]
4444 fn $name() {
4445 let is_split = matches!(stringify!($layout), "split");
4446 let is_proto = matches!(stringify!($output), "proto");
4447
4448 let score_threshold = 0.45;
4449 let iou_threshold = 0.45;
4450 let quant_boxes = (0.021287762_f32, 31_i32);
4451 let quant_protos = (0.02491162_f32, -117_i32);
4452
4453 let raw_boxes = edgefirst_bench::testdata::read("yolov8_boxes_116x8400.bin");
4454 let raw_boxes = unsafe {
4455 std::slice::from_raw_parts(raw_boxes.as_ptr() as *const i8, raw_boxes.len())
4456 };
4457 let boxes_i8 =
4458 ndarray::Array3::from_shape_vec((1, 116, 8400), raw_boxes.to_vec()).unwrap();
4459
4460 let raw_protos = edgefirst_bench::testdata::read("yolov8_protos_160x160x32.bin");
4461 let raw_protos = unsafe {
4462 std::slice::from_raw_parts(raw_protos.as_ptr() as *const i8, raw_protos.len())
4463 };
4464 let protos_i8 =
4465 ndarray::Array4::from_shape_vec((1, 160, 160, 32), raw_protos.to_vec())
4466 .unwrap();
4467
4468 let mask_split = boxes_i8.slice(s![.., 84.., ..]).to_owned();
4470 let mut scores_split = boxes_i8.slice(s![.., 4..84, ..]).to_owned();
4471 let boxes_split = boxes_i8.slice(s![.., ..4, ..]).to_owned();
4472 let mut boxes_combined = boxes_i8;
4473
4474 let decoder = if is_split {
4475 build_yolo_split_segdet_decoder(
4476 score_threshold,
4477 iou_threshold,
4478 quant_boxes,
4479 quant_protos,
4480 )
4481 } else {
4482 build_yolov8_seg_decoder(score_threshold, iou_threshold)
4483 };
4484
4485 let expected = real_data_expected_boxes();
4486 let mut tracker = ByteTrackBuilder::new()
4487 .track_update(0.1)
4488 .track_high_conf(0.7)
4489 .build();
4490 let mut output_boxes = Vec::with_capacity(50);
4491 let mut output_tracks = Vec::with_capacity(50);
4492
4493 if is_proto {
4495 {
4496 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> = if is_split {
4497 vec![
4498 boxes_split.view().into(),
4499 scores_split.view().into(),
4500 mask_split.view().into(),
4501 protos_i8.view().into(),
4502 ]
4503 } else {
4504 vec![boxes_combined.view().into(), protos_i8.view().into()]
4505 };
4506 decoder
4507 .decode_tracked_quantized_proto(
4508 &mut tracker,
4509 0,
4510 &inputs,
4511 &mut output_boxes,
4512 &mut output_tracks,
4513 )
4514 .unwrap();
4515 }
4516 assert_eq!(output_boxes.len(), 2);
4517 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
4518 assert!(output_boxes[1].equal_within_delta(&expected[1], 1.0 / 160.0));
4519
4520 if is_split {
4522 for score in scores_split.iter_mut() {
4523 *score = i8::MIN;
4524 }
4525 } else {
4526 for score in boxes_combined.slice_mut(s![0, 4..84, ..]).iter_mut() {
4527 *score = i8::MIN;
4528 }
4529 }
4530
4531 let proto_result = {
4532 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> = if is_split {
4533 vec![
4534 boxes_split.view().into(),
4535 scores_split.view().into(),
4536 mask_split.view().into(),
4537 protos_i8.view().into(),
4538 ]
4539 } else {
4540 vec![boxes_combined.view().into(), protos_i8.view().into()]
4541 };
4542 decoder
4543 .decode_tracked_quantized_proto(
4544 &mut tracker,
4545 100_000_000 / 3,
4546 &inputs,
4547 &mut output_boxes,
4548 &mut output_tracks,
4549 )
4550 .unwrap()
4551 };
4552 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
4553 assert!(output_boxes[1].equal_within_delta(&expected[1], 1e-6));
4554 assert!(proto_result.is_some_and(|x| x.mask_coefficients.shape()[0] == 0));
4555 } else {
4556 let mut output_masks = Vec::with_capacity(50);
4557 {
4558 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> = if is_split {
4559 vec![
4560 boxes_split.view().into(),
4561 scores_split.view().into(),
4562 mask_split.view().into(),
4563 protos_i8.view().into(),
4564 ]
4565 } else {
4566 vec![boxes_combined.view().into(), protos_i8.view().into()]
4567 };
4568 decoder
4569 .decode_tracked_quantized(
4570 &mut tracker,
4571 0,
4572 &inputs,
4573 &mut output_boxes,
4574 &mut output_masks,
4575 &mut output_tracks,
4576 )
4577 .unwrap();
4578 }
4579 assert_eq!(output_boxes.len(), 2);
4580 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
4581 assert!(output_boxes[1].equal_within_delta(&expected[1], 1.0 / 160.0));
4582
4583 if is_split {
4584 for score in scores_split.iter_mut() {
4585 *score = i8::MIN;
4586 }
4587 } else {
4588 for score in boxes_combined.slice_mut(s![0, 4..84, ..]).iter_mut() {
4589 *score = i8::MIN;
4590 }
4591 }
4592
4593 {
4594 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> = if is_split {
4595 vec![
4596 boxes_split.view().into(),
4597 scores_split.view().into(),
4598 mask_split.view().into(),
4599 protos_i8.view().into(),
4600 ]
4601 } else {
4602 vec![boxes_combined.view().into(), protos_i8.view().into()]
4603 };
4604 decoder
4605 .decode_tracked_quantized(
4606 &mut tracker,
4607 100_000_000 / 3,
4608 &inputs,
4609 &mut output_boxes,
4610 &mut output_masks,
4611 &mut output_tracks,
4612 )
4613 .unwrap();
4614 }
4615 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
4616 assert!(output_boxes[1].equal_within_delta(&expected[1], 1e-6));
4617 assert!(output_masks.is_empty());
4618 }
4619 }
4620 };
4621 ($name:ident, float, $layout:ident, $output:ident) => {
4622 #[test]
4623 fn $name() {
4624 let is_split = matches!(stringify!($layout), "split");
4625 let is_proto = matches!(stringify!($output), "proto");
4626
4627 let score_threshold = 0.45;
4628 let iou_threshold = 0.45;
4629 let quant_boxes = (0.021287762_f32, 31_i32);
4630 let quant_protos = (0.02491162_f32, -117_i32);
4631
4632 let raw_boxes = edgefirst_bench::testdata::read("yolov8_boxes_116x8400.bin");
4633 let raw_boxes = unsafe {
4634 std::slice::from_raw_parts(raw_boxes.as_ptr() as *const i8, raw_boxes.len())
4635 };
4636 let boxes_i8 =
4637 ndarray::Array3::from_shape_vec((1, 116, 8400), raw_boxes.to_vec()).unwrap();
4638 let boxes_f32 = dequantize_ndarray(boxes_i8.view(), quant_boxes.into());
4639
4640 let raw_protos = edgefirst_bench::testdata::read("yolov8_protos_160x160x32.bin");
4641 let raw_protos = unsafe {
4642 std::slice::from_raw_parts(raw_protos.as_ptr() as *const i8, raw_protos.len())
4643 };
4644 let protos_i8 =
4645 ndarray::Array4::from_shape_vec((1, 160, 160, 32), raw_protos.to_vec())
4646 .unwrap();
4647 let protos_f32 = dequantize_ndarray(protos_i8.view(), quant_protos.into());
4648
4649 let mask_split = boxes_f32.slice(s![.., 84.., ..]).to_owned();
4651 let mut scores_split = boxes_f32.slice(s![.., 4..84, ..]).to_owned();
4652 let boxes_split = boxes_f32.slice(s![.., ..4, ..]).to_owned();
4653 let mut boxes_combined = boxes_f32;
4654
4655 let decoder = if is_split {
4656 build_yolo_split_segdet_decoder(
4657 score_threshold,
4658 iou_threshold,
4659 quant_boxes,
4660 quant_protos,
4661 )
4662 } else {
4663 build_yolov8_seg_decoder(score_threshold, iou_threshold)
4664 };
4665
4666 let expected = real_data_expected_boxes();
4667 let mut tracker = ByteTrackBuilder::new()
4668 .track_update(0.1)
4669 .track_high_conf(0.7)
4670 .build();
4671 let mut output_boxes = Vec::with_capacity(50);
4672 let mut output_tracks = Vec::with_capacity(50);
4673
4674 if is_proto {
4675 {
4676 let inputs = if is_split {
4677 vec![
4678 boxes_split.view().into_dyn(),
4679 scores_split.view().into_dyn(),
4680 mask_split.view().into_dyn(),
4681 protos_f32.view().into_dyn(),
4682 ]
4683 } else {
4684 vec![
4685 boxes_combined.view().into_dyn(),
4686 protos_f32.view().into_dyn(),
4687 ]
4688 };
4689 decoder
4690 .decode_tracked_float_proto(
4691 &mut tracker,
4692 0,
4693 &inputs,
4694 &mut output_boxes,
4695 &mut output_tracks,
4696 )
4697 .unwrap();
4698 }
4699 assert_eq!(output_boxes.len(), 2);
4700 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
4701 assert!(output_boxes[1].equal_within_delta(&expected[1], 1.0 / 160.0));
4702
4703 if is_split {
4704 for score in scores_split.iter_mut() {
4705 *score = 0.0;
4706 }
4707 } else {
4708 for score in boxes_combined.slice_mut(s![0, 4..84, ..]).iter_mut() {
4709 *score = 0.0;
4710 }
4711 }
4712
4713 let proto_result = {
4714 let inputs = if is_split {
4715 vec![
4716 boxes_split.view().into_dyn(),
4717 scores_split.view().into_dyn(),
4718 mask_split.view().into_dyn(),
4719 protos_f32.view().into_dyn(),
4720 ]
4721 } else {
4722 vec![
4723 boxes_combined.view().into_dyn(),
4724 protos_f32.view().into_dyn(),
4725 ]
4726 };
4727 decoder
4728 .decode_tracked_float_proto(
4729 &mut tracker,
4730 100_000_000 / 3,
4731 &inputs,
4732 &mut output_boxes,
4733 &mut output_tracks,
4734 )
4735 .unwrap()
4736 };
4737 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
4738 assert!(output_boxes[1].equal_within_delta(&expected[1], 1e-6));
4739 assert!(proto_result.is_some_and(|x| x.mask_coefficients.shape()[0] == 0));
4740 } else {
4741 let mut output_masks = Vec::with_capacity(50);
4742 {
4743 let inputs = if is_split {
4744 vec![
4745 boxes_split.view().into_dyn(),
4746 scores_split.view().into_dyn(),
4747 mask_split.view().into_dyn(),
4748 protos_f32.view().into_dyn(),
4749 ]
4750 } else {
4751 vec![
4752 boxes_combined.view().into_dyn(),
4753 protos_f32.view().into_dyn(),
4754 ]
4755 };
4756 decoder
4757 .decode_tracked_float(
4758 &mut tracker,
4759 0,
4760 &inputs,
4761 &mut output_boxes,
4762 &mut output_masks,
4763 &mut output_tracks,
4764 )
4765 .unwrap();
4766 }
4767 assert_eq!(output_boxes.len(), 2);
4768 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
4769 assert!(output_boxes[1].equal_within_delta(&expected[1], 1.0 / 160.0));
4770
4771 if is_split {
4772 for score in scores_split.iter_mut() {
4773 *score = 0.0;
4774 }
4775 } else {
4776 for score in boxes_combined.slice_mut(s![0, 4..84, ..]).iter_mut() {
4777 *score = 0.0;
4778 }
4779 }
4780
4781 {
4782 let inputs = if is_split {
4783 vec![
4784 boxes_split.view().into_dyn(),
4785 scores_split.view().into_dyn(),
4786 mask_split.view().into_dyn(),
4787 protos_f32.view().into_dyn(),
4788 ]
4789 } else {
4790 vec![
4791 boxes_combined.view().into_dyn(),
4792 protos_f32.view().into_dyn(),
4793 ]
4794 };
4795 decoder
4796 .decode_tracked_float(
4797 &mut tracker,
4798 100_000_000 / 3,
4799 &inputs,
4800 &mut output_boxes,
4801 &mut output_masks,
4802 &mut output_tracks,
4803 )
4804 .unwrap();
4805 }
4806 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
4807 assert!(output_boxes[1].equal_within_delta(&expected[1], 1e-6));
4808 assert!(output_masks.is_empty());
4809 }
4810 }
4811 };
4812 }
4813
4814 real_data_tracked_test!(test_decoder_tracked_segdet, quantized, combined, masks);
4815 real_data_tracked_test!(test_decoder_tracked_segdet_float, float, combined, masks);
4816 real_data_tracked_test!(
4817 test_decoder_tracked_segdet_proto,
4818 quantized,
4819 combined,
4820 proto
4821 );
4822 real_data_tracked_test!(
4823 test_decoder_tracked_segdet_proto_float,
4824 float,
4825 combined,
4826 proto
4827 );
4828 real_data_tracked_test!(test_decoder_tracked_segdet_split, quantized, split, masks);
4829 real_data_tracked_test!(test_decoder_tracked_segdet_split_float, float, split, masks);
4830 real_data_tracked_test!(
4831 test_decoder_tracked_segdet_split_proto,
4832 quantized,
4833 split,
4834 proto
4835 );
4836 real_data_tracked_test!(
4837 test_decoder_tracked_segdet_split_proto_float,
4838 float,
4839 split,
4840 proto
4841 );
4842
4843 const E2E_COMBINED_CONFIG: &str = "
4849decoder_version: yolo26
4850outputs:
4851 - type: detection
4852 decoder: ultralytics
4853 quantization: [0.00784313725490196, 0]
4854 shape: [1, 10, 38]
4855 dshape:
4856 - [batch, 1]
4857 - [num_boxes, 10]
4858 - [num_features, 38]
4859 normalized: true
4860 - type: protos
4861 decoder: ultralytics
4862 quantization: [0.0039215686274509803921568627451, 128]
4863 shape: [1, 160, 160, 32]
4864 dshape:
4865 - [batch, 1]
4866 - [height, 160]
4867 - [width, 160]
4868 - [num_protos, 32]
4869";
4870
4871 const E2E_SPLIT_CONFIG: &str = "
4872decoder_version: yolo26
4873outputs:
4874 - type: boxes
4875 decoder: ultralytics
4876 quantization: [0.00784313725490196, 0]
4877 shape: [1, 10, 4]
4878 dshape:
4879 - [batch, 1]
4880 - [num_boxes, 10]
4881 - [box_coords, 4]
4882 normalized: true
4883 - type: scores
4884 decoder: ultralytics
4885 quantization: [0.00784313725490196, 0]
4886 shape: [1, 10, 1]
4887 dshape:
4888 - [batch, 1]
4889 - [num_boxes, 10]
4890 - [num_classes, 1]
4891 - type: classes
4892 decoder: ultralytics
4893 quantization: [0.00784313725490196, 0]
4894 shape: [1, 10, 1]
4895 dshape:
4896 - [batch, 1]
4897 - [num_boxes, 10]
4898 - [num_classes, 1]
4899 - type: mask_coefficients
4900 decoder: ultralytics
4901 quantization: [0.00784313725490196, 0]
4902 shape: [1, 10, 32]
4903 dshape:
4904 - [batch, 1]
4905 - [num_boxes, 10]
4906 - [num_protos, 32]
4907 - type: protos
4908 decoder: ultralytics
4909 quantization: [0.0039215686274509803921568627451, 128]
4910 shape: [1, 160, 160, 32]
4911 dshape:
4912 - [batch, 1]
4913 - [height, 160]
4914 - [width, 160]
4915 - [num_protos, 32]
4916";
4917
4918 macro_rules! e2e_tracked_test {
4919 ($name:ident, quantized, $layout:ident, $output:ident) => {
4920 #[test]
4921 fn $name() {
4922 let is_split = matches!(stringify!($layout), "split");
4923 let is_proto = matches!(stringify!($output), "proto");
4924
4925 let score_threshold = 0.45;
4926 let iou_threshold = 0.45;
4927
4928 let mut boxes = Array2::zeros((10, 4));
4929 let mut scores = Array2::zeros((10, 1));
4930 let mut classes = Array2::zeros((10, 1));
4931 let mask = Array2::zeros((10, 32));
4932 let protos = Array3::<f64>::zeros((160, 160, 32));
4933 let protos = protos.insert_axis(Axis(0));
4934 let protos_quant = (1.0 / 255.0, 0.0);
4935 let protos: Array4<u8> = quantize_ndarray(protos.view(), protos_quant.into());
4936
4937 boxes
4938 .slice_mut(s![0, ..])
4939 .assign(&array![0.1234, 0.1234, 0.2345, 0.2345]);
4940 scores.slice_mut(s![0, ..]).assign(&array![0.9876]);
4941 classes.slice_mut(s![0, ..]).assign(&array![2.0]);
4942
4943 let detect_quant = (2.0 / 255.0, 0.0);
4944
4945 let decoder = if is_split {
4946 DecoderBuilder::default()
4947 .with_config_yaml_str(E2E_SPLIT_CONFIG.to_string())
4948 .with_score_threshold(score_threshold)
4949 .with_iou_threshold(iou_threshold)
4950 .build()
4951 .unwrap()
4952 } else {
4953 DecoderBuilder::default()
4954 .with_config_yaml_str(E2E_COMBINED_CONFIG.to_string())
4955 .with_score_threshold(score_threshold)
4956 .with_iou_threshold(iou_threshold)
4957 .build()
4958 .unwrap()
4959 };
4960
4961 let expected = e2e_expected_boxes_quant();
4962 let mut tracker = ByteTrackBuilder::new()
4963 .track_update(0.1)
4964 .track_high_conf(0.7)
4965 .build();
4966 let mut output_boxes = Vec::with_capacity(50);
4967 let mut output_tracks = Vec::with_capacity(50);
4968
4969 if is_split {
4970 let boxes = boxes.insert_axis(Axis(0));
4971 let scores = scores.insert_axis(Axis(0));
4972 let classes = classes.insert_axis(Axis(0));
4973 let mask = mask.insert_axis(Axis(0));
4974
4975 let boxes: Array3<u8> = quantize_ndarray(boxes.view(), detect_quant.into());
4976 let mut scores: Array3<u8> =
4977 quantize_ndarray(scores.view(), detect_quant.into());
4978 let classes: Array3<u8> = quantize_ndarray(classes.view(), detect_quant.into());
4979 let mask: Array3<u8> = quantize_ndarray(mask.view(), detect_quant.into());
4980
4981 if is_proto {
4982 {
4983 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> = vec![
4984 boxes.view().into(),
4985 scores.view().into(),
4986 classes.view().into(),
4987 mask.view().into(),
4988 protos.view().into(),
4989 ];
4990 decoder
4991 .decode_tracked_quantized_proto(
4992 &mut tracker,
4993 0,
4994 &inputs,
4995 &mut output_boxes,
4996 &mut output_tracks,
4997 )
4998 .unwrap();
4999 }
5000 assert_eq!(output_boxes.len(), 1);
5001 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
5002
5003 for score in scores.slice_mut(s![.., .., ..]).iter_mut() {
5004 *score = u8::MIN;
5005 }
5006 let proto_result = {
5007 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> = vec![
5008 boxes.view().into(),
5009 scores.view().into(),
5010 classes.view().into(),
5011 mask.view().into(),
5012 protos.view().into(),
5013 ];
5014 decoder
5015 .decode_tracked_quantized_proto(
5016 &mut tracker,
5017 100_000_000 / 3,
5018 &inputs,
5019 &mut output_boxes,
5020 &mut output_tracks,
5021 )
5022 .unwrap()
5023 };
5024 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
5025 assert!(proto_result.is_some_and(|x| x.mask_coefficients.shape()[0] == 0));
5026 } else {
5027 let mut output_masks = Vec::with_capacity(50);
5028 {
5029 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> = vec![
5030 boxes.view().into(),
5031 scores.view().into(),
5032 classes.view().into(),
5033 mask.view().into(),
5034 protos.view().into(),
5035 ];
5036 decoder
5037 .decode_tracked_quantized(
5038 &mut tracker,
5039 0,
5040 &inputs,
5041 &mut output_boxes,
5042 &mut output_masks,
5043 &mut output_tracks,
5044 )
5045 .unwrap();
5046 }
5047 assert_eq!(output_boxes.len(), 1);
5048 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
5049
5050 for score in scores.slice_mut(s![.., .., ..]).iter_mut() {
5051 *score = u8::MIN;
5052 }
5053 {
5054 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> = vec![
5055 boxes.view().into(),
5056 scores.view().into(),
5057 classes.view().into(),
5058 mask.view().into(),
5059 protos.view().into(),
5060 ];
5061 decoder
5062 .decode_tracked_quantized(
5063 &mut tracker,
5064 100_000_000 / 3,
5065 &inputs,
5066 &mut output_boxes,
5067 &mut output_masks,
5068 &mut output_tracks,
5069 )
5070 .unwrap();
5071 }
5072 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
5073 assert!(output_masks.is_empty());
5074 }
5075 } else {
5076 let detect = ndarray::concatenate![
5078 Axis(1),
5079 boxes.view(),
5080 scores.view(),
5081 classes.view(),
5082 mask.view()
5083 ];
5084 let detect = detect.insert_axis(Axis(0));
5085 assert_eq!(detect.shape(), &[1, 10, 38]);
5086 let mut detect: Array3<u8> =
5087 quantize_ndarray(detect.view(), detect_quant.into());
5088
5089 if is_proto {
5090 {
5091 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> =
5092 vec![detect.view().into(), protos.view().into()];
5093 decoder
5094 .decode_tracked_quantized_proto(
5095 &mut tracker,
5096 0,
5097 &inputs,
5098 &mut output_boxes,
5099 &mut output_tracks,
5100 )
5101 .unwrap();
5102 }
5103 assert_eq!(output_boxes.len(), 1);
5104 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
5105
5106 for score in detect.slice_mut(s![.., .., 4]).iter_mut() {
5107 *score = u8::MIN;
5108 }
5109 let proto_result = {
5110 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> =
5111 vec![detect.view().into(), protos.view().into()];
5112 decoder
5113 .decode_tracked_quantized_proto(
5114 &mut tracker,
5115 100_000_000 / 3,
5116 &inputs,
5117 &mut output_boxes,
5118 &mut output_tracks,
5119 )
5120 .unwrap()
5121 };
5122 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
5123 assert!(proto_result.is_some_and(|x| x.mask_coefficients.shape()[0] == 0));
5124 } else {
5125 let mut output_masks = Vec::with_capacity(50);
5126 {
5127 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> =
5128 vec![detect.view().into(), protos.view().into()];
5129 decoder
5130 .decode_tracked_quantized(
5131 &mut tracker,
5132 0,
5133 &inputs,
5134 &mut output_boxes,
5135 &mut output_masks,
5136 &mut output_tracks,
5137 )
5138 .unwrap();
5139 }
5140 assert_eq!(output_boxes.len(), 1);
5141 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
5142
5143 for score in detect.slice_mut(s![.., .., 4]).iter_mut() {
5144 *score = u8::MIN;
5145 }
5146 {
5147 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> =
5148 vec![detect.view().into(), protos.view().into()];
5149 decoder
5150 .decode_tracked_quantized(
5151 &mut tracker,
5152 100_000_000 / 3,
5153 &inputs,
5154 &mut output_boxes,
5155 &mut output_masks,
5156 &mut output_tracks,
5157 )
5158 .unwrap();
5159 }
5160 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
5161 assert!(output_masks.is_empty());
5162 }
5163 }
5164 }
5165 };
5166 ($name:ident, float, $layout:ident, $output:ident) => {
5167 #[test]
5168 fn $name() {
5169 let is_split = matches!(stringify!($layout), "split");
5170 let is_proto = matches!(stringify!($output), "proto");
5171
5172 let score_threshold = 0.45;
5173 let iou_threshold = 0.45;
5174
5175 let mut boxes = Array2::zeros((10, 4));
5176 let mut scores = Array2::zeros((10, 1));
5177 let mut classes = Array2::zeros((10, 1));
5178 let mask: Array2<f64> = Array2::zeros((10, 32));
5179 let protos = Array3::<f64>::zeros((160, 160, 32));
5180 let protos = protos.insert_axis(Axis(0));
5181
5182 boxes
5183 .slice_mut(s![0, ..])
5184 .assign(&array![0.1234, 0.1234, 0.2345, 0.2345]);
5185 scores.slice_mut(s![0, ..]).assign(&array![0.9876]);
5186 classes.slice_mut(s![0, ..]).assign(&array![2.0]);
5187
5188 let decoder = if is_split {
5189 DecoderBuilder::default()
5190 .with_config_yaml_str(E2E_SPLIT_CONFIG.to_string())
5191 .with_score_threshold(score_threshold)
5192 .with_iou_threshold(iou_threshold)
5193 .build()
5194 .unwrap()
5195 } else {
5196 DecoderBuilder::default()
5197 .with_config_yaml_str(E2E_COMBINED_CONFIG.to_string())
5198 .with_score_threshold(score_threshold)
5199 .with_iou_threshold(iou_threshold)
5200 .build()
5201 .unwrap()
5202 };
5203
5204 let expected = e2e_expected_boxes_float();
5205 let mut tracker = ByteTrackBuilder::new()
5206 .track_update(0.1)
5207 .track_high_conf(0.7)
5208 .build();
5209 let mut output_boxes = Vec::with_capacity(50);
5210 let mut output_tracks = Vec::with_capacity(50);
5211
5212 if is_split {
5213 let boxes = boxes.insert_axis(Axis(0));
5214 let mut scores = scores.insert_axis(Axis(0));
5215 let classes = classes.insert_axis(Axis(0));
5216 let mask = mask.insert_axis(Axis(0));
5217
5218 if is_proto {
5219 {
5220 let inputs = vec![
5221 boxes.view().into_dyn(),
5222 scores.view().into_dyn(),
5223 classes.view().into_dyn(),
5224 mask.view().into_dyn(),
5225 protos.view().into_dyn(),
5226 ];
5227 decoder
5228 .decode_tracked_float_proto(
5229 &mut tracker,
5230 0,
5231 &inputs,
5232 &mut output_boxes,
5233 &mut output_tracks,
5234 )
5235 .unwrap();
5236 }
5237 assert_eq!(output_boxes.len(), 1);
5238 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
5239
5240 for score in scores.slice_mut(s![.., .., ..]).iter_mut() {
5241 *score = 0.0;
5242 }
5243 let proto_result = {
5244 let inputs = vec![
5245 boxes.view().into_dyn(),
5246 scores.view().into_dyn(),
5247 classes.view().into_dyn(),
5248 mask.view().into_dyn(),
5249 protos.view().into_dyn(),
5250 ];
5251 decoder
5252 .decode_tracked_float_proto(
5253 &mut tracker,
5254 100_000_000 / 3,
5255 &inputs,
5256 &mut output_boxes,
5257 &mut output_tracks,
5258 )
5259 .unwrap()
5260 };
5261 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
5262 assert!(proto_result.is_some_and(|x| x.mask_coefficients.shape()[0] == 0));
5263 } else {
5264 let mut output_masks = Vec::with_capacity(50);
5265 {
5266 let inputs = vec![
5267 boxes.view().into_dyn(),
5268 scores.view().into_dyn(),
5269 classes.view().into_dyn(),
5270 mask.view().into_dyn(),
5271 protos.view().into_dyn(),
5272 ];
5273 decoder
5274 .decode_tracked_float(
5275 &mut tracker,
5276 0,
5277 &inputs,
5278 &mut output_boxes,
5279 &mut output_masks,
5280 &mut output_tracks,
5281 )
5282 .unwrap();
5283 }
5284 assert_eq!(output_boxes.len(), 1);
5285 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
5286
5287 for score in scores.slice_mut(s![.., .., ..]).iter_mut() {
5288 *score = 0.0;
5289 }
5290 {
5291 let inputs = vec![
5292 boxes.view().into_dyn(),
5293 scores.view().into_dyn(),
5294 classes.view().into_dyn(),
5295 mask.view().into_dyn(),
5296 protos.view().into_dyn(),
5297 ];
5298 decoder
5299 .decode_tracked_float(
5300 &mut tracker,
5301 100_000_000 / 3,
5302 &inputs,
5303 &mut output_boxes,
5304 &mut output_masks,
5305 &mut output_tracks,
5306 )
5307 .unwrap();
5308 }
5309 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
5310 assert!(output_masks.is_empty());
5311 }
5312 } else {
5313 let detect = ndarray::concatenate![
5315 Axis(1),
5316 boxes.view(),
5317 scores.view(),
5318 classes.view(),
5319 mask.view()
5320 ];
5321 let mut detect = detect.insert_axis(Axis(0));
5322 assert_eq!(detect.shape(), &[1, 10, 38]);
5323
5324 if is_proto {
5325 {
5326 let inputs = vec![detect.view().into_dyn(), protos.view().into_dyn()];
5327 decoder
5328 .decode_tracked_float_proto(
5329 &mut tracker,
5330 0,
5331 &inputs,
5332 &mut output_boxes,
5333 &mut output_tracks,
5334 )
5335 .unwrap();
5336 }
5337 assert_eq!(output_boxes.len(), 1);
5338 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
5339
5340 for score in detect.slice_mut(s![.., .., 4]).iter_mut() {
5341 *score = 0.0;
5342 }
5343 let proto_result = {
5344 let inputs = vec![detect.view().into_dyn(), protos.view().into_dyn()];
5345 decoder
5346 .decode_tracked_float_proto(
5347 &mut tracker,
5348 100_000_000 / 3,
5349 &inputs,
5350 &mut output_boxes,
5351 &mut output_tracks,
5352 )
5353 .unwrap()
5354 };
5355 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
5356 assert!(proto_result.is_some_and(|x| x.mask_coefficients.shape()[0] == 0));
5357 } else {
5358 let mut output_masks = Vec::with_capacity(50);
5359 {
5360 let inputs = vec![detect.view().into_dyn(), protos.view().into_dyn()];
5361 decoder
5362 .decode_tracked_float(
5363 &mut tracker,
5364 0,
5365 &inputs,
5366 &mut output_boxes,
5367 &mut output_masks,
5368 &mut output_tracks,
5369 )
5370 .unwrap();
5371 }
5372 assert_eq!(output_boxes.len(), 1);
5373 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
5374
5375 for score in detect.slice_mut(s![.., .., 4]).iter_mut() {
5376 *score = 0.0;
5377 }
5378 {
5379 let inputs = vec![detect.view().into_dyn(), protos.view().into_dyn()];
5380 decoder
5381 .decode_tracked_float(
5382 &mut tracker,
5383 100_000_000 / 3,
5384 &inputs,
5385 &mut output_boxes,
5386 &mut output_masks,
5387 &mut output_tracks,
5388 )
5389 .unwrap();
5390 }
5391 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
5392 assert!(output_masks.is_empty());
5393 }
5394 }
5395 }
5396 };
5397 }
5398
5399 e2e_tracked_test!(
5400 test_decoder_tracked_end_to_end_segdet,
5401 quantized,
5402 combined,
5403 masks
5404 );
5405 e2e_tracked_test!(
5406 test_decoder_tracked_end_to_end_segdet_float,
5407 float,
5408 combined,
5409 masks
5410 );
5411 e2e_tracked_test!(
5412 test_decoder_tracked_end_to_end_segdet_proto,
5413 quantized,
5414 combined,
5415 proto
5416 );
5417 e2e_tracked_test!(
5418 test_decoder_tracked_end_to_end_segdet_proto_float,
5419 float,
5420 combined,
5421 proto
5422 );
5423 e2e_tracked_test!(
5424 test_decoder_tracked_end_to_end_segdet_split,
5425 quantized,
5426 split,
5427 masks
5428 );
5429 e2e_tracked_test!(
5430 test_decoder_tracked_end_to_end_segdet_split_float,
5431 float,
5432 split,
5433 masks
5434 );
5435 e2e_tracked_test!(
5436 test_decoder_tracked_end_to_end_segdet_split_proto,
5437 quantized,
5438 split,
5439 proto
5440 );
5441 e2e_tracked_test!(
5442 test_decoder_tracked_end_to_end_segdet_split_proto_float,
5443 float,
5444 split,
5445 proto
5446 );
5447
5448 macro_rules! e2e_tracked_tensor_test {
5454 ($name:ident, quantized, $layout:ident, $output:ident) => {
5455 #[test]
5456 fn $name() {
5457 use edgefirst_tensor::{Tensor, TensorMapTrait, TensorTrait};
5458
5459 let is_split = matches!(stringify!($layout), "split");
5460 let is_proto = matches!(stringify!($output), "proto");
5461
5462 let score_threshold = 0.45;
5463 let iou_threshold = 0.45;
5464
5465 let mut boxes = Array2::zeros((10, 4));
5466 let mut scores = Array2::zeros((10, 1));
5467 let mut classes = Array2::zeros((10, 1));
5468 let mask = Array2::zeros((10, 32));
5469 let protos_f64 = Array3::<f64>::zeros((160, 160, 32));
5470 let protos_f64 = protos_f64.insert_axis(Axis(0));
5471 let protos_quant = (1.0 / 255.0, 0.0);
5472 let protos_u8: Array4<u8> =
5473 quantize_ndarray(protos_f64.view(), protos_quant.into());
5474
5475 boxes
5476 .slice_mut(s![0, ..])
5477 .assign(&array![0.1234, 0.1234, 0.2345, 0.2345]);
5478 scores.slice_mut(s![0, ..]).assign(&array![0.9876]);
5479 classes.slice_mut(s![0, ..]).assign(&array![2.0]);
5480
5481 let detect_quant = (2.0 / 255.0, 0.0);
5482
5483 let decoder = if is_split {
5484 DecoderBuilder::default()
5485 .with_config_yaml_str(E2E_SPLIT_CONFIG.to_string())
5486 .with_score_threshold(score_threshold)
5487 .with_iou_threshold(iou_threshold)
5488 .build()
5489 .unwrap()
5490 } else {
5491 DecoderBuilder::default()
5492 .with_config_yaml_str(E2E_COMBINED_CONFIG.to_string())
5493 .with_score_threshold(score_threshold)
5494 .with_iou_threshold(iou_threshold)
5495 .build()
5496 .unwrap()
5497 };
5498
5499 let make_u8_tensor =
5501 |shape: &[usize], data: &[u8]| -> edgefirst_tensor::TensorDyn {
5502 let t = Tensor::<u8>::new(shape, None, None).unwrap();
5503 t.map().unwrap().as_mut_slice()[..data.len()].copy_from_slice(data);
5504 t.into()
5505 };
5506
5507 let expected = e2e_expected_boxes_quant();
5508 let mut tracker = ByteTrackBuilder::new()
5509 .track_update(0.1)
5510 .track_high_conf(0.7)
5511 .build();
5512 let mut output_boxes = Vec::with_capacity(50);
5513 let mut output_tracks = Vec::with_capacity(50);
5514
5515 let protos_td = make_u8_tensor(protos_u8.shape(), protos_u8.as_slice().unwrap());
5516
5517 if is_split {
5518 let boxes = boxes.insert_axis(Axis(0));
5519 let scores = scores.insert_axis(Axis(0));
5520 let classes = classes.insert_axis(Axis(0));
5521 let mask = mask.insert_axis(Axis(0));
5522
5523 let boxes_q: Array3<u8> = quantize_ndarray(boxes.view(), detect_quant.into());
5524 let mut scores_q: Array3<u8> =
5525 quantize_ndarray(scores.view(), detect_quant.into());
5526 let classes_q: Array3<u8> =
5527 quantize_ndarray(classes.view(), detect_quant.into());
5528 let mask_q: Array3<u8> = quantize_ndarray(mask.view(), detect_quant.into());
5529
5530 let boxes_td = make_u8_tensor(boxes_q.shape(), boxes_q.as_slice().unwrap());
5531 let classes_td =
5532 make_u8_tensor(classes_q.shape(), classes_q.as_slice().unwrap());
5533 let mask_td = make_u8_tensor(mask_q.shape(), mask_q.as_slice().unwrap());
5534
5535 if is_proto {
5536 let scores_td =
5537 make_u8_tensor(scores_q.shape(), scores_q.as_slice().unwrap());
5538 decoder
5539 .decode_proto_tracked(
5540 &mut tracker,
5541 0,
5542 &[&boxes_td, &scores_td, &classes_td, &mask_td, &protos_td],
5543 &mut output_boxes,
5544 &mut output_tracks,
5545 )
5546 .unwrap();
5547
5548 assert_eq!(output_boxes.len(), 1);
5549 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
5550
5551 for score in scores_q.slice_mut(s![.., .., ..]).iter_mut() {
5552 *score = u8::MIN;
5553 }
5554 let scores_td =
5555 make_u8_tensor(scores_q.shape(), scores_q.as_slice().unwrap());
5556 let proto_result = decoder
5557 .decode_proto_tracked(
5558 &mut tracker,
5559 100_000_000 / 3,
5560 &[&boxes_td, &scores_td, &classes_td, &mask_td, &protos_td],
5561 &mut output_boxes,
5562 &mut output_tracks,
5563 )
5564 .unwrap();
5565 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
5566 assert!(proto_result.is_some_and(|x| x.mask_coefficients.shape()[0] == 0));
5567 } else {
5568 let scores_td =
5569 make_u8_tensor(scores_q.shape(), scores_q.as_slice().unwrap());
5570 let mut output_masks = Vec::with_capacity(50);
5571 decoder
5572 .decode_tracked(
5573 &mut tracker,
5574 0,
5575 &[&boxes_td, &scores_td, &classes_td, &mask_td, &protos_td],
5576 &mut output_boxes,
5577 &mut output_masks,
5578 &mut output_tracks,
5579 )
5580 .unwrap();
5581
5582 assert_eq!(output_boxes.len(), 1);
5583 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
5584
5585 for score in scores_q.slice_mut(s![.., .., ..]).iter_mut() {
5586 *score = u8::MIN;
5587 }
5588 let scores_td =
5589 make_u8_tensor(scores_q.shape(), scores_q.as_slice().unwrap());
5590 decoder
5591 .decode_tracked(
5592 &mut tracker,
5593 100_000_000 / 3,
5594 &[&boxes_td, &scores_td, &classes_td, &mask_td, &protos_td],
5595 &mut output_boxes,
5596 &mut output_masks,
5597 &mut output_tracks,
5598 )
5599 .unwrap();
5600 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
5601 assert!(output_masks.is_empty());
5602 }
5603 } else {
5604 let detect = ndarray::concatenate![
5606 Axis(1),
5607 boxes.view(),
5608 scores.view(),
5609 classes.view(),
5610 mask.view()
5611 ];
5612 let detect = detect.insert_axis(Axis(0));
5613 assert_eq!(detect.shape(), &[1, 10, 38]);
5614 let detect =
5616 Array3::from_shape_vec(detect.raw_dim(), detect.iter().copied().collect())
5617 .unwrap();
5618 let mut detect_q: Array3<u8> =
5619 quantize_ndarray(detect.view(), detect_quant.into());
5620
5621 if is_proto {
5622 let detect_td =
5623 make_u8_tensor(detect_q.shape(), detect_q.as_slice().unwrap());
5624 decoder
5625 .decode_proto_tracked(
5626 &mut tracker,
5627 0,
5628 &[&detect_td, &protos_td],
5629 &mut output_boxes,
5630 &mut output_tracks,
5631 )
5632 .unwrap();
5633
5634 assert_eq!(output_boxes.len(), 1);
5635 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
5636
5637 for score in detect_q.slice_mut(s![.., .., 4]).iter_mut() {
5638 *score = u8::MIN;
5639 }
5640 let detect_td =
5641 make_u8_tensor(detect_q.shape(), detect_q.as_slice().unwrap());
5642 let proto_result = decoder
5643 .decode_proto_tracked(
5644 &mut tracker,
5645 100_000_000 / 3,
5646 &[&detect_td, &protos_td],
5647 &mut output_boxes,
5648 &mut output_tracks,
5649 )
5650 .unwrap();
5651 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
5652 assert!(proto_result.is_some_and(|x| x.mask_coefficients.shape()[0] == 0));
5653 } else {
5654 let detect_td =
5655 make_u8_tensor(detect_q.shape(), detect_q.as_slice().unwrap());
5656 let mut output_masks = Vec::with_capacity(50);
5657 decoder
5658 .decode_tracked(
5659 &mut tracker,
5660 0,
5661 &[&detect_td, &protos_td],
5662 &mut output_boxes,
5663 &mut output_masks,
5664 &mut output_tracks,
5665 )
5666 .unwrap();
5667
5668 assert_eq!(output_boxes.len(), 1);
5669 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
5670
5671 for score in detect_q.slice_mut(s![.., .., 4]).iter_mut() {
5672 *score = u8::MIN;
5673 }
5674 let detect_td =
5675 make_u8_tensor(detect_q.shape(), detect_q.as_slice().unwrap());
5676 decoder
5677 .decode_tracked(
5678 &mut tracker,
5679 100_000_000 / 3,
5680 &[&detect_td, &protos_td],
5681 &mut output_boxes,
5682 &mut output_masks,
5683 &mut output_tracks,
5684 )
5685 .unwrap();
5686 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
5687 assert!(output_masks.is_empty());
5688 }
5689 }
5690 }
5691 };
5692 ($name:ident, float, $layout:ident, $output:ident) => {
5693 #[test]
5694 fn $name() {
5695 use edgefirst_tensor::{Tensor, TensorMapTrait, TensorTrait};
5696
5697 let is_split = matches!(stringify!($layout), "split");
5698 let is_proto = matches!(stringify!($output), "proto");
5699
5700 let score_threshold = 0.45;
5701 let iou_threshold = 0.45;
5702
5703 let mut boxes = Array2::zeros((10, 4));
5704 let mut scores = Array2::zeros((10, 1));
5705 let mut classes = Array2::zeros((10, 1));
5706 let mask: Array2<f64> = Array2::zeros((10, 32));
5707 let protos = Array3::<f64>::zeros((160, 160, 32));
5708 let protos = protos.insert_axis(Axis(0));
5709
5710 boxes
5711 .slice_mut(s![0, ..])
5712 .assign(&array![0.1234, 0.1234, 0.2345, 0.2345]);
5713 scores.slice_mut(s![0, ..]).assign(&array![0.9876]);
5714 classes.slice_mut(s![0, ..]).assign(&array![2.0]);
5715
5716 let decoder = if is_split {
5717 DecoderBuilder::default()
5718 .with_config_yaml_str(E2E_SPLIT_CONFIG.to_string())
5719 .with_score_threshold(score_threshold)
5720 .with_iou_threshold(iou_threshold)
5721 .build()
5722 .unwrap()
5723 } else {
5724 DecoderBuilder::default()
5725 .with_config_yaml_str(E2E_COMBINED_CONFIG.to_string())
5726 .with_score_threshold(score_threshold)
5727 .with_iou_threshold(iou_threshold)
5728 .build()
5729 .unwrap()
5730 };
5731
5732 let make_f64_tensor =
5734 |shape: &[usize], data: &[f64]| -> edgefirst_tensor::TensorDyn {
5735 let t = Tensor::<f64>::new(shape, None, None).unwrap();
5736 t.map().unwrap().as_mut_slice()[..data.len()].copy_from_slice(data);
5737 t.into()
5738 };
5739
5740 let expected = e2e_expected_boxes_float();
5741 let mut tracker = ByteTrackBuilder::new()
5742 .track_update(0.1)
5743 .track_high_conf(0.7)
5744 .build();
5745 let mut output_boxes = Vec::with_capacity(50);
5746 let mut output_tracks = Vec::with_capacity(50);
5747
5748 let protos_td = make_f64_tensor(protos.shape(), protos.as_slice().unwrap());
5749
5750 if is_split {
5751 let boxes = boxes.insert_axis(Axis(0));
5752 let mut scores = scores.insert_axis(Axis(0));
5753 let classes = classes.insert_axis(Axis(0));
5754 let mask = mask.insert_axis(Axis(0));
5755
5756 let boxes_td = make_f64_tensor(boxes.shape(), boxes.as_slice().unwrap());
5757 let classes_td = make_f64_tensor(classes.shape(), classes.as_slice().unwrap());
5758 let mask_td = make_f64_tensor(mask.shape(), mask.as_slice().unwrap());
5759
5760 if is_proto {
5761 let scores_td = make_f64_tensor(scores.shape(), scores.as_slice().unwrap());
5762 decoder
5763 .decode_proto_tracked(
5764 &mut tracker,
5765 0,
5766 &[&boxes_td, &scores_td, &classes_td, &mask_td, &protos_td],
5767 &mut output_boxes,
5768 &mut output_tracks,
5769 )
5770 .unwrap();
5771
5772 assert_eq!(output_boxes.len(), 1);
5773 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
5774
5775 for score in scores.slice_mut(s![.., .., ..]).iter_mut() {
5776 *score = 0.0;
5777 }
5778 let scores_td = make_f64_tensor(scores.shape(), scores.as_slice().unwrap());
5779 let proto_result = decoder
5780 .decode_proto_tracked(
5781 &mut tracker,
5782 100_000_000 / 3,
5783 &[&boxes_td, &scores_td, &classes_td, &mask_td, &protos_td],
5784 &mut output_boxes,
5785 &mut output_tracks,
5786 )
5787 .unwrap();
5788 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
5789 assert!(proto_result.is_some_and(|x| x.mask_coefficients.shape()[0] == 0));
5790 } else {
5791 let scores_td = make_f64_tensor(scores.shape(), scores.as_slice().unwrap());
5792 let mut output_masks = Vec::with_capacity(50);
5793 decoder
5794 .decode_tracked(
5795 &mut tracker,
5796 0,
5797 &[&boxes_td, &scores_td, &classes_td, &mask_td, &protos_td],
5798 &mut output_boxes,
5799 &mut output_masks,
5800 &mut output_tracks,
5801 )
5802 .unwrap();
5803
5804 assert_eq!(output_boxes.len(), 1);
5805 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
5806
5807 for score in scores.slice_mut(s![.., .., ..]).iter_mut() {
5808 *score = 0.0;
5809 }
5810 let scores_td = make_f64_tensor(scores.shape(), scores.as_slice().unwrap());
5811 decoder
5812 .decode_tracked(
5813 &mut tracker,
5814 100_000_000 / 3,
5815 &[&boxes_td, &scores_td, &classes_td, &mask_td, &protos_td],
5816 &mut output_boxes,
5817 &mut output_masks,
5818 &mut output_tracks,
5819 )
5820 .unwrap();
5821 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
5822 assert!(output_masks.is_empty());
5823 }
5824 } else {
5825 let detect = ndarray::concatenate![
5827 Axis(1),
5828 boxes.view(),
5829 scores.view(),
5830 classes.view(),
5831 mask.view()
5832 ];
5833 let detect = detect.insert_axis(Axis(0));
5834 assert_eq!(detect.shape(), &[1, 10, 38]);
5835 let mut detect =
5837 Array3::from_shape_vec(detect.raw_dim(), detect.iter().copied().collect())
5838 .unwrap();
5839
5840 if is_proto {
5841 let detect_td = make_f64_tensor(detect.shape(), detect.as_slice().unwrap());
5842 decoder
5843 .decode_proto_tracked(
5844 &mut tracker,
5845 0,
5846 &[&detect_td, &protos_td],
5847 &mut output_boxes,
5848 &mut output_tracks,
5849 )
5850 .unwrap();
5851
5852 assert_eq!(output_boxes.len(), 1);
5853 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
5854
5855 for score in detect.slice_mut(s![.., .., 4]).iter_mut() {
5856 *score = 0.0;
5857 }
5858 let detect_td = make_f64_tensor(detect.shape(), detect.as_slice().unwrap());
5859 let proto_result = decoder
5860 .decode_proto_tracked(
5861 &mut tracker,
5862 100_000_000 / 3,
5863 &[&detect_td, &protos_td],
5864 &mut output_boxes,
5865 &mut output_tracks,
5866 )
5867 .unwrap();
5868 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
5869 assert!(proto_result.is_some_and(|x| x.mask_coefficients.shape()[0] == 0));
5870 } else {
5871 let detect_td = make_f64_tensor(detect.shape(), detect.as_slice().unwrap());
5872 let mut output_masks = Vec::with_capacity(50);
5873 decoder
5874 .decode_tracked(
5875 &mut tracker,
5876 0,
5877 &[&detect_td, &protos_td],
5878 &mut output_boxes,
5879 &mut output_masks,
5880 &mut output_tracks,
5881 )
5882 .unwrap();
5883
5884 assert_eq!(output_boxes.len(), 1);
5885 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
5886
5887 for score in detect.slice_mut(s![.., .., 4]).iter_mut() {
5888 *score = 0.0;
5889 }
5890 let detect_td = make_f64_tensor(detect.shape(), detect.as_slice().unwrap());
5891 decoder
5892 .decode_tracked(
5893 &mut tracker,
5894 100_000_000 / 3,
5895 &[&detect_td, &protos_td],
5896 &mut output_boxes,
5897 &mut output_masks,
5898 &mut output_tracks,
5899 )
5900 .unwrap();
5901 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
5902 assert!(output_masks.is_empty());
5903 }
5904 }
5905 }
5906 };
5907 }
5908
5909 e2e_tracked_tensor_test!(
5910 test_decoder_tracked_tensor_end_to_end_segdet,
5911 quantized,
5912 combined,
5913 masks
5914 );
5915 e2e_tracked_tensor_test!(
5916 test_decoder_tracked_tensor_end_to_end_segdet_float,
5917 float,
5918 combined,
5919 masks
5920 );
5921 e2e_tracked_tensor_test!(
5922 test_decoder_tracked_tensor_end_to_end_segdet_proto,
5923 quantized,
5924 combined,
5925 proto
5926 );
5927 e2e_tracked_tensor_test!(
5928 test_decoder_tracked_tensor_end_to_end_segdet_proto_float,
5929 float,
5930 combined,
5931 proto
5932 );
5933 e2e_tracked_tensor_test!(
5934 test_decoder_tracked_tensor_end_to_end_segdet_split,
5935 quantized,
5936 split,
5937 masks
5938 );
5939 e2e_tracked_tensor_test!(
5940 test_decoder_tracked_tensor_end_to_end_segdet_split_float,
5941 float,
5942 split,
5943 masks
5944 );
5945 e2e_tracked_tensor_test!(
5946 test_decoder_tracked_tensor_end_to_end_segdet_split_proto,
5947 quantized,
5948 split,
5949 proto
5950 );
5951 e2e_tracked_tensor_test!(
5952 test_decoder_tracked_tensor_end_to_end_segdet_split_proto_float,
5953 float,
5954 split,
5955 proto
5956 );
5957
5958 #[test]
5959 fn test_decoder_tracked_linear_motion() {
5960 use crate::configs::{DecoderType, Nms};
5961 use crate::DecoderBuilder;
5962
5963 let score_threshold = 0.25;
5964 let iou_threshold = 0.1;
5965 let out = edgefirst_bench::testdata::read("yolov8s_80_classes.bin");
5966 let out = unsafe { std::slice::from_raw_parts(out.as_ptr() as *const i8, out.len()) };
5967 let mut out = Array3::from_shape_vec((1, 84, 8400), out.to_vec()).unwrap();
5968 let quant = (0.0040811873, -123).into();
5969
5970 let decoder = DecoderBuilder::default()
5971 .with_config_yolo_det(
5972 crate::configs::Detection {
5973 decoder: DecoderType::Ultralytics,
5974 shape: vec![1, 84, 8400],
5975 anchors: None,
5976 quantization: Some(quant),
5977 dshape: vec![
5978 (crate::configs::DimName::Batch, 1),
5979 (crate::configs::DimName::NumFeatures, 84),
5980 (crate::configs::DimName::NumBoxes, 8400),
5981 ],
5982 normalized: Some(true),
5983 },
5984 None,
5985 )
5986 .with_score_threshold(score_threshold)
5987 .with_iou_threshold(iou_threshold)
5988 .with_nms(Some(Nms::ClassAgnostic))
5989 .build()
5990 .unwrap();
5991
5992 let mut expected_boxes = [
5993 DetectBox {
5994 bbox: BoundingBox {
5995 xmin: 0.5285137,
5996 ymin: 0.05305544,
5997 xmax: 0.87541467,
5998 ymax: 0.9998909,
5999 },
6000 score: 0.5591227,
6001 label: 0,
6002 },
6003 DetectBox {
6004 bbox: BoundingBox {
6005 xmin: 0.130598,
6006 ymin: 0.43260583,
6007 xmax: 0.35098213,
6008 ymax: 0.9958097,
6009 },
6010 score: 0.33057618,
6011 label: 75,
6012 },
6013 ];
6014
6015 let mut tracker = ByteTrackBuilder::new()
6016 .track_update(0.1)
6017 .track_high_conf(0.3)
6018 .build();
6019
6020 let mut output_boxes = Vec::with_capacity(50);
6021 let mut output_masks = Vec::with_capacity(50);
6022 let mut output_tracks = Vec::with_capacity(50);
6023
6024 decoder
6025 .decode_tracked_quantized(
6026 &mut tracker,
6027 0,
6028 &[out.view().into()],
6029 &mut output_boxes,
6030 &mut output_masks,
6031 &mut output_tracks,
6032 )
6033 .unwrap();
6034
6035 assert_eq!(output_boxes.len(), 2);
6036 assert!(output_boxes[0].equal_within_delta(&expected_boxes[0], 1e-6));
6037 assert!(output_boxes[1].equal_within_delta(&expected_boxes[1], 1e-6));
6038
6039 for i in 1..=100 {
6040 let mut out = out.clone();
6041 let mut x_values = out.slice_mut(s![0, 0, ..]);
6043 for x in x_values.iter_mut() {
6044 *x = x.saturating_add((i as f32 * 1e-3 / quant.0).round() as i8);
6045 }
6046
6047 decoder
6048 .decode_tracked_quantized(
6049 &mut tracker,
6050 100_000_000 * i / 3, &[out.view().into()],
6052 &mut output_boxes,
6053 &mut output_masks,
6054 &mut output_tracks,
6055 )
6056 .unwrap();
6057
6058 assert_eq!(output_boxes.len(), 2);
6059 }
6060 let tracks = tracker.get_active_tracks();
6061 let predicted_boxes: Vec<_> = tracks
6062 .iter()
6063 .map(|track| {
6064 let mut l = track.last_box;
6065 l.bbox = track.info.tracked_location.into();
6066 l
6067 })
6068 .collect();
6069 expected_boxes[0].bbox.xmin += 0.1; expected_boxes[0].bbox.xmax += 0.1;
6071 expected_boxes[1].bbox.xmin += 0.1;
6072 expected_boxes[1].bbox.xmax += 0.1;
6073
6074 assert!(predicted_boxes[0].equal_within_delta(&expected_boxes[0], 1e-3));
6075 assert!(predicted_boxes[1].equal_within_delta(&expected_boxes[1], 1e-3));
6076
6077 let mut scores_values = out.slice_mut(s![0, 4.., ..]);
6079 for score in scores_values.iter_mut() {
6080 *score = i8::MIN; }
6082 decoder
6083 .decode_tracked_quantized(
6084 &mut tracker,
6085 100_000_000 * 101 / 3,
6086 &[out.view().into()],
6087 &mut output_boxes,
6088 &mut output_masks,
6089 &mut output_tracks,
6090 )
6091 .unwrap();
6092 expected_boxes[0].bbox.xmin += 0.001; expected_boxes[0].bbox.xmax += 0.001;
6094 expected_boxes[1].bbox.xmin += 0.001;
6095 expected_boxes[1].bbox.xmax += 0.001;
6096
6097 assert!(output_boxes[0].equal_within_delta(&expected_boxes[0], 1e-3));
6098 assert!(output_boxes[1].equal_within_delta(&expected_boxes[1], 1e-3));
6099 }
6100
6101 #[test]
6102 fn test_decoder_tracked_end_to_end_float() {
6103 let score_threshold = 0.45;
6104 let iou_threshold = 0.45;
6105
6106 let mut boxes = Array2::zeros((10, 4));
6107 let mut scores = Array2::zeros((10, 1));
6108 let mut classes = Array2::zeros((10, 1));
6109
6110 boxes
6111 .slice_mut(s![0, ..,])
6112 .assign(&array![0.1234, 0.1234, 0.2345, 0.2345]);
6113 scores.slice_mut(s![0, ..]).assign(&array![0.9876]);
6114 classes.slice_mut(s![0, ..]).assign(&array![2.0]);
6115
6116 let detect = ndarray::concatenate![Axis(1), boxes.view(), scores.view(), classes.view(),];
6117 let mut detect = detect.insert_axis(Axis(0));
6118 assert_eq!(detect.shape(), &[1, 10, 6]);
6119 let config = "
6120decoder_version: yolo26
6121outputs:
6122 - type: detection
6123 decoder: ultralytics
6124 quantization: [0.00784313725490196, 0]
6125 shape: [1, 10, 6]
6126 dshape:
6127 - [batch, 1]
6128 - [num_boxes, 10]
6129 - [num_features, 6]
6130 normalized: true
6131";
6132
6133 let decoder = DecoderBuilder::default()
6134 .with_config_yaml_str(config.to_string())
6135 .with_score_threshold(score_threshold)
6136 .with_iou_threshold(iou_threshold)
6137 .build()
6138 .unwrap();
6139
6140 let expected_boxes = [DetectBox {
6141 bbox: BoundingBox {
6142 xmin: 0.1234,
6143 ymin: 0.1234,
6144 xmax: 0.2345,
6145 ymax: 0.2345,
6146 },
6147 score: 0.9876,
6148 label: 2,
6149 }];
6150
6151 let mut tracker = ByteTrackBuilder::new()
6152 .track_update(0.1)
6153 .track_high_conf(0.7)
6154 .build();
6155
6156 let mut output_boxes = Vec::with_capacity(50);
6157 let mut output_masks = Vec::with_capacity(50);
6158 let mut output_tracks = Vec::with_capacity(50);
6159
6160 decoder
6161 .decode_tracked_float(
6162 &mut tracker,
6163 0,
6164 &[detect.view().into_dyn()],
6165 &mut output_boxes,
6166 &mut output_masks,
6167 &mut output_tracks,
6168 )
6169 .unwrap();
6170
6171 assert_eq!(output_boxes.len(), 1);
6172 assert!(output_boxes[0].equal_within_delta(&expected_boxes[0], 1e-6));
6173
6174 for score in detect.slice_mut(s![.., .., 4]).iter_mut() {
6177 *score = 0.0; }
6179
6180 decoder
6181 .decode_tracked_float(
6182 &mut tracker,
6183 100_000_000 / 3,
6184 &[detect.view().into_dyn()],
6185 &mut output_boxes,
6186 &mut output_masks,
6187 &mut output_tracks,
6188 )
6189 .unwrap();
6190 assert!(output_boxes[0].equal_within_delta(&expected_boxes[0], 1e-6));
6191 }
6192}