1#![cfg_attr(coverage_nightly, feature(coverage_attribute))]
68
69use ndarray::{Array, Array2, ArrayView, ArrayView1, ArrayView3, Dimension};
70use num_traits::{AsPrimitive, Float, PrimInt};
71
72pub mod byte;
73pub mod error;
74pub mod float;
75pub mod infer;
76pub mod modelpack;
77pub mod per_scale;
78pub mod schema;
79pub mod tiling;
80pub mod yolo;
81
82pub use infer::{
83 infer_ultralytics_schema, InferError, InferredSchema, ModelSignals, ModelSource, TensorInfo,
84};
85
86mod decoder;
87pub use decoder::*;
88#[cfg(test)]
89mod test_support;
90
91pub use edgefirst_tensor::{BoundingBox, DetectBox, ProtoData, ProtoLayout, Segmentation};
95
96pub use edgefirst_decoder_abi::TilePlacement;
101
102pub use configs::{DecoderVersion, Nms};
103pub use error::{DecoderError, DecoderResult};
104pub use per_scale::DecodeDtype;
105
106use crate::{
107 decoder::configs::QuantTuple, modelpack::modelpack_segmentation_to_mask,
108 yolo::yolo_segmentation_to_mask,
109};
110
111fn mask_to_tensor(mask: ndarray::ArrayView3<'_, u8>) -> DecoderResult<edgefirst_tensor::TensorDyn> {
120 edgefirst_tensor::Tensor::from_arrayview3(mask)
121 .map(edgefirst_tensor::TensorDyn::from)
122 .map_err(|e| DecoderError::Internal(format!("mask -> TensorDyn: {e}")))
123}
124
125#[cfg(test)]
133pub(crate) trait DetectBoxTestExt {
134 fn equal_within_delta(&self, rhs: &DetectBox, eps: f32) -> bool;
135}
136
137#[cfg(test)]
138impl DetectBoxTestExt for DetectBox {
139 fn equal_within_delta(&self, rhs: &DetectBox, eps: f32) -> bool {
140 let eq_delta = |a: f32, b: f32| (a - b).abs() <= eps;
141 self.label == rhs.label
142 && eq_delta(self.score, rhs.score)
143 && eq_delta(self.bbox.xmin, rhs.bbox.xmin)
144 && eq_delta(self.bbox.ymin, rhs.bbox.ymin)
145 && eq_delta(self.bbox.xmax, rhs.bbox.xmax)
146 && eq_delta(self.bbox.ymax, rhs.bbox.ymax)
147 }
148}
149
150pub trait BBoxTypeTrait {
152 fn to_xyxy_float<A: Float + 'static, B: AsPrimitive<A>>(input: &[B; 4]) -> [A; 4];
154
155 fn to_xyxy_dequant<A: Float + 'static, B: AsPrimitive<A>>(
157 input: &[B; 4],
158 quant: Quantization,
159 ) -> [A; 4]
160 where
161 f32: AsPrimitive<A>,
162 i32: AsPrimitive<A>;
163
164 #[inline(always)]
175 fn ndarray_to_xyxy_float<A: Float + 'static, B: AsPrimitive<A>>(
176 input: ArrayView1<B>,
177 ) -> [A; 4] {
178 Self::to_xyxy_float(&[input[0], input[1], input[2], input[3]])
179 }
180
181 #[inline(always)]
182 fn ndarray_to_xyxy_dequant<A: Float + 'static, B: AsPrimitive<A>>(
184 input: ArrayView1<B>,
185 quant: Quantization,
186 ) -> [A; 4]
187 where
188 f32: AsPrimitive<A>,
189 i32: AsPrimitive<A>,
190 {
191 Self::to_xyxy_dequant(&[input[0], input[1], input[2], input[3]], quant)
192 }
193}
194
195#[derive(Debug, Clone, Copy, PartialEq, Eq)]
197pub struct XYXY {}
198
199impl BBoxTypeTrait for XYXY {
200 fn to_xyxy_float<A: Float + 'static, B: AsPrimitive<A>>(input: &[B; 4]) -> [A; 4] {
201 input.map(|b| b.as_())
202 }
203
204 fn to_xyxy_dequant<A: Float + 'static, B: AsPrimitive<A>>(
205 input: &[B; 4],
206 quant: Quantization,
207 ) -> [A; 4]
208 where
209 f32: AsPrimitive<A>,
210 i32: AsPrimitive<A>,
211 {
212 let scale = quant.scale.as_();
213 let zp = quant.zero_point.as_();
214 input.map(|b| (b.as_() - zp) * scale)
215 }
216}
217
218#[derive(Debug, Clone, Copy, PartialEq, Eq)]
221pub struct XYWH {}
222
223impl BBoxTypeTrait for XYWH {
224 #[inline(always)]
225 fn to_xyxy_float<A: Float + 'static, B: AsPrimitive<A>>(input: &[B; 4]) -> [A; 4] {
226 let half = A::one() / (A::one() + A::one());
227 [
228 (input[0].as_()) - (input[2].as_() * half),
229 (input[1].as_()) - (input[3].as_() * half),
230 (input[0].as_()) + (input[2].as_() * half),
231 (input[1].as_()) + (input[3].as_() * half),
232 ]
233 }
234
235 #[inline(always)]
236 fn to_xyxy_dequant<A: Float + 'static, B: AsPrimitive<A>>(
237 input: &[B; 4],
238 quant: Quantization,
239 ) -> [A; 4]
240 where
241 f32: AsPrimitive<A>,
242 i32: AsPrimitive<A>,
243 {
244 let scale = quant.scale.as_();
245 let half_scale = (quant.scale * 0.5).as_();
246 let zp = quant.zero_point.as_();
247 let [x, y, w, h] = [
248 (input[0].as_() - zp) * scale,
249 (input[1].as_() - zp) * scale,
250 (input[2].as_() - zp) * half_scale,
251 (input[3].as_() - zp) * half_scale,
252 ];
253
254 [x - w, y - h, x + w, y + h]
255 }
256}
257
258#[derive(Debug, Clone, Copy, PartialEq)]
260pub struct Quantization {
261 pub scale: f32,
262 pub zero_point: i32,
263}
264
265impl Quantization {
266 pub fn new(scale: f32, zero_point: i32) -> Self {
275 Self { scale, zero_point }
276 }
277
278 pub fn identity() -> Self {
290 Self {
291 scale: 1.0,
292 zero_point: 0,
293 }
294 }
295}
296
297impl From<QuantTuple> for Quantization {
298 fn from(quant_tuple: QuantTuple) -> Quantization {
309 Quantization {
310 scale: quant_tuple.0,
311 zero_point: quant_tuple.1,
312 }
313 }
314}
315
316impl<S, Z> From<(S, Z)> for Quantization
317where
318 S: AsPrimitive<f32>,
319 Z: AsPrimitive<i32>,
320{
321 fn from((scale, zp): (S, Z)) -> Quantization {
330 Self {
331 scale: scale.as_(),
332 zero_point: zp.as_(),
333 }
334 }
335}
336
337impl Default for Quantization {
338 fn default() -> Self {
347 Self {
348 scale: 1.0,
349 zero_point: 0,
350 }
351 }
352}
353
354pub fn dequant_detect_box<SCORE: PrimInt + AsPrimitive<f32>>(
372 detect: &DetectBoxQuantized<SCORE>,
373 quant_scores: Quantization,
374) -> DetectBox {
375 let scaled_zp = -quant_scores.scale * quant_scores.zero_point as f32;
376 DetectBox {
377 bbox: detect.bbox,
378 score: quant_scores.scale * detect.score.as_() + scaled_zp,
379 label: detect.label,
380 }
381}
382#[derive(Debug, Clone, Copy, PartialEq)]
384pub struct DetectBoxQuantized<
385 SCORE: PrimInt + AsPrimitive<f32>,
387> {
388 pub bbox: BoundingBox,
390 pub score: SCORE,
393 pub label: usize,
395}
396
397pub fn dequantize_ndarray<T: AsPrimitive<F>, D: Dimension, F: Float + 'static>(
410 input: ArrayView<T, D>,
411 quant: Quantization,
412) -> Array<F, D>
413where
414 i32: num_traits::AsPrimitive<F>,
415 f32: num_traits::AsPrimitive<F>,
416{
417 let zero_point = quant.zero_point.as_();
418 let scale = quant.scale.as_();
419 if zero_point != F::zero() {
420 let scaled_zero = -zero_point * scale;
421 input.mapv(|d| d.as_() * scale + scaled_zero)
422 } else {
423 input.mapv(|d| d.as_() * scale)
424 }
425}
426
427pub fn dequantize_cpu<T: AsPrimitive<F>, F: Float + 'static>(
440 input: &[T],
441 quant: Quantization,
442 output: &mut [F],
443) where
444 f32: num_traits::AsPrimitive<F>,
445 i32: num_traits::AsPrimitive<F>,
446{
447 assert!(input.len() == output.len());
448 let zero_point = quant.zero_point.as_();
449 let scale = quant.scale.as_();
450 if zero_point != F::zero() {
451 let scaled_zero = -zero_point * scale; input
453 .iter()
454 .zip(output)
455 .for_each(|(d, deq)| *deq = d.as_() * scale + scaled_zero);
456 } else {
457 input
458 .iter()
459 .zip(output)
460 .for_each(|(d, deq)| *deq = d.as_() * scale);
461 }
462}
463
464pub fn dequantize_cpu_chunked<T: AsPrimitive<F>, F: Float + 'static>(
478 input: &[T],
479 quant: Quantization,
480 output: &mut [F],
481) where
482 f32: num_traits::AsPrimitive<F>,
483 i32: num_traits::AsPrimitive<F>,
484{
485 assert!(input.len() == output.len());
486 let zero_point = quant.zero_point.as_();
487 let scale = quant.scale.as_();
488
489 let input = input.as_chunks::<4>();
490 let output = output.as_chunks_mut::<4>();
491
492 if zero_point != F::zero() {
493 let scaled_zero = -zero_point * scale; input
496 .0
497 .iter()
498 .zip(output.0)
499 .for_each(|(d, deq)| *deq = d.map(|d| d.as_() * scale + scaled_zero));
500 input
501 .1
502 .iter()
503 .zip(output.1)
504 .for_each(|(d, deq)| *deq = d.as_() * scale + scaled_zero);
505 } else {
506 input
507 .0
508 .iter()
509 .zip(output.0)
510 .for_each(|(d, deq)| *deq = d.map(|d| d.as_() * scale));
511 input
512 .1
513 .iter()
514 .zip(output.1)
515 .for_each(|(d, deq)| *deq = d.as_() * scale);
516 }
517}
518
519pub fn segmentation_to_mask(segmentation: ArrayView3<u8>) -> Result<Array2<u8>, DecoderError> {
537 if segmentation.shape()[2] == 0 {
538 return Err(DecoderError::InvalidShape(
539 "Segmentation tensor must have non-zero depth".to_string(),
540 ));
541 }
542 if segmentation.shape()[2] == 1 {
543 yolo_segmentation_to_mask(segmentation, 128)
544 } else {
545 Ok(modelpack_segmentation_to_mask(segmentation))
546 }
547}
548
549fn arg_max<T: PartialOrd + Copy>(score: ArrayView1<T>) -> (T, usize) {
551 score
552 .iter()
553 .enumerate()
554 .fold((score[0], 0), |(max, arg_max), (ind, s)| {
555 if max > *s {
556 (max, arg_max)
557 } else {
558 (*s, ind)
559 }
560 })
561}
562
563#[cfg(target_arch = "aarch64")]
569pub(crate) fn arg_max_i8(scores: &[i8]) -> (i8, usize) {
570 use std::arch::aarch64::*; let n = scores.len();
573 if n < 16 {
574 let mut max = scores[0];
576 let mut idx = 0;
577 for (i, &s) in scores.iter().enumerate().skip(1) {
578 if s >= max {
579 max = s;
580 idx = i;
581 }
582 }
583 return (max, idx);
584 }
585
586 unsafe {
587 let chunks = n / 16;
589 let mut vmax = vld1q_s8(scores.as_ptr());
590 for i in 1..chunks {
591 let v = vld1q_s8(scores.as_ptr().add(i * 16));
592 vmax = vmaxq_s8(vmax, v);
593 }
594 let global_max = vmaxvq_s8(vmax);
595
596 let remainder_start = chunks * 16;
598 let mut final_max = global_max;
599 for &s in &scores[remainder_start..] {
600 if s > final_max {
601 final_max = s;
602 }
603 }
604
605 let mut idx = 0;
608 for i in (0..n).rev() {
609 if scores[i] == final_max {
610 idx = i;
611 break;
612 }
613 }
614 (final_max, idx)
615 }
616}
617#[cfg(test)]
618#[cfg_attr(coverage_nightly, coverage(off))]
619mod decoder_tests {
620 #![allow(clippy::excessive_precision)]
621 use crate::DetectBoxTestExt;
622 use crate::{
623 configs::{DecoderType, DimName, Protos},
624 modelpack::{decode_modelpack_det, decode_modelpack_split_quant},
625 yolo::{
626 decode_yolo_det, decode_yolo_det_float, decode_yolo_segdet_float,
627 decode_yolo_segdet_quant,
628 },
629 *,
630 };
631 use edgefirst_tensor::{Tensor, TensorMapTrait, TensorTrait};
632 use ndarray::Dimension;
633 use ndarray::{array, s, Array2, Array3, Array4, Axis};
634 use num_traits::{AsPrimitive, PrimInt};
635
636 fn compare_outputs(
637 boxes: (&[DetectBox], &[DetectBox]),
638 masks: (&[Segmentation], &[Segmentation]),
639 ) {
640 let (boxes0, boxes1) = boxes;
641 let (masks0, masks1) = masks;
642
643 assert_eq!(boxes0.len(), boxes1.len());
644 assert_eq!(masks0.len(), masks1.len());
645
646 for (b_i8, b_f32) in boxes0.iter().zip(boxes1) {
647 assert!(
648 b_i8.equal_within_delta(b_f32, 1e-6),
649 "{b_i8:?} is not equal to {b_f32:?}"
650 );
651 }
652
653 for (m_i8, m_f32) in masks0.iter().zip(masks1) {
654 assert_eq!(
655 [m_i8.xmin, m_i8.ymin, m_i8.xmax, m_i8.ymax],
656 [m_f32.xmin, m_f32.ymin, m_f32.xmax, m_f32.ymax],
657 );
658 let shape = m_i8.segmentation.shape().to_vec();
659 assert_eq!(shape, m_f32.segmentation.shape());
660 let (t_i8, t_f32) = (
663 m_i8.segmentation.as_u8().expect("mask must be U8"),
664 m_f32.segmentation.as_u8().expect("mask must be U8"),
665 );
666 let (map_i8, map_f32) = (
667 t_i8.map_read().expect("map i8 mask"),
668 t_f32.map_read().expect("map f32 mask"),
669 );
670 let (a, b) = (map_i8.as_slice(), map_f32.as_slice());
671 let (w, c) = (shape[1], shape[2]);
672 let mut sq_err = 0f64;
673 for (i, (&va, &vb)) in a.iter().zip(b.iter()).enumerate() {
674 let val = va as i32 - vb as i32;
675 assert!(
676 val.abs() <= 1,
677 "Difference between mask0 and mask1 is greater than 1 at ({}, {}, {}): {}",
678 i / (w * c),
679 (i / c) % w,
680 i % c,
681 val
682 );
683 sq_err += (val as f64) * (val as f64);
684 }
685 let mean_sq_err = sq_err / a.len() as f64;
686 assert!(
687 mean_sq_err < 1e-2,
688 "Mean Square Error between masks was greater than 1%: {:.2}%",
689 mean_sq_err * 100.0
690 );
691 }
692 }
693
694 fn seg_view(s: &Segmentation) -> ndarray::Array3<u8> {
697 let t = s.segmentation.as_u8().expect("mask must be U8");
698 let m = t.map_read().expect("map mask");
699 let sh = s.segmentation.shape();
700 ndarray::Array3::from_shape_vec((sh[0], sh[1], sh[2]), m.as_slice().to_vec())
701 .expect("mask shape")
702 }
703
704 fn load_yolov8_boxes() -> Array3<i8> {
707 let raw = edgefirst_bench::testdata::read("yolov8_boxes_116x8400.bin");
708 let raw = unsafe { std::slice::from_raw_parts(raw.as_ptr() as *const i8, raw.len()) };
709 Array3::from_shape_vec((1, 116, 8400), raw.to_vec()).unwrap()
710 }
711
712 fn load_yolov8_protos() -> Array4<i8> {
713 let raw = edgefirst_bench::testdata::read("yolov8_protos_160x160x32.bin");
714 let raw = unsafe { std::slice::from_raw_parts(raw.as_ptr() as *const i8, raw.len()) };
715 Array4::from_shape_vec((1, 160, 160, 32), raw.to_vec()).unwrap()
716 }
717
718 fn load_yolov8s_det() -> Array3<i8> {
719 let raw = edgefirst_bench::testdata::read("yolov8s_80_classes.bin");
720 let raw = unsafe { std::slice::from_raw_parts(raw.as_ptr() as *const i8, raw.len()) };
721 Array3::from_shape_vec((1, 84, 8400), raw.to_vec()).unwrap()
722 }
723
724 #[test]
725 fn test_decoder_modelpack() {
726 let score_threshold = 0.45;
727 let iou_threshold = 0.45;
728 let boxes = edgefirst_bench::testdata::read("modelpack_boxes_1935x1x4.bin");
729 let boxes = ndarray::Array4::from_shape_vec((1, 1935, 1, 4), boxes.to_vec()).unwrap();
730
731 let scores = edgefirst_bench::testdata::read("modelpack_scores_1935x1.bin");
732 let scores = ndarray::Array3::from_shape_vec((1, 1935, 1), scores.to_vec()).unwrap();
733
734 let quant_boxes = (0.004656755365431309, 21).into();
735 let quant_scores = (0.0019603664986789227, 0).into();
736
737 let decoder = DecoderBuilder::default()
738 .with_config_modelpack_det(
739 configs::Boxes {
740 decoder: DecoderType::ModelPack,
741 quantization: Some(quant_boxes),
742 shape: vec![1, 1935, 1, 4],
743 dshape: vec![
744 (DimName::Batch, 1),
745 (DimName::NumBoxes, 1935),
746 (DimName::Padding, 1),
747 (DimName::BoxCoords, 4),
748 ],
749 normalized: Some(true),
750 },
751 configs::Scores {
752 decoder: DecoderType::ModelPack,
753 quantization: Some(quant_scores),
754 shape: vec![1, 1935, 1],
755 dshape: vec![
756 (DimName::Batch, 1),
757 (DimName::NumBoxes, 1935),
758 (DimName::NumClasses, 1),
759 ],
760 },
761 )
762 .with_score_threshold(score_threshold)
763 .with_iou_threshold(iou_threshold)
764 .build()
765 .unwrap();
766
767 let quant_boxes = quant_boxes.into();
768 let quant_scores = quant_scores.into();
769
770 let mut output_boxes: Vec<_> = Vec::with_capacity(50);
771 decode_modelpack_det(
772 (boxes.slice(s![0, .., 0, ..]), quant_boxes),
773 (scores.slice(s![0, .., ..]), quant_scores),
774 score_threshold,
775 iou_threshold,
776 Some(configs::Nms::ClassAgnostic),
777 0,
778 300,
779 false,
780 &mut output_boxes,
781 )
782 .unwrap();
783 assert!(output_boxes[0].equal_within_delta(
784 &DetectBox {
785 bbox: BoundingBox {
786 xmin: 0.40513772,
787 ymin: 0.6379755,
788 xmax: 0.5122431,
789 ymax: 0.7730214,
790 },
791 score: 0.4861709,
792 label: 0
793 },
794 1e-6
795 ));
796
797 let mut output_boxes1 = Vec::with_capacity(50);
798 let mut output_masks1 = Vec::with_capacity(50);
799
800 decoder
801 .decode_quantized(
802 &[boxes.view().into(), scores.view().into()],
803 &mut output_boxes1,
804 &mut output_masks1,
805 )
806 .unwrap();
807
808 let mut output_boxes_float = Vec::with_capacity(50);
809 let mut output_masks_float = Vec::with_capacity(50);
810
811 let boxes = dequantize_ndarray(boxes.view(), quant_boxes);
812 let scores = dequantize_ndarray(scores.view(), quant_scores);
813
814 decoder
815 .decode_float::<f32>(
816 &[boxes.view().into_dyn(), scores.view().into_dyn()],
817 &mut output_boxes_float,
818 &mut output_masks_float,
819 )
820 .unwrap();
821
822 compare_outputs((&output_boxes, &output_boxes1), (&[], &output_masks1));
823 compare_outputs(
824 (&output_boxes, &output_boxes_float),
825 (&[], &output_masks_float),
826 );
827 }
828
829 #[test]
830 fn test_decoder_modelpack_split_u8() {
831 let score_threshold = 0.45;
832 let iou_threshold = 0.45;
833 let detect0 = edgefirst_bench::testdata::read("modelpack_split_9x15x18.bin");
834 let detect0 = ndarray::Array4::from_shape_vec((1, 9, 15, 18), detect0.to_vec()).unwrap();
835
836 let detect1 = edgefirst_bench::testdata::read("modelpack_split_17x30x18.bin");
837 let detect1 = ndarray::Array4::from_shape_vec((1, 17, 30, 18), detect1.to_vec()).unwrap();
838
839 let quant0 = (0.08547406643629074, 174).into();
840 let quant1 = (0.09929127991199493, 183).into();
841 let anchors0 = vec![
842 [0.36666667461395264, 0.31481480598449707],
843 [0.38749998807907104, 0.4740740656852722],
844 [0.5333333611488342, 0.644444465637207],
845 ];
846 let anchors1 = vec![
847 [0.13750000298023224, 0.2074074000120163],
848 [0.2541666626930237, 0.21481481194496155],
849 [0.23125000298023224, 0.35185185074806213],
850 ];
851
852 let detect_config0 = configs::Detection {
853 decoder: DecoderType::ModelPack,
854 shape: vec![1, 9, 15, 18],
855 anchors: Some(anchors0.clone()),
856 quantization: Some(quant0),
857 dshape: vec![
858 (DimName::Batch, 1),
859 (DimName::Height, 9),
860 (DimName::Width, 15),
861 (DimName::NumAnchorsXFeatures, 18),
862 ],
863 normalized: Some(true),
864 };
865
866 let detect_config1 = configs::Detection {
867 decoder: DecoderType::ModelPack,
868 shape: vec![1, 17, 30, 18],
869 anchors: Some(anchors1.clone()),
870 quantization: Some(quant1),
871 dshape: vec![
872 (DimName::Batch, 1),
873 (DimName::Height, 17),
874 (DimName::Width, 30),
875 (DimName::NumAnchorsXFeatures, 18),
876 ],
877 normalized: Some(true),
878 };
879
880 let config0 = (&detect_config0).try_into().unwrap();
881 let config1 = (&detect_config1).try_into().unwrap();
882
883 let decoder = DecoderBuilder::default()
884 .with_config_modelpack_det_split(vec![detect_config1, detect_config0])
885 .with_score_threshold(score_threshold)
886 .with_iou_threshold(iou_threshold)
887 .build()
888 .unwrap();
889
890 let quant0 = quant0.into();
891 let quant1 = quant1.into();
892
893 let mut output_boxes: Vec<_> = Vec::with_capacity(10);
894 decode_modelpack_split_quant(
895 &[
896 detect0.slice(s![0, .., .., ..]),
897 detect1.slice(s![0, .., .., ..]),
898 ],
899 &[config0, config1],
900 score_threshold,
901 iou_threshold,
902 Some(configs::Nms::ClassAgnostic),
903 0,
904 300,
905 false,
906 &mut output_boxes,
907 )
908 .unwrap();
909 assert!(output_boxes[0].equal_within_delta(
910 &DetectBox {
911 bbox: BoundingBox {
912 xmin: 0.43171933,
913 ymin: 0.68243736,
914 xmax: 0.5626645,
915 ymax: 0.808863,
916 },
917 score: 0.99240804,
918 label: 0
919 },
920 1e-6
921 ));
922
923 let mut output_boxes1: Vec<_> = Vec::with_capacity(10);
924 let mut output_masks1: Vec<_> = Vec::with_capacity(10);
925 decoder
926 .decode_quantized(
927 &[detect0.view().into(), detect1.view().into()],
928 &mut output_boxes1,
929 &mut output_masks1,
930 )
931 .unwrap();
932
933 let mut output_boxes1_f32: Vec<_> = Vec::with_capacity(10);
934 let mut output_masks1_f32: Vec<_> = Vec::with_capacity(10);
935
936 let detect0 = dequantize_ndarray(detect0.view(), quant0);
937 let detect1 = dequantize_ndarray(detect1.view(), quant1);
938 decoder
939 .decode_float::<f32>(
940 &[detect0.view().into_dyn(), detect1.view().into_dyn()],
941 &mut output_boxes1_f32,
942 &mut output_masks1_f32,
943 )
944 .unwrap();
945
946 compare_outputs((&output_boxes, &output_boxes1), (&[], &output_masks1));
947 compare_outputs(
948 (&output_boxes, &output_boxes1_f32),
949 (&[], &output_masks1_f32),
950 );
951 }
952
953 #[test]
954 fn test_decoder_parse_config_modelpack_split_u8() {
955 let score_threshold = 0.45;
956 let iou_threshold = 0.45;
957 let detect0 = edgefirst_bench::testdata::read("modelpack_split_9x15x18.bin");
958 let detect0 = ndarray::Array4::from_shape_vec((1, 9, 15, 18), detect0.to_vec()).unwrap();
959
960 let detect1 = edgefirst_bench::testdata::read("modelpack_split_17x30x18.bin");
961 let detect1 = ndarray::Array4::from_shape_vec((1, 17, 30, 18), detect1.to_vec()).unwrap();
962
963 let decoder = DecoderBuilder::default()
964 .with_config_yaml_str(
965 edgefirst_bench::testdata::read_to_string("modelpack_split.yaml").to_string(),
966 )
967 .with_score_threshold(score_threshold)
968 .with_iou_threshold(iou_threshold)
969 .build()
970 .unwrap();
971
972 let mut output_boxes: Vec<_> = Vec::with_capacity(10);
973 let mut output_masks: Vec<_> = Vec::with_capacity(10);
974 decoder
975 .decode_quantized(
976 &[
977 ArrayViewDQuantized::from(detect1.view()),
978 ArrayViewDQuantized::from(detect0.view()),
979 ],
980 &mut output_boxes,
981 &mut output_masks,
982 )
983 .unwrap();
984 assert!(output_boxes[0].equal_within_delta(
985 &DetectBox {
986 bbox: BoundingBox {
987 xmin: 0.43171933,
988 ymin: 0.68243736,
989 xmax: 0.5626645,
990 ymax: 0.808863,
991 },
992 score: 0.99240804,
993 label: 0
994 },
995 1e-6
996 ));
997 }
998
999 #[test]
1000 fn test_modelpack_seg() {
1001 let out = edgefirst_bench::testdata::read("modelpack_seg_2x160x160.bin");
1002 let out = ndarray::Array4::from_shape_vec((1, 2, 160, 160), out.to_vec()).unwrap();
1003 let quant = (1.0 / 255.0, 0).into();
1004
1005 let decoder = DecoderBuilder::default()
1006 .with_config_modelpack_seg(configs::Segmentation {
1007 decoder: DecoderType::ModelPack,
1008 quantization: Some(quant),
1009 shape: vec![1, 2, 160, 160],
1010 dshape: vec![
1011 (DimName::Batch, 1),
1012 (DimName::NumClasses, 2),
1013 (DimName::Height, 160),
1014 (DimName::Width, 160),
1015 ],
1016 })
1017 .build()
1018 .unwrap();
1019 let mut output_boxes: Vec<_> = Vec::with_capacity(10);
1020 let mut output_masks: Vec<_> = Vec::with_capacity(10);
1021 decoder
1022 .decode_quantized(&[out.view().into()], &mut output_boxes, &mut output_masks)
1023 .unwrap();
1024
1025 let mut mask = out.slice(s![0, .., .., ..]);
1026 mask.swap_axes(0, 1);
1027 mask.swap_axes(1, 2);
1028 let mask = [Segmentation {
1029 xmin: 0.0,
1030 ymin: 0.0,
1031 xmax: 1.0,
1032 ymax: 1.0,
1033 segmentation: crate::mask_to_tensor(mask.view()).unwrap(),
1034 }];
1035 compare_outputs((&[], &output_boxes), (&mask, &output_masks));
1036
1037 decoder
1038 .decode_float::<f32>(
1039 &[dequantize_ndarray(out.view(), quant.into())
1040 .view()
1041 .into_dyn()],
1042 &mut output_boxes,
1043 &mut output_masks,
1044 )
1045 .unwrap();
1046
1047 compare_outputs((&[], &output_boxes), (&[], &[]));
1053 let mask0 = segmentation_to_mask(seg_view(&mask[0]).view()).unwrap();
1054 let mask1 = segmentation_to_mask(seg_view(&output_masks[0]).view()).unwrap();
1055
1056 assert_eq!(mask0, mask1);
1057 }
1058 #[test]
1059 fn test_modelpack_seg_quant() {
1060 let out = edgefirst_bench::testdata::read("modelpack_seg_2x160x160.bin");
1061 let out_u8 = ndarray::Array4::from_shape_vec((1, 2, 160, 160), out.to_vec()).unwrap();
1062 let out_i8 = out_u8.mapv(|x| (x as i16 - 128) as i8);
1063 let out_u16 = out_u8.mapv(|x| (x as u16) << 8);
1064 let out_i16 = out_u8.mapv(|x| (((x as i32) << 8) - 32768) as i16);
1065 let out_u32 = out_u8.mapv(|x| (x as u32) << 24);
1066 let out_i32 = out_u8.mapv(|x| (((x as i64) << 24) - 2147483648) as i32);
1067
1068 let quant = (1.0 / 255.0, 0).into();
1069
1070 let decoder = DecoderBuilder::default()
1071 .with_config_modelpack_seg(configs::Segmentation {
1072 decoder: DecoderType::ModelPack,
1073 quantization: Some(quant),
1074 shape: vec![1, 2, 160, 160],
1075 dshape: vec![
1076 (DimName::Batch, 1),
1077 (DimName::NumClasses, 2),
1078 (DimName::Height, 160),
1079 (DimName::Width, 160),
1080 ],
1081 })
1082 .build()
1083 .unwrap();
1084 let mut output_boxes: Vec<_> = Vec::with_capacity(10);
1085 let mut output_masks_u8: Vec<_> = Vec::with_capacity(10);
1086 decoder
1087 .decode_quantized(
1088 &[out_u8.view().into()],
1089 &mut output_boxes,
1090 &mut output_masks_u8,
1091 )
1092 .unwrap();
1093
1094 let mut output_masks_i8: Vec<_> = Vec::with_capacity(10);
1095 decoder
1096 .decode_quantized(
1097 &[out_i8.view().into()],
1098 &mut output_boxes,
1099 &mut output_masks_i8,
1100 )
1101 .unwrap();
1102
1103 let mut output_masks_u16: Vec<_> = Vec::with_capacity(10);
1104 decoder
1105 .decode_quantized(
1106 &[out_u16.view().into()],
1107 &mut output_boxes,
1108 &mut output_masks_u16,
1109 )
1110 .unwrap();
1111
1112 let mut output_masks_i16: Vec<_> = Vec::with_capacity(10);
1113 decoder
1114 .decode_quantized(
1115 &[out_i16.view().into()],
1116 &mut output_boxes,
1117 &mut output_masks_i16,
1118 )
1119 .unwrap();
1120
1121 let mut output_masks_u32: Vec<_> = Vec::with_capacity(10);
1122 decoder
1123 .decode_quantized(
1124 &[out_u32.view().into()],
1125 &mut output_boxes,
1126 &mut output_masks_u32,
1127 )
1128 .unwrap();
1129
1130 let mut output_masks_i32: Vec<_> = Vec::with_capacity(10);
1131 decoder
1132 .decode_quantized(
1133 &[out_i32.view().into()],
1134 &mut output_boxes,
1135 &mut output_masks_i32,
1136 )
1137 .unwrap();
1138
1139 compare_outputs((&[], &output_boxes), (&[], &[]));
1140 let mask_u8 = segmentation_to_mask(seg_view(&output_masks_u8[0]).view()).unwrap();
1141 let mask_i8 = segmentation_to_mask(seg_view(&output_masks_i8[0]).view()).unwrap();
1142 let mask_u16 = segmentation_to_mask(seg_view(&output_masks_u16[0]).view()).unwrap();
1143 let mask_i16 = segmentation_to_mask(seg_view(&output_masks_i16[0]).view()).unwrap();
1144 let mask_u32 = segmentation_to_mask(seg_view(&output_masks_u32[0]).view()).unwrap();
1145 let mask_i32 = segmentation_to_mask(seg_view(&output_masks_i32[0]).view()).unwrap();
1146 assert_eq!(mask_u8, mask_i8);
1147 assert_eq!(mask_u8, mask_u16);
1148 assert_eq!(mask_u8, mask_i16);
1149 assert_eq!(mask_u8, mask_u32);
1150 assert_eq!(mask_u8, mask_i32);
1151 }
1152
1153 #[test]
1154 fn test_modelpack_segdet() {
1155 let score_threshold = 0.45;
1156 let iou_threshold = 0.45;
1157
1158 let boxes = edgefirst_bench::testdata::read("modelpack_boxes_1935x1x4.bin");
1159 let boxes = Array4::from_shape_vec((1, 1935, 1, 4), boxes.to_vec()).unwrap();
1160
1161 let scores = edgefirst_bench::testdata::read("modelpack_scores_1935x1.bin");
1162 let scores = Array3::from_shape_vec((1, 1935, 1), scores.to_vec()).unwrap();
1163
1164 let seg = edgefirst_bench::testdata::read("modelpack_seg_2x160x160.bin");
1165 let seg = Array4::from_shape_vec((1, 2, 160, 160), seg.to_vec()).unwrap();
1166
1167 let quant_boxes = (0.004656755365431309, 21).into();
1168 let quant_scores = (0.0019603664986789227, 0).into();
1169 let quant_seg = (1.0 / 255.0, 0).into();
1170
1171 let decoder = DecoderBuilder::default()
1172 .with_config_modelpack_segdet(
1173 configs::Boxes {
1174 decoder: DecoderType::ModelPack,
1175 quantization: Some(quant_boxes),
1176 shape: vec![1, 1935, 1, 4],
1177 dshape: vec![
1178 (DimName::Batch, 1),
1179 (DimName::NumBoxes, 1935),
1180 (DimName::Padding, 1),
1181 (DimName::BoxCoords, 4),
1182 ],
1183 normalized: Some(true),
1184 },
1185 configs::Scores {
1186 decoder: DecoderType::ModelPack,
1187 quantization: Some(quant_scores),
1188 shape: vec![1, 1935, 1],
1189 dshape: vec![
1190 (DimName::Batch, 1),
1191 (DimName::NumBoxes, 1935),
1192 (DimName::NumClasses, 1),
1193 ],
1194 },
1195 configs::Segmentation {
1196 decoder: DecoderType::ModelPack,
1197 quantization: Some(quant_seg),
1198 shape: vec![1, 2, 160, 160],
1199 dshape: vec![
1200 (DimName::Batch, 1),
1201 (DimName::NumClasses, 2),
1202 (DimName::Height, 160),
1203 (DimName::Width, 160),
1204 ],
1205 },
1206 )
1207 .with_iou_threshold(iou_threshold)
1208 .with_score_threshold(score_threshold)
1209 .build()
1210 .unwrap();
1211 let mut output_boxes: Vec<_> = Vec::with_capacity(10);
1212 let mut output_masks: Vec<_> = Vec::with_capacity(10);
1213 decoder
1214 .decode_quantized(
1215 &[scores.view().into(), boxes.view().into(), seg.view().into()],
1216 &mut output_boxes,
1217 &mut output_masks,
1218 )
1219 .unwrap();
1220
1221 let mut mask = seg.slice(s![0, .., .., ..]);
1222 mask.swap_axes(0, 1);
1223 mask.swap_axes(1, 2);
1224 let mask = [Segmentation {
1225 xmin: 0.0,
1226 ymin: 0.0,
1227 xmax: 1.0,
1228 ymax: 1.0,
1229 segmentation: crate::mask_to_tensor(mask.view()).unwrap(),
1230 }];
1231 let correct_boxes = [DetectBox {
1232 bbox: BoundingBox {
1233 xmin: 0.40513772,
1234 ymin: 0.6379755,
1235 xmax: 0.5122431,
1236 ymax: 0.7730214,
1237 },
1238 score: 0.4861709,
1239 label: 0,
1240 }];
1241 compare_outputs((&correct_boxes, &output_boxes), (&mask, &output_masks));
1242
1243 let scores = dequantize_ndarray(scores.view(), quant_scores.into());
1244 let boxes = dequantize_ndarray(boxes.view(), quant_boxes.into());
1245 let seg = dequantize_ndarray(seg.view(), quant_seg.into());
1246 decoder
1247 .decode_float::<f32>(
1248 &[
1249 scores.view().into_dyn(),
1250 boxes.view().into_dyn(),
1251 seg.view().into_dyn(),
1252 ],
1253 &mut output_boxes,
1254 &mut output_masks,
1255 )
1256 .unwrap();
1257
1258 compare_outputs((&correct_boxes, &output_boxes), (&[], &[]));
1264 let mask0 = segmentation_to_mask(seg_view(&mask[0]).view()).unwrap();
1265 let mask1 = segmentation_to_mask(seg_view(&output_masks[0]).view()).unwrap();
1266
1267 assert_eq!(mask0, mask1);
1268 }
1269
1270 #[test]
1271 fn test_modelpack_segdet_split() {
1272 let score_threshold = 0.8;
1273 let iou_threshold = 0.5;
1274
1275 let seg = edgefirst_bench::testdata::read("modelpack_seg_2x160x160.bin");
1276 let seg = ndarray::Array4::from_shape_vec((1, 2, 160, 160), seg.to_vec()).unwrap();
1277
1278 let detect0 = edgefirst_bench::testdata::read("modelpack_split_9x15x18.bin");
1279 let detect0 = ndarray::Array4::from_shape_vec((1, 9, 15, 18), detect0.to_vec()).unwrap();
1280
1281 let detect1 = edgefirst_bench::testdata::read("modelpack_split_17x30x18.bin");
1282 let detect1 = ndarray::Array4::from_shape_vec((1, 17, 30, 18), detect1.to_vec()).unwrap();
1283
1284 let quant0 = (0.08547406643629074, 174).into();
1285 let quant1 = (0.09929127991199493, 183).into();
1286 let quant_seg = (1.0 / 255.0, 0).into();
1287
1288 let anchors0 = vec![
1289 [0.36666667461395264, 0.31481480598449707],
1290 [0.38749998807907104, 0.4740740656852722],
1291 [0.5333333611488342, 0.644444465637207],
1292 ];
1293 let anchors1 = vec![
1294 [0.13750000298023224, 0.2074074000120163],
1295 [0.2541666626930237, 0.21481481194496155],
1296 [0.23125000298023224, 0.35185185074806213],
1297 ];
1298
1299 let decoder = DecoderBuilder::default()
1300 .with_config_modelpack_segdet_split(
1301 vec![
1302 configs::Detection {
1303 decoder: DecoderType::ModelPack,
1304 shape: vec![1, 17, 30, 18],
1305 anchors: Some(anchors1),
1306 quantization: Some(quant1),
1307 dshape: vec![
1308 (DimName::Batch, 1),
1309 (DimName::Height, 17),
1310 (DimName::Width, 30),
1311 (DimName::NumAnchorsXFeatures, 18),
1312 ],
1313 normalized: Some(true),
1314 },
1315 configs::Detection {
1316 decoder: DecoderType::ModelPack,
1317 shape: vec![1, 9, 15, 18],
1318 anchors: Some(anchors0),
1319 quantization: Some(quant0),
1320 dshape: vec![
1321 (DimName::Batch, 1),
1322 (DimName::Height, 9),
1323 (DimName::Width, 15),
1324 (DimName::NumAnchorsXFeatures, 18),
1325 ],
1326 normalized: Some(true),
1327 },
1328 ],
1329 configs::Segmentation {
1330 decoder: DecoderType::ModelPack,
1331 quantization: Some(quant_seg),
1332 shape: vec![1, 2, 160, 160],
1333 dshape: vec![
1334 (DimName::Batch, 1),
1335 (DimName::NumClasses, 2),
1336 (DimName::Height, 160),
1337 (DimName::Width, 160),
1338 ],
1339 },
1340 )
1341 .with_score_threshold(score_threshold)
1342 .with_iou_threshold(iou_threshold)
1343 .build()
1344 .unwrap();
1345 let mut output_boxes: Vec<_> = Vec::with_capacity(10);
1346 let mut output_masks: Vec<_> = Vec::with_capacity(10);
1347 decoder
1348 .decode_quantized(
1349 &[
1350 detect0.view().into(),
1351 detect1.view().into(),
1352 seg.view().into(),
1353 ],
1354 &mut output_boxes,
1355 &mut output_masks,
1356 )
1357 .unwrap();
1358
1359 let mut mask = seg.slice(s![0, .., .., ..]);
1360 mask.swap_axes(0, 1);
1361 mask.swap_axes(1, 2);
1362 let mask = [Segmentation {
1363 xmin: 0.0,
1364 ymin: 0.0,
1365 xmax: 1.0,
1366 ymax: 1.0,
1367 segmentation: crate::mask_to_tensor(mask.view()).unwrap(),
1368 }];
1369 let correct_boxes = [DetectBox {
1370 bbox: BoundingBox {
1371 xmin: 0.43171933,
1372 ymin: 0.68243736,
1373 xmax: 0.5626645,
1374 ymax: 0.808863,
1375 },
1376 score: 0.99240804,
1377 label: 0,
1378 }];
1379 println!("Output Boxes: {:?}", output_boxes);
1380 compare_outputs((&correct_boxes, &output_boxes), (&mask, &output_masks));
1381
1382 let detect0 = dequantize_ndarray(detect0.view(), quant0.into());
1383 let detect1 = dequantize_ndarray(detect1.view(), quant1.into());
1384 let seg = dequantize_ndarray(seg.view(), quant_seg.into());
1385 decoder
1386 .decode_float::<f32>(
1387 &[
1388 detect0.view().into_dyn(),
1389 detect1.view().into_dyn(),
1390 seg.view().into_dyn(),
1391 ],
1392 &mut output_boxes,
1393 &mut output_masks,
1394 )
1395 .unwrap();
1396
1397 compare_outputs((&correct_boxes, &output_boxes), (&[], &[]));
1403 let mask0 = segmentation_to_mask(seg_view(&mask[0]).view()).unwrap();
1404 let mask1 = segmentation_to_mask(seg_view(&output_masks[0]).view()).unwrap();
1405
1406 assert_eq!(mask0, mask1);
1407 }
1408
1409 #[test]
1410 fn test_dequant_chunked() {
1411 let mut out = load_yolov8s_det().into_raw_vec_and_offset().0;
1412 out.push(123); let mut out_dequant = vec![0.0; 84 * 8400 + 1];
1415 let mut out_dequant_simd = vec![0.0; 84 * 8400 + 1];
1416 let quant = Quantization::new(0.0040811873, -123);
1417 dequantize_cpu(&out, quant, &mut out_dequant);
1418
1419 dequantize_cpu_chunked(&out, quant, &mut out_dequant_simd);
1420 assert_eq!(out_dequant, out_dequant_simd);
1421
1422 let quant = Quantization::new(0.0040811873, 0);
1423 dequantize_cpu(&out, quant, &mut out_dequant);
1424
1425 dequantize_cpu_chunked(&out, quant, &mut out_dequant_simd);
1426 assert_eq!(out_dequant, out_dequant_simd);
1427 }
1428
1429 #[test]
1430 fn test_dequant_ground_truth() {
1431 let quant = Quantization::new(0.1, -128);
1436 let input: Vec<i8> = vec![0, 127, -128, 64];
1437 let mut output = vec![0.0f32; 4];
1438 let mut output_chunked = vec![0.0f32; 4];
1439 dequantize_cpu(&input, quant, &mut output);
1440 dequantize_cpu_chunked(&input, quant, &mut output_chunked);
1441 let expected: Vec<f32> = vec![12.8, 25.5, 0.0, 19.2];
1446 for (i, (&out, &exp)) in output.iter().zip(expected.iter()).enumerate() {
1447 assert!((out - exp).abs() < 1e-5, "cpu[{i}]: {out} != {exp}");
1448 }
1449 for (i, (&out, &exp)) in output_chunked.iter().zip(expected.iter()).enumerate() {
1450 assert!((out - exp).abs() < 1e-5, "chunked[{i}]: {out} != {exp}");
1451 }
1452
1453 let quant = Quantization::new(1.0, 0);
1455 dequantize_cpu(&input, quant, &mut output);
1456 dequantize_cpu_chunked(&input, quant, &mut output_chunked);
1457 let expected: Vec<f32> = vec![0.0, 127.0, -128.0, 64.0];
1458 assert_eq!(output, expected);
1459 assert_eq!(output_chunked, expected);
1460
1461 let quant = Quantization::new(0.5, 0);
1463 dequantize_cpu(&input, quant, &mut output);
1464 dequantize_cpu_chunked(&input, quant, &mut output_chunked);
1465 let expected: Vec<f32> = vec![0.0, 63.5, -64.0, 32.0];
1466 assert_eq!(output, expected);
1467 assert_eq!(output_chunked, expected);
1468
1469 let quant = Quantization::new(0.021287762, 31);
1471 let input: Vec<i8> = vec![-128, -1, 0, 1, 31, 127];
1472 let mut output = vec![0.0f32; 6];
1473 let mut output_chunked = vec![0.0f32; 6];
1474 dequantize_cpu(&input, quant, &mut output);
1475 dequantize_cpu_chunked(&input, quant, &mut output_chunked);
1476 for i in 0..6 {
1477 let expected = (input[i] as f32 - 31.0) * 0.021287762;
1478 assert!(
1479 (output[i] - expected).abs() < 1e-5,
1480 "cpu[{i}]: {} != {expected}",
1481 output[i]
1482 );
1483 assert!(
1484 (output_chunked[i] - expected).abs() < 1e-5,
1485 "chunked[{i}]: {} != {expected}",
1486 output_chunked[i]
1487 );
1488 }
1489 }
1490
1491 #[test]
1492 fn test_decoder_yolo_det() {
1493 let score_threshold = 0.25;
1494 let iou_threshold = 0.7;
1495 let out = load_yolov8s_det();
1496 let quant = (0.0040811873, -123).into();
1497
1498 let decoder = DecoderBuilder::default()
1499 .with_config_yolo_det(
1500 configs::Detection {
1501 decoder: DecoderType::Ultralytics,
1502 shape: vec![1, 84, 8400],
1503 anchors: None,
1504 quantization: Some(quant),
1505 dshape: vec![
1506 (DimName::Batch, 1),
1507 (DimName::NumFeatures, 84),
1508 (DimName::NumBoxes, 8400),
1509 ],
1510 normalized: Some(true),
1511 },
1512 Some(DecoderVersion::Yolo11),
1513 )
1514 .with_score_threshold(score_threshold)
1515 .with_iou_threshold(iou_threshold)
1516 .build()
1517 .unwrap();
1518
1519 let mut output_boxes: Vec<_> = Vec::with_capacity(50);
1520 decode_yolo_det(
1521 (out.slice(s![0, .., ..]), quant.into()),
1522 score_threshold,
1523 iou_threshold,
1524 Some(configs::Nms::ClassAgnostic),
1525 0,
1526 crate::yolo::cap_or_default(&output_boxes),
1527 false,
1528 &mut output_boxes,
1529 )
1530 .unwrap();
1531 assert!(output_boxes[0].equal_within_delta(
1532 &DetectBox {
1533 bbox: BoundingBox {
1534 xmin: 0.5285137,
1535 ymin: 0.05305544,
1536 xmax: 0.87541467,
1537 ymax: 0.9998909,
1538 },
1539 score: 0.5591227,
1540 label: 0
1541 },
1542 1e-6
1543 ));
1544
1545 assert!(output_boxes[1].equal_within_delta(
1546 &DetectBox {
1547 bbox: BoundingBox {
1548 xmin: 0.130598,
1549 ymin: 0.43260583,
1550 xmax: 0.35098213,
1551 ymax: 0.9958097,
1552 },
1553 score: 0.33057618,
1554 label: 75
1555 },
1556 1e-6
1557 ));
1558
1559 let mut output_boxes1: Vec<_> = Vec::with_capacity(50);
1560 let mut output_masks1: Vec<_> = Vec::with_capacity(50);
1561 decoder
1562 .decode_quantized(&[out.view().into()], &mut output_boxes1, &mut output_masks1)
1563 .unwrap();
1564
1565 let out = dequantize_ndarray(out.view(), quant.into());
1566 let mut output_boxes_f32: Vec<_> = Vec::with_capacity(50);
1567 let mut output_masks_f32: Vec<_> = Vec::with_capacity(50);
1568 decoder
1569 .decode_float::<f32>(
1570 &[out.view().into_dyn()],
1571 &mut output_boxes_f32,
1572 &mut output_masks_f32,
1573 )
1574 .unwrap();
1575
1576 compare_outputs((&output_boxes, &output_boxes1), (&[], &output_masks1));
1577 compare_outputs((&output_boxes, &output_boxes_f32), (&[], &output_masks_f32));
1578 }
1579
1580 #[test]
1581 fn test_decoder_masks() {
1582 let score_threshold = 0.45;
1583 let iou_threshold = 0.45;
1584 let boxes = load_yolov8_boxes();
1585 let quant_boxes = Quantization::new(0.021287761628627777, 31);
1586
1587 let protos = load_yolov8_protos();
1588 let quant_protos = Quantization::new(0.02491161972284317, -117);
1589 let protos = dequantize_ndarray::<_, _, f32>(protos.view(), quant_protos);
1590 let seg = dequantize_ndarray::<_, _, f32>(boxes.view(), quant_boxes);
1591 let mut output_boxes: Vec<_> = Vec::with_capacity(10);
1592 let mut output_masks: Vec<_> = Vec::with_capacity(10);
1593 decode_yolo_segdet_float(
1594 seg.slice(s![0, .., ..]),
1595 protos.slice(s![0, .., .., ..]),
1596 score_threshold,
1597 iou_threshold,
1598 Some(configs::Nms::ClassAgnostic),
1599 &mut output_boxes,
1600 &mut output_masks,
1601 )
1602 .unwrap();
1603 assert_eq!(output_boxes.len(), 2);
1604 assert_eq!(output_boxes.len(), output_masks.len());
1605
1606 for (b, m) in output_boxes.iter().zip(&output_masks) {
1607 assert!(b.bbox.xmin >= m.xmin);
1610 assert!(b.bbox.ymin >= m.ymin);
1611 assert!(b.bbox.xmax <= m.xmax);
1612 assert!(b.bbox.ymax <= m.ymax);
1613 }
1614 assert!(output_boxes[0].equal_within_delta(
1615 &DetectBox {
1616 bbox: BoundingBox {
1617 xmin: 0.08515105,
1618 ymin: 0.7131401,
1619 xmax: 0.29802868,
1620 ymax: 0.8195788,
1621 },
1622 score: 0.91537374,
1623 label: 23
1624 },
1625 1.0 / 160.0, ));
1627
1628 assert!(output_boxes[1].equal_within_delta(
1629 &DetectBox {
1630 bbox: BoundingBox {
1631 xmin: 0.59605736,
1632 ymin: 0.25545314,
1633 xmax: 0.93666154,
1634 ymax: 0.72378385,
1635 },
1636 score: 0.91537374,
1637 label: 23
1638 },
1639 1.0 / 160.0, ));
1641
1642 let full_mask = edgefirst_bench::testdata::read("yolov8_mask_results.bin");
1643 let full_mask = ndarray::Array2::from_shape_vec((160, 160), full_mask.to_vec()).unwrap();
1644
1645 let cropped_mask = full_mask.slice(ndarray::s![
1646 (output_masks[1].ymin * 160.0) as usize..(output_masks[1].ymax * 160.0) as usize,
1647 (output_masks[1].xmin * 160.0) as usize..(output_masks[1].xmax * 160.0) as usize,
1648 ]);
1649
1650 assert_eq!(
1651 cropped_mask,
1652 segmentation_to_mask(seg_view(&output_masks[1]).view()).unwrap()
1653 );
1654 }
1655
1656 #[test]
1666 fn test_decoder_masks_nchw_protos() {
1667 let score_threshold = 0.45;
1668 let iou_threshold = 0.45;
1669
1670 let boxes_2d = load_yolov8_boxes().slice_move(s![0, .., ..]);
1672 let quant_boxes = Quantization::new(0.021287761628627777, 31);
1673
1674 let protos_hwc = load_yolov8_protos().slice_move(s![0, .., .., ..]);
1676 let quant_protos = Quantization::new(0.02491161972284317, -117);
1677 let protos_f32_hwc = dequantize_ndarray::<_, _, f32>(protos_hwc.view(), quant_protos);
1678
1679 let seg = dequantize_ndarray::<_, _, f32>(boxes_2d.view(), quant_boxes);
1681 let mut ref_boxes: Vec<_> = Vec::with_capacity(10);
1682 let mut ref_masks: Vec<_> = Vec::with_capacity(10);
1683 decode_yolo_segdet_float(
1684 seg.view(),
1685 protos_f32_hwc.view(),
1686 score_threshold,
1687 iou_threshold,
1688 Some(configs::Nms::ClassAgnostic),
1689 &mut ref_boxes,
1690 &mut ref_masks,
1691 )
1692 .unwrap();
1693 assert_eq!(ref_boxes.len(), 2);
1694
1695 let protos_f32_chw_view = protos_f32_hwc.view().permuted_axes([2, 0, 1]); let protos_f32_chw = protos_f32_chw_view.to_owned();
1702 let protos_nchw = protos_f32_chw.insert_axis(ndarray::Axis(0)); let seg_3d = seg.insert_axis(ndarray::Axis(0)); let decoder = DecoderBuilder::default()
1711 .with_config_yolo_segdet(
1712 configs::Detection {
1713 decoder: configs::DecoderType::Ultralytics,
1714 quantization: None,
1715 shape: vec![1, 116, 8400],
1716 dshape: vec![
1717 (configs::DimName::Batch, 1),
1718 (configs::DimName::NumFeatures, 116),
1719 (configs::DimName::NumBoxes, 8400),
1720 ],
1721 normalized: Some(true),
1722 anchors: None,
1723 },
1724 configs::Protos {
1725 decoder: configs::DecoderType::Ultralytics,
1726 quantization: None,
1727 shape: vec![1, 32, 160, 160],
1728 dshape: vec![
1729 (configs::DimName::Batch, 1),
1730 (configs::DimName::NumProtos, 32),
1731 (configs::DimName::Height, 160),
1732 (configs::DimName::Width, 160),
1733 ],
1734 },
1735 None, )
1737 .with_score_threshold(score_threshold)
1738 .with_iou_threshold(iou_threshold)
1739 .build()
1740 .unwrap();
1741
1742 let mut cfg_boxes: Vec<_> = Vec::with_capacity(10);
1743 let mut cfg_masks: Vec<_> = Vec::with_capacity(10);
1744 decoder
1745 .decode_float(
1746 &[seg_3d.view().into_dyn(), protos_nchw.view().into_dyn()],
1747 &mut cfg_boxes,
1748 &mut cfg_masks,
1749 )
1750 .unwrap();
1751
1752 assert_eq!(
1754 cfg_boxes.len(),
1755 ref_boxes.len(),
1756 "config path produced {} boxes, reference produced {}",
1757 cfg_boxes.len(),
1758 ref_boxes.len()
1759 );
1760
1761 for (i, (cb, rb)) in cfg_boxes.iter().zip(&ref_boxes).enumerate() {
1763 assert!(
1764 cb.equal_within_delta(rb, 0.01),
1765 "box {i} mismatch: config={cb:?}, reference={rb:?}"
1766 );
1767 }
1768
1769 for (i, (cm, rm)) in cfg_masks.iter().zip(&ref_masks).enumerate() {
1771 let cm_arr = segmentation_to_mask(seg_view(cm).view()).unwrap();
1772 let rm_arr = segmentation_to_mask(seg_view(rm).view()).unwrap();
1773 assert_eq!(
1774 cm_arr, rm_arr,
1775 "mask {i} pixel mismatch between config-driven and reference paths"
1776 );
1777 }
1778 }
1779
1780 #[test]
1781 fn test_decoder_masks_i8() {
1782 let score_threshold = 0.45;
1783 let iou_threshold = 0.45;
1784 let boxes = load_yolov8_boxes();
1785 let quant_boxes = (0.021287761628627777, 31).into();
1786
1787 let protos = load_yolov8_protos();
1788 let quant_protos = (0.02491161972284317, -117).into();
1789 let mut output_boxes: Vec<_> = Vec::with_capacity(500);
1790 let mut output_masks: Vec<_> = Vec::with_capacity(500);
1791
1792 let decoder = DecoderBuilder::default()
1793 .with_config_yolo_segdet(
1794 configs::Detection {
1795 decoder: configs::DecoderType::Ultralytics,
1796 quantization: Some(quant_boxes),
1797 shape: vec![1, 116, 8400],
1798 anchors: None,
1799 dshape: vec![
1800 (DimName::Batch, 1),
1801 (DimName::NumFeatures, 116),
1802 (DimName::NumBoxes, 8400),
1803 ],
1804 normalized: Some(true),
1805 },
1806 Protos {
1807 decoder: configs::DecoderType::Ultralytics,
1808 quantization: Some(quant_protos),
1809 shape: vec![1, 160, 160, 32],
1810 dshape: vec![
1811 (DimName::Batch, 1),
1812 (DimName::Height, 160),
1813 (DimName::Width, 160),
1814 (DimName::NumProtos, 32),
1815 ],
1816 },
1817 Some(DecoderVersion::Yolo11),
1818 )
1819 .with_score_threshold(score_threshold)
1820 .with_iou_threshold(iou_threshold)
1821 .build()
1822 .unwrap();
1823
1824 let quant_boxes = quant_boxes.into();
1825 let quant_protos = quant_protos.into();
1826
1827 decode_yolo_segdet_quant(
1828 (boxes.slice(s![0, .., ..]), quant_boxes),
1829 (protos.slice(s![0, .., .., ..]), quant_protos),
1830 score_threshold,
1831 iou_threshold,
1832 Some(configs::Nms::ClassAgnostic),
1833 &mut output_boxes,
1834 &mut output_masks,
1835 )
1836 .unwrap();
1837
1838 let mut output_boxes1: Vec<_> = Vec::with_capacity(500);
1839 let mut output_masks1: Vec<_> = Vec::with_capacity(500);
1840
1841 decoder
1842 .decode_quantized(
1843 &[boxes.view().into(), protos.view().into()],
1844 &mut output_boxes1,
1845 &mut output_masks1,
1846 )
1847 .unwrap();
1848
1849 let protos = dequantize_ndarray::<_, _, f32>(protos.view(), quant_protos);
1850 let seg = dequantize_ndarray::<_, _, f32>(boxes.view(), quant_boxes);
1851
1852 let mut output_boxes_f32: Vec<_> = Vec::with_capacity(500);
1853 let mut output_masks_f32: Vec<_> = Vec::with_capacity(500);
1854 decode_yolo_segdet_float(
1855 seg.slice(s![0, .., ..]),
1856 protos.slice(s![0, .., .., ..]),
1857 score_threshold,
1858 iou_threshold,
1859 Some(configs::Nms::ClassAgnostic),
1860 &mut output_boxes_f32,
1861 &mut output_masks_f32,
1862 )
1863 .unwrap();
1864
1865 let mut output_boxes1_f32: Vec<_> = Vec::with_capacity(500);
1866 let mut output_masks1_f32: Vec<_> = Vec::with_capacity(500);
1867
1868 decoder
1869 .decode_float(
1870 &[seg.view().into_dyn(), protos.view().into_dyn()],
1871 &mut output_boxes1_f32,
1872 &mut output_masks1_f32,
1873 )
1874 .unwrap();
1875
1876 compare_outputs(
1877 (&output_boxes, &output_boxes1),
1878 (&output_masks, &output_masks1),
1879 );
1880
1881 compare_outputs(
1882 (&output_boxes, &output_boxes_f32),
1883 (&output_masks, &output_masks_f32),
1884 );
1885
1886 compare_outputs(
1887 (&output_boxes_f32, &output_boxes1_f32),
1888 (&output_masks_f32, &output_masks1_f32),
1889 );
1890 }
1891
1892 #[test]
1893 fn test_decoder_yolo_split() {
1894 let score_threshold = 0.45;
1895 let iou_threshold = 0.45;
1896 let boxes = load_yolov8_boxes();
1897 let boxes: Vec<_> = boxes.iter().map(|x| *x as i16 * 256).collect();
1898 let boxes = ndarray::Array3::from_shape_vec((1, 116, 8400), boxes).unwrap();
1899
1900 let quant_boxes = Quantization::new(0.021287761628627777 / 256.0, 31 * 256);
1901
1902 let decoder = DecoderBuilder::default()
1903 .with_config_yolo_split_det(
1904 configs::Boxes {
1905 decoder: configs::DecoderType::Ultralytics,
1906 quantization: Some(QuantTuple(quant_boxes.scale, quant_boxes.zero_point)),
1907 shape: vec![1, 4, 8400],
1908 dshape: vec![
1909 (DimName::Batch, 1),
1910 (DimName::BoxCoords, 4),
1911 (DimName::NumBoxes, 8400),
1912 ],
1913 normalized: Some(true),
1914 },
1915 configs::Scores {
1916 decoder: configs::DecoderType::Ultralytics,
1917 quantization: Some(QuantTuple(quant_boxes.scale, quant_boxes.zero_point)),
1918 shape: vec![1, 80, 8400],
1919 dshape: vec![
1920 (DimName::Batch, 1),
1921 (DimName::NumClasses, 80),
1922 (DimName::NumBoxes, 8400),
1923 ],
1924 },
1925 )
1926 .with_score_threshold(score_threshold)
1927 .with_iou_threshold(iou_threshold)
1928 .build()
1929 .unwrap();
1930
1931 let mut output_boxes: Vec<_> = Vec::with_capacity(500);
1932 let mut output_masks: Vec<_> = Vec::with_capacity(500);
1933
1934 decoder
1935 .decode_quantized(
1936 &[
1937 boxes.slice(s![.., ..4, ..]).into(),
1938 boxes.slice(s![.., 4..84, ..]).into(),
1939 ],
1940 &mut output_boxes,
1941 &mut output_masks,
1942 )
1943 .unwrap();
1944
1945 let seg = dequantize_ndarray::<_, _, f32>(boxes.view(), quant_boxes);
1946 let mut output_boxes_f32: Vec<_> = Vec::with_capacity(500);
1947 decode_yolo_det_float(
1948 seg.slice(s![0, ..84, ..]),
1949 score_threshold,
1950 iou_threshold,
1951 Some(configs::Nms::ClassAgnostic),
1952 0,
1953 crate::yolo::cap_or_default(&output_boxes_f32),
1954 false,
1955 &mut output_boxes_f32,
1956 )
1957 .unwrap();
1958
1959 let mut output_boxes1: Vec<_> = Vec::with_capacity(500);
1960 let mut output_masks1: Vec<_> = Vec::with_capacity(500);
1961
1962 decoder
1963 .decode_float(
1964 &[
1965 seg.slice(s![.., ..4, ..]).into_dyn(),
1966 seg.slice(s![.., 4..84, ..]).into_dyn(),
1967 ],
1968 &mut output_boxes1,
1969 &mut output_masks1,
1970 )
1971 .unwrap();
1972 compare_outputs((&output_boxes, &output_boxes_f32), (&output_masks, &[]));
1973 compare_outputs((&output_boxes_f32, &output_boxes1), (&[], &output_masks1));
1974 }
1975
1976 #[test]
1977 fn test_decoder_masks_config_mixed() {
1978 let score_threshold = 0.45;
1979 let iou_threshold = 0.45;
1980 let boxes_raw = load_yolov8_boxes();
1981 let boxes: Vec<_> = boxes_raw.iter().map(|x| *x as i16 * 256).collect();
1982 let boxes = ndarray::Array3::from_shape_vec((1, 116, 8400), boxes).unwrap();
1983
1984 let quant_boxes = (0.021287761628627777 / 256.0, 31 * 256);
1985
1986 let protos = load_yolov8_protos();
1987 let quant_protos = (0.02491161972284317, -117);
1988
1989 let decoder = build_yolo_split_segdet_decoder(
1990 score_threshold,
1991 iou_threshold,
1992 quant_boxes,
1993 quant_protos,
1994 );
1995 let mut output_boxes: Vec<_> = Vec::with_capacity(500);
1996 let mut output_masks: Vec<_> = Vec::with_capacity(500);
1997
1998 decoder
1999 .decode_quantized(
2000 &[
2001 boxes.slice(s![.., ..4, ..]).into(),
2002 boxes.slice(s![.., 4..84, ..]).into(),
2003 boxes.slice(s![.., 84.., ..]).into(),
2004 protos.view().into(),
2005 ],
2006 &mut output_boxes,
2007 &mut output_masks,
2008 )
2009 .unwrap();
2010
2011 let protos = dequantize_ndarray::<_, _, f32>(protos.view(), quant_protos.into());
2012 let seg = dequantize_ndarray::<_, _, f32>(boxes.view(), quant_boxes.into());
2013 let mut output_boxes_f32: Vec<_> = Vec::with_capacity(500);
2014 let mut output_masks_f32: Vec<_> = Vec::with_capacity(500);
2015 decode_yolo_segdet_float(
2016 seg.slice(s![0, .., ..]),
2017 protos.slice(s![0, .., .., ..]),
2018 score_threshold,
2019 iou_threshold,
2020 Some(configs::Nms::ClassAgnostic),
2021 &mut output_boxes_f32,
2022 &mut output_masks_f32,
2023 )
2024 .unwrap();
2025
2026 let mut output_boxes1: Vec<_> = Vec::with_capacity(500);
2027 let mut output_masks1: Vec<_> = Vec::with_capacity(500);
2028
2029 decoder
2030 .decode_float(
2031 &[
2032 seg.slice(s![.., ..4, ..]).into_dyn(),
2033 seg.slice(s![.., 4..84, ..]).into_dyn(),
2034 seg.slice(s![.., 84.., ..]).into_dyn(),
2035 protos.view().into_dyn(),
2036 ],
2037 &mut output_boxes1,
2038 &mut output_masks1,
2039 )
2040 .unwrap();
2041 compare_outputs(
2042 (&output_boxes, &output_boxes_f32),
2043 (&output_masks, &output_masks_f32),
2044 );
2045 compare_outputs(
2046 (&output_boxes_f32, &output_boxes1),
2047 (&output_masks_f32, &output_masks1),
2048 );
2049 }
2050
2051 fn build_yolo_split_segdet_decoder(
2052 score_threshold: f32,
2053 iou_threshold: f32,
2054 quant_boxes: (f32, i32),
2055 quant_protos: (f32, i32),
2056 ) -> crate::Decoder {
2057 DecoderBuilder::default()
2058 .with_config_yolo_split_segdet(
2059 configs::Boxes {
2060 decoder: configs::DecoderType::Ultralytics,
2061 quantization: Some(quant_boxes.into()),
2062 shape: vec![1, 4, 8400],
2063 dshape: vec![
2064 (DimName::Batch, 1),
2065 (DimName::BoxCoords, 4),
2066 (DimName::NumBoxes, 8400),
2067 ],
2068 normalized: Some(true),
2069 },
2070 configs::Scores {
2071 decoder: configs::DecoderType::Ultralytics,
2072 quantization: Some(quant_boxes.into()),
2073 shape: vec![1, 80, 8400],
2074 dshape: vec![
2075 (DimName::Batch, 1),
2076 (DimName::NumClasses, 80),
2077 (DimName::NumBoxes, 8400),
2078 ],
2079 },
2080 configs::MaskCoefficients {
2081 decoder: configs::DecoderType::Ultralytics,
2082 quantization: Some(quant_boxes.into()),
2083 shape: vec![1, 32, 8400],
2084 dshape: vec![
2085 (DimName::Batch, 1),
2086 (DimName::NumProtos, 32),
2087 (DimName::NumBoxes, 8400),
2088 ],
2089 },
2090 configs::Protos {
2091 decoder: configs::DecoderType::Ultralytics,
2092 quantization: Some(quant_protos.into()),
2093 shape: vec![1, 160, 160, 32],
2094 dshape: vec![
2095 (DimName::Batch, 1),
2096 (DimName::Height, 160),
2097 (DimName::Width, 160),
2098 (DimName::NumProtos, 32),
2099 ],
2100 },
2101 )
2102 .with_score_threshold(score_threshold)
2103 .with_iou_threshold(iou_threshold)
2104 .build()
2105 .unwrap()
2106 }
2107
2108 fn build_yolov8_seg_decoder(score_threshold: f32, iou_threshold: f32) -> crate::Decoder {
2109 let config_yaml = edgefirst_bench::testdata::read_to_string("yolov8_seg.yaml");
2110 DecoderBuilder::default()
2111 .with_config_yaml_str(config_yaml.to_string())
2112 .with_score_threshold(score_threshold)
2113 .with_iou_threshold(iou_threshold)
2114 .build()
2115 .unwrap()
2116 }
2117 #[test]
2118 fn test_decoder_masks_config_i32() {
2119 let score_threshold = 0.45;
2120 let iou_threshold = 0.45;
2121 let boxes_raw = load_yolov8_boxes();
2122 let scale = 1 << 23;
2123 let boxes: Vec<_> = boxes_raw.iter().map(|x| *x as i32 * scale).collect();
2124 let boxes = ndarray::Array3::from_shape_vec((1, 116, 8400), boxes).unwrap();
2125
2126 let quant_boxes = (0.021287761628627777 / scale as f32, 31 * scale);
2127
2128 let protos_raw = load_yolov8_protos();
2129 let protos: Vec<_> = protos_raw.iter().map(|x| *x as i32 * scale).collect();
2130 let protos = ndarray::Array4::from_shape_vec((1, 160, 160, 32), protos).unwrap();
2131 let quant_protos = (0.02491161972284317 / scale as f32, -117 * scale);
2132
2133 let decoder = build_yolo_split_segdet_decoder(
2134 score_threshold,
2135 iou_threshold,
2136 quant_boxes,
2137 quant_protos,
2138 );
2139
2140 let mut output_boxes: Vec<_> = Vec::with_capacity(500);
2141 let mut output_masks: Vec<_> = Vec::with_capacity(500);
2142
2143 decoder
2144 .decode_quantized(
2145 &[
2146 boxes.slice(s![.., ..4, ..]).into(),
2147 boxes.slice(s![.., 4..84, ..]).into(),
2148 boxes.slice(s![.., 84.., ..]).into(),
2149 protos.view().into(),
2150 ],
2151 &mut output_boxes,
2152 &mut output_masks,
2153 )
2154 .unwrap();
2155
2156 let protos = dequantize_ndarray::<_, _, f32>(protos.view(), quant_protos.into());
2157 let seg = dequantize_ndarray::<_, _, f32>(boxes.view(), quant_boxes.into());
2158 let mut output_boxes_f32: Vec<_> = Vec::with_capacity(500);
2159 let mut output_masks_f32: Vec<Segmentation> = Vec::with_capacity(500);
2160 decode_yolo_segdet_float(
2161 seg.slice(s![0, .., ..]),
2162 protos.slice(s![0, .., .., ..]),
2163 score_threshold,
2164 iou_threshold,
2165 Some(configs::Nms::ClassAgnostic),
2166 &mut output_boxes_f32,
2167 &mut output_masks_f32,
2168 )
2169 .unwrap();
2170
2171 assert_eq!(output_boxes.len(), output_boxes_f32.len());
2172 assert_eq!(output_masks.len(), output_masks_f32.len());
2173
2174 compare_outputs(
2175 (&output_boxes, &output_boxes_f32),
2176 (&output_masks, &output_masks_f32),
2177 );
2178 }
2179
2180 #[test]
2182 fn test_context_switch() {
2183 let yolo_det = || {
2184 let score_threshold = 0.25;
2185 let iou_threshold = 0.7;
2186 let out = load_yolov8s_det();
2187 let quant = (0.0040811873, -123).into();
2188
2189 let decoder = DecoderBuilder::default()
2190 .with_config_yolo_det(
2191 configs::Detection {
2192 decoder: DecoderType::Ultralytics,
2193 shape: vec![1, 84, 8400],
2194 anchors: None,
2195 quantization: Some(quant),
2196 dshape: vec![
2197 (DimName::Batch, 1),
2198 (DimName::NumFeatures, 84),
2199 (DimName::NumBoxes, 8400),
2200 ],
2201 normalized: None,
2202 },
2203 None,
2204 )
2205 .with_score_threshold(score_threshold)
2206 .with_iou_threshold(iou_threshold)
2207 .build()
2208 .unwrap();
2209
2210 let mut output_boxes: Vec<_> = Vec::with_capacity(50);
2211 let mut output_masks: Vec<_> = Vec::with_capacity(50);
2212
2213 for _ in 0..100 {
2214 decoder
2215 .decode_quantized(&[out.view().into()], &mut output_boxes, &mut output_masks)
2216 .unwrap();
2217
2218 assert!(output_boxes[0].equal_within_delta(
2219 &DetectBox {
2220 bbox: BoundingBox {
2221 xmin: 0.5285137,
2222 ymin: 0.05305544,
2223 xmax: 0.87541467,
2224 ymax: 0.9998909,
2225 },
2226 score: 0.5591227,
2227 label: 0
2228 },
2229 1e-6
2230 ));
2231
2232 assert!(output_boxes[1].equal_within_delta(
2233 &DetectBox {
2234 bbox: BoundingBox {
2235 xmin: 0.130598,
2236 ymin: 0.43260583,
2237 xmax: 0.35098213,
2238 ymax: 0.9958097,
2239 },
2240 score: 0.33057618,
2241 label: 75
2242 },
2243 1e-6
2244 ));
2245 assert!(output_masks.is_empty());
2246 }
2247 };
2248
2249 let modelpack_det_split = || {
2250 let score_threshold = 0.8;
2251 let iou_threshold = 0.5;
2252
2253 let seg = edgefirst_bench::testdata::read("modelpack_seg_2x160x160.bin");
2254 let seg = ndarray::Array4::from_shape_vec((1, 2, 160, 160), seg.to_vec()).unwrap();
2255
2256 let detect0 = edgefirst_bench::testdata::read("modelpack_split_9x15x18.bin");
2257 let detect0 =
2258 ndarray::Array4::from_shape_vec((1, 9, 15, 18), detect0.to_vec()).unwrap();
2259
2260 let detect1 = edgefirst_bench::testdata::read("modelpack_split_17x30x18.bin");
2261 let detect1 =
2262 ndarray::Array4::from_shape_vec((1, 17, 30, 18), detect1.to_vec()).unwrap();
2263
2264 let mut mask = seg.slice(s![0, .., .., ..]);
2265 mask.swap_axes(0, 1);
2266 mask.swap_axes(1, 2);
2267 let mask = [Segmentation {
2268 xmin: 0.0,
2269 ymin: 0.0,
2270 xmax: 1.0,
2271 ymax: 1.0,
2272 segmentation: crate::mask_to_tensor(mask.view()).unwrap(),
2273 }];
2274 let correct_boxes = [DetectBox {
2275 bbox: BoundingBox {
2276 xmin: 0.43171933,
2277 ymin: 0.68243736,
2278 xmax: 0.5626645,
2279 ymax: 0.808863,
2280 },
2281 score: 0.99240804,
2282 label: 0,
2283 }];
2284
2285 let quant0 = (0.08547406643629074, 174).into();
2286 let quant1 = (0.09929127991199493, 183).into();
2287 let quant_seg = (1.0 / 255.0, 0).into();
2288
2289 let anchors0 = vec![
2290 [0.36666667461395264, 0.31481480598449707],
2291 [0.38749998807907104, 0.4740740656852722],
2292 [0.5333333611488342, 0.644444465637207],
2293 ];
2294 let anchors1 = vec![
2295 [0.13750000298023224, 0.2074074000120163],
2296 [0.2541666626930237, 0.21481481194496155],
2297 [0.23125000298023224, 0.35185185074806213],
2298 ];
2299
2300 let decoder = DecoderBuilder::default()
2301 .with_config_modelpack_segdet_split(
2302 vec![
2303 configs::Detection {
2304 decoder: DecoderType::ModelPack,
2305 shape: vec![1, 17, 30, 18],
2306 anchors: Some(anchors1),
2307 quantization: Some(quant1),
2308 dshape: vec![
2309 (DimName::Batch, 1),
2310 (DimName::Height, 17),
2311 (DimName::Width, 30),
2312 (DimName::NumAnchorsXFeatures, 18),
2313 ],
2314 normalized: None,
2315 },
2316 configs::Detection {
2317 decoder: DecoderType::ModelPack,
2318 shape: vec![1, 9, 15, 18],
2319 anchors: Some(anchors0),
2320 quantization: Some(quant0),
2321 dshape: vec![
2322 (DimName::Batch, 1),
2323 (DimName::Height, 9),
2324 (DimName::Width, 15),
2325 (DimName::NumAnchorsXFeatures, 18),
2326 ],
2327 normalized: None,
2328 },
2329 ],
2330 configs::Segmentation {
2331 decoder: DecoderType::ModelPack,
2332 quantization: Some(quant_seg),
2333 shape: vec![1, 2, 160, 160],
2334 dshape: vec![
2335 (DimName::Batch, 1),
2336 (DimName::NumClasses, 2),
2337 (DimName::Height, 160),
2338 (DimName::Width, 160),
2339 ],
2340 },
2341 )
2342 .with_score_threshold(score_threshold)
2343 .with_iou_threshold(iou_threshold)
2344 .build()
2345 .unwrap();
2346 let mut output_boxes: Vec<_> = Vec::with_capacity(10);
2347 let mut output_masks: Vec<_> = Vec::with_capacity(10);
2348
2349 for _ in 0..100 {
2350 decoder
2351 .decode_quantized(
2352 &[
2353 detect0.view().into(),
2354 detect1.view().into(),
2355 seg.view().into(),
2356 ],
2357 &mut output_boxes,
2358 &mut output_masks,
2359 )
2360 .unwrap();
2361
2362 compare_outputs((&correct_boxes, &output_boxes), (&mask, &output_masks));
2363 }
2364 };
2365
2366 let handles = vec![
2367 std::thread::spawn(yolo_det),
2368 std::thread::spawn(modelpack_det_split),
2369 std::thread::spawn(yolo_det),
2370 std::thread::spawn(modelpack_det_split),
2371 std::thread::spawn(yolo_det),
2372 std::thread::spawn(modelpack_det_split),
2373 std::thread::spawn(yolo_det),
2374 std::thread::spawn(modelpack_det_split),
2375 ];
2376 for handle in handles {
2377 handle.join().unwrap();
2378 }
2379 }
2380
2381 #[test]
2382 fn test_ndarray_to_xyxy_float() {
2383 let arr = array![10.0_f32, 20.0, 20.0, 20.0];
2384 let xyxy: [f32; 4] = XYWH::ndarray_to_xyxy_float(arr.view());
2385 assert_eq!(xyxy, [0.0_f32, 10.0, 20.0, 30.0]);
2386
2387 let arr = array![10.0_f32, 20.0, 20.0, 20.0];
2388 let xyxy: [f32; 4] = XYXY::ndarray_to_xyxy_float(arr.view());
2389 assert_eq!(xyxy, [10.0_f32, 20.0, 20.0, 20.0]);
2390 }
2391
2392 #[test]
2401 fn to_xyxy_float_converts_centre_and_size_to_corners() {
2402 let xyxy: [f32; 4] = XYWH::to_xyxy_float(&[10.0_f32, 20.0, 6.0, 8.0]);
2403 assert_eq!(xyxy, [7.0_f32, 16.0, 13.0, 24.0]);
2404
2405 let xyxy: [f32; 4] = XYXY::to_xyxy_float(&[7.0_f32, 16.0, 13.0, 24.0]);
2406 assert_eq!(xyxy, [7.0_f32, 16.0, 13.0, 24.0]);
2407 }
2408
2409 #[test]
2413 fn arg_max_returns_the_last_index_on_a_tie() {
2414 let scores = array![1_i32, 5, 5, 2];
2415 assert_eq!(crate::arg_max(scores.view()), (5, 2));
2416
2417 let single = array![7_i32];
2418 assert_eq!(crate::arg_max(single.view()), (7, 0));
2419 }
2420
2421 #[test]
2422 fn test_class_aware_nms_float() {
2423 use crate::float::nms_class_aware_float;
2424
2425 let boxes = vec![
2427 DetectBox {
2428 bbox: BoundingBox {
2429 xmin: 0.0,
2430 ymin: 0.0,
2431 xmax: 0.5,
2432 ymax: 0.5,
2433 },
2434 score: 0.9,
2435 label: 0, },
2437 DetectBox {
2438 bbox: BoundingBox {
2439 xmin: 0.1,
2440 ymin: 0.1,
2441 xmax: 0.6,
2442 ymax: 0.6,
2443 },
2444 score: 0.8,
2445 label: 1, },
2447 ];
2448
2449 let result = nms_class_aware_float(0.3, None, boxes.clone());
2452 assert_eq!(
2453 result.len(),
2454 2,
2455 "Class-aware NMS should keep both boxes with different classes"
2456 );
2457
2458 let same_class_boxes = vec![
2460 DetectBox {
2461 bbox: BoundingBox {
2462 xmin: 0.0,
2463 ymin: 0.0,
2464 xmax: 0.5,
2465 ymax: 0.5,
2466 },
2467 score: 0.9,
2468 label: 0,
2469 },
2470 DetectBox {
2471 bbox: BoundingBox {
2472 xmin: 0.1,
2473 ymin: 0.1,
2474 xmax: 0.6,
2475 ymax: 0.6,
2476 },
2477 score: 0.8,
2478 label: 0, },
2480 ];
2481
2482 let result = nms_class_aware_float(0.3, None, same_class_boxes);
2483 assert_eq!(
2484 result.len(),
2485 1,
2486 "Class-aware NMS should suppress overlapping box with same class"
2487 );
2488 assert_eq!(result[0].label, 0);
2489 assert!((result[0].score - 0.9).abs() < 1e-6);
2490 }
2491
2492 #[test]
2493 fn test_class_agnostic_vs_aware_nms() {
2494 use crate::float::{nms_class_aware_float, nms_float};
2495
2496 let boxes = vec![
2498 DetectBox {
2499 bbox: BoundingBox {
2500 xmin: 0.0,
2501 ymin: 0.0,
2502 xmax: 0.5,
2503 ymax: 0.5,
2504 },
2505 score: 0.9,
2506 label: 0,
2507 },
2508 DetectBox {
2509 bbox: BoundingBox {
2510 xmin: 0.1,
2511 ymin: 0.1,
2512 xmax: 0.6,
2513 ymax: 0.6,
2514 },
2515 score: 0.8,
2516 label: 1,
2517 },
2518 ];
2519
2520 let agnostic_result = nms_float(0.3, None, boxes.clone());
2522 assert_eq!(
2523 agnostic_result.len(),
2524 1,
2525 "Class-agnostic NMS should suppress overlapping boxes"
2526 );
2527
2528 let aware_result = nms_class_aware_float(0.3, None, boxes);
2530 assert_eq!(
2531 aware_result.len(),
2532 2,
2533 "Class-aware NMS should keep boxes with different classes"
2534 );
2535 }
2536
2537 #[test]
2538 fn test_class_aware_nms_int() {
2539 use crate::byte::nms_class_aware_int;
2540
2541 let boxes = vec![
2543 DetectBoxQuantized {
2544 bbox: BoundingBox {
2545 xmin: 0.0,
2546 ymin: 0.0,
2547 xmax: 0.5,
2548 ymax: 0.5,
2549 },
2550 score: 200_u8,
2551 label: 0,
2552 },
2553 DetectBoxQuantized {
2554 bbox: BoundingBox {
2555 xmin: 0.1,
2556 ymin: 0.1,
2557 xmax: 0.6,
2558 ymax: 0.6,
2559 },
2560 score: 180_u8,
2561 label: 1, },
2563 ];
2564
2565 let result = nms_class_aware_int(0.5, None, boxes);
2567 assert_eq!(
2568 result.len(),
2569 2,
2570 "Class-aware NMS (int) should keep boxes with different classes"
2571 );
2572 }
2573
2574 #[test]
2575 fn test_nms_enum_default() {
2576 let default_nms: configs::Nms = Default::default();
2578 assert_eq!(default_nms, configs::Nms::ClassAware);
2579 }
2580
2581 #[test]
2582 fn test_decoder_nms_mode() {
2583 let decoder = DecoderBuilder::default()
2585 .with_config_yolo_det(
2586 configs::Detection {
2587 anchors: None,
2588 decoder: DecoderType::Ultralytics,
2589 quantization: None,
2590 shape: vec![1, 84, 8400],
2591 dshape: Vec::new(),
2592 normalized: Some(true),
2593 },
2594 None,
2595 )
2596 .with_nms(Some(configs::Nms::ClassAware))
2597 .build()
2598 .unwrap();
2599
2600 assert_eq!(decoder.nms, Some(configs::Nms::ClassAware));
2601 }
2602
2603 #[test]
2604 fn test_decoder_nms_bypass() {
2605 let decoder = DecoderBuilder::default()
2607 .with_config_yolo_det(
2608 configs::Detection {
2609 anchors: None,
2610 decoder: DecoderType::Ultralytics,
2611 quantization: None,
2612 shape: vec![1, 84, 8400],
2613 dshape: Vec::new(),
2614 normalized: Some(true),
2615 },
2616 None,
2617 )
2618 .with_nms(None)
2619 .build()
2620 .unwrap();
2621
2622 assert_eq!(decoder.nms, None);
2623 }
2624
2625 #[test]
2626 fn test_decoder_normalized_boxes_true() {
2627 let decoder = DecoderBuilder::default()
2629 .with_config_yolo_det(
2630 configs::Detection {
2631 anchors: None,
2632 decoder: DecoderType::Ultralytics,
2633 quantization: None,
2634 shape: vec![1, 84, 8400],
2635 dshape: Vec::new(),
2636 normalized: Some(true),
2637 },
2638 None,
2639 )
2640 .build()
2641 .unwrap();
2642
2643 assert_eq!(decoder.normalized_boxes(), Some(true));
2644 }
2645
2646 #[test]
2647 fn test_decoder_normalized_boxes_false() {
2648 let decoder = DecoderBuilder::default()
2651 .with_config_yolo_det(
2652 configs::Detection {
2653 anchors: None,
2654 decoder: DecoderType::Ultralytics,
2655 quantization: None,
2656 shape: vec![1, 84, 8400],
2657 dshape: Vec::new(),
2658 normalized: Some(false),
2659 },
2660 None,
2661 )
2662 .build()
2663 .unwrap();
2664
2665 assert_eq!(decoder.normalized_boxes(), Some(false));
2666 }
2667
2668 #[test]
2669 fn test_decoder_normalized_boxes_unknown() {
2670 let decoder = DecoderBuilder::default()
2672 .with_config_yolo_det(
2673 configs::Detection {
2674 anchors: None,
2675 decoder: DecoderType::Ultralytics,
2676 quantization: None,
2677 shape: vec![1, 84, 8400],
2678 dshape: Vec::new(),
2679 normalized: None,
2680 },
2681 Some(DecoderVersion::Yolo11),
2682 )
2683 .build()
2684 .unwrap();
2685
2686 assert_eq!(decoder.normalized_boxes(), None);
2687 }
2688
2689 pub fn quantize_ndarray<T: PrimInt + 'static, D: Dimension, F: Float + AsPrimitive<T>>(
2690 input: ArrayView<F, D>,
2691 quant: Quantization,
2692 ) -> Array<T, D>
2693 where
2694 i32: num_traits::AsPrimitive<F>,
2695 f32: num_traits::AsPrimitive<F>,
2696 {
2697 let zero_point = quant.zero_point.as_();
2698 let div_scale = F::one() / quant.scale.as_();
2699 if zero_point != F::zero() {
2700 input.mapv(|d| (d * div_scale + zero_point).round().as_())
2701 } else {
2702 input.mapv(|d| (d * div_scale).round().as_())
2703 }
2704 }
2705
2706 fn real_data_expected_boxes() -> [DetectBox; 2] {
2707 [
2708 DetectBox {
2709 bbox: BoundingBox {
2710 xmin: 0.08515105,
2711 ymin: 0.7131401,
2712 xmax: 0.29802868,
2713 ymax: 0.8195788,
2714 },
2715 score: 0.91537374,
2716 label: 23,
2717 },
2718 DetectBox {
2719 bbox: BoundingBox {
2720 xmin: 0.59605736,
2721 ymin: 0.25545314,
2722 xmax: 0.93666154,
2723 ymax: 0.72378385,
2724 },
2725 score: 0.91537374,
2726 label: 23,
2727 },
2728 ]
2729 }
2730
2731 fn e2e_expected_boxes_quant() -> [DetectBox; 1] {
2732 [DetectBox {
2733 bbox: BoundingBox {
2734 xmin: 0.12549022,
2735 ymin: 0.12549022,
2736 xmax: 0.23529413,
2737 ymax: 0.23529413,
2738 },
2739 score: 0.98823535,
2740 label: 2,
2741 }]
2742 }
2743
2744 fn e2e_expected_boxes_float() -> [DetectBox; 1] {
2745 [DetectBox {
2746 bbox: BoundingBox {
2747 xmin: 0.1234,
2748 ymin: 0.1234,
2749 xmax: 0.2345,
2750 ymax: 0.2345,
2751 },
2752 score: 0.9876,
2753 label: 2,
2754 }]
2755 }
2756
2757 macro_rules! real_data_proto_test {
2758 ($name:ident, quantized, $layout:ident) => {
2759 #[test]
2760 fn $name() {
2761 let is_split = matches!(stringify!($layout), "split");
2762
2763 let score_threshold = 0.45;
2764 let iou_threshold = 0.45;
2765 let quant_boxes = (0.021287762_f32, 31_i32);
2766 let quant_protos = (0.02491162_f32, -117_i32);
2767
2768 let raw_boxes = edgefirst_bench::testdata::read("yolov8_boxes_116x8400.bin");
2769 let raw_boxes = unsafe {
2770 std::slice::from_raw_parts(raw_boxes.as_ptr() as *const i8, raw_boxes.len())
2771 };
2772 let boxes_i8 =
2773 ndarray::Array3::from_shape_vec((1, 116, 8400), raw_boxes.to_vec()).unwrap();
2774
2775 let raw_protos = edgefirst_bench::testdata::read("yolov8_protos_160x160x32.bin");
2776 let raw_protos = unsafe {
2777 std::slice::from_raw_parts(raw_protos.as_ptr() as *const i8, raw_protos.len())
2778 };
2779 let protos_i8 =
2780 ndarray::Array4::from_shape_vec((1, 160, 160, 32), raw_protos.to_vec())
2781 .unwrap();
2782
2783 let mask_split = boxes_i8.slice(s![.., 84.., ..]).to_owned();
2785 let scores_split = boxes_i8.slice(s![.., 4..84, ..]).to_owned();
2786 let boxes_split = boxes_i8.slice(s![.., ..4, ..]).to_owned();
2787 let boxes_combined = boxes_i8;
2788
2789 let decoder = if is_split {
2790 build_yolo_split_segdet_decoder(
2791 score_threshold,
2792 iou_threshold,
2793 quant_boxes,
2794 quant_protos,
2795 )
2796 } else {
2797 build_yolov8_seg_decoder(score_threshold, iou_threshold)
2798 };
2799
2800 let expected = real_data_expected_boxes();
2801 let mut output_boxes = Vec::with_capacity(50);
2802
2803 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> = if is_split {
2804 vec![
2805 boxes_split.view().into(),
2806 scores_split.view().into(),
2807 mask_split.view().into(),
2808 protos_i8.view().into(),
2809 ]
2810 } else {
2811 vec![boxes_combined.view().into(), protos_i8.view().into()]
2812 };
2813 decoder
2814 .decode_quantized_proto(&inputs, &mut output_boxes)
2815 .unwrap();
2816
2817 assert_eq!(output_boxes.len(), 2);
2818 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
2819 assert!(output_boxes[1].equal_within_delta(&expected[1], 1.0 / 160.0));
2820 }
2821 };
2822 ($name:ident, float, $layout:ident) => {
2823 #[test]
2824 fn $name() {
2825 let is_split = matches!(stringify!($layout), "split");
2826
2827 let score_threshold = 0.45;
2828 let iou_threshold = 0.45;
2829 let quant_boxes = (0.021287762_f32, 31_i32);
2830 let quant_protos = (0.02491162_f32, -117_i32);
2831
2832 let raw_boxes = edgefirst_bench::testdata::read("yolov8_boxes_116x8400.bin");
2833 let raw_boxes = unsafe {
2834 std::slice::from_raw_parts(raw_boxes.as_ptr() as *const i8, raw_boxes.len())
2835 };
2836 let boxes_i8 =
2837 ndarray::Array3::from_shape_vec((1, 116, 8400), raw_boxes.to_vec()).unwrap();
2838 let boxes_f32: Array3<f32> =
2839 dequantize_ndarray(boxes_i8.view(), quant_boxes.into());
2840
2841 let raw_protos = edgefirst_bench::testdata::read("yolov8_protos_160x160x32.bin");
2842 let raw_protos = unsafe {
2843 std::slice::from_raw_parts(raw_protos.as_ptr() as *const i8, raw_protos.len())
2844 };
2845 let protos_i8 =
2846 ndarray::Array4::from_shape_vec((1, 160, 160, 32), raw_protos.to_vec())
2847 .unwrap();
2848 let protos_f32: Array4<f32> =
2849 dequantize_ndarray(protos_i8.view(), quant_protos.into());
2850
2851 let mask_split = boxes_f32.slice(s![.., 84.., ..]).to_owned();
2853 let scores_split = boxes_f32.slice(s![.., 4..84, ..]).to_owned();
2854 let boxes_split = boxes_f32.slice(s![.., ..4, ..]).to_owned();
2855 let boxes_combined = boxes_f32;
2856
2857 let decoder = if is_split {
2858 build_yolo_split_segdet_decoder(
2859 score_threshold,
2860 iou_threshold,
2861 quant_boxes,
2862 quant_protos,
2863 )
2864 } else {
2865 build_yolov8_seg_decoder(score_threshold, iou_threshold)
2866 };
2867
2868 let expected = real_data_expected_boxes();
2869 let mut output_boxes = Vec::with_capacity(50);
2870
2871 let inputs = if is_split {
2872 vec![
2873 boxes_split.view().into_dyn(),
2874 scores_split.view().into_dyn(),
2875 mask_split.view().into_dyn(),
2876 protos_f32.view().into_dyn(),
2877 ]
2878 } else {
2879 vec![
2880 boxes_combined.view().into_dyn(),
2881 protos_f32.view().into_dyn(),
2882 ]
2883 };
2884 decoder
2885 .decode_float_proto(&inputs, &mut output_boxes)
2886 .unwrap();
2887
2888 assert_eq!(output_boxes.len(), 2);
2889 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
2890 assert!(output_boxes[1].equal_within_delta(&expected[1], 1.0 / 160.0));
2891 }
2892 };
2893 }
2894
2895 real_data_proto_test!(test_decoder_segdet_proto, quantized, combined);
2896 real_data_proto_test!(test_decoder_segdet_proto_float, float, combined);
2897 real_data_proto_test!(test_decoder_segdet_split_proto, quantized, split);
2898 real_data_proto_test!(test_decoder_segdet_split_proto_float, float, split);
2899
2900 const E2E_COMBINED_DET_CONFIG: &str = "
2901decoder_version: yolo26
2902outputs:
2903 - type: detection
2904 decoder: ultralytics
2905 quantization: [0.00784313725490196, 0]
2906 shape: [1, 10, 6]
2907 dshape:
2908 - [batch, 1]
2909 - [num_boxes, 10]
2910 - [num_features, 6]
2911 normalized: true
2912";
2913
2914 const E2E_COMBINED_SEGDET_CONFIG: &str = "
2915decoder_version: yolo26
2916outputs:
2917 - type: detection
2918 decoder: ultralytics
2919 quantization: [0.00784313725490196, 0]
2920 shape: [1, 10, 38]
2921 dshape:
2922 - [batch, 1]
2923 - [num_boxes, 10]
2924 - [num_features, 38]
2925 normalized: true
2926 - type: protos
2927 decoder: ultralytics
2928 quantization: [0.0039215686274509803921568627451, 128]
2929 shape: [1, 160, 160, 32]
2930 dshape:
2931 - [batch, 1]
2932 - [height, 160]
2933 - [width, 160]
2934 - [num_protos, 32]
2935";
2936
2937 const E2E_SPLIT_DET_CONFIG: &str = "
2938decoder_version: yolo26
2939outputs:
2940 - type: boxes
2941 decoder: ultralytics
2942 quantization: [0.00784313725490196, 0]
2943 shape: [1, 10, 4]
2944 dshape:
2945 - [batch, 1]
2946 - [num_boxes, 10]
2947 - [box_coords, 4]
2948 normalized: true
2949 - type: scores
2950 decoder: ultralytics
2951 quantization: [0.00784313725490196, 0]
2952 shape: [1, 10, 1]
2953 dshape:
2954 - [batch, 1]
2955 - [num_boxes, 10]
2956 - [num_classes, 1]
2957 - type: classes
2958 decoder: ultralytics
2959 quantization: [0.00784313725490196, 0]
2960 shape: [1, 10, 1]
2961 dshape:
2962 - [batch, 1]
2963 - [num_boxes, 10]
2964 - [num_classes, 1]
2965";
2966
2967 const E2E_SPLIT_SEGDET_CONFIG: &str = "
2968decoder_version: yolo26
2969outputs:
2970 - type: boxes
2971 decoder: ultralytics
2972 quantization: [0.00784313725490196, 0]
2973 shape: [1, 10, 4]
2974 dshape:
2975 - [batch, 1]
2976 - [num_boxes, 10]
2977 - [box_coords, 4]
2978 normalized: true
2979 - type: scores
2980 decoder: ultralytics
2981 quantization: [0.00784313725490196, 0]
2982 shape: [1, 10, 1]
2983 dshape:
2984 - [batch, 1]
2985 - [num_boxes, 10]
2986 - [num_classes, 1]
2987 - type: classes
2988 decoder: ultralytics
2989 quantization: [0.00784313725490196, 0]
2990 shape: [1, 10, 1]
2991 dshape:
2992 - [batch, 1]
2993 - [num_boxes, 10]
2994 - [num_classes, 1]
2995 - type: mask_coefficients
2996 decoder: ultralytics
2997 quantization: [0.00784313725490196, 0]
2998 shape: [1, 10, 32]
2999 dshape:
3000 - [batch, 1]
3001 - [num_boxes, 10]
3002 - [num_protos, 32]
3003 - type: protos
3004 decoder: ultralytics
3005 quantization: [0.0039215686274509803921568627451, 128]
3006 shape: [1, 160, 160, 32]
3007 dshape:
3008 - [batch, 1]
3009 - [height, 160]
3010 - [width, 160]
3011 - [num_protos, 32]
3012";
3013
3014 macro_rules! e2e_segdet_test {
3015 ($name:ident, quantized, $layout:ident, $output:ident) => {
3016 #[test]
3017 fn $name() {
3018 let is_split = matches!(stringify!($layout), "split");
3019 let is_proto = matches!(stringify!($output), "proto");
3020
3021 let score_threshold = 0.45;
3022 let iou_threshold = 0.45;
3023
3024 let mut boxes = Array2::zeros((10, 4));
3025 let mut scores = Array2::zeros((10, 1));
3026 let mut classes = Array2::zeros((10, 1));
3027 let mask = Array2::zeros((10, 32));
3028 let protos = Array3::<f64>::zeros((160, 160, 32));
3029 let protos = protos.insert_axis(Axis(0));
3030 let protos_quant = (1.0 / 255.0, 0.0);
3031 let protos: Array4<u8> = quantize_ndarray(protos.view(), protos_quant.into());
3032
3033 boxes
3034 .slice_mut(s![0, ..])
3035 .assign(&array![0.1234, 0.1234, 0.2345, 0.2345]);
3036 scores.slice_mut(s![0, ..]).assign(&array![0.9876]);
3037 classes.slice_mut(s![0, ..]).assign(&array![2.0]);
3038
3039 let detect_quant = (2.0 / 255.0, 0.0);
3040
3041 let decoder = if is_split {
3042 DecoderBuilder::default()
3043 .with_config_yaml_str(E2E_SPLIT_SEGDET_CONFIG.to_string())
3044 .with_score_threshold(score_threshold)
3045 .with_iou_threshold(iou_threshold)
3046 .build()
3047 .unwrap()
3048 } else {
3049 DecoderBuilder::default()
3050 .with_config_yaml_str(E2E_COMBINED_SEGDET_CONFIG.to_string())
3051 .with_score_threshold(score_threshold)
3052 .with_iou_threshold(iou_threshold)
3053 .build()
3054 .unwrap()
3055 };
3056
3057 let expected = e2e_expected_boxes_quant();
3058 let mut output_boxes = Vec::with_capacity(50);
3059
3060 if is_split {
3061 let boxes = boxes.insert_axis(Axis(0));
3062 let scores = scores.insert_axis(Axis(0));
3063 let classes = classes.insert_axis(Axis(0));
3064 let mask = mask.insert_axis(Axis(0));
3065
3066 let boxes: Array3<u8> = quantize_ndarray(boxes.view(), detect_quant.into());
3067 let scores: Array3<u8> = quantize_ndarray(scores.view(), detect_quant.into());
3068 let classes: Array3<u8> = quantize_ndarray(classes.view(), detect_quant.into());
3069 let mask: Array3<u8> = quantize_ndarray(mask.view(), detect_quant.into());
3070
3071 if is_proto {
3072 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> = vec![
3073 boxes.view().into(),
3074 scores.view().into(),
3075 classes.view().into(),
3076 mask.view().into(),
3077 protos.view().into(),
3078 ];
3079 decoder
3080 .decode_quantized_proto(&inputs, &mut output_boxes)
3081 .unwrap();
3082
3083 assert_eq!(output_boxes.len(), 1);
3084 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
3085 } else {
3086 let mut output_masks = Vec::with_capacity(50);
3087 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> = vec![
3088 boxes.view().into(),
3089 scores.view().into(),
3090 classes.view().into(),
3091 mask.view().into(),
3092 protos.view().into(),
3093 ];
3094 decoder
3095 .decode_quantized(&inputs, &mut output_boxes, &mut output_masks)
3096 .unwrap();
3097
3098 assert_eq!(output_boxes.len(), 1);
3099 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
3100 }
3101 } else {
3102 let detect = ndarray::concatenate![
3104 Axis(1),
3105 boxes.view(),
3106 scores.view(),
3107 classes.view(),
3108 mask.view()
3109 ];
3110 let detect = detect.insert_axis(Axis(0));
3111 assert_eq!(detect.shape(), &[1, 10, 38]);
3112 let detect: Array3<u8> = quantize_ndarray(detect.view(), detect_quant.into());
3113
3114 if is_proto {
3115 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> =
3116 vec![detect.view().into(), protos.view().into()];
3117 decoder
3118 .decode_quantized_proto(&inputs, &mut output_boxes)
3119 .unwrap();
3120
3121 assert_eq!(output_boxes.len(), 1);
3122 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
3123 } else {
3124 let mut output_masks = Vec::with_capacity(50);
3125 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> =
3126 vec![detect.view().into(), protos.view().into()];
3127 decoder
3128 .decode_quantized(&inputs, &mut output_boxes, &mut output_masks)
3129 .unwrap();
3130
3131 assert_eq!(output_boxes.len(), 1);
3132 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
3133 }
3134 }
3135 }
3136 };
3137 ($name:ident, float, $layout:ident, $output:ident) => {
3138 #[test]
3139 fn $name() {
3140 let is_split = matches!(stringify!($layout), "split");
3141 let is_proto = matches!(stringify!($output), "proto");
3142
3143 let score_threshold = 0.45;
3144 let iou_threshold = 0.45;
3145
3146 let mut boxes = Array2::zeros((10, 4));
3147 let mut scores = Array2::zeros((10, 1));
3148 let mut classes = Array2::zeros((10, 1));
3149 let mask: Array2<f64> = Array2::zeros((10, 32));
3150 let protos = Array3::<f64>::zeros((160, 160, 32));
3151 let protos = protos.insert_axis(Axis(0));
3152
3153 boxes
3154 .slice_mut(s![0, ..])
3155 .assign(&array![0.1234, 0.1234, 0.2345, 0.2345]);
3156 scores.slice_mut(s![0, ..]).assign(&array![0.9876]);
3157 classes.slice_mut(s![0, ..]).assign(&array![2.0]);
3158
3159 let decoder = if is_split {
3160 DecoderBuilder::default()
3161 .with_config_yaml_str(E2E_SPLIT_SEGDET_CONFIG.to_string())
3162 .with_score_threshold(score_threshold)
3163 .with_iou_threshold(iou_threshold)
3164 .build()
3165 .unwrap()
3166 } else {
3167 DecoderBuilder::default()
3168 .with_config_yaml_str(E2E_COMBINED_SEGDET_CONFIG.to_string())
3169 .with_score_threshold(score_threshold)
3170 .with_iou_threshold(iou_threshold)
3171 .build()
3172 .unwrap()
3173 };
3174
3175 let expected = e2e_expected_boxes_float();
3176 let mut output_boxes = Vec::with_capacity(50);
3177
3178 if is_split {
3179 let boxes = boxes.insert_axis(Axis(0));
3180 let scores = scores.insert_axis(Axis(0));
3181 let classes = classes.insert_axis(Axis(0));
3182 let mask = mask.insert_axis(Axis(0));
3183
3184 if is_proto {
3185 let inputs = vec![
3186 boxes.view().into_dyn(),
3187 scores.view().into_dyn(),
3188 classes.view().into_dyn(),
3189 mask.view().into_dyn(),
3190 protos.view().into_dyn(),
3191 ];
3192 decoder
3193 .decode_float_proto(&inputs, &mut output_boxes)
3194 .unwrap();
3195
3196 assert_eq!(output_boxes.len(), 1);
3197 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
3198 } else {
3199 let mut output_masks = Vec::with_capacity(50);
3200 let inputs = vec![
3201 boxes.view().into_dyn(),
3202 scores.view().into_dyn(),
3203 classes.view().into_dyn(),
3204 mask.view().into_dyn(),
3205 protos.view().into_dyn(),
3206 ];
3207 decoder
3208 .decode_float(&inputs, &mut output_boxes, &mut output_masks)
3209 .unwrap();
3210
3211 assert_eq!(output_boxes.len(), 1);
3212 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
3213 }
3214 } else {
3215 let detect = ndarray::concatenate![
3217 Axis(1),
3218 boxes.view(),
3219 scores.view(),
3220 classes.view(),
3221 mask.view()
3222 ];
3223 let detect = detect.insert_axis(Axis(0));
3224 assert_eq!(detect.shape(), &[1, 10, 38]);
3225
3226 if is_proto {
3227 let inputs = vec![detect.view().into_dyn(), protos.view().into_dyn()];
3228 decoder
3229 .decode_float_proto(&inputs, &mut output_boxes)
3230 .unwrap();
3231
3232 assert_eq!(output_boxes.len(), 1);
3233 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
3234 } else {
3235 let mut output_masks = Vec::with_capacity(50);
3236 let inputs = vec![detect.view().into_dyn(), protos.view().into_dyn()];
3237 decoder
3238 .decode_float(&inputs, &mut output_boxes, &mut output_masks)
3239 .unwrap();
3240
3241 assert_eq!(output_boxes.len(), 1);
3242 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
3243 }
3244 }
3245 }
3246 };
3247 }
3248
3249 e2e_segdet_test!(test_decoder_end_to_end_segdet, quantized, combined, masks);
3250 e2e_segdet_test!(test_decoder_end_to_end_segdet_float, float, combined, masks);
3251 e2e_segdet_test!(
3252 test_decoder_end_to_end_segdet_proto,
3253 quantized,
3254 combined,
3255 proto
3256 );
3257 e2e_segdet_test!(
3258 test_decoder_end_to_end_segdet_proto_float,
3259 float,
3260 combined,
3261 proto
3262 );
3263 e2e_segdet_test!(
3264 test_decoder_end_to_end_segdet_split,
3265 quantized,
3266 split,
3267 masks
3268 );
3269 e2e_segdet_test!(
3270 test_decoder_end_to_end_segdet_split_float,
3271 float,
3272 split,
3273 masks
3274 );
3275 e2e_segdet_test!(
3276 test_decoder_end_to_end_segdet_split_proto,
3277 quantized,
3278 split,
3279 proto
3280 );
3281 e2e_segdet_test!(
3282 test_decoder_end_to_end_segdet_split_proto_float,
3283 float,
3284 split,
3285 proto
3286 );
3287
3288 macro_rules! e2e_det_test {
3289 ($name:ident, quantized, $layout:ident) => {
3290 #[test]
3291 fn $name() {
3292 let is_split = matches!(stringify!($layout), "split");
3293
3294 let score_threshold = 0.45;
3295 let iou_threshold = 0.45;
3296
3297 let mut boxes = Array3::zeros((1, 10, 4));
3298 let mut scores = Array3::zeros((1, 10, 1));
3299 let mut classes = Array3::zeros((1, 10, 1));
3300
3301 boxes
3302 .slice_mut(s![0, 0, ..])
3303 .assign(&array![0.1234, 0.1234, 0.2345, 0.2345]);
3304 scores.slice_mut(s![0, 0, ..]).assign(&array![0.9876]);
3305 classes.slice_mut(s![0, 0, ..]).assign(&array![2.0]);
3306
3307 let detect_quant = (2.0 / 255.0, 0_i32);
3308
3309 let decoder = if is_split {
3310 DecoderBuilder::default()
3311 .with_config_yaml_str(E2E_SPLIT_DET_CONFIG.to_string())
3312 .with_score_threshold(score_threshold)
3313 .with_iou_threshold(iou_threshold)
3314 .build()
3315 .unwrap()
3316 } else {
3317 DecoderBuilder::default()
3318 .with_config_yaml_str(E2E_COMBINED_DET_CONFIG.to_string())
3319 .with_score_threshold(score_threshold)
3320 .with_iou_threshold(iou_threshold)
3321 .build()
3322 .unwrap()
3323 };
3324
3325 let expected = e2e_expected_boxes_quant();
3326 let mut output_boxes = Vec::with_capacity(50);
3327
3328 if is_split {
3329 let boxes: Array<u8, _> = quantize_ndarray(boxes.view(), detect_quant.into());
3330 let scores: Array<u8, _> = quantize_ndarray(scores.view(), detect_quant.into());
3331 let classes: Array<u8, _> =
3332 quantize_ndarray(classes.view(), detect_quant.into());
3333 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> = vec![
3334 boxes.view().into(),
3335 scores.view().into(),
3336 classes.view().into(),
3337 ];
3338 decoder
3339 .decode_quantized(&inputs, &mut output_boxes, &mut Vec::new())
3340 .unwrap();
3341 } else {
3342 let detect =
3343 ndarray::concatenate![Axis(2), boxes.view(), scores.view(), classes.view()];
3344 assert_eq!(detect.shape(), &[1, 10, 6]);
3345 let detect: Array3<u8> = quantize_ndarray(detect.view(), detect_quant.into());
3346 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> =
3347 vec![detect.view().into()];
3348 decoder
3349 .decode_quantized(&inputs, &mut output_boxes, &mut Vec::new())
3350 .unwrap();
3351 }
3352
3353 assert_eq!(output_boxes.len(), 1);
3354 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
3355 }
3356 };
3357 ($name:ident, float, $layout:ident) => {
3358 #[test]
3359 fn $name() {
3360 let is_split = matches!(stringify!($layout), "split");
3361
3362 let score_threshold = 0.45;
3363 let iou_threshold = 0.45;
3364
3365 let mut boxes = Array3::zeros((1, 10, 4));
3366 let mut scores = Array3::zeros((1, 10, 1));
3367 let mut classes = Array3::zeros((1, 10, 1));
3368
3369 boxes
3370 .slice_mut(s![0, 0, ..])
3371 .assign(&array![0.1234, 0.1234, 0.2345, 0.2345]);
3372 scores.slice_mut(s![0, 0, ..]).assign(&array![0.9876]);
3373 classes.slice_mut(s![0, 0, ..]).assign(&array![2.0]);
3374
3375 let decoder = if is_split {
3376 DecoderBuilder::default()
3377 .with_config_yaml_str(E2E_SPLIT_DET_CONFIG.to_string())
3378 .with_score_threshold(score_threshold)
3379 .with_iou_threshold(iou_threshold)
3380 .build()
3381 .unwrap()
3382 } else {
3383 DecoderBuilder::default()
3384 .with_config_yaml_str(E2E_COMBINED_DET_CONFIG.to_string())
3385 .with_score_threshold(score_threshold)
3386 .with_iou_threshold(iou_threshold)
3387 .build()
3388 .unwrap()
3389 };
3390
3391 let expected = e2e_expected_boxes_float();
3392 let mut output_boxes = Vec::with_capacity(50);
3393
3394 if is_split {
3395 let inputs = vec![
3396 boxes.view().into_dyn(),
3397 scores.view().into_dyn(),
3398 classes.view().into_dyn(),
3399 ];
3400 decoder
3401 .decode_float(&inputs, &mut output_boxes, &mut Vec::new())
3402 .unwrap();
3403 } else {
3404 let detect =
3405 ndarray::concatenate![Axis(2), boxes.view(), scores.view(), classes.view()];
3406 assert_eq!(detect.shape(), &[1, 10, 6]);
3407 let inputs = vec![detect.view().into_dyn()];
3408 decoder
3409 .decode_float(&inputs, &mut output_boxes, &mut Vec::new())
3410 .unwrap();
3411 }
3412
3413 assert_eq!(output_boxes.len(), 1);
3414 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
3415 }
3416 };
3417 }
3418
3419 e2e_det_test!(test_decoder_end_to_end_combined_det, quantized, combined);
3420 e2e_det_test!(test_decoder_end_to_end_combined_det_float, float, combined);
3421 e2e_det_test!(test_decoder_end_to_end_split_det, quantized, split);
3422 e2e_det_test!(test_decoder_end_to_end_split_det_float, float, split);
3423
3424 #[test]
3425 fn test_decode_tensor() {
3426 let score_threshold = 0.45;
3427 let iou_threshold = 0.45;
3428
3429 let raw_boxes = edgefirst_bench::testdata::read("yolov8_boxes_116x8400.bin");
3430 let raw_boxes =
3431 unsafe { std::slice::from_raw_parts(raw_boxes.as_ptr() as *const i8, raw_boxes.len()) };
3432 let boxes_i8: Tensor<i8> = Tensor::new(&[1, 116, 8400], None, None).unwrap();
3433 boxes_i8
3434 .map()
3435 .unwrap()
3436 .as_mut_slice()
3437 .copy_from_slice(raw_boxes);
3438 let boxes_i8 = boxes_i8.into();
3439
3440 let raw_protos = edgefirst_bench::testdata::read("yolov8_protos_160x160x32.bin");
3441 let raw_protos = unsafe {
3442 std::slice::from_raw_parts(raw_protos.as_ptr() as *const i8, raw_protos.len())
3443 };
3444 let protos_i8: Tensor<i8> = Tensor::new(&[1, 160, 160, 32], None, None).unwrap();
3445 protos_i8
3446 .map()
3447 .unwrap()
3448 .as_mut_slice()
3449 .copy_from_slice(raw_protos);
3450 let protos_i8 = protos_i8.into();
3451
3452 let decoder = build_yolov8_seg_decoder(score_threshold, iou_threshold);
3453 let expected = real_data_expected_boxes();
3454 let mut output_boxes = Vec::with_capacity(50);
3455
3456 decoder
3457 .decode(&[&boxes_i8, &protos_i8], &mut output_boxes, &mut Vec::new())
3458 .unwrap();
3459
3460 assert_eq!(output_boxes.len(), 2);
3461 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
3462 assert!(output_boxes[1].equal_within_delta(&expected[1], 1.0 / 160.0));
3463 }
3464
3465 #[test]
3466 fn test_decode_tensor_f32() {
3467 let score_threshold = 0.45;
3468 let iou_threshold = 0.45;
3469
3470 let quant_boxes = (0.021287762_f32, 31_i32);
3471 let quant_protos = (0.02491162_f32, -117_i32);
3472 let raw_boxes = edgefirst_bench::testdata::read("yolov8_boxes_116x8400.bin");
3473 let raw_boxes =
3474 unsafe { std::slice::from_raw_parts(raw_boxes.as_ptr() as *const i8, raw_boxes.len()) };
3475 let mut raw_boxes_f32 = vec![0f32; raw_boxes.len()];
3476 dequantize_cpu(raw_boxes, quant_boxes.into(), &mut raw_boxes_f32);
3477 let boxes_f32: Tensor<f32> = Tensor::new(&[1, 116, 8400], None, None).unwrap();
3478 boxes_f32
3479 .map()
3480 .unwrap()
3481 .as_mut_slice()
3482 .copy_from_slice(&raw_boxes_f32);
3483 let boxes_f32 = boxes_f32.into();
3484
3485 let raw_protos = edgefirst_bench::testdata::read("yolov8_protos_160x160x32.bin");
3486 let raw_protos = unsafe {
3487 std::slice::from_raw_parts(raw_protos.as_ptr() as *const i8, raw_protos.len())
3488 };
3489 let mut raw_protos_f32 = vec![0f32; raw_protos.len()];
3490 dequantize_cpu(raw_protos, quant_protos.into(), &mut raw_protos_f32);
3491 let protos_f32: Tensor<f32> = Tensor::new(&[1, 160, 160, 32], None, None).unwrap();
3492 protos_f32
3493 .map()
3494 .unwrap()
3495 .as_mut_slice()
3496 .copy_from_slice(&raw_protos_f32);
3497 let protos_f32 = protos_f32.into();
3498
3499 let decoder = build_yolov8_seg_decoder(score_threshold, iou_threshold);
3500
3501 let expected = real_data_expected_boxes();
3502 let mut output_boxes = Vec::with_capacity(50);
3503
3504 decoder
3505 .decode(
3506 &[&boxes_f32, &protos_f32],
3507 &mut output_boxes,
3508 &mut Vec::new(),
3509 )
3510 .unwrap();
3511
3512 assert_eq!(output_boxes.len(), 2);
3513 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
3514 assert!(output_boxes[1].equal_within_delta(&expected[1], 1.0 / 160.0));
3515 }
3516
3517 #[test]
3518 fn test_decode_tensor_f64() {
3519 let score_threshold = 0.45;
3520 let iou_threshold = 0.45;
3521
3522 let quant_boxes = (0.021287762_f32, 31_i32);
3523 let quant_protos = (0.02491162_f32, -117_i32);
3524 let raw_boxes = edgefirst_bench::testdata::read("yolov8_boxes_116x8400.bin");
3525 let raw_boxes =
3526 unsafe { std::slice::from_raw_parts(raw_boxes.as_ptr() as *const i8, raw_boxes.len()) };
3527 let mut raw_boxes_f64 = vec![0f64; raw_boxes.len()];
3528 dequantize_cpu(raw_boxes, quant_boxes.into(), &mut raw_boxes_f64);
3529 let boxes_f64: Tensor<f64> = Tensor::new(&[1, 116, 8400], None, None).unwrap();
3530 boxes_f64
3531 .map()
3532 .unwrap()
3533 .as_mut_slice()
3534 .copy_from_slice(&raw_boxes_f64);
3535 let boxes_f64 = boxes_f64.into();
3536
3537 let raw_protos = edgefirst_bench::testdata::read("yolov8_protos_160x160x32.bin");
3538 let raw_protos = unsafe {
3539 std::slice::from_raw_parts(raw_protos.as_ptr() as *const i8, raw_protos.len())
3540 };
3541 let mut raw_protos_f64 = vec![0f64; raw_protos.len()];
3542 dequantize_cpu(raw_protos, quant_protos.into(), &mut raw_protos_f64);
3543 let protos_f64: Tensor<f64> = Tensor::new(&[1, 160, 160, 32], None, None).unwrap();
3544 protos_f64
3545 .map()
3546 .unwrap()
3547 .as_mut_slice()
3548 .copy_from_slice(&raw_protos_f64);
3549 let protos_f64 = protos_f64.into();
3550
3551 let decoder = build_yolov8_seg_decoder(score_threshold, iou_threshold);
3552
3553 let expected = real_data_expected_boxes();
3554 let mut output_boxes = Vec::with_capacity(50);
3555
3556 decoder
3557 .decode(
3558 &[&boxes_f64, &protos_f64],
3559 &mut output_boxes,
3560 &mut Vec::new(),
3561 )
3562 .unwrap();
3563
3564 assert_eq!(output_boxes.len(), 2);
3565 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
3566 assert!(output_boxes[1].equal_within_delta(&expected[1], 1.0 / 160.0));
3567 }
3568
3569 #[test]
3570 fn test_decode_tensor_proto() {
3571 let score_threshold = 0.45;
3572 let iou_threshold = 0.45;
3573
3574 let raw_boxes = edgefirst_bench::testdata::read("yolov8_boxes_116x8400.bin");
3575 let raw_boxes =
3576 unsafe { std::slice::from_raw_parts(raw_boxes.as_ptr() as *const i8, raw_boxes.len()) };
3577 let boxes_i8: Tensor<i8> = Tensor::new(&[1, 116, 8400], None, None).unwrap();
3578 boxes_i8
3579 .map()
3580 .unwrap()
3581 .as_mut_slice()
3582 .copy_from_slice(raw_boxes);
3583 let boxes_i8 = boxes_i8.into();
3584
3585 let raw_protos = edgefirst_bench::testdata::read("yolov8_protos_160x160x32.bin");
3586 let raw_protos = unsafe {
3587 std::slice::from_raw_parts(raw_protos.as_ptr() as *const i8, raw_protos.len())
3588 };
3589 let protos_i8: Tensor<i8> = Tensor::new(&[1, 160, 160, 32], None, None).unwrap();
3590 protos_i8
3591 .map()
3592 .unwrap()
3593 .as_mut_slice()
3594 .copy_from_slice(raw_protos);
3595 let protos_i8 = protos_i8.into();
3596
3597 let decoder = build_yolov8_seg_decoder(score_threshold, iou_threshold);
3598
3599 let expected = real_data_expected_boxes();
3600 let mut output_boxes = Vec::with_capacity(50);
3601
3602 let proto_data = decoder
3603 .decode_proto(&[&boxes_i8, &protos_i8], &mut output_boxes)
3604 .unwrap();
3605
3606 assert_eq!(output_boxes.len(), 2);
3607 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
3608 assert!(output_boxes[1].equal_within_delta(&expected[1], 1.0 / 160.0));
3609
3610 let proto_data = proto_data.expect("segmentation model should return ProtoData");
3611 let coeffs_shape = proto_data.mask_coefficients.shape();
3612 assert_eq!(
3613 coeffs_shape[0],
3614 output_boxes.len(),
3615 "mask_coefficients count must match detection count"
3616 );
3617 assert_eq!(
3618 coeffs_shape[1], 32,
3619 "each detection should have 32 mask coefficients"
3620 );
3621 }
3622
3623 #[test]
3641 fn test_physical_order_tflite_nhwc_protos() {
3642 let score_threshold = 0.45;
3643 let iou_threshold = 0.45;
3644
3645 let protos_hwc = load_yolov8_protos().slice_move(s![0, .., .., ..]);
3647 let quant_protos = Quantization::new(0.02491161972284317, -117);
3648 let protos_f32_hwc = dequantize_ndarray::<_, _, f32>(protos_hwc.view(), quant_protos);
3649
3650 let boxes_2d = load_yolov8_boxes().slice_move(s![0, .., ..]);
3651 let quant_boxes = Quantization::new(0.021287761628627777, 31);
3652 let seg = dequantize_ndarray::<_, _, f32>(boxes_2d.view(), quant_boxes);
3653
3654 let mut ref_boxes: Vec<_> = Vec::with_capacity(10);
3656 let mut ref_masks: Vec<_> = Vec::with_capacity(10);
3657 decode_yolo_segdet_float(
3658 seg.view(),
3659 protos_f32_hwc.view(),
3660 score_threshold,
3661 iou_threshold,
3662 Some(configs::Nms::ClassAgnostic),
3663 &mut ref_boxes,
3664 &mut ref_masks,
3665 )
3666 .unwrap();
3667
3668 let protos_nhwc = protos_f32_hwc.clone().insert_axis(Axis(0)); let seg_3d = seg.insert_axis(Axis(0)); let decoder = DecoderBuilder::default()
3674 .with_config_yolo_segdet(
3675 configs::Detection {
3676 decoder: configs::DecoderType::Ultralytics,
3677 quantization: None,
3678 shape: vec![1, 116, 8400],
3679 dshape: vec![
3680 (DimName::Batch, 1),
3681 (DimName::NumFeatures, 116),
3682 (DimName::NumBoxes, 8400),
3683 ],
3684 normalized: Some(true),
3685 anchors: None,
3686 },
3687 configs::Protos {
3688 decoder: configs::DecoderType::Ultralytics,
3689 quantization: None,
3690 shape: vec![1, 160, 160, 32],
3691 dshape: vec![
3693 (DimName::Batch, 1),
3694 (DimName::Height, 160),
3695 (DimName::Width, 160),
3696 (DimName::NumProtos, 32),
3697 ],
3698 },
3699 None,
3700 )
3701 .with_score_threshold(score_threshold)
3702 .with_iou_threshold(iou_threshold)
3703 .build()
3704 .expect("config with NHWC protos dshape must build");
3705
3706 let mut cfg_boxes = Vec::with_capacity(10);
3707 let mut cfg_masks = Vec::with_capacity(10);
3708 decoder
3709 .decode_float(
3710 &[seg_3d.view().into_dyn(), protos_nhwc.view().into_dyn()],
3711 &mut cfg_boxes,
3712 &mut cfg_masks,
3713 )
3714 .unwrap();
3715
3716 assert_eq!(cfg_boxes.len(), ref_boxes.len(), "box count mismatch");
3717 for (c, r) in cfg_boxes.iter().zip(&ref_boxes) {
3718 assert!(
3719 c.equal_within_delta(r, 0.01),
3720 "NHWC-declared box does not match reference: {c:?} vs {r:?}"
3721 );
3722 }
3723 for (cm, rm) in cfg_masks.iter().zip(&ref_masks) {
3724 let cm_arr = segmentation_to_mask(seg_view(cm).view()).unwrap();
3725 let rm_arr = segmentation_to_mask(seg_view(rm).view()).unwrap();
3726 assert_eq!(
3727 cm_arr, rm_arr,
3728 "NHWC-declared mask must match reference pixel-for-pixel"
3729 );
3730 }
3731 }
3732
3733 #[test]
3742 fn test_physical_order_ara2_anchor_first_split_boxes() {
3743 use configs::{Boxes, Scores};
3744
3745 const N: usize = 8400;
3748 let mut boxes_canonical = Array3::<f32>::zeros((1, 4, N));
3749 let target_anchor = 42usize;
3750 boxes_canonical[[0, 0, target_anchor]] = 0.4; boxes_canonical[[0, 1, target_anchor]] = 0.5; boxes_canonical[[0, 2, target_anchor]] = 0.2; boxes_canonical[[0, 3, target_anchor]] = 0.2; let mut scores_canonical = Array3::<f32>::zeros((1, 80, N));
3757 scores_canonical[[0, 0, target_anchor]] = 0.9;
3758
3759 let ref_decoder = DecoderBuilder::default()
3761 .with_config_yolo_split_det(
3762 Boxes {
3763 decoder: configs::DecoderType::Ultralytics,
3764 quantization: None,
3765 shape: vec![1, 4, N],
3766 dshape: vec![
3767 (DimName::Batch, 1),
3768 (DimName::BoxCoords, 4),
3769 (DimName::NumBoxes, N),
3770 ],
3771 normalized: Some(true),
3772 },
3773 Scores {
3774 decoder: configs::DecoderType::Ultralytics,
3775 quantization: None,
3776 shape: vec![1, 80, N],
3777 dshape: vec![
3778 (DimName::Batch, 1),
3779 (DimName::NumClasses, 80),
3780 (DimName::NumBoxes, N),
3781 ],
3782 },
3783 )
3784 .with_score_threshold(0.5)
3785 .with_iou_threshold(0.5)
3786 .with_nms(Some(configs::Nms::ClassAgnostic))
3787 .build()
3788 .expect("reference canonical split decoder must build");
3789
3790 let mut ref_boxes = Vec::with_capacity(4);
3791 let mut ref_masks = Vec::with_capacity(0);
3792 ref_decoder
3793 .decode_float(
3794 &[
3795 boxes_canonical.view().into_dyn(),
3796 scores_canonical.view().into_dyn(),
3797 ],
3798 &mut ref_boxes,
3799 &mut ref_masks,
3800 )
3801 .unwrap();
3802 assert_eq!(ref_boxes.len(), 1, "reference should produce one box");
3803
3804 let boxes_ara2 = boxes_canonical.view().permuted_axes([0, 2, 1]).to_owned(); let scores_ara2 = scores_canonical.view().permuted_axes([0, 2, 1]).to_owned(); let ara2_decoder = DecoderBuilder::default()
3812 .with_config_yolo_split_det(
3813 Boxes {
3814 decoder: configs::DecoderType::Ultralytics,
3815 quantization: None,
3816 shape: vec![1, N, 4],
3817 dshape: vec![
3818 (DimName::Batch, 1),
3819 (DimName::NumBoxes, N),
3820 (DimName::BoxCoords, 4),
3821 ],
3822 normalized: Some(true),
3823 },
3824 Scores {
3825 decoder: configs::DecoderType::Ultralytics,
3826 quantization: None,
3827 shape: vec![1, N, 80],
3828 dshape: vec![
3829 (DimName::Batch, 1),
3830 (DimName::NumBoxes, N),
3831 (DimName::NumClasses, 80),
3832 ],
3833 },
3834 )
3835 .with_score_threshold(0.5)
3836 .with_iou_threshold(0.5)
3837 .with_nms(Some(configs::Nms::ClassAgnostic))
3838 .build()
3839 .expect("Ara-2 anchor-first decoder must build");
3840
3841 let mut ara2_boxes = Vec::with_capacity(4);
3842 let mut ara2_masks = Vec::with_capacity(0);
3843 ara2_decoder
3844 .decode_float(
3845 &[boxes_ara2.view().into_dyn(), scores_ara2.view().into_dyn()],
3846 &mut ara2_boxes,
3847 &mut ara2_masks,
3848 )
3849 .unwrap();
3850
3851 assert_eq!(
3852 ara2_boxes.len(),
3853 ref_boxes.len(),
3854 "Ara-2 anchor-first declaration must produce the same number \
3855 of boxes as the canonical features-first reference"
3856 );
3857 for (a, r) in ara2_boxes.iter().zip(&ref_boxes) {
3858 assert!(
3859 a.equal_within_delta(r, 1e-4),
3860 "Ara-2 box differs from reference: {a:?} vs {r:?}"
3861 );
3862 }
3863 }
3864
3865 #[test]
3869 fn test_physical_order_rejects_shape_dshape_mismatch() {
3870 let result = DecoderBuilder::default()
3871 .with_config_yolo_segdet(
3872 configs::Detection {
3873 decoder: configs::DecoderType::Ultralytics,
3874 quantization: None,
3875 shape: vec![1, 116, 8400],
3876 dshape: vec![
3877 (DimName::Batch, 1),
3878 (DimName::NumFeatures, 116),
3879 (DimName::NumBoxes, 8400),
3880 ],
3881 normalized: Some(true),
3882 anchors: None,
3883 },
3884 configs::Protos {
3885 decoder: configs::DecoderType::Ultralytics,
3886 quantization: None,
3887 shape: vec![1, 32, 160, 160],
3889 dshape: vec![
3892 (DimName::Batch, 1),
3893 (DimName::Height, 160),
3894 (DimName::Width, 160),
3895 (DimName::NumProtos, 32),
3896 ],
3897 },
3898 None,
3899 )
3900 .build();
3901
3902 match result {
3903 Err(DecoderError::InvalidConfig(msg)) => {
3904 assert!(
3905 msg.contains("does not match shape"),
3906 "expected shape/dshape size mismatch error, got: {msg}"
3907 );
3908 }
3909 other => panic!("expected InvalidConfig, got {other:?}"),
3910 }
3911 }
3912
3913 #[test]
3916 fn test_physical_order_rejects_duplicate_dshape_axis() {
3917 let result = DecoderBuilder::default()
3918 .with_config_yolo_split_det(
3919 configs::Boxes {
3920 decoder: configs::DecoderType::Ultralytics,
3921 quantization: None,
3922 shape: vec![1, 4, 8400],
3923 dshape: vec![
3924 (DimName::Batch, 1),
3925 (DimName::BoxCoords, 4),
3926 (DimName::BoxCoords, 4), ],
3928 normalized: Some(true),
3929 },
3930 configs::Scores {
3931 decoder: configs::DecoderType::Ultralytics,
3932 quantization: None,
3933 shape: vec![1, 80, 8400],
3934 dshape: vec![
3935 (DimName::Batch, 1),
3936 (DimName::NumClasses, 80),
3937 (DimName::NumBoxes, 8400),
3938 ],
3939 },
3940 )
3941 .build();
3942
3943 match result {
3948 Err(DecoderError::InvalidConfig(msg)) => {
3949 assert!(
3950 msg.contains("appears at both index") || msg.contains("does not match shape"),
3951 "expected positional or duplicate-axis error, got: {msg}"
3952 );
3953 }
3954 other => panic!("expected InvalidConfig, got {other:?}"),
3955 }
3956
3957 let result = DecoderBuilder::default()
3962 .with_config_yolo_split_det(
3963 configs::Boxes {
3964 decoder: configs::DecoderType::Ultralytics,
3965 quantization: None,
3966 shape: vec![1, 1, 4, 8400],
3967 dshape: vec![
3968 (DimName::Batch, 1),
3969 (DimName::Batch, 1), (DimName::BoxCoords, 4),
3971 (DimName::NumBoxes, 8400),
3972 ],
3973 normalized: Some(true),
3974 },
3975 configs::Scores {
3976 decoder: configs::DecoderType::Ultralytics,
3977 quantization: None,
3978 shape: vec![1, 80, 8400],
3979 dshape: vec![
3980 (DimName::Batch, 1),
3981 (DimName::NumClasses, 80),
3982 (DimName::NumBoxes, 8400),
3983 ],
3984 },
3985 )
3986 .build();
3987 match result {
3988 Err(DecoderError::InvalidConfig(msg)) => {
3989 assert!(
3990 msg.contains("appears at both index"),
3991 "expected duplicate-axis error, got: {msg}"
3992 );
3993 }
3994 other => panic!("expected InvalidConfig, got {other:?}"),
3995 }
3996 }
3997
3998 #[test]
4004 fn test_physical_order_dshape_omitted_decodes_numerically() {
4005 let score_threshold = 0.45;
4006 let iou_threshold = 0.45;
4007
4008 let protos_hwc = load_yolov8_protos().slice_move(s![0, .., .., ..]);
4009 let quant_protos = Quantization::new(0.02491161972284317, -117);
4010 let protos_f32_hwc = dequantize_ndarray::<_, _, f32>(protos_hwc.view(), quant_protos);
4011
4012 let boxes_2d = load_yolov8_boxes().slice_move(s![0, .., ..]);
4013 let quant_boxes = Quantization::new(0.021287761628627777, 31);
4014 let seg = dequantize_ndarray::<_, _, f32>(boxes_2d.view(), quant_boxes);
4015
4016 let protos_nhwc = protos_f32_hwc.clone().insert_axis(Axis(0));
4017 let seg_3d = seg.insert_axis(Axis(0));
4018
4019 let build_decoder = |det_dshape: Vec<(DimName, usize)>,
4020 proto_dshape: Vec<(DimName, usize)>| {
4021 DecoderBuilder::default()
4022 .with_config_yolo_segdet(
4023 configs::Detection {
4024 decoder: configs::DecoderType::Ultralytics,
4025 quantization: None,
4026 shape: vec![1, 116, 8400],
4027 dshape: det_dshape,
4028 normalized: Some(true),
4029 anchors: None,
4030 },
4031 configs::Protos {
4032 decoder: configs::DecoderType::Ultralytics,
4033 quantization: None,
4034 shape: vec![1, 160, 160, 32],
4035 dshape: proto_dshape,
4036 },
4037 None,
4038 )
4039 .with_score_threshold(score_threshold)
4040 .with_iou_threshold(iou_threshold)
4041 .build()
4042 .unwrap()
4043 };
4044
4045 let dshaped = build_decoder(
4047 vec![
4048 (DimName::Batch, 1),
4049 (DimName::NumFeatures, 116),
4050 (DimName::NumBoxes, 8400),
4051 ],
4052 vec![
4053 (DimName::Batch, 1),
4054 (DimName::Height, 160),
4055 (DimName::Width, 160),
4056 (DimName::NumProtos, 32),
4057 ],
4058 );
4059 let mut dshaped_boxes = Vec::new();
4060 let mut dshaped_masks = Vec::new();
4061 dshaped
4062 .decode_float(
4063 &[seg_3d.view().into_dyn(), protos_nhwc.view().into_dyn()],
4064 &mut dshaped_boxes,
4065 &mut dshaped_masks,
4066 )
4067 .unwrap();
4068
4069 let bare = build_decoder(vec![], vec![]);
4072 let mut bare_boxes = Vec::new();
4073 let mut bare_masks = Vec::new();
4074 bare.decode_float(
4075 &[seg_3d.view().into_dyn(), protos_nhwc.view().into_dyn()],
4076 &mut bare_boxes,
4077 &mut bare_masks,
4078 )
4079 .unwrap();
4080
4081 assert_eq!(bare_boxes.len(), dshaped_boxes.len());
4082 for (b, d) in bare_boxes.iter().zip(&dshaped_boxes) {
4083 assert!(
4084 b.equal_within_delta(d, 1e-4),
4085 "dshape-omitted box {b:?} differs from dshape-populated {d:?}"
4086 );
4087 }
4088 for (bm, dm) in bare_masks.iter().zip(&dshaped_masks) {
4089 let bm_arr = segmentation_to_mask(seg_view(bm).view()).unwrap();
4090 let dm_arr = segmentation_to_mask(seg_view(dm).view()).unwrap();
4091 assert_eq!(
4092 bm_arr, dm_arr,
4093 "dshape-omitted mask must match dshape-populated pixel-for-pixel"
4094 );
4095 }
4096 }
4097
4098 #[test]
4108 fn test_physical_order_ara2_4d_anchor_first_with_padding() {
4109 const N: usize = 8400;
4114 let mut boxes = Array3::<f32>::zeros((1, N, 4));
4115 let target = 42usize;
4116 boxes[[0, target, 0]] = 0.4;
4117 boxes[[0, target, 1]] = 0.5;
4118 boxes[[0, target, 2]] = 0.2;
4119 boxes[[0, target, 3]] = 0.2;
4120 let mut scores = Array3::<f32>::zeros((1, N, 80));
4121 scores[[0, target, 0]] = 0.9;
4122
4123 let json = r#"{
4129 "schema_version": 2,
4130 "decoder_version": "yolov8",
4131 "nms": "class_agnostic",
4132 "outputs": [
4133 {"name": "boxes", "type": "boxes",
4134 "shape": [1, 8400, 1, 4],
4135 "dshape": [{"batch":1},{"num_boxes":8400},{"padding":1},{"box_coords":4}],
4136 "encoding": "direct",
4137 "decoder": "ultralytics",
4138 "normalized": true},
4139 {"name": "scores", "type": "scores",
4140 "shape": [1, 8400, 1, 80],
4141 "dshape": [{"batch":1},{"num_boxes":8400},{"padding":1},{"num_classes":80}],
4142 "decoder": "ultralytics",
4143 "score_format": "per_class"}
4144 ]
4145 }"#;
4146 let decoder = DecoderBuilder::default()
4147 .with_config_json_str(json.to_string())
4148 .with_score_threshold(0.5)
4149 .with_iou_threshold(0.5)
4150 .build()
4151 .expect("4D anchor-first schema should build via squeeze_padding_dims");
4152
4153 let mut out_boxes = Vec::with_capacity(4);
4154 let mut out_masks = Vec::with_capacity(0);
4155 decoder
4156 .decode_float(
4157 &[boxes.view().into_dyn(), scores.view().into_dyn()],
4158 &mut out_boxes,
4159 &mut out_masks,
4160 )
4161 .unwrap();
4162
4163 assert_eq!(
4164 out_boxes.len(),
4165 1,
4166 "4D anchor-first with padding should decode exactly one box from the seeded anchor"
4167 );
4168 let b = &out_boxes[0];
4169 assert!((b.bbox.xmin - 0.3).abs() < 1e-3, "xmin wrong: {b:?}");
4171 assert!((b.bbox.ymin - 0.4).abs() < 1e-3, "ymin wrong: {b:?}");
4172 assert!((b.bbox.xmax - 0.5).abs() < 1e-3, "xmax wrong: {b:?}");
4173 assert!((b.bbox.ymax - 0.6).abs() < 1e-3, "ymax wrong: {b:?}");
4174 assert_eq!(b.label, 0);
4175 assert!(b.score > 0.85, "score {}: {b:?}", b.score);
4176 }
4177}
4178
4179#[cfg(feature = "tracker")]
4180#[cfg(test)]
4181#[cfg_attr(coverage_nightly, coverage(off))]
4182mod decoder_tracked_tests {
4183
4184 use crate::DetectBoxTestExt;
4185 use edgefirst_tracker::{ByteTrackBuilder, Tracker};
4186 use ndarray::{array, s, Array, Array2, Array3, Array4, ArrayView, Axis, Dimension};
4187 use num_traits::{AsPrimitive, Float, PrimInt};
4188 use rand::{RngExt, SeedableRng};
4189 use rand_distr::StandardNormal;
4190
4191 use crate::{
4192 configs::{self, DimName},
4193 dequantize_ndarray, BoundingBox, DecoderBuilder, DetectBox, Quantization,
4194 };
4195
4196 pub fn quantize_ndarray<T: PrimInt + 'static, D: Dimension, F: Float + AsPrimitive<T>>(
4197 input: ArrayView<F, D>,
4198 quant: Quantization,
4199 ) -> Array<T, D>
4200 where
4201 i32: num_traits::AsPrimitive<F>,
4202 f32: num_traits::AsPrimitive<F>,
4203 {
4204 let zero_point = quant.zero_point.as_();
4205 let div_scale = F::one() / quant.scale.as_();
4206 if zero_point != F::zero() {
4207 input.mapv(|d| (d * div_scale + zero_point).round().as_())
4208 } else {
4209 input.mapv(|d| (d * div_scale).round().as_())
4210 }
4211 }
4212
4213 #[test]
4214 fn test_decoder_tracked_random_jitter() {
4215 use crate::configs::{DecoderType, Nms};
4216 use crate::DecoderBuilder;
4217
4218 let score_threshold = 0.25;
4219 let iou_threshold = 0.1;
4220 let out = edgefirst_bench::testdata::read("yolov8s_80_classes.bin");
4221 let out = unsafe { std::slice::from_raw_parts(out.as_ptr() as *const i8, out.len()) };
4222 let out = Array3::from_shape_vec((1, 84, 8400), out.to_vec()).unwrap();
4223 let quant = (0.0040811873, -123).into();
4224
4225 let decoder = DecoderBuilder::default()
4226 .with_config_yolo_det(
4227 crate::configs::Detection {
4228 decoder: DecoderType::Ultralytics,
4229 shape: vec![1, 84, 8400],
4230 anchors: None,
4231 quantization: Some(quant),
4232 dshape: vec![
4233 (crate::configs::DimName::Batch, 1),
4234 (crate::configs::DimName::NumFeatures, 84),
4235 (crate::configs::DimName::NumBoxes, 8400),
4236 ],
4237 normalized: Some(true),
4238 },
4239 None,
4240 )
4241 .with_score_threshold(score_threshold)
4242 .with_iou_threshold(iou_threshold)
4243 .with_nms(Some(Nms::ClassAgnostic))
4244 .build()
4245 .unwrap();
4246 let mut rng = rand::rngs::StdRng::seed_from_u64(0xAB_BEEF); let expected_boxes = [
4249 crate::DetectBox {
4250 bbox: crate::BoundingBox {
4251 xmin: 0.5285137,
4252 ymin: 0.05305544,
4253 xmax: 0.87541467,
4254 ymax: 0.9998909,
4255 },
4256 score: 0.5591227,
4257 label: 0,
4258 },
4259 crate::DetectBox {
4260 bbox: crate::BoundingBox {
4261 xmin: 0.130598,
4262 ymin: 0.43260583,
4263 xmax: 0.35098213,
4264 ymax: 0.9958097,
4265 },
4266 score: 0.33057618,
4267 label: 75,
4268 },
4269 ];
4270
4271 let mut tracker = ByteTrackBuilder::new()
4272 .track_update(0.1)
4273 .track_high_conf(0.3)
4274 .build();
4275
4276 let mut output_boxes = Vec::with_capacity(50);
4277 let mut output_masks = Vec::with_capacity(50);
4278 let mut output_tracks = Vec::with_capacity(50);
4279
4280 decoder
4281 .decode_tracked_quantized(
4282 &mut tracker,
4283 0,
4284 &[out.view().into()],
4285 &mut output_boxes,
4286 &mut output_masks,
4287 &mut output_tracks,
4288 )
4289 .unwrap();
4290
4291 assert_eq!(output_boxes.len(), 2);
4292 assert!(output_boxes[0].equal_within_delta(&expected_boxes[0], 1e-6));
4293 assert!(output_boxes[1].equal_within_delta(&expected_boxes[1], 1e-6));
4294
4295 let mut last_boxes = output_boxes.clone();
4296
4297 for i in 1..=100 {
4298 let mut out = out.clone();
4299 let mut x_values = out.slice_mut(s![0, 0, ..]);
4301 for x in x_values.iter_mut() {
4302 let r: f32 = rng.sample(StandardNormal);
4303 let r = r.clamp(-2.0, 2.0) / 2.0;
4304 *x = x.saturating_add((r * 1e-2 / quant.0) as i8);
4305 }
4306
4307 let mut y_values = out.slice_mut(s![0, 1, ..]);
4308 for y in y_values.iter_mut() {
4309 let r: f32 = rng.sample(StandardNormal);
4310 let r = r.clamp(-2.0, 2.0) / 2.0;
4311 *y = y.saturating_add((r * 1e-2 / quant.0) as i8);
4312 }
4313
4314 decoder
4315 .decode_tracked_quantized(
4316 &mut tracker,
4317 100_000_000 * i / 3, &[out.view().into()],
4319 &mut output_boxes,
4320 &mut output_masks,
4321 &mut output_tracks,
4322 )
4323 .unwrap();
4324
4325 assert_eq!(output_boxes.len(), 2);
4326 assert!(output_boxes[0].equal_within_delta(&expected_boxes[0], 5e-3));
4327 assert!(output_boxes[1].equal_within_delta(&expected_boxes[1], 5e-3));
4328
4329 assert!(output_boxes[0].equal_within_delta(&last_boxes[0], 1e-3));
4330 assert!(output_boxes[1].equal_within_delta(&last_boxes[1], 1e-3));
4331 last_boxes = output_boxes.clone();
4332 }
4333 }
4334
4335 fn real_data_expected_boxes() -> [DetectBox; 2] {
4338 [
4339 DetectBox {
4340 bbox: BoundingBox {
4341 xmin: 0.08515105,
4342 ymin: 0.7131401,
4343 xmax: 0.29802868,
4344 ymax: 0.8195788,
4345 },
4346 score: 0.91537374,
4347 label: 23,
4348 },
4349 DetectBox {
4350 bbox: BoundingBox {
4351 xmin: 0.59605736,
4352 ymin: 0.25545314,
4353 xmax: 0.93666154,
4354 ymax: 0.72378385,
4355 },
4356 score: 0.91537374,
4357 label: 23,
4358 },
4359 ]
4360 }
4361
4362 fn e2e_expected_boxes_quant() -> [DetectBox; 1] {
4363 [DetectBox {
4364 bbox: BoundingBox {
4365 xmin: 0.12549022,
4366 ymin: 0.12549022,
4367 xmax: 0.23529413,
4368 ymax: 0.23529413,
4369 },
4370 score: 0.98823535,
4371 label: 2,
4372 }]
4373 }
4374
4375 fn e2e_expected_boxes_float() -> [DetectBox; 1] {
4376 [DetectBox {
4377 bbox: BoundingBox {
4378 xmin: 0.1234,
4379 ymin: 0.1234,
4380 xmax: 0.2345,
4381 ymax: 0.2345,
4382 },
4383 score: 0.9876,
4384 label: 2,
4385 }]
4386 }
4387
4388 fn build_yolo_split_segdet_decoder(
4389 score_threshold: f32,
4390 iou_threshold: f32,
4391 quant_boxes: (f32, i32),
4392 quant_protos: (f32, i32),
4393 ) -> crate::Decoder {
4394 DecoderBuilder::default()
4395 .with_config_yolo_split_segdet(
4396 configs::Boxes {
4397 decoder: configs::DecoderType::Ultralytics,
4398 quantization: Some(quant_boxes.into()),
4399 shape: vec![1, 4, 8400],
4400 dshape: vec![
4401 (DimName::Batch, 1),
4402 (DimName::BoxCoords, 4),
4403 (DimName::NumBoxes, 8400),
4404 ],
4405 normalized: Some(true),
4406 },
4407 configs::Scores {
4408 decoder: configs::DecoderType::Ultralytics,
4409 quantization: Some(quant_boxes.into()),
4410 shape: vec![1, 80, 8400],
4411 dshape: vec![
4412 (DimName::Batch, 1),
4413 (DimName::NumClasses, 80),
4414 (DimName::NumBoxes, 8400),
4415 ],
4416 },
4417 configs::MaskCoefficients {
4418 decoder: configs::DecoderType::Ultralytics,
4419 quantization: Some(quant_boxes.into()),
4420 shape: vec![1, 32, 8400],
4421 dshape: vec![
4422 (DimName::Batch, 1),
4423 (DimName::NumProtos, 32),
4424 (DimName::NumBoxes, 8400),
4425 ],
4426 },
4427 configs::Protos {
4428 decoder: configs::DecoderType::Ultralytics,
4429 quantization: Some(quant_protos.into()),
4430 shape: vec![1, 160, 160, 32],
4431 dshape: vec![
4432 (DimName::Batch, 1),
4433 (DimName::Height, 160),
4434 (DimName::Width, 160),
4435 (DimName::NumProtos, 32),
4436 ],
4437 },
4438 )
4439 .with_score_threshold(score_threshold)
4440 .with_iou_threshold(iou_threshold)
4441 .build()
4442 .unwrap()
4443 }
4444
4445 fn build_yolov8_seg_decoder(score_threshold: f32, iou_threshold: f32) -> crate::Decoder {
4446 let config_yaml = edgefirst_bench::testdata::read_to_string("yolov8_seg.yaml");
4447 DecoderBuilder::default()
4448 .with_config_yaml_str(config_yaml.to_string())
4449 .with_score_threshold(score_threshold)
4450 .with_iou_threshold(iou_threshold)
4451 .build()
4452 .unwrap()
4453 }
4454
4455 macro_rules! real_data_tracked_test {
4462 ($name:ident, quantized, $layout:ident, $output:ident) => {
4463 #[test]
4464 fn $name() {
4465 let is_split = matches!(stringify!($layout), "split");
4466 let is_proto = matches!(stringify!($output), "proto");
4467
4468 let score_threshold = 0.45;
4469 let iou_threshold = 0.45;
4470 let quant_boxes = (0.021287762_f32, 31_i32);
4471 let quant_protos = (0.02491162_f32, -117_i32);
4472
4473 let raw_boxes = edgefirst_bench::testdata::read("yolov8_boxes_116x8400.bin");
4474 let raw_boxes = unsafe {
4475 std::slice::from_raw_parts(raw_boxes.as_ptr() as *const i8, raw_boxes.len())
4476 };
4477 let boxes_i8 =
4478 ndarray::Array3::from_shape_vec((1, 116, 8400), raw_boxes.to_vec()).unwrap();
4479
4480 let raw_protos = edgefirst_bench::testdata::read("yolov8_protos_160x160x32.bin");
4481 let raw_protos = unsafe {
4482 std::slice::from_raw_parts(raw_protos.as_ptr() as *const i8, raw_protos.len())
4483 };
4484 let protos_i8 =
4485 ndarray::Array4::from_shape_vec((1, 160, 160, 32), raw_protos.to_vec())
4486 .unwrap();
4487
4488 let mask_split = boxes_i8.slice(s![.., 84.., ..]).to_owned();
4490 let mut scores_split = boxes_i8.slice(s![.., 4..84, ..]).to_owned();
4491 let boxes_split = boxes_i8.slice(s![.., ..4, ..]).to_owned();
4492 let mut boxes_combined = boxes_i8;
4493
4494 let decoder = if is_split {
4495 build_yolo_split_segdet_decoder(
4496 score_threshold,
4497 iou_threshold,
4498 quant_boxes,
4499 quant_protos,
4500 )
4501 } else {
4502 build_yolov8_seg_decoder(score_threshold, iou_threshold)
4503 };
4504
4505 let expected = real_data_expected_boxes();
4506 let mut tracker = ByteTrackBuilder::new()
4507 .track_update(0.1)
4508 .track_high_conf(0.7)
4509 .build();
4510 let mut output_boxes = Vec::with_capacity(50);
4511 let mut output_tracks = Vec::with_capacity(50);
4512
4513 if is_proto {
4515 {
4516 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> = if is_split {
4517 vec![
4518 boxes_split.view().into(),
4519 scores_split.view().into(),
4520 mask_split.view().into(),
4521 protos_i8.view().into(),
4522 ]
4523 } else {
4524 vec![boxes_combined.view().into(), protos_i8.view().into()]
4525 };
4526 decoder
4527 .decode_tracked_quantized_proto(
4528 &mut tracker,
4529 0,
4530 &inputs,
4531 &mut output_boxes,
4532 &mut output_tracks,
4533 )
4534 .unwrap();
4535 }
4536 assert_eq!(output_boxes.len(), 2);
4537 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
4538 assert!(output_boxes[1].equal_within_delta(&expected[1], 1.0 / 160.0));
4539
4540 if is_split {
4542 for score in scores_split.iter_mut() {
4543 *score = i8::MIN;
4544 }
4545 } else {
4546 for score in boxes_combined.slice_mut(s![0, 4..84, ..]).iter_mut() {
4547 *score = i8::MIN;
4548 }
4549 }
4550
4551 let proto_result = {
4552 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> = if is_split {
4553 vec![
4554 boxes_split.view().into(),
4555 scores_split.view().into(),
4556 mask_split.view().into(),
4557 protos_i8.view().into(),
4558 ]
4559 } else {
4560 vec![boxes_combined.view().into(), protos_i8.view().into()]
4561 };
4562 decoder
4563 .decode_tracked_quantized_proto(
4564 &mut tracker,
4565 100_000_000 / 3,
4566 &inputs,
4567 &mut output_boxes,
4568 &mut output_tracks,
4569 )
4570 .unwrap()
4571 };
4572 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
4573 assert!(output_boxes[1].equal_within_delta(&expected[1], 1e-6));
4574 assert!(proto_result.is_some_and(|x| x.mask_coefficients.shape()[0] == 0));
4575 } else {
4576 let mut output_masks = Vec::with_capacity(50);
4577 {
4578 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> = if is_split {
4579 vec![
4580 boxes_split.view().into(),
4581 scores_split.view().into(),
4582 mask_split.view().into(),
4583 protos_i8.view().into(),
4584 ]
4585 } else {
4586 vec![boxes_combined.view().into(), protos_i8.view().into()]
4587 };
4588 decoder
4589 .decode_tracked_quantized(
4590 &mut tracker,
4591 0,
4592 &inputs,
4593 &mut output_boxes,
4594 &mut output_masks,
4595 &mut output_tracks,
4596 )
4597 .unwrap();
4598 }
4599 assert_eq!(output_boxes.len(), 2);
4600 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
4601 assert!(output_boxes[1].equal_within_delta(&expected[1], 1.0 / 160.0));
4602
4603 if is_split {
4604 for score in scores_split.iter_mut() {
4605 *score = i8::MIN;
4606 }
4607 } else {
4608 for score in boxes_combined.slice_mut(s![0, 4..84, ..]).iter_mut() {
4609 *score = i8::MIN;
4610 }
4611 }
4612
4613 {
4614 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> = if is_split {
4615 vec![
4616 boxes_split.view().into(),
4617 scores_split.view().into(),
4618 mask_split.view().into(),
4619 protos_i8.view().into(),
4620 ]
4621 } else {
4622 vec![boxes_combined.view().into(), protos_i8.view().into()]
4623 };
4624 decoder
4625 .decode_tracked_quantized(
4626 &mut tracker,
4627 100_000_000 / 3,
4628 &inputs,
4629 &mut output_boxes,
4630 &mut output_masks,
4631 &mut output_tracks,
4632 )
4633 .unwrap();
4634 }
4635 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
4636 assert!(output_boxes[1].equal_within_delta(&expected[1], 1e-6));
4637 assert!(output_masks.is_empty());
4638 }
4639 }
4640 };
4641 ($name:ident, float, $layout:ident, $output:ident) => {
4642 #[test]
4643 fn $name() {
4644 let is_split = matches!(stringify!($layout), "split");
4645 let is_proto = matches!(stringify!($output), "proto");
4646
4647 let score_threshold = 0.45;
4648 let iou_threshold = 0.45;
4649 let quant_boxes = (0.021287762_f32, 31_i32);
4650 let quant_protos = (0.02491162_f32, -117_i32);
4651
4652 let raw_boxes = edgefirst_bench::testdata::read("yolov8_boxes_116x8400.bin");
4653 let raw_boxes = unsafe {
4654 std::slice::from_raw_parts(raw_boxes.as_ptr() as *const i8, raw_boxes.len())
4655 };
4656 let boxes_i8 =
4657 ndarray::Array3::from_shape_vec((1, 116, 8400), raw_boxes.to_vec()).unwrap();
4658 let boxes_f32 = dequantize_ndarray(boxes_i8.view(), quant_boxes.into());
4659
4660 let raw_protos = edgefirst_bench::testdata::read("yolov8_protos_160x160x32.bin");
4661 let raw_protos = unsafe {
4662 std::slice::from_raw_parts(raw_protos.as_ptr() as *const i8, raw_protos.len())
4663 };
4664 let protos_i8 =
4665 ndarray::Array4::from_shape_vec((1, 160, 160, 32), raw_protos.to_vec())
4666 .unwrap();
4667 let protos_f32 = dequantize_ndarray(protos_i8.view(), quant_protos.into());
4668
4669 let mask_split = boxes_f32.slice(s![.., 84.., ..]).to_owned();
4671 let mut scores_split = boxes_f32.slice(s![.., 4..84, ..]).to_owned();
4672 let boxes_split = boxes_f32.slice(s![.., ..4, ..]).to_owned();
4673 let mut boxes_combined = boxes_f32;
4674
4675 let decoder = if is_split {
4676 build_yolo_split_segdet_decoder(
4677 score_threshold,
4678 iou_threshold,
4679 quant_boxes,
4680 quant_protos,
4681 )
4682 } else {
4683 build_yolov8_seg_decoder(score_threshold, iou_threshold)
4684 };
4685
4686 let expected = real_data_expected_boxes();
4687 let mut tracker = ByteTrackBuilder::new()
4688 .track_update(0.1)
4689 .track_high_conf(0.7)
4690 .build();
4691 let mut output_boxes = Vec::with_capacity(50);
4692 let mut output_tracks = Vec::with_capacity(50);
4693
4694 if is_proto {
4695 {
4696 let inputs = if is_split {
4697 vec![
4698 boxes_split.view().into_dyn(),
4699 scores_split.view().into_dyn(),
4700 mask_split.view().into_dyn(),
4701 protos_f32.view().into_dyn(),
4702 ]
4703 } else {
4704 vec![
4705 boxes_combined.view().into_dyn(),
4706 protos_f32.view().into_dyn(),
4707 ]
4708 };
4709 decoder
4710 .decode_tracked_float_proto(
4711 &mut tracker,
4712 0,
4713 &inputs,
4714 &mut output_boxes,
4715 &mut output_tracks,
4716 )
4717 .unwrap();
4718 }
4719 assert_eq!(output_boxes.len(), 2);
4720 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
4721 assert!(output_boxes[1].equal_within_delta(&expected[1], 1.0 / 160.0));
4722
4723 if is_split {
4724 for score in scores_split.iter_mut() {
4725 *score = 0.0;
4726 }
4727 } else {
4728 for score in boxes_combined.slice_mut(s![0, 4..84, ..]).iter_mut() {
4729 *score = 0.0;
4730 }
4731 }
4732
4733 let proto_result = {
4734 let inputs = if is_split {
4735 vec![
4736 boxes_split.view().into_dyn(),
4737 scores_split.view().into_dyn(),
4738 mask_split.view().into_dyn(),
4739 protos_f32.view().into_dyn(),
4740 ]
4741 } else {
4742 vec![
4743 boxes_combined.view().into_dyn(),
4744 protos_f32.view().into_dyn(),
4745 ]
4746 };
4747 decoder
4748 .decode_tracked_float_proto(
4749 &mut tracker,
4750 100_000_000 / 3,
4751 &inputs,
4752 &mut output_boxes,
4753 &mut output_tracks,
4754 )
4755 .unwrap()
4756 };
4757 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
4758 assert!(output_boxes[1].equal_within_delta(&expected[1], 1e-6));
4759 assert!(proto_result.is_some_and(|x| x.mask_coefficients.shape()[0] == 0));
4760 } else {
4761 let mut output_masks = Vec::with_capacity(50);
4762 {
4763 let inputs = if is_split {
4764 vec![
4765 boxes_split.view().into_dyn(),
4766 scores_split.view().into_dyn(),
4767 mask_split.view().into_dyn(),
4768 protos_f32.view().into_dyn(),
4769 ]
4770 } else {
4771 vec![
4772 boxes_combined.view().into_dyn(),
4773 protos_f32.view().into_dyn(),
4774 ]
4775 };
4776 decoder
4777 .decode_tracked_float(
4778 &mut tracker,
4779 0,
4780 &inputs,
4781 &mut output_boxes,
4782 &mut output_masks,
4783 &mut output_tracks,
4784 )
4785 .unwrap();
4786 }
4787 assert_eq!(output_boxes.len(), 2);
4788 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
4789 assert!(output_boxes[1].equal_within_delta(&expected[1], 1.0 / 160.0));
4790
4791 if is_split {
4792 for score in scores_split.iter_mut() {
4793 *score = 0.0;
4794 }
4795 } else {
4796 for score in boxes_combined.slice_mut(s![0, 4..84, ..]).iter_mut() {
4797 *score = 0.0;
4798 }
4799 }
4800
4801 {
4802 let inputs = if is_split {
4803 vec![
4804 boxes_split.view().into_dyn(),
4805 scores_split.view().into_dyn(),
4806 mask_split.view().into_dyn(),
4807 protos_f32.view().into_dyn(),
4808 ]
4809 } else {
4810 vec![
4811 boxes_combined.view().into_dyn(),
4812 protos_f32.view().into_dyn(),
4813 ]
4814 };
4815 decoder
4816 .decode_tracked_float(
4817 &mut tracker,
4818 100_000_000 / 3,
4819 &inputs,
4820 &mut output_boxes,
4821 &mut output_masks,
4822 &mut output_tracks,
4823 )
4824 .unwrap();
4825 }
4826 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
4827 assert!(output_boxes[1].equal_within_delta(&expected[1], 1e-6));
4828 assert!(output_masks.is_empty());
4829 }
4830 }
4831 };
4832 }
4833
4834 real_data_tracked_test!(test_decoder_tracked_segdet, quantized, combined, masks);
4835 real_data_tracked_test!(test_decoder_tracked_segdet_float, float, combined, masks);
4836 real_data_tracked_test!(
4837 test_decoder_tracked_segdet_proto,
4838 quantized,
4839 combined,
4840 proto
4841 );
4842 real_data_tracked_test!(
4843 test_decoder_tracked_segdet_proto_float,
4844 float,
4845 combined,
4846 proto
4847 );
4848 real_data_tracked_test!(test_decoder_tracked_segdet_split, quantized, split, masks);
4849 real_data_tracked_test!(test_decoder_tracked_segdet_split_float, float, split, masks);
4850 real_data_tracked_test!(
4851 test_decoder_tracked_segdet_split_proto,
4852 quantized,
4853 split,
4854 proto
4855 );
4856 real_data_tracked_test!(
4857 test_decoder_tracked_segdet_split_proto_float,
4858 float,
4859 split,
4860 proto
4861 );
4862
4863 const E2E_COMBINED_CONFIG: &str = "
4869decoder_version: yolo26
4870outputs:
4871 - type: detection
4872 decoder: ultralytics
4873 quantization: [0.00784313725490196, 0]
4874 shape: [1, 10, 38]
4875 dshape:
4876 - [batch, 1]
4877 - [num_boxes, 10]
4878 - [num_features, 38]
4879 normalized: true
4880 - type: protos
4881 decoder: ultralytics
4882 quantization: [0.0039215686274509803921568627451, 128]
4883 shape: [1, 160, 160, 32]
4884 dshape:
4885 - [batch, 1]
4886 - [height, 160]
4887 - [width, 160]
4888 - [num_protos, 32]
4889";
4890
4891 const E2E_SPLIT_CONFIG: &str = "
4892decoder_version: yolo26
4893outputs:
4894 - type: boxes
4895 decoder: ultralytics
4896 quantization: [0.00784313725490196, 0]
4897 shape: [1, 10, 4]
4898 dshape:
4899 - [batch, 1]
4900 - [num_boxes, 10]
4901 - [box_coords, 4]
4902 normalized: true
4903 - type: scores
4904 decoder: ultralytics
4905 quantization: [0.00784313725490196, 0]
4906 shape: [1, 10, 1]
4907 dshape:
4908 - [batch, 1]
4909 - [num_boxes, 10]
4910 - [num_classes, 1]
4911 - type: classes
4912 decoder: ultralytics
4913 quantization: [0.00784313725490196, 0]
4914 shape: [1, 10, 1]
4915 dshape:
4916 - [batch, 1]
4917 - [num_boxes, 10]
4918 - [num_classes, 1]
4919 - type: mask_coefficients
4920 decoder: ultralytics
4921 quantization: [0.00784313725490196, 0]
4922 shape: [1, 10, 32]
4923 dshape:
4924 - [batch, 1]
4925 - [num_boxes, 10]
4926 - [num_protos, 32]
4927 - type: protos
4928 decoder: ultralytics
4929 quantization: [0.0039215686274509803921568627451, 128]
4930 shape: [1, 160, 160, 32]
4931 dshape:
4932 - [batch, 1]
4933 - [height, 160]
4934 - [width, 160]
4935 - [num_protos, 32]
4936";
4937
4938 macro_rules! e2e_tracked_test {
4939 ($name:ident, quantized, $layout:ident, $output:ident) => {
4940 #[test]
4941 fn $name() {
4942 let is_split = matches!(stringify!($layout), "split");
4943 let is_proto = matches!(stringify!($output), "proto");
4944
4945 let score_threshold = 0.45;
4946 let iou_threshold = 0.45;
4947
4948 let mut boxes = Array2::zeros((10, 4));
4949 let mut scores = Array2::zeros((10, 1));
4950 let mut classes = Array2::zeros((10, 1));
4951 let mask = Array2::zeros((10, 32));
4952 let protos = Array3::<f64>::zeros((160, 160, 32));
4953 let protos = protos.insert_axis(Axis(0));
4954 let protos_quant = (1.0 / 255.0, 0.0);
4955 let protos: Array4<u8> = quantize_ndarray(protos.view(), protos_quant.into());
4956
4957 boxes
4958 .slice_mut(s![0, ..])
4959 .assign(&array![0.1234, 0.1234, 0.2345, 0.2345]);
4960 scores.slice_mut(s![0, ..]).assign(&array![0.9876]);
4961 classes.slice_mut(s![0, ..]).assign(&array![2.0]);
4962
4963 let detect_quant = (2.0 / 255.0, 0.0);
4964
4965 let decoder = if is_split {
4966 DecoderBuilder::default()
4967 .with_config_yaml_str(E2E_SPLIT_CONFIG.to_string())
4968 .with_score_threshold(score_threshold)
4969 .with_iou_threshold(iou_threshold)
4970 .build()
4971 .unwrap()
4972 } else {
4973 DecoderBuilder::default()
4974 .with_config_yaml_str(E2E_COMBINED_CONFIG.to_string())
4975 .with_score_threshold(score_threshold)
4976 .with_iou_threshold(iou_threshold)
4977 .build()
4978 .unwrap()
4979 };
4980
4981 let expected = e2e_expected_boxes_quant();
4982 let mut tracker = ByteTrackBuilder::new()
4983 .track_update(0.1)
4984 .track_high_conf(0.7)
4985 .build();
4986 let mut output_boxes = Vec::with_capacity(50);
4987 let mut output_tracks = Vec::with_capacity(50);
4988
4989 if is_split {
4990 let boxes = boxes.insert_axis(Axis(0));
4991 let scores = scores.insert_axis(Axis(0));
4992 let classes = classes.insert_axis(Axis(0));
4993 let mask = mask.insert_axis(Axis(0));
4994
4995 let boxes: Array3<u8> = quantize_ndarray(boxes.view(), detect_quant.into());
4996 let mut scores: Array3<u8> =
4997 quantize_ndarray(scores.view(), detect_quant.into());
4998 let classes: Array3<u8> = quantize_ndarray(classes.view(), detect_quant.into());
4999 let mask: Array3<u8> = quantize_ndarray(mask.view(), detect_quant.into());
5000
5001 if is_proto {
5002 {
5003 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> = vec![
5004 boxes.view().into(),
5005 scores.view().into(),
5006 classes.view().into(),
5007 mask.view().into(),
5008 protos.view().into(),
5009 ];
5010 decoder
5011 .decode_tracked_quantized_proto(
5012 &mut tracker,
5013 0,
5014 &inputs,
5015 &mut output_boxes,
5016 &mut output_tracks,
5017 )
5018 .unwrap();
5019 }
5020 assert_eq!(output_boxes.len(), 1);
5021 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
5022
5023 for score in scores.slice_mut(s![.., .., ..]).iter_mut() {
5024 *score = u8::MIN;
5025 }
5026 let proto_result = {
5027 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> = vec![
5028 boxes.view().into(),
5029 scores.view().into(),
5030 classes.view().into(),
5031 mask.view().into(),
5032 protos.view().into(),
5033 ];
5034 decoder
5035 .decode_tracked_quantized_proto(
5036 &mut tracker,
5037 100_000_000 / 3,
5038 &inputs,
5039 &mut output_boxes,
5040 &mut output_tracks,
5041 )
5042 .unwrap()
5043 };
5044 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
5045 assert!(proto_result.is_some_and(|x| x.mask_coefficients.shape()[0] == 0));
5046 } else {
5047 let mut output_masks = Vec::with_capacity(50);
5048 {
5049 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> = vec![
5050 boxes.view().into(),
5051 scores.view().into(),
5052 classes.view().into(),
5053 mask.view().into(),
5054 protos.view().into(),
5055 ];
5056 decoder
5057 .decode_tracked_quantized(
5058 &mut tracker,
5059 0,
5060 &inputs,
5061 &mut output_boxes,
5062 &mut output_masks,
5063 &mut output_tracks,
5064 )
5065 .unwrap();
5066 }
5067 assert_eq!(output_boxes.len(), 1);
5068 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
5069
5070 for score in scores.slice_mut(s![.., .., ..]).iter_mut() {
5071 *score = u8::MIN;
5072 }
5073 {
5074 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> = vec![
5075 boxes.view().into(),
5076 scores.view().into(),
5077 classes.view().into(),
5078 mask.view().into(),
5079 protos.view().into(),
5080 ];
5081 decoder
5082 .decode_tracked_quantized(
5083 &mut tracker,
5084 100_000_000 / 3,
5085 &inputs,
5086 &mut output_boxes,
5087 &mut output_masks,
5088 &mut output_tracks,
5089 )
5090 .unwrap();
5091 }
5092 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
5093 assert!(output_masks.is_empty());
5094 }
5095 } else {
5096 let detect = ndarray::concatenate![
5098 Axis(1),
5099 boxes.view(),
5100 scores.view(),
5101 classes.view(),
5102 mask.view()
5103 ];
5104 let detect = detect.insert_axis(Axis(0));
5105 assert_eq!(detect.shape(), &[1, 10, 38]);
5106 let mut detect: Array3<u8> =
5107 quantize_ndarray(detect.view(), detect_quant.into());
5108
5109 if is_proto {
5110 {
5111 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> =
5112 vec![detect.view().into(), protos.view().into()];
5113 decoder
5114 .decode_tracked_quantized_proto(
5115 &mut tracker,
5116 0,
5117 &inputs,
5118 &mut output_boxes,
5119 &mut output_tracks,
5120 )
5121 .unwrap();
5122 }
5123 assert_eq!(output_boxes.len(), 1);
5124 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
5125
5126 for score in detect.slice_mut(s![.., .., 4]).iter_mut() {
5127 *score = u8::MIN;
5128 }
5129 let proto_result = {
5130 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> =
5131 vec![detect.view().into(), protos.view().into()];
5132 decoder
5133 .decode_tracked_quantized_proto(
5134 &mut tracker,
5135 100_000_000 / 3,
5136 &inputs,
5137 &mut output_boxes,
5138 &mut output_tracks,
5139 )
5140 .unwrap()
5141 };
5142 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
5143 assert!(proto_result.is_some_and(|x| x.mask_coefficients.shape()[0] == 0));
5144 } else {
5145 let mut output_masks = Vec::with_capacity(50);
5146 {
5147 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> =
5148 vec![detect.view().into(), protos.view().into()];
5149 decoder
5150 .decode_tracked_quantized(
5151 &mut tracker,
5152 0,
5153 &inputs,
5154 &mut output_boxes,
5155 &mut output_masks,
5156 &mut output_tracks,
5157 )
5158 .unwrap();
5159 }
5160 assert_eq!(output_boxes.len(), 1);
5161 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
5162
5163 for score in detect.slice_mut(s![.., .., 4]).iter_mut() {
5164 *score = u8::MIN;
5165 }
5166 {
5167 let inputs: Vec<crate::decoder::ArrayViewDQuantized<'_>> =
5168 vec![detect.view().into(), protos.view().into()];
5169 decoder
5170 .decode_tracked_quantized(
5171 &mut tracker,
5172 100_000_000 / 3,
5173 &inputs,
5174 &mut output_boxes,
5175 &mut output_masks,
5176 &mut output_tracks,
5177 )
5178 .unwrap();
5179 }
5180 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
5181 assert!(output_masks.is_empty());
5182 }
5183 }
5184 }
5185 };
5186 ($name:ident, float, $layout:ident, $output:ident) => {
5187 #[test]
5188 fn $name() {
5189 let is_split = matches!(stringify!($layout), "split");
5190 let is_proto = matches!(stringify!($output), "proto");
5191
5192 let score_threshold = 0.45;
5193 let iou_threshold = 0.45;
5194
5195 let mut boxes = Array2::zeros((10, 4));
5196 let mut scores = Array2::zeros((10, 1));
5197 let mut classes = Array2::zeros((10, 1));
5198 let mask: Array2<f64> = Array2::zeros((10, 32));
5199 let protos = Array3::<f64>::zeros((160, 160, 32));
5200 let protos = protos.insert_axis(Axis(0));
5201
5202 boxes
5203 .slice_mut(s![0, ..])
5204 .assign(&array![0.1234, 0.1234, 0.2345, 0.2345]);
5205 scores.slice_mut(s![0, ..]).assign(&array![0.9876]);
5206 classes.slice_mut(s![0, ..]).assign(&array![2.0]);
5207
5208 let decoder = if is_split {
5209 DecoderBuilder::default()
5210 .with_config_yaml_str(E2E_SPLIT_CONFIG.to_string())
5211 .with_score_threshold(score_threshold)
5212 .with_iou_threshold(iou_threshold)
5213 .build()
5214 .unwrap()
5215 } else {
5216 DecoderBuilder::default()
5217 .with_config_yaml_str(E2E_COMBINED_CONFIG.to_string())
5218 .with_score_threshold(score_threshold)
5219 .with_iou_threshold(iou_threshold)
5220 .build()
5221 .unwrap()
5222 };
5223
5224 let expected = e2e_expected_boxes_float();
5225 let mut tracker = ByteTrackBuilder::new()
5226 .track_update(0.1)
5227 .track_high_conf(0.7)
5228 .build();
5229 let mut output_boxes = Vec::with_capacity(50);
5230 let mut output_tracks = Vec::with_capacity(50);
5231
5232 if is_split {
5233 let boxes = boxes.insert_axis(Axis(0));
5234 let mut scores = scores.insert_axis(Axis(0));
5235 let classes = classes.insert_axis(Axis(0));
5236 let mask = mask.insert_axis(Axis(0));
5237
5238 if is_proto {
5239 {
5240 let inputs = vec![
5241 boxes.view().into_dyn(),
5242 scores.view().into_dyn(),
5243 classes.view().into_dyn(),
5244 mask.view().into_dyn(),
5245 protos.view().into_dyn(),
5246 ];
5247 decoder
5248 .decode_tracked_float_proto(
5249 &mut tracker,
5250 0,
5251 &inputs,
5252 &mut output_boxes,
5253 &mut output_tracks,
5254 )
5255 .unwrap();
5256 }
5257 assert_eq!(output_boxes.len(), 1);
5258 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
5259
5260 for score in scores.slice_mut(s![.., .., ..]).iter_mut() {
5261 *score = 0.0;
5262 }
5263 let proto_result = {
5264 let inputs = vec![
5265 boxes.view().into_dyn(),
5266 scores.view().into_dyn(),
5267 classes.view().into_dyn(),
5268 mask.view().into_dyn(),
5269 protos.view().into_dyn(),
5270 ];
5271 decoder
5272 .decode_tracked_float_proto(
5273 &mut tracker,
5274 100_000_000 / 3,
5275 &inputs,
5276 &mut output_boxes,
5277 &mut output_tracks,
5278 )
5279 .unwrap()
5280 };
5281 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
5282 assert!(proto_result.is_some_and(|x| x.mask_coefficients.shape()[0] == 0));
5283 } else {
5284 let mut output_masks = Vec::with_capacity(50);
5285 {
5286 let inputs = vec![
5287 boxes.view().into_dyn(),
5288 scores.view().into_dyn(),
5289 classes.view().into_dyn(),
5290 mask.view().into_dyn(),
5291 protos.view().into_dyn(),
5292 ];
5293 decoder
5294 .decode_tracked_float(
5295 &mut tracker,
5296 0,
5297 &inputs,
5298 &mut output_boxes,
5299 &mut output_masks,
5300 &mut output_tracks,
5301 )
5302 .unwrap();
5303 }
5304 assert_eq!(output_boxes.len(), 1);
5305 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
5306
5307 for score in scores.slice_mut(s![.., .., ..]).iter_mut() {
5308 *score = 0.0;
5309 }
5310 {
5311 let inputs = vec![
5312 boxes.view().into_dyn(),
5313 scores.view().into_dyn(),
5314 classes.view().into_dyn(),
5315 mask.view().into_dyn(),
5316 protos.view().into_dyn(),
5317 ];
5318 decoder
5319 .decode_tracked_float(
5320 &mut tracker,
5321 100_000_000 / 3,
5322 &inputs,
5323 &mut output_boxes,
5324 &mut output_masks,
5325 &mut output_tracks,
5326 )
5327 .unwrap();
5328 }
5329 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
5330 assert!(output_masks.is_empty());
5331 }
5332 } else {
5333 let detect = ndarray::concatenate![
5335 Axis(1),
5336 boxes.view(),
5337 scores.view(),
5338 classes.view(),
5339 mask.view()
5340 ];
5341 let mut detect = detect.insert_axis(Axis(0));
5342 assert_eq!(detect.shape(), &[1, 10, 38]);
5343
5344 if is_proto {
5345 {
5346 let inputs = vec![detect.view().into_dyn(), protos.view().into_dyn()];
5347 decoder
5348 .decode_tracked_float_proto(
5349 &mut tracker,
5350 0,
5351 &inputs,
5352 &mut output_boxes,
5353 &mut output_tracks,
5354 )
5355 .unwrap();
5356 }
5357 assert_eq!(output_boxes.len(), 1);
5358 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
5359
5360 for score in detect.slice_mut(s![.., .., 4]).iter_mut() {
5361 *score = 0.0;
5362 }
5363 let proto_result = {
5364 let inputs = vec![detect.view().into_dyn(), protos.view().into_dyn()];
5365 decoder
5366 .decode_tracked_float_proto(
5367 &mut tracker,
5368 100_000_000 / 3,
5369 &inputs,
5370 &mut output_boxes,
5371 &mut output_tracks,
5372 )
5373 .unwrap()
5374 };
5375 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
5376 assert!(proto_result.is_some_and(|x| x.mask_coefficients.shape()[0] == 0));
5377 } else {
5378 let mut output_masks = Vec::with_capacity(50);
5379 {
5380 let inputs = vec![detect.view().into_dyn(), protos.view().into_dyn()];
5381 decoder
5382 .decode_tracked_float(
5383 &mut tracker,
5384 0,
5385 &inputs,
5386 &mut output_boxes,
5387 &mut output_masks,
5388 &mut output_tracks,
5389 )
5390 .unwrap();
5391 }
5392 assert_eq!(output_boxes.len(), 1);
5393 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
5394
5395 for score in detect.slice_mut(s![.., .., 4]).iter_mut() {
5396 *score = 0.0;
5397 }
5398 {
5399 let inputs = vec![detect.view().into_dyn(), protos.view().into_dyn()];
5400 decoder
5401 .decode_tracked_float(
5402 &mut tracker,
5403 100_000_000 / 3,
5404 &inputs,
5405 &mut output_boxes,
5406 &mut output_masks,
5407 &mut output_tracks,
5408 )
5409 .unwrap();
5410 }
5411 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
5412 assert!(output_masks.is_empty());
5413 }
5414 }
5415 }
5416 };
5417 }
5418
5419 e2e_tracked_test!(
5420 test_decoder_tracked_end_to_end_segdet,
5421 quantized,
5422 combined,
5423 masks
5424 );
5425 e2e_tracked_test!(
5426 test_decoder_tracked_end_to_end_segdet_float,
5427 float,
5428 combined,
5429 masks
5430 );
5431 e2e_tracked_test!(
5432 test_decoder_tracked_end_to_end_segdet_proto,
5433 quantized,
5434 combined,
5435 proto
5436 );
5437 e2e_tracked_test!(
5438 test_decoder_tracked_end_to_end_segdet_proto_float,
5439 float,
5440 combined,
5441 proto
5442 );
5443 e2e_tracked_test!(
5444 test_decoder_tracked_end_to_end_segdet_split,
5445 quantized,
5446 split,
5447 masks
5448 );
5449 e2e_tracked_test!(
5450 test_decoder_tracked_end_to_end_segdet_split_float,
5451 float,
5452 split,
5453 masks
5454 );
5455 e2e_tracked_test!(
5456 test_decoder_tracked_end_to_end_segdet_split_proto,
5457 quantized,
5458 split,
5459 proto
5460 );
5461 e2e_tracked_test!(
5462 test_decoder_tracked_end_to_end_segdet_split_proto_float,
5463 float,
5464 split,
5465 proto
5466 );
5467
5468 macro_rules! e2e_tracked_tensor_test {
5474 ($name:ident, quantized, $layout:ident, $output:ident) => {
5475 #[test]
5476 fn $name() {
5477 use edgefirst_tensor::{Tensor, TensorMapTrait, TensorTrait};
5478
5479 let is_split = matches!(stringify!($layout), "split");
5480 let is_proto = matches!(stringify!($output), "proto");
5481
5482 let score_threshold = 0.45;
5483 let iou_threshold = 0.45;
5484
5485 let mut boxes = Array2::zeros((10, 4));
5486 let mut scores = Array2::zeros((10, 1));
5487 let mut classes = Array2::zeros((10, 1));
5488 let mask = Array2::zeros((10, 32));
5489 let protos_f64 = Array3::<f64>::zeros((160, 160, 32));
5490 let protos_f64 = protos_f64.insert_axis(Axis(0));
5491 let protos_quant = (1.0 / 255.0, 0.0);
5492 let protos_u8: Array4<u8> =
5493 quantize_ndarray(protos_f64.view(), protos_quant.into());
5494
5495 boxes
5496 .slice_mut(s![0, ..])
5497 .assign(&array![0.1234, 0.1234, 0.2345, 0.2345]);
5498 scores.slice_mut(s![0, ..]).assign(&array![0.9876]);
5499 classes.slice_mut(s![0, ..]).assign(&array![2.0]);
5500
5501 let detect_quant = (2.0 / 255.0, 0.0);
5502
5503 let decoder = if is_split {
5504 DecoderBuilder::default()
5505 .with_config_yaml_str(E2E_SPLIT_CONFIG.to_string())
5506 .with_score_threshold(score_threshold)
5507 .with_iou_threshold(iou_threshold)
5508 .build()
5509 .unwrap()
5510 } else {
5511 DecoderBuilder::default()
5512 .with_config_yaml_str(E2E_COMBINED_CONFIG.to_string())
5513 .with_score_threshold(score_threshold)
5514 .with_iou_threshold(iou_threshold)
5515 .build()
5516 .unwrap()
5517 };
5518
5519 let make_u8_tensor =
5521 |shape: &[usize], data: &[u8]| -> edgefirst_tensor::TensorDyn {
5522 let t = Tensor::<u8>::new(shape, None, None).unwrap();
5523 t.map().unwrap().as_mut_slice()[..data.len()].copy_from_slice(data);
5524 t.into()
5525 };
5526
5527 let expected = e2e_expected_boxes_quant();
5528 let mut tracker = ByteTrackBuilder::new()
5529 .track_update(0.1)
5530 .track_high_conf(0.7)
5531 .build();
5532 let mut output_boxes = Vec::with_capacity(50);
5533 let mut output_tracks = Vec::with_capacity(50);
5534
5535 let protos_td = make_u8_tensor(protos_u8.shape(), protos_u8.as_slice().unwrap());
5536
5537 if is_split {
5538 let boxes = boxes.insert_axis(Axis(0));
5539 let scores = scores.insert_axis(Axis(0));
5540 let classes = classes.insert_axis(Axis(0));
5541 let mask = mask.insert_axis(Axis(0));
5542
5543 let boxes_q: Array3<u8> = quantize_ndarray(boxes.view(), detect_quant.into());
5544 let mut scores_q: Array3<u8> =
5545 quantize_ndarray(scores.view(), detect_quant.into());
5546 let classes_q: Array3<u8> =
5547 quantize_ndarray(classes.view(), detect_quant.into());
5548 let mask_q: Array3<u8> = quantize_ndarray(mask.view(), detect_quant.into());
5549
5550 let boxes_td = make_u8_tensor(boxes_q.shape(), boxes_q.as_slice().unwrap());
5551 let classes_td =
5552 make_u8_tensor(classes_q.shape(), classes_q.as_slice().unwrap());
5553 let mask_td = make_u8_tensor(mask_q.shape(), mask_q.as_slice().unwrap());
5554
5555 if is_proto {
5556 let scores_td =
5557 make_u8_tensor(scores_q.shape(), scores_q.as_slice().unwrap());
5558 decoder
5559 .decode_proto_tracked(
5560 &mut tracker,
5561 0,
5562 &[&boxes_td, &scores_td, &classes_td, &mask_td, &protos_td],
5563 &mut output_boxes,
5564 &mut output_tracks,
5565 )
5566 .unwrap();
5567
5568 assert_eq!(output_boxes.len(), 1);
5569 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
5570
5571 for score in scores_q.slice_mut(s![.., .., ..]).iter_mut() {
5572 *score = u8::MIN;
5573 }
5574 let scores_td =
5575 make_u8_tensor(scores_q.shape(), scores_q.as_slice().unwrap());
5576 let proto_result = decoder
5577 .decode_proto_tracked(
5578 &mut tracker,
5579 100_000_000 / 3,
5580 &[&boxes_td, &scores_td, &classes_td, &mask_td, &protos_td],
5581 &mut output_boxes,
5582 &mut output_tracks,
5583 )
5584 .unwrap();
5585 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
5586 assert!(proto_result.is_some_and(|x| x.mask_coefficients.shape()[0] == 0));
5587 } else {
5588 let scores_td =
5589 make_u8_tensor(scores_q.shape(), scores_q.as_slice().unwrap());
5590 let mut output_masks = Vec::with_capacity(50);
5591 decoder
5592 .decode_tracked(
5593 &mut tracker,
5594 0,
5595 &[&boxes_td, &scores_td, &classes_td, &mask_td, &protos_td],
5596 &mut output_boxes,
5597 &mut output_masks,
5598 &mut output_tracks,
5599 )
5600 .unwrap();
5601
5602 assert_eq!(output_boxes.len(), 1);
5603 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
5604
5605 for score in scores_q.slice_mut(s![.., .., ..]).iter_mut() {
5606 *score = u8::MIN;
5607 }
5608 let scores_td =
5609 make_u8_tensor(scores_q.shape(), scores_q.as_slice().unwrap());
5610 decoder
5611 .decode_tracked(
5612 &mut tracker,
5613 100_000_000 / 3,
5614 &[&boxes_td, &scores_td, &classes_td, &mask_td, &protos_td],
5615 &mut output_boxes,
5616 &mut output_masks,
5617 &mut output_tracks,
5618 )
5619 .unwrap();
5620 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
5621 assert!(output_masks.is_empty());
5622 }
5623 } else {
5624 let detect = ndarray::concatenate![
5626 Axis(1),
5627 boxes.view(),
5628 scores.view(),
5629 classes.view(),
5630 mask.view()
5631 ];
5632 let detect = detect.insert_axis(Axis(0));
5633 assert_eq!(detect.shape(), &[1, 10, 38]);
5634 let detect =
5636 Array3::from_shape_vec(detect.raw_dim(), detect.iter().copied().collect())
5637 .unwrap();
5638 let mut detect_q: Array3<u8> =
5639 quantize_ndarray(detect.view(), detect_quant.into());
5640
5641 if is_proto {
5642 let detect_td =
5643 make_u8_tensor(detect_q.shape(), detect_q.as_slice().unwrap());
5644 decoder
5645 .decode_proto_tracked(
5646 &mut tracker,
5647 0,
5648 &[&detect_td, &protos_td],
5649 &mut output_boxes,
5650 &mut output_tracks,
5651 )
5652 .unwrap();
5653
5654 assert_eq!(output_boxes.len(), 1);
5655 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
5656
5657 for score in detect_q.slice_mut(s![.., .., 4]).iter_mut() {
5658 *score = u8::MIN;
5659 }
5660 let detect_td =
5661 make_u8_tensor(detect_q.shape(), detect_q.as_slice().unwrap());
5662 let proto_result = decoder
5663 .decode_proto_tracked(
5664 &mut tracker,
5665 100_000_000 / 3,
5666 &[&detect_td, &protos_td],
5667 &mut output_boxes,
5668 &mut output_tracks,
5669 )
5670 .unwrap();
5671 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
5672 assert!(proto_result.is_some_and(|x| x.mask_coefficients.shape()[0] == 0));
5673 } else {
5674 let detect_td =
5675 make_u8_tensor(detect_q.shape(), detect_q.as_slice().unwrap());
5676 let mut output_masks = Vec::with_capacity(50);
5677 decoder
5678 .decode_tracked(
5679 &mut tracker,
5680 0,
5681 &[&detect_td, &protos_td],
5682 &mut output_boxes,
5683 &mut output_masks,
5684 &mut output_tracks,
5685 )
5686 .unwrap();
5687
5688 assert_eq!(output_boxes.len(), 1);
5689 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
5690
5691 for score in detect_q.slice_mut(s![.., .., 4]).iter_mut() {
5692 *score = u8::MIN;
5693 }
5694 let detect_td =
5695 make_u8_tensor(detect_q.shape(), detect_q.as_slice().unwrap());
5696 decoder
5697 .decode_tracked(
5698 &mut tracker,
5699 100_000_000 / 3,
5700 &[&detect_td, &protos_td],
5701 &mut output_boxes,
5702 &mut output_masks,
5703 &mut output_tracks,
5704 )
5705 .unwrap();
5706 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
5707 assert!(output_masks.is_empty());
5708 }
5709 }
5710 }
5711 };
5712 ($name:ident, float, $layout:ident, $output:ident) => {
5713 #[test]
5714 fn $name() {
5715 use edgefirst_tensor::{Tensor, TensorMapTrait, TensorTrait};
5716
5717 let is_split = matches!(stringify!($layout), "split");
5718 let is_proto = matches!(stringify!($output), "proto");
5719
5720 let score_threshold = 0.45;
5721 let iou_threshold = 0.45;
5722
5723 let mut boxes = Array2::zeros((10, 4));
5724 let mut scores = Array2::zeros((10, 1));
5725 let mut classes = Array2::zeros((10, 1));
5726 let mask: Array2<f64> = Array2::zeros((10, 32));
5727 let protos = Array3::<f64>::zeros((160, 160, 32));
5728 let protos = protos.insert_axis(Axis(0));
5729
5730 boxes
5731 .slice_mut(s![0, ..])
5732 .assign(&array![0.1234, 0.1234, 0.2345, 0.2345]);
5733 scores.slice_mut(s![0, ..]).assign(&array![0.9876]);
5734 classes.slice_mut(s![0, ..]).assign(&array![2.0]);
5735
5736 let decoder = if is_split {
5737 DecoderBuilder::default()
5738 .with_config_yaml_str(E2E_SPLIT_CONFIG.to_string())
5739 .with_score_threshold(score_threshold)
5740 .with_iou_threshold(iou_threshold)
5741 .build()
5742 .unwrap()
5743 } else {
5744 DecoderBuilder::default()
5745 .with_config_yaml_str(E2E_COMBINED_CONFIG.to_string())
5746 .with_score_threshold(score_threshold)
5747 .with_iou_threshold(iou_threshold)
5748 .build()
5749 .unwrap()
5750 };
5751
5752 let make_f64_tensor =
5754 |shape: &[usize], data: &[f64]| -> edgefirst_tensor::TensorDyn {
5755 let t = Tensor::<f64>::new(shape, None, None).unwrap();
5756 t.map().unwrap().as_mut_slice()[..data.len()].copy_from_slice(data);
5757 t.into()
5758 };
5759
5760 let expected = e2e_expected_boxes_float();
5761 let mut tracker = ByteTrackBuilder::new()
5762 .track_update(0.1)
5763 .track_high_conf(0.7)
5764 .build();
5765 let mut output_boxes = Vec::with_capacity(50);
5766 let mut output_tracks = Vec::with_capacity(50);
5767
5768 let protos_td = make_f64_tensor(protos.shape(), protos.as_slice().unwrap());
5769
5770 if is_split {
5771 let boxes = boxes.insert_axis(Axis(0));
5772 let mut scores = scores.insert_axis(Axis(0));
5773 let classes = classes.insert_axis(Axis(0));
5774 let mask = mask.insert_axis(Axis(0));
5775
5776 let boxes_td = make_f64_tensor(boxes.shape(), boxes.as_slice().unwrap());
5777 let classes_td = make_f64_tensor(classes.shape(), classes.as_slice().unwrap());
5778 let mask_td = make_f64_tensor(mask.shape(), mask.as_slice().unwrap());
5779
5780 if is_proto {
5781 let scores_td = make_f64_tensor(scores.shape(), scores.as_slice().unwrap());
5782 decoder
5783 .decode_proto_tracked(
5784 &mut tracker,
5785 0,
5786 &[&boxes_td, &scores_td, &classes_td, &mask_td, &protos_td],
5787 &mut output_boxes,
5788 &mut output_tracks,
5789 )
5790 .unwrap();
5791
5792 assert_eq!(output_boxes.len(), 1);
5793 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
5794
5795 for score in scores.slice_mut(s![.., .., ..]).iter_mut() {
5796 *score = 0.0;
5797 }
5798 let scores_td = make_f64_tensor(scores.shape(), scores.as_slice().unwrap());
5799 let proto_result = decoder
5800 .decode_proto_tracked(
5801 &mut tracker,
5802 100_000_000 / 3,
5803 &[&boxes_td, &scores_td, &classes_td, &mask_td, &protos_td],
5804 &mut output_boxes,
5805 &mut output_tracks,
5806 )
5807 .unwrap();
5808 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
5809 assert!(proto_result.is_some_and(|x| x.mask_coefficients.shape()[0] == 0));
5810 } else {
5811 let scores_td = make_f64_tensor(scores.shape(), scores.as_slice().unwrap());
5812 let mut output_masks = Vec::with_capacity(50);
5813 decoder
5814 .decode_tracked(
5815 &mut tracker,
5816 0,
5817 &[&boxes_td, &scores_td, &classes_td, &mask_td, &protos_td],
5818 &mut output_boxes,
5819 &mut output_masks,
5820 &mut output_tracks,
5821 )
5822 .unwrap();
5823
5824 assert_eq!(output_boxes.len(), 1);
5825 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
5826
5827 for score in scores.slice_mut(s![.., .., ..]).iter_mut() {
5828 *score = 0.0;
5829 }
5830 let scores_td = make_f64_tensor(scores.shape(), scores.as_slice().unwrap());
5831 decoder
5832 .decode_tracked(
5833 &mut tracker,
5834 100_000_000 / 3,
5835 &[&boxes_td, &scores_td, &classes_td, &mask_td, &protos_td],
5836 &mut output_boxes,
5837 &mut output_masks,
5838 &mut output_tracks,
5839 )
5840 .unwrap();
5841 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
5842 assert!(output_masks.is_empty());
5843 }
5844 } else {
5845 let detect = ndarray::concatenate![
5847 Axis(1),
5848 boxes.view(),
5849 scores.view(),
5850 classes.view(),
5851 mask.view()
5852 ];
5853 let detect = detect.insert_axis(Axis(0));
5854 assert_eq!(detect.shape(), &[1, 10, 38]);
5855 let mut detect =
5857 Array3::from_shape_vec(detect.raw_dim(), detect.iter().copied().collect())
5858 .unwrap();
5859
5860 if is_proto {
5861 let detect_td = make_f64_tensor(detect.shape(), detect.as_slice().unwrap());
5862 decoder
5863 .decode_proto_tracked(
5864 &mut tracker,
5865 0,
5866 &[&detect_td, &protos_td],
5867 &mut output_boxes,
5868 &mut output_tracks,
5869 )
5870 .unwrap();
5871
5872 assert_eq!(output_boxes.len(), 1);
5873 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
5874
5875 for score in detect.slice_mut(s![.., .., 4]).iter_mut() {
5876 *score = 0.0;
5877 }
5878 let detect_td = make_f64_tensor(detect.shape(), detect.as_slice().unwrap());
5879 let proto_result = decoder
5880 .decode_proto_tracked(
5881 &mut tracker,
5882 100_000_000 / 3,
5883 &[&detect_td, &protos_td],
5884 &mut output_boxes,
5885 &mut output_tracks,
5886 )
5887 .unwrap();
5888 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
5889 assert!(proto_result.is_some_and(|x| x.mask_coefficients.shape()[0] == 0));
5890 } else {
5891 let detect_td = make_f64_tensor(detect.shape(), detect.as_slice().unwrap());
5892 let mut output_masks = Vec::with_capacity(50);
5893 decoder
5894 .decode_tracked(
5895 &mut tracker,
5896 0,
5897 &[&detect_td, &protos_td],
5898 &mut output_boxes,
5899 &mut output_masks,
5900 &mut output_tracks,
5901 )
5902 .unwrap();
5903
5904 assert_eq!(output_boxes.len(), 1);
5905 assert!(output_boxes[0].equal_within_delta(&expected[0], 1.0 / 160.0));
5906
5907 for score in detect.slice_mut(s![.., .., 4]).iter_mut() {
5908 *score = 0.0;
5909 }
5910 let detect_td = make_f64_tensor(detect.shape(), detect.as_slice().unwrap());
5911 decoder
5912 .decode_tracked(
5913 &mut tracker,
5914 100_000_000 / 3,
5915 &[&detect_td, &protos_td],
5916 &mut output_boxes,
5917 &mut output_masks,
5918 &mut output_tracks,
5919 )
5920 .unwrap();
5921 assert!(output_boxes[0].equal_within_delta(&expected[0], 1e-6));
5922 assert!(output_masks.is_empty());
5923 }
5924 }
5925 }
5926 };
5927 }
5928
5929 e2e_tracked_tensor_test!(
5930 test_decoder_tracked_tensor_end_to_end_segdet,
5931 quantized,
5932 combined,
5933 masks
5934 );
5935 e2e_tracked_tensor_test!(
5936 test_decoder_tracked_tensor_end_to_end_segdet_float,
5937 float,
5938 combined,
5939 masks
5940 );
5941 e2e_tracked_tensor_test!(
5942 test_decoder_tracked_tensor_end_to_end_segdet_proto,
5943 quantized,
5944 combined,
5945 proto
5946 );
5947 e2e_tracked_tensor_test!(
5948 test_decoder_tracked_tensor_end_to_end_segdet_proto_float,
5949 float,
5950 combined,
5951 proto
5952 );
5953 e2e_tracked_tensor_test!(
5954 test_decoder_tracked_tensor_end_to_end_segdet_split,
5955 quantized,
5956 split,
5957 masks
5958 );
5959 e2e_tracked_tensor_test!(
5960 test_decoder_tracked_tensor_end_to_end_segdet_split_float,
5961 float,
5962 split,
5963 masks
5964 );
5965 e2e_tracked_tensor_test!(
5966 test_decoder_tracked_tensor_end_to_end_segdet_split_proto,
5967 quantized,
5968 split,
5969 proto
5970 );
5971 e2e_tracked_tensor_test!(
5972 test_decoder_tracked_tensor_end_to_end_segdet_split_proto_float,
5973 float,
5974 split,
5975 proto
5976 );
5977
5978 #[test]
5979 fn test_decoder_tracked_linear_motion() {
5980 use crate::configs::{DecoderType, Nms};
5981 use crate::DecoderBuilder;
5982
5983 let score_threshold = 0.25;
5984 let iou_threshold = 0.1;
5985 let out = edgefirst_bench::testdata::read("yolov8s_80_classes.bin");
5986 let out = unsafe { std::slice::from_raw_parts(out.as_ptr() as *const i8, out.len()) };
5987 let mut out = Array3::from_shape_vec((1, 84, 8400), out.to_vec()).unwrap();
5988 let quant = (0.0040811873, -123).into();
5989
5990 let decoder = DecoderBuilder::default()
5991 .with_config_yolo_det(
5992 crate::configs::Detection {
5993 decoder: DecoderType::Ultralytics,
5994 shape: vec![1, 84, 8400],
5995 anchors: None,
5996 quantization: Some(quant),
5997 dshape: vec![
5998 (crate::configs::DimName::Batch, 1),
5999 (crate::configs::DimName::NumFeatures, 84),
6000 (crate::configs::DimName::NumBoxes, 8400),
6001 ],
6002 normalized: Some(true),
6003 },
6004 None,
6005 )
6006 .with_score_threshold(score_threshold)
6007 .with_iou_threshold(iou_threshold)
6008 .with_nms(Some(Nms::ClassAgnostic))
6009 .build()
6010 .unwrap();
6011
6012 let mut expected_boxes = [
6013 DetectBox {
6014 bbox: BoundingBox {
6015 xmin: 0.5285137,
6016 ymin: 0.05305544,
6017 xmax: 0.87541467,
6018 ymax: 0.9998909,
6019 },
6020 score: 0.5591227,
6021 label: 0,
6022 },
6023 DetectBox {
6024 bbox: BoundingBox {
6025 xmin: 0.130598,
6026 ymin: 0.43260583,
6027 xmax: 0.35098213,
6028 ymax: 0.9958097,
6029 },
6030 score: 0.33057618,
6031 label: 75,
6032 },
6033 ];
6034
6035 let mut tracker = ByteTrackBuilder::new()
6036 .track_update(0.1)
6037 .track_high_conf(0.3)
6038 .build();
6039
6040 let mut output_boxes = Vec::with_capacity(50);
6041 let mut output_masks = Vec::with_capacity(50);
6042 let mut output_tracks = Vec::with_capacity(50);
6043
6044 decoder
6045 .decode_tracked_quantized(
6046 &mut tracker,
6047 0,
6048 &[out.view().into()],
6049 &mut output_boxes,
6050 &mut output_masks,
6051 &mut output_tracks,
6052 )
6053 .unwrap();
6054
6055 assert_eq!(output_boxes.len(), 2);
6056 assert!(output_boxes[0].equal_within_delta(&expected_boxes[0], 1e-6));
6057 assert!(output_boxes[1].equal_within_delta(&expected_boxes[1], 1e-6));
6058
6059 for i in 1..=100 {
6060 let mut out = out.clone();
6061 let mut x_values = out.slice_mut(s![0, 0, ..]);
6063 for x in x_values.iter_mut() {
6064 *x = x.saturating_add((i as f32 * 1e-3 / quant.0).round() as i8);
6065 }
6066
6067 decoder
6068 .decode_tracked_quantized(
6069 &mut tracker,
6070 100_000_000 * i / 3, &[out.view().into()],
6072 &mut output_boxes,
6073 &mut output_masks,
6074 &mut output_tracks,
6075 )
6076 .unwrap();
6077
6078 assert_eq!(output_boxes.len(), 2);
6079 }
6080 let tracks = tracker.get_active_tracks();
6081 let predicted_boxes: Vec<_> = tracks
6082 .iter()
6083 .map(|track| {
6084 let mut l = track.last_box;
6085 l.bbox = track.info.tracked_location.into();
6086 l
6087 })
6088 .collect();
6089 expected_boxes[0].bbox.xmin += 0.1; expected_boxes[0].bbox.xmax += 0.1;
6091 expected_boxes[1].bbox.xmin += 0.1;
6092 expected_boxes[1].bbox.xmax += 0.1;
6093
6094 assert!(predicted_boxes[0].equal_within_delta(&expected_boxes[0], 1e-3));
6095 assert!(predicted_boxes[1].equal_within_delta(&expected_boxes[1], 1e-3));
6096
6097 let mut scores_values = out.slice_mut(s![0, 4.., ..]);
6099 for score in scores_values.iter_mut() {
6100 *score = i8::MIN; }
6102 decoder
6103 .decode_tracked_quantized(
6104 &mut tracker,
6105 100_000_000 * 101 / 3,
6106 &[out.view().into()],
6107 &mut output_boxes,
6108 &mut output_masks,
6109 &mut output_tracks,
6110 )
6111 .unwrap();
6112 expected_boxes[0].bbox.xmin += 0.001; expected_boxes[0].bbox.xmax += 0.001;
6114 expected_boxes[1].bbox.xmin += 0.001;
6115 expected_boxes[1].bbox.xmax += 0.001;
6116
6117 assert!(output_boxes[0].equal_within_delta(&expected_boxes[0], 1e-3));
6118 assert!(output_boxes[1].equal_within_delta(&expected_boxes[1], 1e-3));
6119 }
6120
6121 #[test]
6122 fn test_decoder_tracked_end_to_end_float() {
6123 let score_threshold = 0.45;
6124 let iou_threshold = 0.45;
6125
6126 let mut boxes = Array2::zeros((10, 4));
6127 let mut scores = Array2::zeros((10, 1));
6128 let mut classes = Array2::zeros((10, 1));
6129
6130 boxes
6131 .slice_mut(s![0, ..,])
6132 .assign(&array![0.1234, 0.1234, 0.2345, 0.2345]);
6133 scores.slice_mut(s![0, ..]).assign(&array![0.9876]);
6134 classes.slice_mut(s![0, ..]).assign(&array![2.0]);
6135
6136 let detect = ndarray::concatenate![Axis(1), boxes.view(), scores.view(), classes.view(),];
6137 let mut detect = detect.insert_axis(Axis(0));
6138 assert_eq!(detect.shape(), &[1, 10, 6]);
6139 let config = "
6140decoder_version: yolo26
6141outputs:
6142 - type: detection
6143 decoder: ultralytics
6144 quantization: [0.00784313725490196, 0]
6145 shape: [1, 10, 6]
6146 dshape:
6147 - [batch, 1]
6148 - [num_boxes, 10]
6149 - [num_features, 6]
6150 normalized: true
6151";
6152
6153 let decoder = DecoderBuilder::default()
6154 .with_config_yaml_str(config.to_string())
6155 .with_score_threshold(score_threshold)
6156 .with_iou_threshold(iou_threshold)
6157 .build()
6158 .unwrap();
6159
6160 let expected_boxes = [DetectBox {
6161 bbox: BoundingBox {
6162 xmin: 0.1234,
6163 ymin: 0.1234,
6164 xmax: 0.2345,
6165 ymax: 0.2345,
6166 },
6167 score: 0.9876,
6168 label: 2,
6169 }];
6170
6171 let mut tracker = ByteTrackBuilder::new()
6172 .track_update(0.1)
6173 .track_high_conf(0.7)
6174 .build();
6175
6176 let mut output_boxes = Vec::with_capacity(50);
6177 let mut output_masks = Vec::with_capacity(50);
6178 let mut output_tracks = Vec::with_capacity(50);
6179
6180 decoder
6181 .decode_tracked_float(
6182 &mut tracker,
6183 0,
6184 &[detect.view().into_dyn()],
6185 &mut output_boxes,
6186 &mut output_masks,
6187 &mut output_tracks,
6188 )
6189 .unwrap();
6190
6191 assert_eq!(output_boxes.len(), 1);
6192 assert!(output_boxes[0].equal_within_delta(&expected_boxes[0], 1e-6));
6193
6194 for score in detect.slice_mut(s![.., .., 4]).iter_mut() {
6197 *score = 0.0; }
6199
6200 decoder
6201 .decode_tracked_float(
6202 &mut tracker,
6203 100_000_000 / 3,
6204 &[detect.view().into_dyn()],
6205 &mut output_boxes,
6206 &mut output_masks,
6207 &mut output_tracks,
6208 )
6209 .unwrap();
6210 assert!(output_boxes[0].equal_within_delta(&expected_boxes[0], 1e-6));
6211 }
6212}