1use ndarray::{Array2, ArrayView2, ArrayViewMut2};
87
88use crate::accumulate::PerImageEval;
89use crate::dataset::{
90 Bbox, CategoryId, CocoAnnotation, CocoDataset, CocoDetection, CocoDetections, EvalDataset,
91 ImageId, ImageMeta,
92};
93use crate::error::EvalError;
94use crate::matching::{match_image, MatchResult};
95use crate::parity::ParityMode;
96use crate::segmentation::Segmentation;
97use crate::similarity::{
98 boundary_iou_compute, segm_iou_compute, BboxAnn, BboxIou, BoundaryComputeScratch,
99 BoundaryGtCache, BoundaryIou, OksAnn, OksSimilarity, SegmAnn, SegmComputeScratch, SegmGtCache,
100 SegmIou, Similarity,
101};
102use std::collections::{HashMap, HashSet};
103use std::sync::{Arc, Mutex};
104use vernier_mask::Rle;
105
106#[derive(Clone)]
116pub enum GtCacheRef<'a, T: ?Sized> {
117 Borrowed(&'a T),
120 Owned(Arc<T>),
124}
125
126impl<T: ?Sized> GtCacheRef<'_, T> {
127 pub fn get(&self) -> &T {
129 match self {
130 GtCacheRef::Borrowed(r) => r,
131 GtCacheRef::Owned(a) => a.as_ref(),
132 }
133 }
134}
135
136pub const COLLAPSED_CATEGORY_SENTINEL: i64 = -1;
139
140pub const AREA_UNBOUNDED: f64 = 1e10;
143
144#[derive(Debug, Clone, Copy, PartialEq, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)]
152#[rkyv(derive(Debug))]
153pub struct AreaRange {
154 pub index: usize,
157 pub lo: f64,
159 pub hi: f64,
162}
163
164impl AreaRange {
165 pub fn coco_default() -> [Self; 4] {
169 [
170 Self {
171 index: 0,
172 lo: 0.0,
173 hi: AREA_UNBOUNDED,
174 },
175 Self {
176 index: 1,
177 lo: 0.0,
178 hi: 32.0 * 32.0,
179 },
180 Self {
181 index: 2,
182 lo: 32.0 * 32.0,
183 hi: 96.0 * 96.0,
184 },
185 Self {
186 index: 3,
187 lo: 96.0 * 96.0,
188 hi: AREA_UNBOUNDED,
189 },
190 ]
191 }
192
193 pub fn keypoints_default() -> [Self; 3] {
200 [
201 Self {
202 index: 0,
203 lo: 0.0,
204 hi: AREA_UNBOUNDED,
205 },
206 Self {
207 index: 1,
208 lo: 32.0 * 32.0,
209 hi: 96.0 * 96.0,
210 },
211 Self {
212 index: 2,
213 lo: 96.0 * 96.0,
214 hi: AREA_UNBOUNDED,
215 },
216 ]
217 }
218
219 fn contains(&self, area: f64) -> bool {
220 area >= self.lo && area <= self.hi
225 }
226}
227
228#[derive(Debug, Clone, Copy)]
232pub struct EvaluateParams<'p> {
233 pub iou_thresholds: &'p [f64],
236 pub area_ranges: &'p [AreaRange],
241 pub max_dets_per_image: usize,
246 pub use_cats: bool,
250 pub retain_iou: bool,
256}
257
258#[derive(Debug, Clone, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)]
266#[rkyv(derive(Debug))]
267pub struct OwnedEvaluateParams {
268 pub iou_thresholds: Vec<f64>,
270 pub area_ranges: Vec<AreaRange>,
272 pub max_dets_per_image: usize,
274 pub use_cats: bool,
276 pub retain_iou: bool,
278}
279
280impl OwnedEvaluateParams {
281 pub fn borrow(&self) -> EvaluateParams<'_> {
283 EvaluateParams {
284 iou_thresholds: &self.iou_thresholds,
285 area_ranges: &self.area_ranges,
286 max_dets_per_image: self.max_dets_per_image,
287 use_cats: self.use_cats,
288 retain_iou: self.retain_iou,
289 }
290 }
291
292 pub fn params_hash(&self) -> Result<[u8; 32], EvalError> {
308 let bytes =
309 rkyv::to_bytes::<rkyv::rancor::Error>(self).map_err(|e| EvalError::InvalidConfig {
310 detail: format!("rkyv serialization of OwnedEvaluateParams failed: {e}"),
311 })?;
312 Ok(*blake3::hash(&bytes).as_bytes())
313 }
314}
315
316#[derive(
326 Debug, Clone, Copy, PartialEq, Eq, Hash, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize,
327)]
328#[rkyv(derive(Debug, PartialEq, Eq))]
329pub enum KernelKind {
330 Bbox,
332 Segm,
334 Boundary,
336 Keypoints,
338}
339
340impl KernelKind {
341 pub const fn discriminator(self) -> u32 {
346 match self {
347 Self::Bbox => 0,
348 Self::Segm => 1,
349 Self::Boundary => 2,
350 Self::Keypoints => 3,
351 }
352 }
353}
354
355pub trait EvalKernel: Similarity {
366 fn kind(&self) -> KernelKind;
370
371 fn build_gt_anns(
375 &self,
376 gt_anns: &[CocoAnnotation],
377 indices: &[usize],
378 image: &ImageMeta,
379 ) -> Result<Vec<Self::Annotation>, EvalError>;
380
381 fn build_dt_anns(
389 &self,
390 dt_anns: &[CocoDetection],
391 indices: &[usize],
392 image: &ImageMeta,
393 parity_mode: ParityMode,
394 ) -> Result<Vec<Self::Annotation>, EvalError>;
395
396 fn extra_gt_ignore(&self, _ann: &CocoAnnotation) -> bool {
407 false
408 }
409
410 fn is_keypoints(&self) -> bool {
419 false
420 }
421}
422
423impl EvalKernel for BboxIou {
424 fn kind(&self) -> KernelKind {
425 KernelKind::Bbox
426 }
427
428 fn build_gt_anns(
429 &self,
430 gt_anns: &[CocoAnnotation],
431 indices: &[usize],
432 _image: &ImageMeta,
433 ) -> Result<Vec<BboxAnn>, EvalError> {
434 Ok(indices
435 .iter()
436 .map(|&j| BboxAnn {
437 bbox: gt_anns[j].bbox,
438 is_crowd: gt_anns[j].is_crowd,
439 })
440 .collect())
441 }
442
443 fn build_dt_anns(
444 &self,
445 dt_anns: &[CocoDetection],
446 indices: &[usize],
447 _image: &ImageMeta,
448 _parity_mode: ParityMode,
449 ) -> Result<Vec<BboxAnn>, EvalError> {
450 Ok(indices
452 .iter()
453 .map(|&j| BboxAnn {
454 bbox: dt_anns[j].bbox,
455 is_crowd: false,
456 })
457 .collect())
458 }
459}
460
461impl EvalKernel for SegmIou {
462 fn kind(&self) -> KernelKind {
463 KernelKind::Segm
464 }
465
466 fn build_gt_anns(
467 &self,
468 gt_anns: &[CocoAnnotation],
469 indices: &[usize],
470 image: &ImageMeta,
471 ) -> Result<Vec<SegmAnn>, EvalError> {
472 build_segm_gt_anns(gt_anns, indices, image)
473 }
474
475 fn build_dt_anns(
476 &self,
477 dt_anns: &[CocoDetection],
478 indices: &[usize],
479 image: &ImageMeta,
480 parity_mode: ParityMode,
481 ) -> Result<Vec<SegmAnn>, EvalError> {
482 build_segm_dt_anns(dt_anns, indices, image, parity_mode)
483 }
484}
485
486impl EvalKernel for BoundaryIou {
487 fn kind(&self) -> KernelKind {
488 KernelKind::Boundary
489 }
490
491 fn build_gt_anns(
492 &self,
493 gt_anns: &[CocoAnnotation],
494 indices: &[usize],
495 image: &ImageMeta,
496 ) -> Result<Vec<SegmAnn>, EvalError> {
497 build_segm_gt_anns(gt_anns, indices, image)
498 }
499
500 fn build_dt_anns(
501 &self,
502 dt_anns: &[CocoDetection],
503 indices: &[usize],
504 image: &ImageMeta,
505 parity_mode: ParityMode,
506 ) -> Result<Vec<SegmAnn>, EvalError> {
507 build_segm_dt_anns(dt_anns, indices, image, parity_mode)
508 }
509}
510
511impl EvalKernel for OksSimilarity {
512 fn kind(&self) -> KernelKind {
513 KernelKind::Keypoints
514 }
515
516 fn build_gt_anns(
517 &self,
518 gt_anns: &[CocoAnnotation],
519 indices: &[usize],
520 _image: &ImageMeta,
521 ) -> Result<Vec<OksAnn>, EvalError> {
522 indices
523 .iter()
524 .map(|&j| {
525 let ann = >_anns[j];
526 let kps = ann
527 .keypoints
528 .as_deref()
529 .ok_or_else(|| missing_keypoints_err("GT", ann.id.0, ann.image_id.0))?;
530 let num_keypoints = ann
531 .num_keypoints
532 .unwrap_or_else(|| count_visible_keypoints(kps));
533 Ok(OksAnn {
534 category_id: ann.category_id.0,
535 keypoints: kps.to_vec(),
536 num_keypoints,
537 bbox: ann.bbox.into(),
538 area: ann.area,
539 })
540 })
541 .collect()
542 }
543
544 fn build_dt_anns(
545 &self,
546 dt_anns: &[CocoDetection],
547 indices: &[usize],
548 _image: &ImageMeta,
549 _parity_mode: ParityMode,
550 ) -> Result<Vec<OksAnn>, EvalError> {
551 indices
556 .iter()
557 .map(|&j| {
558 let dt = &dt_anns[j];
559 let kps = dt
560 .keypoints
561 .as_deref()
562 .ok_or_else(|| missing_keypoints_err("DT", dt.id.0, dt.image_id.0))?;
563 let num_keypoints = dt
564 .num_keypoints
565 .unwrap_or_else(|| count_visible_keypoints(kps));
566 Ok(OksAnn {
567 category_id: dt.category_id.0,
568 keypoints: kps.to_vec(),
569 num_keypoints,
570 bbox: dt.bbox.into(),
571 area: dt.area,
572 })
573 })
574 .collect()
575 }
576
577 fn extra_gt_ignore(&self, ann: &CocoAnnotation) -> bool {
578 let visible = ann
584 .num_keypoints
585 .or_else(|| ann.keypoints.as_deref().map(count_visible_keypoints))
586 .unwrap_or(0);
587 visible == 0
588 }
589
590 fn is_keypoints(&self) -> bool {
591 true
592 }
593}
594
595fn count_visible_keypoints(kps: &[f64]) -> u32 {
599 kps.chunks_exact(3).filter(|t| t[2] > 0.0).count() as u32
600}
601
602fn missing_keypoints_err(kind: &str, ann_id: i64, image_id: i64) -> EvalError {
606 EvalError::InvalidAnnotation {
607 detail: format!(
608 "{kind} id={ann_id} on image {image_id} has no `keypoints` field; \
609 OKS eval requires keypoints on every entry. There is no \
610 pycocotools-equivalent bbox-synthesis fallback for keypoints \
611 (unlike segm quirk J2)."
612 ),
613 }
614}
615
616fn build_segm_gt_anns(
617 gt_anns: &[CocoAnnotation],
618 indices: &[usize],
619 image: &ImageMeta,
620) -> Result<Vec<SegmAnn>, EvalError> {
621 indices
622 .iter()
623 .map(|&j| {
624 let ann = >_anns[j];
625 let seg = ann
626 .segmentation
627 .as_ref()
628 .ok_or_else(|| missing_segmentation_err("GT", ann.id.0, image.id.0))?;
629 Ok(SegmAnn {
630 rle: seg.to_rle(image.height, image.width)?,
631 is_crowd: ann.is_crowd,
632 ann_id: ann.id.0,
633 })
634 })
635 .collect()
636}
637
638fn build_segm_dt_anns(
639 dt_anns: &[CocoDetection],
640 indices: &[usize],
641 image: &ImageMeta,
642 parity_mode: ParityMode,
643) -> Result<Vec<SegmAnn>, EvalError> {
644 indices
645 .iter()
646 .map(|&j| {
647 let dt = &dt_anns[j];
648 let rle = match (&dt.segmentation, parity_mode) {
649 (Some(seg), _) => seg.to_rle(image.height, image.width)?,
650 (None, ParityMode::Strict) => {
657 synthesize_dt_segm_from_bbox(&dt.bbox, image.height, image.width)?
658 }
659 (None, ParityMode::Corrected) => {
667 return Err(missing_segmentation_err("DT", dt.id.0, image.id.0));
668 }
669 };
670 Ok(SegmAnn {
671 rle,
672 is_crowd: false,
673 ann_id: dt.id.0,
674 })
675 })
676 .collect()
677}
678
679fn synthesize_dt_segm_from_bbox(bbox: &Bbox, h: u32, w: u32) -> Result<Rle, EvalError> {
685 let x1 = bbox.x;
686 let y1 = bbox.y;
687 let x2 = bbox.x + bbox.w;
688 let y2 = bbox.y + bbox.h;
689 let polygon = vec![x1, y1, x1, y2, x2, y2, x2, y1];
690 let segm = Segmentation::Polygons(vec![polygon]);
691 segm.to_rle(h, w)
692}
693
694fn missing_segmentation_err(kind: &str, ann_id: i64, image_id: i64) -> EvalError {
700 EvalError::InvalidAnnotation {
701 detail: format!(
702 "{kind} id={ann_id} on image {image_id} has no `segmentation` field; \
703 segm eval in corrected mode requires one on every entry. \
704 pycocotools synthesizes a bbox-rectangle polygon here \
705 (quirks J2/J6); pass `ParityMode::Strict` to opt into that \
706 behavior."
707 ),
708 }
709}
710
711#[derive(Debug, Clone)]
722pub struct EvalImageMeta {
723 pub image_id: i64,
725 pub category_id: i64,
728 pub area_rng: [f64; 2],
730 pub max_det: usize,
732 pub dt_ids: Vec<i64>,
734 pub gt_ids: Vec<i64>,
736 pub dt_matches: Array2<i64>,
740 pub gt_matches: Array2<i64>,
743}
744
745#[derive(Debug, Clone)]
750pub struct EvalGrid {
751 pub eval_imgs: Vec<Option<Box<PerImageEval>>>,
764 pub eval_imgs_meta: Vec<Option<Box<EvalImageMeta>>>,
768 pub n_categories: usize,
771 pub n_area_ranges: usize,
773 pub n_images: usize,
776 pub retained_ious: Option<crate::tables::RetainedIous>,
780}
781
782impl EvalGrid {
783 pub fn cell(&self, k: usize, a: usize, i: usize) -> Option<&PerImageEval> {
788 let idx = self.flat_index(k, a, i)?;
789 self.eval_imgs.get(idx).and_then(Option::as_deref)
790 }
791
792 pub fn cell_meta(&self, k: usize, a: usize, i: usize) -> Option<&EvalImageMeta> {
795 let idx = self.flat_index(k, a, i)?;
796 self.eval_imgs_meta.get(idx).and_then(Option::as_deref)
797 }
798
799 fn flat_index(&self, k: usize, a: usize, i: usize) -> Option<usize> {
800 if k >= self.n_categories || a >= self.n_area_ranges || i >= self.n_images {
801 return None;
802 }
803 Some(k * self.n_area_ranges * self.n_images + a * self.n_images + i)
804 }
805}
806
807pub fn evaluate_with<K: EvalKernel>(
824 gt: &CocoDataset,
825 dt: &CocoDetections,
826 params: EvaluateParams<'_>,
827 parity_mode: ParityMode,
828 kernel: &K,
829) -> Result<EvalGrid, EvalError> {
830 let mut images: Vec<&ImageMeta> = gt.images().iter().collect();
832 images.sort_unstable_by_key(|im| im.id.0);
833 let n_i = images.len();
834 let n_a = params.area_ranges.len();
835
836 let category_buckets: Vec<Option<CategoryId>> = if params.use_cats {
838 let mut cats: Vec<_> = gt.categories().iter().map(|c| c.id).collect();
839 cats.sort_unstable_by_key(|id| id.0);
840 cats.into_iter().map(Some).collect()
841 } else {
842 vec![None]
843 };
844 let n_k = category_buckets.len();
845
846 let mut eval_imgs: Vec<Option<Box<PerImageEval>>> = vec![None; n_k * n_a * n_i];
847 let mut eval_imgs_meta: Vec<Option<Box<EvalImageMeta>>> = vec![None; n_k * n_a * n_i];
848 let mut retained_ious_map: Option<std::collections::HashMap<(usize, usize), Array2<f64>>> =
851 if params.retain_iou {
852 Some(std::collections::HashMap::new())
853 } else {
854 None
855 };
856
857 let federated_per_image: Vec<Option<(&HashSet<CategoryId>, &HashSet<CategoryId>)>> =
867 match (params.use_cats, gt.federated()) {
868 (true, Some(fed)) => images
869 .iter()
870 .map(|im| {
871 let neg = fed.neg_category_ids.get(&im.id)?;
872 let nel = fed.not_exhaustive_category_ids.get(&im.id)?;
873 Some((neg, nel))
874 })
875 .collect(),
876 _ => Vec::new(),
877 };
878
879 let mut scratch = CellScratch::new();
883 let gt_anns = gt.annotations();
884 let dt_anns = dt.detections();
885
886 let strict_lvis_zero_area_filter =
898 matches!(parity_mode, ParityMode::Strict) && gt.federated().is_some();
899
900 for (k, cat) in category_buckets.iter().enumerate() {
901 let nk = k * n_a * n_i;
902 let category_id = cat.map_or(COLLAPSED_CATEGORY_SENTINEL, |c| c.0);
903 for (i, image) in images.iter().enumerate() {
904 let image_id = image.id;
905 let gt_indices_raw = gt_indices_for_cell(gt, image_id, *cat);
906 let gt_indices_buf: Vec<usize>;
907 let gt_indices: &[usize] = if strict_lvis_zero_area_filter
908 && gt_indices_raw.iter().any(|&j| gt_anns[j].area <= 0.0)
909 {
910 gt_indices_buf = gt_indices_raw
911 .iter()
912 .copied()
913 .filter(|&j| gt_anns[j].area > 0.0)
914 .collect();
915 >_indices_buf
916 } else {
917 gt_indices_raw
918 };
919 let raw_dt_indices = raw_dt_indices_for_cell(dt, image_id, *cat);
920 if gt_indices.is_empty() && raw_dt_indices.is_empty() {
921 continue;
922 }
923
924 let mut not_exhaustive_for_cell = false;
929 if let (Some(c), Some(Some((neg_set, nel_set)))) = (cat, federated_per_image.get(i)) {
930 if gt_indices.is_empty() && !neg_set.contains(c) {
934 continue;
935 }
936 not_exhaustive_for_cell = nel_set.contains(c);
937 }
938
939 dt_top_indices_for_cell_into(
942 &mut scratch.dt_indices,
943 &mut scratch.dt_score_buf,
944 &mut scratch.dt_perm_buf,
945 dt_anns,
946 raw_dt_indices,
947 params.max_dets_per_image,
948 );
949
950 scratch.gt_areas.clear();
955 scratch
956 .gt_areas
957 .extend(gt_indices.iter().map(|&j| gt_anns[j].area));
958 scratch.gt_iscrowd.clear();
959 scratch
960 .gt_iscrowd
961 .extend(gt_indices.iter().map(|&j| gt_anns[j].is_crowd));
962 scratch.gt_base_ignore.clear();
966 scratch.gt_base_ignore.extend(gt_indices.iter().map(|&j| {
967 gt_anns[j].effective_ignore(parity_mode) || kernel.extra_gt_ignore(>_anns[j])
968 }));
969 scratch.gt_ids.clear();
970 scratch
971 .gt_ids
972 .extend(gt_indices.iter().map(|&j| gt_anns[j].id.0));
973 scratch.dt_areas.clear();
974 scratch
975 .dt_areas
976 .extend(scratch.dt_indices.iter().map(|&j| dt_anns[j].area));
977 scratch.dt_scores.clear();
978 scratch
979 .dt_scores
980 .extend(scratch.dt_indices.iter().map(|&j| dt_anns[j].score));
981 scratch.dt_ids.clear();
982 scratch
983 .dt_ids
984 .extend(scratch.dt_indices.iter().map(|&j| dt_anns[j].id.0));
985
986 let gt_kernel = kernel.build_gt_anns(gt_anns, gt_indices, image)?;
987 let dt_kernel =
988 kernel.build_dt_anns(dt_anns, &scratch.dt_indices, image, parity_mode)?;
989
990 let g = gt_kernel.len();
994 let d = dt_kernel.len();
995 scratch.iou_buf.clear();
996 scratch.iou_buf.resize(g * d, 0.0);
997 if g > 0 && d > 0 {
998 let mut iou_view = ArrayViewMut2::from_shape((g, d), &mut scratch.iou_buf[..])
999 .map_err(|e| EvalError::DimensionMismatch {
1000 detail: format!("iou scratch view: {e}"),
1001 })?;
1002 kernel.compute(>_kernel, &dt_kernel, &mut iou_view)?;
1003 }
1004
1005 let iou_view = ArrayView2::from_shape((g, d), &scratch.iou_buf[..]).map_err(|e| {
1006 EvalError::DimensionMismatch {
1007 detail: format!("iou scratch view: {e}"),
1008 }
1009 })?;
1010 let buffers = CellBuffers {
1011 image_id: image_id.0,
1012 category_id,
1013 max_det: params.max_dets_per_image,
1014 gt_areas: &scratch.gt_areas,
1015 gt_iscrowd: &scratch.gt_iscrowd,
1016 gt_base_ignore: &scratch.gt_base_ignore,
1017 gt_ids: &scratch.gt_ids,
1018 dt_areas: &scratch.dt_areas,
1019 dt_scores: &scratch.dt_scores,
1020 dt_ids: &scratch.dt_ids,
1021 iou: iou_view,
1022 not_exhaustive: not_exhaustive_for_cell,
1023 };
1024 for (a, area) in params.area_ranges.iter().enumerate() {
1025 let (cell, meta) = evaluate_cell(
1026 &mut scratch.gt_ignore_buf,
1027 &buffers,
1028 area,
1029 params.iou_thresholds,
1030 parity_mode,
1031 )?;
1032 let flat = nk + a * n_i + i;
1033 eval_imgs[flat] = Some(Box::new(cell));
1034 eval_imgs_meta[flat] = Some(Box::new(meta));
1035 }
1036
1037 if let Some(map) = retained_ious_map.as_mut() {
1042 let cloned =
1043 Array2::from_shape_vec((g, d), scratch.iou_buf.clone()).map_err(|e| {
1044 EvalError::DimensionMismatch {
1045 detail: format!("retained iou clone: {e}"),
1046 }
1047 })?;
1048 map.insert((k, i), cloned);
1049 }
1050 }
1051 }
1052
1053 Ok(EvalGrid {
1054 eval_imgs,
1055 eval_imgs_meta,
1056 n_categories: n_k,
1057 n_area_ranges: n_a,
1058 n_images: n_i,
1059 retained_ious: retained_ious_map.map(crate::tables::RetainedIous::from_map),
1060 })
1061}
1062
1063pub(crate) fn evaluate_with_retention<K: EvalKernel>(
1082 gt: &CocoDataset,
1083 dt: &CocoDetections,
1084 params: EvaluateParams<'_>,
1085 parity_mode: ParityMode,
1086 kernel: &K,
1087) -> Result<(EvalGrid, crate::tables::CrossClassIous), EvalError> {
1088 let grid = evaluate_with(gt, dt, params, parity_mode, kernel)?;
1089 let cross_class = crate::tide::compute_cross_class_ious(
1090 gt,
1091 dt,
1092 kernel,
1093 parity_mode,
1094 params.max_dets_per_image,
1095 )?;
1096 Ok((grid, cross_class))
1097}
1098
1099pub fn evaluate_bbox(
1107 gt: &CocoDataset,
1108 dt: &CocoDetections,
1109 params: EvaluateParams<'_>,
1110 parity_mode: ParityMode,
1111) -> Result<EvalGrid, EvalError> {
1112 evaluate_with(gt, dt, params, parity_mode, &BboxIou)
1113}
1114
1115pub fn evaluate_segm(
1135 gt: &CocoDataset,
1136 dt: &CocoDetections,
1137 params: EvaluateParams<'_>,
1138 parity_mode: ParityMode,
1139) -> Result<EvalGrid, EvalError> {
1140 evaluate_with(gt, dt, params, parity_mode, &segm_kernel(None))
1141}
1142
1143pub fn evaluate_segm_cached(
1159 gt: &CocoDataset,
1160 dt: &CocoDetections,
1161 params: EvaluateParams<'_>,
1162 parity_mode: ParityMode,
1163 cache: &SegmGtCache,
1164) -> Result<EvalGrid, EvalError> {
1165 evaluate_with(gt, dt, params, parity_mode, &segm_kernel(Some(cache)))
1166}
1167
1168fn segm_kernel(gt_cache: Option<&SegmGtCache>) -> SegmIouCached<'_> {
1169 SegmIouCached {
1170 scratch: Mutex::new(SegmComputeScratch::new()),
1171 gt_cache: gt_cache.map(GtCacheRef::Borrowed),
1172 }
1173}
1174
1175pub struct SegmIouCached<'a> {
1189 scratch: Mutex<SegmComputeScratch>,
1190 gt_cache: Option<GtCacheRef<'a, SegmGtCache>>,
1191}
1192
1193impl SegmIouCached<'static> {
1194 pub fn with_arc_cache(cache: Arc<SegmGtCache>) -> Self {
1201 Self {
1202 scratch: Mutex::new(SegmComputeScratch::new()),
1203 gt_cache: Some(GtCacheRef::Owned(cache)),
1204 }
1205 }
1206}
1207
1208impl Similarity for SegmIouCached<'_> {
1209 type Annotation = SegmAnn;
1210
1211 fn compute(
1212 &self,
1213 gts: &[SegmAnn],
1214 dts: &[SegmAnn],
1215 out: &mut ArrayViewMut2<'_, f64>,
1216 ) -> Result<(), EvalError> {
1217 let mut scratch = self
1218 .scratch
1219 .lock()
1220 .unwrap_or_else(|poisoned| poisoned.into_inner());
1221 segm_iou_compute(
1222 gts,
1223 dts,
1224 out,
1225 &mut scratch,
1226 self.gt_cache.as_ref().map(GtCacheRef::get),
1227 )
1228 }
1229}
1230
1231impl EvalKernel for SegmIouCached<'_> {
1232 fn kind(&self) -> KernelKind {
1233 KernelKind::Segm
1234 }
1235
1236 fn build_gt_anns(
1237 &self,
1238 gt_anns: &[CocoAnnotation],
1239 indices: &[usize],
1240 image: &ImageMeta,
1241 ) -> Result<Vec<SegmAnn>, EvalError> {
1242 build_segm_gt_anns(gt_anns, indices, image)
1243 }
1244
1245 fn build_dt_anns(
1246 &self,
1247 dt_anns: &[CocoDetection],
1248 indices: &[usize],
1249 image: &ImageMeta,
1250 parity_mode: ParityMode,
1251 ) -> Result<Vec<SegmAnn>, EvalError> {
1252 build_segm_dt_anns(dt_anns, indices, image, parity_mode)
1253 }
1254}
1255
1256pub fn evaluate_boundary(
1271 gt: &CocoDataset,
1272 dt: &CocoDetections,
1273 params: EvaluateParams<'_>,
1274 parity_mode: ParityMode,
1275 dilation_ratio: f64,
1276) -> Result<EvalGrid, EvalError> {
1277 evaluate_with(gt, dt, params, parity_mode, &kernel(dilation_ratio, None))
1278}
1279
1280pub fn evaluate_boundary_cached(
1298 gt: &CocoDataset,
1299 dt: &CocoDetections,
1300 params: EvaluateParams<'_>,
1301 parity_mode: ParityMode,
1302 dilation_ratio: f64,
1303 cache: &BoundaryGtCache,
1304) -> Result<EvalGrid, EvalError> {
1305 cache.align_ratio(dilation_ratio);
1306 evaluate_with(
1307 gt,
1308 dt,
1309 params,
1310 parity_mode,
1311 &kernel(dilation_ratio, Some(cache)),
1312 )
1313}
1314
1315fn kernel(dilation_ratio: f64, gt_cache: Option<&BoundaryGtCache>) -> BoundaryIouCached<'_> {
1316 BoundaryIouCached {
1317 dilation_ratio,
1318 scratch: Mutex::new(BoundaryComputeScratch::new()),
1319 gt_cache: gt_cache.map(GtCacheRef::Borrowed),
1320 }
1321}
1322
1323pub struct BoundaryIouCached<'a> {
1337 dilation_ratio: f64,
1338 scratch: Mutex<BoundaryComputeScratch>,
1339 gt_cache: Option<GtCacheRef<'a, BoundaryGtCache>>,
1340}
1341
1342impl BoundaryIouCached<'static> {
1343 pub fn with_arc_cache(dilation_ratio: f64, cache: Arc<BoundaryGtCache>) -> Self {
1354 cache.align_ratio(dilation_ratio);
1355 Self {
1356 dilation_ratio,
1357 scratch: Mutex::new(BoundaryComputeScratch::new()),
1358 gt_cache: Some(GtCacheRef::Owned(cache)),
1359 }
1360 }
1361}
1362
1363impl Similarity for BoundaryIouCached<'_> {
1364 type Annotation = SegmAnn;
1365
1366 fn compute(
1367 &self,
1368 gts: &[SegmAnn],
1369 dts: &[SegmAnn],
1370 out: &mut ArrayViewMut2<'_, f64>,
1371 ) -> Result<(), EvalError> {
1372 let mut scratch = self
1373 .scratch
1374 .lock()
1375 .unwrap_or_else(|poisoned| poisoned.into_inner());
1376 boundary_iou_compute(
1377 self.dilation_ratio,
1378 gts,
1379 dts,
1380 out,
1381 &mut scratch,
1382 self.gt_cache.as_ref().map(GtCacheRef::get),
1383 )
1384 }
1385}
1386
1387impl EvalKernel for BoundaryIouCached<'_> {
1388 fn kind(&self) -> KernelKind {
1389 KernelKind::Boundary
1390 }
1391
1392 fn build_gt_anns(
1393 &self,
1394 gt_anns: &[CocoAnnotation],
1395 indices: &[usize],
1396 image: &ImageMeta,
1397 ) -> Result<Vec<SegmAnn>, EvalError> {
1398 build_segm_gt_anns(gt_anns, indices, image)
1399 }
1400
1401 fn build_dt_anns(
1402 &self,
1403 dt_anns: &[CocoDetection],
1404 indices: &[usize],
1405 image: &ImageMeta,
1406 parity_mode: ParityMode,
1407 ) -> Result<Vec<SegmAnn>, EvalError> {
1408 build_segm_dt_anns(dt_anns, indices, image, parity_mode)
1409 }
1410}
1411
1412pub fn evaluate_keypoints(
1453 gt: &CocoDataset,
1454 dt: &CocoDetections,
1455 params: EvaluateParams<'_>,
1456 parity_mode: ParityMode,
1457 sigmas: HashMap<i64, Vec<f64>>,
1458) -> Result<EvalGrid, EvalError> {
1459 evaluate_with(gt, dt, params, parity_mode, &OksSimilarity::new(sigmas))
1460}
1461
1462fn gt_indices_for_cell(gt: &CocoDataset, image: ImageId, cat: Option<CategoryId>) -> &[usize] {
1463 match cat {
1464 Some(c) => gt.ann_indices_for(image, c),
1465 None => gt.ann_indices_for_image(image),
1466 }
1467}
1468
1469fn raw_dt_indices_for_cell(
1474 dt: &CocoDetections,
1475 image: ImageId,
1476 cat: Option<CategoryId>,
1477) -> &[usize] {
1478 match cat {
1479 Some(c) => dt.indices_for(image, c),
1480 None => dt.indices_for_image(image),
1481 }
1482}
1483
1484pub(crate) fn dt_top_indices_for_cell(
1485 dt: &CocoDetections,
1486 image: ImageId,
1487 cat: Option<CategoryId>,
1488 max_dets: usize,
1489) -> Vec<usize> {
1490 let raw_indices = raw_dt_indices_for_cell(dt, image, cat);
1491 let mut out = Vec::new();
1492 let mut score_buf = Vec::new();
1493 let mut perm_buf = Vec::new();
1494 dt_top_indices_for_cell_into(
1495 &mut out,
1496 &mut score_buf,
1497 &mut perm_buf,
1498 dt.detections(),
1499 raw_indices,
1500 max_dets,
1501 );
1502 out
1503}
1504
1505fn dt_top_indices_for_cell_into(
1513 out: &mut Vec<usize>,
1514 score_buf: &mut Vec<f64>,
1515 perm_buf: &mut Vec<usize>,
1516 dts: &[CocoDetection],
1517 raw_indices: &[usize],
1518 max_dets: usize,
1519) {
1520 score_buf.clear();
1521 score_buf.extend(raw_indices.iter().map(|&i| dts[i].score));
1522 perm_buf.clear();
1523 perm_buf.extend(0..score_buf.len());
1524 perm_buf.sort_by(|&a, &b| {
1527 score_buf[b]
1528 .partial_cmp(&score_buf[a])
1529 .unwrap_or(std::cmp::Ordering::Equal)
1530 });
1531 out.clear();
1532 out.extend(perm_buf.iter().take(max_dets).map(|&k| raw_indices[k]));
1533}
1534
1535#[derive(Default)]
1542struct CellScratch {
1543 gt_areas: Vec<f64>,
1545 gt_iscrowd: Vec<bool>,
1546 gt_base_ignore: Vec<bool>,
1547 gt_ids: Vec<i64>,
1548 dt_indices: Vec<usize>,
1551 dt_areas: Vec<f64>,
1553 dt_scores: Vec<f64>,
1554 dt_ids: Vec<i64>,
1555 iou_buf: Vec<f64>,
1559 dt_score_buf: Vec<f64>,
1561 dt_perm_buf: Vec<usize>,
1563 gt_ignore_buf: Vec<bool>,
1568}
1569
1570impl CellScratch {
1571 fn new() -> Self {
1572 Self::default()
1573 }
1574}
1575
1576struct CellBuffers<'a> {
1578 image_id: i64,
1579 category_id: i64,
1580 max_det: usize,
1581 gt_areas: &'a [f64],
1582 gt_iscrowd: &'a [bool],
1583 gt_base_ignore: &'a [bool],
1584 gt_ids: &'a [i64],
1585 dt_areas: &'a [f64],
1586 dt_scores: &'a [f64],
1587 dt_ids: &'a [i64],
1588 iou: ArrayView2<'a, f64>,
1589 not_exhaustive: bool,
1594}
1595
1596fn evaluate_cell(
1597 gt_ignore_buf: &mut Vec<bool>,
1598 buf: &CellBuffers<'_>,
1599 area: &AreaRange,
1600 iou_thresholds: &[f64],
1601 parity_mode: ParityMode,
1602) -> Result<(PerImageEval, EvalImageMeta), EvalError> {
1603 gt_ignore_buf.clear();
1609 gt_ignore_buf.extend(
1610 buf.gt_base_ignore
1611 .iter()
1612 .zip(buf.gt_areas)
1613 .map(|(&base, &a)| base || !area.contains(a)),
1614 );
1615 let gt_ignore: &[bool] = gt_ignore_buf.as_slice();
1616
1617 let MatchResult {
1618 dt_perm,
1619 gt_perm,
1620 dt_matches: dt_matches_pos,
1621 gt_matches: gt_matches_pos,
1622 mut dt_ignore,
1623 } = match_image(
1624 buf.iou,
1625 gt_ignore,
1626 buf.gt_iscrowd,
1627 buf.dt_scores,
1628 iou_thresholds,
1629 parity_mode,
1630 )?;
1631
1632 let n_t = iou_thresholds.len();
1633 let n_d = buf.dt_scores.len();
1634 let n_g = gt_ignore.len();
1635
1636 let dt_scores_sorted: Vec<f64> = dt_perm.iter().map(|&k| buf.dt_scores[k]).collect();
1637 let gt_ignore_sorted: Vec<bool> = gt_perm.iter().map(|&k| gt_ignore[k]).collect();
1638 let dt_ids_sorted: Vec<i64> = dt_perm.iter().map(|&k| buf.dt_ids[k]).collect();
1639 let gt_ids_sorted: Vec<i64> = gt_perm.iter().map(|&k| buf.gt_ids[k]).collect();
1640
1641 let mut dt_matched = Array2::<bool>::default((n_t, n_d));
1642 let mut dt_matches_id = Array2::<i64>::zeros((n_t, n_d));
1643 let mut gt_matches_id = Array2::<i64>::zeros((n_t, n_g));
1644 for d in 0..n_d {
1651 let in_range = area.contains(buf.dt_areas[dt_perm[d]]);
1652 for t in 0..n_t {
1653 let m = dt_matches_pos[(t, d)];
1654 let matched = m >= 0;
1655 dt_matched[(t, d)] = matched;
1656 if matched {
1657 dt_matches_id[(t, d)] = gt_ids_sorted[m as usize];
1658 }
1659 if !matched && (!in_range || buf.not_exhaustive) {
1664 dt_ignore[(t, d)] = true;
1665 }
1666 }
1667 }
1668 for t in 0..n_t {
1669 for g in 0..n_g {
1670 let p = gt_matches_pos[(t, g)];
1671 if p >= 0 {
1672 gt_matches_id[(t, g)] = dt_ids_sorted[p as usize];
1673 }
1674 }
1675 }
1676
1677 let cell = PerImageEval {
1678 dt_scores: dt_scores_sorted,
1679 dt_matched,
1680 dt_ignore,
1681 gt_ignore: gt_ignore_sorted,
1682 };
1683 let meta = EvalImageMeta {
1684 image_id: buf.image_id,
1685 category_id: buf.category_id,
1686 area_rng: [area.lo, area.hi],
1687 max_det: buf.max_det,
1688 dt_ids: dt_ids_sorted,
1689 gt_ids: gt_ids_sorted,
1690 dt_matches: dt_matches_id,
1691 gt_matches: gt_matches_id,
1692 };
1693 Ok((cell, meta))
1694}
1695
1696#[cfg(test)]
1697mod tests {
1698 use super::*;
1699 use crate::accumulate::{accumulate, AccumulateParams};
1700 use crate::dataset::{AnnId, Bbox, CategoryMeta, CocoAnnotation, DetectionInput, ImageMeta};
1701 use crate::parity::{iou_thresholds, recall_thresholds};
1702 use crate::summarize::summarize_detection;
1703
1704 fn img(id: i64, w: u32, h: u32) -> ImageMeta {
1705 ImageMeta {
1706 id: ImageId(id),
1707 width: w,
1708 height: h,
1709 file_name: None,
1710 }
1711 }
1712
1713 fn cat(id: i64, name: &str) -> CategoryMeta {
1714 CategoryMeta {
1715 id: CategoryId(id),
1716 name: name.into(),
1717 supercategory: None,
1718 }
1719 }
1720
1721 fn ann(id: i64, image: i64, cat: i64, bbox: (f64, f64, f64, f64)) -> CocoAnnotation {
1722 CocoAnnotation {
1723 id: AnnId(id),
1724 image_id: ImageId(image),
1725 category_id: CategoryId(cat),
1726 area: bbox.2 * bbox.3,
1727 is_crowd: false,
1728 ignore_flag: None,
1729 bbox: Bbox {
1730 x: bbox.0,
1731 y: bbox.1,
1732 w: bbox.2,
1733 h: bbox.3,
1734 },
1735 segmentation: None,
1736 keypoints: None,
1737 num_keypoints: None,
1738 }
1739 }
1740
1741 fn dt_input(image: i64, cat: i64, score: f64, bbox: (f64, f64, f64, f64)) -> DetectionInput {
1742 DetectionInput {
1743 id: None,
1744 image_id: ImageId(image),
1745 category_id: CategoryId(cat),
1746 score,
1747 bbox: Bbox {
1748 x: bbox.0,
1749 y: bbox.1,
1750 w: bbox.2,
1751 h: bbox.3,
1752 },
1753 segmentation: None,
1754 keypoints: None,
1755 num_keypoints: None,
1756 }
1757 }
1758
1759 fn perfect_match_grid() -> EvalGrid {
1760 let images = vec![img(1, 100, 100)];
1761 let cats = vec![cat(1, "thing")];
1762 let anns = vec![
1763 ann(1, 1, 1, (0.0, 0.0, 10.0, 10.0)),
1764 ann(2, 1, 1, (50.0, 50.0, 10.0, 10.0)),
1765 ];
1766 let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
1767 let dts = CocoDetections::from_inputs(vec![
1768 dt_input(1, 1, 0.9, (0.0, 0.0, 10.0, 10.0)),
1769 dt_input(1, 1, 0.8, (50.0, 50.0, 10.0, 10.0)),
1770 ])
1771 .unwrap();
1772 let area = AreaRange::coco_default();
1773 let params = EvaluateParams {
1774 iou_thresholds: iou_thresholds(),
1775 area_ranges: &area,
1776 max_dets_per_image: 100,
1777 use_cats: true,
1778 retain_iou: false,
1779 };
1780 evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap()
1781 }
1782
1783 #[test]
1784 fn d4_coco_default_area_ranges_pin_literal_values() {
1785 let ranges = AreaRange::coco_default();
1792 assert_eq!(ranges.len(), 4);
1793 assert_eq!(
1794 (ranges[0].lo, ranges[0].hi),
1795 (0.0, 1e10),
1796 "all bucket bounds"
1797 );
1798 assert_eq!(
1799 (ranges[1].lo, ranges[1].hi),
1800 (0.0, 1024.0),
1801 "small bucket bounds"
1802 );
1803 assert_eq!(
1804 (ranges[2].lo, ranges[2].hi),
1805 (1024.0, 9216.0),
1806 "medium bucket bounds"
1807 );
1808 assert_eq!(
1809 (ranges[3].lo, ranges[3].hi),
1810 (9216.0, 1e10),
1811 "large bucket bounds"
1812 );
1813
1814 use crate::summarize::AreaRng;
1818 assert_eq!(ranges[0].index, AreaRng::ALL.index);
1819 assert_eq!(AreaRng::ALL.label.as_ref(), "all");
1820 assert_eq!(ranges[1].index, AreaRng::SMALL.index);
1821 assert_eq!(AreaRng::SMALL.label.as_ref(), "small");
1822 assert_eq!(ranges[2].index, AreaRng::MEDIUM.index);
1823 assert_eq!(AreaRng::MEDIUM.label.as_ref(), "medium");
1824 assert_eq!(ranges[3].index, AreaRng::LARGE.index);
1825 assert_eq!(AreaRng::LARGE.label.as_ref(), "large");
1826
1827 let pyco_unbounded: f64 = 1e5_f64.powi(2);
1831 assert_eq!(pyco_unbounded.to_bits(), 1e10_f64.to_bits());
1832 assert_eq!(ranges[0].hi.to_bits(), 1e10_f64.to_bits());
1833 assert_eq!(ranges[3].hi.to_bits(), 1e10_f64.to_bits());
1834 }
1835
1836 #[test]
1837 fn perfect_match_produces_one_cell_per_area_range() {
1838 let grid = perfect_match_grid();
1839 assert_eq!(grid.n_categories, 1);
1840 assert_eq!(grid.n_area_ranges, 4);
1841 assert_eq!(grid.n_images, 1);
1842 let cells: Vec<_> = grid.eval_imgs.iter().filter(|c| c.is_some()).collect();
1844 assert_eq!(cells.len(), 4);
1845 let all_cell = grid.cell(0, 0, 0).unwrap();
1847 assert_eq!(all_cell.dt_scores.len(), 2);
1848 assert!(all_cell.dt_matched.iter().all(|&m| m));
1849 assert!(all_cell.dt_ignore.iter().all(|&ig| !ig));
1850 }
1851
1852 #[test]
1853 fn perfect_match_summarizes_to_one() {
1854 let grid = perfect_match_grid();
1855 let max_dets = vec![1usize, 10, 100];
1856 let acc = accumulate(
1857 &grid.eval_imgs,
1858 AccumulateParams {
1859 iou_thresholds: iou_thresholds(),
1860 recall_thresholds: recall_thresholds(),
1861 max_dets: &max_dets,
1862 n_categories: grid.n_categories,
1863 n_area_ranges: grid.n_area_ranges,
1864 n_images: grid.n_images,
1865 },
1866 ParityMode::Strict,
1867 )
1868 .unwrap();
1869 let summary = summarize_detection(&acc, iou_thresholds(), &max_dets).unwrap();
1870 let stats = summary.stats();
1871 assert!((stats[0] - 1.0).abs() < 1e-12, "AP={}", stats[0]);
1875 assert!((stats[3] - 1.0).abs() < 1e-12, "AP_S={}", stats[3]);
1876 assert_eq!(stats[4], -1.0, "AP_M should be -1 with no medium GTs");
1877 assert_eq!(stats[5], -1.0, "AP_L should be -1 with no large GTs");
1878 assert!((stats[8] - 1.0).abs() < 1e-12, "AR@100={}", stats[8]);
1879 }
1880
1881 #[test]
1882 fn b7_unmatched_dt_outside_area_range_is_ignored() {
1883 let images = vec![img(1, 300, 300)];
1887 let cats = vec![cat(1, "thing")];
1888 let anns = vec![ann(1, 1, 1, (0.0, 0.0, 200.0, 200.0))];
1889 let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
1890 let dts =
1891 CocoDetections::from_inputs(vec![dt_input(1, 1, 0.5, (200.0, 200.0, 50.0, 50.0))])
1892 .unwrap();
1893 let area = AreaRange::coco_default();
1894 let params = EvaluateParams {
1895 iou_thresholds: iou_thresholds(),
1896 area_ranges: &area,
1897 max_dets_per_image: 100,
1898 use_cats: true,
1899 retain_iou: false,
1900 };
1901 let grid = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
1902 let small = grid.cell(0, 1, 0).unwrap();
1903 assert_eq!(small.gt_ignore, vec![true]);
1905 assert!(small.dt_ignore.iter().all(|&ig| ig));
1907 assert!(small.dt_matched.iter().all(|&m| !m));
1908 }
1909
1910 #[test]
1911 fn d6_boundary_area_lands_in_both_buckets() {
1912 let images = vec![img(1, 100, 100)];
1916 let cats = vec![cat(1, "thing")];
1917 let anns = vec![ann(1, 1, 1, (0.0, 0.0, 32.0, 32.0))];
1919 let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
1920 let dts =
1921 CocoDetections::from_inputs(vec![dt_input(1, 1, 0.5, (0.0, 0.0, 32.0, 32.0))]).unwrap();
1922 let area = AreaRange::coco_default();
1923 let params = EvaluateParams {
1924 iou_thresholds: iou_thresholds(),
1925 area_ranges: &area,
1926 max_dets_per_image: 100,
1927 use_cats: true,
1928 retain_iou: false,
1929 };
1930 let grid = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
1931 let small = grid.cell(0, 1, 0).unwrap();
1933 assert_eq!(small.gt_ignore, vec![false]);
1934 let medium = grid.cell(0, 2, 0).unwrap();
1936 assert_eq!(medium.gt_ignore, vec![false]);
1937 let all = grid.cell(0, 0, 0).unwrap();
1939 assert_eq!(all.gt_ignore, vec![false]);
1940 let large = grid.cell(0, 3, 0).unwrap();
1942 assert_eq!(large.gt_ignore, vec![true]);
1943 }
1944
1945 #[test]
1946 fn l4_use_cats_false_collapses_categories() {
1947 let images = vec![img(1, 100, 100)];
1948 let cats = vec![cat(1, "a"), cat(2, "b")];
1949 let anns = vec![
1950 ann(1, 1, 1, (0.0, 0.0, 10.0, 10.0)),
1951 ann(2, 1, 2, (50.0, 50.0, 10.0, 10.0)),
1952 ];
1953 let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
1954 let dts = CocoDetections::from_inputs(vec![dt_input(1, 1, 0.9, (50.0, 50.0, 10.0, 10.0))])
1957 .unwrap();
1958 let area = AreaRange::coco_default();
1959 let params = EvaluateParams {
1960 iou_thresholds: iou_thresholds(),
1961 area_ranges: &area,
1962 max_dets_per_image: 100,
1963 use_cats: false,
1964 retain_iou: false,
1965 };
1966 let grid = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
1967 assert_eq!(grid.n_categories, 1);
1968 let all = grid.cell(0, 0, 0).unwrap();
1969 assert_eq!(all.gt_ignore.len(), 2);
1971 assert_eq!(all.dt_scores.len(), 1);
1972 assert!(all.dt_matched.iter().all(|&m| m));
1973 }
1974
1975 #[test]
1976 fn max_dets_per_image_caps_top_n_by_score() {
1977 let images = vec![img(1, 100, 100)];
1978 let cats = vec![cat(1, "thing")];
1979 let anns = vec![ann(1, 1, 1, (0.0, 0.0, 10.0, 10.0))];
1980 let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
1981 let dts = CocoDetections::from_inputs(vec![
1982 dt_input(1, 1, 0.1, (50.0, 50.0, 5.0, 5.0)),
1983 dt_input(1, 1, 0.9, (0.0, 0.0, 10.0, 10.0)),
1984 dt_input(1, 1, 0.5, (50.0, 50.0, 5.0, 5.0)),
1985 ])
1986 .unwrap();
1987 let area = AreaRange::coco_default();
1988 let params = EvaluateParams {
1989 iou_thresholds: iou_thresholds(),
1990 area_ranges: &area,
1991 max_dets_per_image: 2,
1992 use_cats: true,
1993 retain_iou: false,
1994 };
1995 let grid = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
1996 let all = grid.cell(0, 0, 0).unwrap();
1997 assert_eq!(all.dt_scores.len(), 2);
1999 assert_eq!(all.dt_scores[0], 0.9);
2000 assert_eq!(all.dt_scores[1], 0.5);
2001 }
2002
2003 #[test]
2004 fn d1_parity_mode_propagates_to_base_ignore() {
2005 const ANN_JSON: &str = r#"{
2011 "images": [{"id": 1, "width": 100, "height": 100}],
2012 "annotations": [
2013 {"id": 1, "image_id": 1, "category_id": 1,
2014 "bbox": [0, 0, 10, 10], "area": 100,
2015 "iscrowd": 0, "ignore": 1}
2016 ],
2017 "categories": [{"id": 1, "name": "thing"}]
2018 }"#;
2019 let gt = CocoDataset::from_json_bytes(ANN_JSON.as_bytes()).unwrap();
2020 let dts =
2021 CocoDetections::from_inputs(vec![dt_input(1, 1, 0.9, (0.0, 0.0, 10.0, 10.0))]).unwrap();
2022 let area = AreaRange::coco_default();
2023 let params = EvaluateParams {
2024 iou_thresholds: iou_thresholds(),
2025 area_ranges: &area,
2026 max_dets_per_image: 100,
2027 use_cats: true,
2028 retain_iou: false,
2029 };
2030
2031 let strict = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
2032 let strict_all = strict.cell(0, 0, 0).unwrap();
2033 assert_eq!(strict_all.gt_ignore, vec![false]);
2034 assert!(strict_all.dt_ignore.iter().all(|&ig| !ig));
2035
2036 let corrected = evaluate_bbox(>, &dts, params, ParityMode::Corrected).unwrap();
2037 let corrected_all = corrected.cell(0, 0, 0).unwrap();
2038 assert_eq!(corrected_all.gt_ignore, vec![true]);
2039 assert!(corrected_all.dt_ignore.iter().all(|&ig| ig));
2041 }
2042
2043 #[test]
2044 fn cell_meta_carries_pycocotools_shape() {
2045 let grid = perfect_match_grid();
2046 let meta = grid.cell_meta(0, 0, 0).unwrap();
2048 assert_eq!(meta.image_id, 1);
2049 assert_eq!(meta.category_id, 1);
2050 assert_eq!(meta.area_rng, [0.0, AREA_UNBOUNDED]);
2051 assert_eq!(meta.max_det, 100);
2052 assert_eq!(meta.dt_ids, vec![1, 2]);
2054 assert_eq!(meta.gt_ids, vec![1, 2]);
2056 let n_t = iou_thresholds().len();
2057 assert_eq!(meta.dt_matches.shape(), &[n_t, 2]);
2058 assert_eq!(meta.gt_matches.shape(), &[n_t, 2]);
2059 for t in 0..n_t {
2062 assert_eq!(meta.dt_matches[(t, 0)], 1, "dt[0] -> gt[1] at t={t}");
2063 assert_eq!(meta.dt_matches[(t, 1)], 2, "dt[1] -> gt[2] at t={t}");
2064 assert_eq!(meta.gt_matches[(t, 0)], 1, "gt[1] -> dt[1] at t={t}");
2065 assert_eq!(meta.gt_matches[(t, 1)], 2, "gt[2] -> dt[2] at t={t}");
2066 }
2067 }
2068
2069 #[test]
2070 fn cell_meta_unmatched_dt_uses_zero_sentinel() {
2071 let images = vec![img(1, 100, 100)];
2073 let cats = vec![cat(1, "thing")];
2074 let anns = vec![ann(7, 1, 1, (0.0, 0.0, 10.0, 10.0))];
2075 let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
2076 let dts = CocoDetections::from_inputs(vec![dt_input(1, 1, 0.5, (50.0, 50.0, 10.0, 10.0))])
2077 .unwrap();
2078 let area = AreaRange::coco_default();
2079 let params = EvaluateParams {
2080 iou_thresholds: iou_thresholds(),
2081 area_ranges: &area,
2082 max_dets_per_image: 100,
2083 use_cats: true,
2084 retain_iou: false,
2085 };
2086 let grid = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
2087 let meta = grid.cell_meta(0, 0, 0).unwrap();
2088 assert_eq!(meta.gt_ids, vec![7]);
2089 assert_eq!(meta.dt_ids.len(), 1);
2091 assert!(meta.dt_matches.iter().all(|&x| x == 0));
2092 assert!(meta.gt_matches.iter().all(|&x| x == 0));
2093 }
2094
2095 #[test]
2096 fn cell_meta_use_cats_false_emits_sentinel_category() {
2097 let images = vec![img(1, 100, 100)];
2098 let cats = vec![cat(1, "a"), cat(2, "b")];
2099 let anns = vec![ann(1, 1, 1, (0.0, 0.0, 10.0, 10.0))];
2100 let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
2101 let dts =
2102 CocoDetections::from_inputs(vec![dt_input(1, 1, 0.9, (0.0, 0.0, 10.0, 10.0))]).unwrap();
2103 let area = AreaRange::coco_default();
2104 let params = EvaluateParams {
2105 iou_thresholds: iou_thresholds(),
2106 area_ranges: &area,
2107 max_dets_per_image: 100,
2108 use_cats: false,
2109 retain_iou: false,
2110 };
2111 let grid = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
2112 let meta = grid.cell_meta(0, 0, 0).unwrap();
2113 assert_eq!(meta.category_id, COLLAPSED_CATEGORY_SENTINEL);
2114 }
2115
2116 #[test]
2117 fn missing_dt_image_yields_none_cells() {
2118 let images = vec![img(1, 100, 100), img(2, 100, 100)];
2121 let cats = vec![cat(1, "thing")];
2122 let anns = vec![ann(1, 1, 1, (0.0, 0.0, 10.0, 10.0))];
2123 let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
2124 let dts = CocoDetections::from_inputs(vec![]).unwrap();
2125 let area = AreaRange::coco_default();
2126 let params = EvaluateParams {
2127 iou_thresholds: iou_thresholds(),
2128 area_ranges: &area,
2129 max_dets_per_image: 100,
2130 use_cats: true,
2131 retain_iou: false,
2132 };
2133 let grid = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
2134 for a in 0..4 {
2135 assert!(grid.cell(0, a, 0).is_some(), "image 1 area {a}");
2136 assert!(grid.cell(0, a, 1).is_none(), "image 2 area {a}");
2137 }
2138 }
2139
2140 fn square_polygon(x: f64, y: f64, side: f64) -> Segmentation {
2141 Segmentation::Polygons(vec![vec![
2142 x,
2143 y,
2144 x + side,
2145 y,
2146 x + side,
2147 y + side,
2148 x,
2149 y + side,
2150 ]])
2151 }
2152
2153 fn ann_with_segm(
2154 id: i64,
2155 image: i64,
2156 cat: i64,
2157 bbox: (f64, f64, f64, f64),
2158 segm: Segmentation,
2159 ) -> CocoAnnotation {
2160 CocoAnnotation {
2161 id: AnnId(id),
2162 image_id: ImageId(image),
2163 category_id: CategoryId(cat),
2164 area: bbox.2 * bbox.3,
2165 is_crowd: false,
2166 ignore_flag: None,
2167 bbox: Bbox {
2168 x: bbox.0,
2169 y: bbox.1,
2170 w: bbox.2,
2171 h: bbox.3,
2172 },
2173 segmentation: Some(segm),
2174 keypoints: None,
2175 num_keypoints: None,
2176 }
2177 }
2178
2179 fn dt_input_with_segm(
2180 image: i64,
2181 cat: i64,
2182 score: f64,
2183 bbox: (f64, f64, f64, f64),
2184 segm: Segmentation,
2185 ) -> DetectionInput {
2186 DetectionInput {
2187 id: None,
2188 image_id: ImageId(image),
2189 category_id: CategoryId(cat),
2190 score,
2191 bbox: Bbox {
2192 x: bbox.0,
2193 y: bbox.1,
2194 w: bbox.2,
2195 h: bbox.3,
2196 },
2197 segmentation: Some(segm),
2198 keypoints: None,
2199 num_keypoints: None,
2200 }
2201 }
2202
2203 #[test]
2204 fn segm_perfect_overlap_summarizes_to_one() {
2205 let images = vec![img(1, 100, 100)];
2206 let cats = vec![cat(1, "thing")];
2207 let anns = vec![ann_with_segm(
2208 1,
2209 1,
2210 1,
2211 (10.0, 10.0, 20.0, 20.0),
2212 square_polygon(10.0, 10.0, 20.0),
2213 )];
2214 let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
2215 let dts = CocoDetections::from_inputs(vec![dt_input_with_segm(
2216 1,
2217 1,
2218 0.9,
2219 (10.0, 10.0, 20.0, 20.0),
2220 square_polygon(10.0, 10.0, 20.0),
2221 )])
2222 .unwrap();
2223 let area = AreaRange::coco_default();
2224 let params = EvaluateParams {
2225 iou_thresholds: iou_thresholds(),
2226 area_ranges: &area,
2227 max_dets_per_image: 100,
2228 use_cats: true,
2229 retain_iou: false,
2230 };
2231 let grid = evaluate_segm(>, &dts, params, ParityMode::Strict).unwrap();
2232 let max_dets = vec![1usize, 10, 100];
2233 let acc = accumulate(
2234 &grid.eval_imgs,
2235 AccumulateParams {
2236 iou_thresholds: iou_thresholds(),
2237 recall_thresholds: recall_thresholds(),
2238 max_dets: &max_dets,
2239 n_categories: grid.n_categories,
2240 n_area_ranges: grid.n_area_ranges,
2241 n_images: grid.n_images,
2242 },
2243 ParityMode::Strict,
2244 )
2245 .unwrap();
2246 let summary = summarize_detection(&acc, iou_thresholds(), &max_dets).unwrap();
2247 let stats = summary.stats();
2248 assert!((stats[0] - 1.0).abs() < 1e-12, "AP={}", stats[0]);
2249 }
2250
2251 #[test]
2252 fn segm_disjoint_masks_summarize_to_zero() {
2253 let images = vec![img(1, 100, 100)];
2254 let cats = vec![cat(1, "thing")];
2255 let anns = vec![ann_with_segm(
2256 1,
2257 1,
2258 1,
2259 (0.0, 0.0, 10.0, 10.0),
2260 square_polygon(0.0, 0.0, 10.0),
2261 )];
2262 let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
2263 let dts = CocoDetections::from_inputs(vec![dt_input_with_segm(
2264 1,
2265 1,
2266 0.9,
2267 (50.0, 50.0, 10.0, 10.0),
2268 square_polygon(50.0, 50.0, 10.0),
2269 )])
2270 .unwrap();
2271 let area = AreaRange::coco_default();
2272 let params = EvaluateParams {
2273 iou_thresholds: iou_thresholds(),
2274 area_ranges: &area,
2275 max_dets_per_image: 100,
2276 use_cats: true,
2277 retain_iou: false,
2278 };
2279 let grid = evaluate_segm(>, &dts, params, ParityMode::Strict).unwrap();
2280 let all = grid.cell(0, 0, 0).unwrap();
2281 assert!(all.dt_matched.iter().all(|&m| !m));
2283 }
2284
2285 #[test]
2286 fn segm_missing_gt_segmentation_surfaces_typed_error() {
2287 let images = vec![img(1, 100, 100)];
2290 let cats = vec![cat(1, "thing")];
2291 let anns = vec![ann(7, 1, 1, (0.0, 0.0, 10.0, 10.0))];
2292 let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
2293 let dts = CocoDetections::from_inputs(vec![dt_input_with_segm(
2294 1,
2295 1,
2296 0.9,
2297 (0.0, 0.0, 10.0, 10.0),
2298 square_polygon(0.0, 0.0, 10.0),
2299 )])
2300 .unwrap();
2301 let area = AreaRange::coco_default();
2302 let params = EvaluateParams {
2303 iou_thresholds: iou_thresholds(),
2304 area_ranges: &area,
2305 max_dets_per_image: 100,
2306 use_cats: true,
2307 retain_iou: false,
2308 };
2309 let err = evaluate_segm(>, &dts, params, ParityMode::Strict).unwrap_err();
2310 match err {
2311 EvalError::InvalidAnnotation { detail } => {
2312 assert!(detail.contains("GT id=7"), "msg: {detail}");
2313 }
2314 other => panic!("expected InvalidAnnotation, got {other:?}"),
2315 }
2316 }
2317
2318 #[test]
2319 fn j2_bbox_only_dt_under_segm_iou_type_raises_in_corrected_mode() {
2320 let images = vec![img(1, 100, 100)];
2324 let cats = vec![cat(1, "thing")];
2325 let anns = vec![ann_with_segm(
2326 1,
2327 1,
2328 1,
2329 (0.0, 0.0, 10.0, 10.0),
2330 square_polygon(0.0, 0.0, 10.0),
2331 )];
2332 let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
2333 let dts =
2335 CocoDetections::from_inputs(vec![dt_input(1, 1, 0.9, (0.0, 0.0, 10.0, 10.0))]).unwrap();
2336 let area = AreaRange::coco_default();
2337 let params = EvaluateParams {
2338 iou_thresholds: iou_thresholds(),
2339 area_ranges: &area,
2340 max_dets_per_image: 100,
2341 use_cats: true,
2342 retain_iou: false,
2343 };
2344 let err = evaluate_segm(>, &dts, params, ParityMode::Corrected).unwrap_err();
2345 match err {
2346 EvalError::InvalidAnnotation { detail } => {
2347 assert!(detail.contains("DT"), "expected DT in msg: {detail}");
2348 assert!(detail.contains("J2"), "expected J2 cite in msg: {detail}");
2349 }
2350 other => panic!("expected InvalidAnnotation, got {other:?}"),
2351 }
2352 }
2353
2354 #[test]
2355 fn j2_bbox_only_dt_under_segm_iou_type_synthesizes_in_strict_mode() {
2356 let images = vec![img(1, 100, 100)];
2362 let cats = vec![cat(1, "thing")];
2363 let anns = vec![ann_with_segm(
2365 1,
2366 1,
2367 1,
2368 (0.0, 0.0, 10.0, 10.0),
2369 square_polygon(0.0, 0.0, 10.0),
2370 )];
2371 let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
2372 let dts =
2374 CocoDetections::from_inputs(vec![dt_input(1, 1, 0.9, (0.0, 0.0, 10.0, 10.0))]).unwrap();
2375 let area = AreaRange::coco_default();
2376 let params = EvaluateParams {
2377 iou_thresholds: iou_thresholds(),
2378 area_ranges: &area,
2379 max_dets_per_image: 100,
2380 use_cats: true,
2381 retain_iou: false,
2382 };
2383 let grid = evaluate_segm(>, &dts, params, ParityMode::Strict).unwrap();
2384 let all = grid.cell(0, 0, 0).unwrap();
2385 assert!(all.dt_matched.iter().all(|&m| m), "expected matches");
2388 }
2389
2390 #[test]
2391 fn j6_heterogeneous_dt_list_first_with_segm_second_without_raises_in_corrected_mode() {
2392 let images = vec![img(1, 100, 100)];
2399 let cats = vec![cat(1, "thing")];
2400 let anns = vec![ann_with_segm(
2401 1,
2402 1,
2403 1,
2404 (0.0, 0.0, 10.0, 10.0),
2405 square_polygon(0.0, 0.0, 10.0),
2406 )];
2407 let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
2408 let dts = CocoDetections::from_inputs(vec![
2413 dt_input_with_segm(
2414 1,
2415 1,
2416 0.9,
2417 (0.0, 0.0, 10.0, 10.0),
2418 square_polygon(0.0, 0.0, 10.0),
2419 ),
2420 dt_input(1, 1, 0.8, (50.0, 50.0, 10.0, 10.0)),
2421 ])
2422 .unwrap();
2423 let area = AreaRange::coco_default();
2424 let params = EvaluateParams {
2425 iou_thresholds: iou_thresholds(),
2426 area_ranges: &area,
2427 max_dets_per_image: 100,
2428 use_cats: true,
2429 retain_iou: false,
2430 };
2431 let err = evaluate_segm(>, &dts, params, ParityMode::Corrected).unwrap_err();
2432 assert!(matches!(err, EvalError::InvalidAnnotation { .. }));
2433 }
2434
2435 #[test]
2436 fn j6_heterogeneous_dt_list_first_without_segm_second_with_raises_in_corrected_mode() {
2437 let images = vec![img(1, 100, 100)];
2444 let cats = vec![cat(1, "thing")];
2445 let anns = vec![ann_with_segm(
2446 1,
2447 1,
2448 1,
2449 (0.0, 0.0, 10.0, 10.0),
2450 square_polygon(0.0, 0.0, 10.0),
2451 )];
2452 let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
2453 let dts = CocoDetections::from_inputs(vec![
2454 dt_input(1, 1, 0.9, (0.0, 0.0, 10.0, 10.0)),
2455 dt_input_with_segm(
2456 1,
2457 1,
2458 0.8,
2459 (50.0, 50.0, 10.0, 10.0),
2460 square_polygon(50.0, 50.0, 10.0),
2461 ),
2462 ])
2463 .unwrap();
2464 let area = AreaRange::coco_default();
2465 let params = EvaluateParams {
2466 iou_thresholds: iou_thresholds(),
2467 area_ranges: &area,
2468 max_dets_per_image: 100,
2469 use_cats: true,
2470 retain_iou: false,
2471 };
2472 let err = evaluate_segm(>, &dts, params, ParityMode::Corrected).unwrap_err();
2473 assert!(matches!(err, EvalError::InvalidAnnotation { .. }));
2474 }
2475
2476 #[test]
2477 fn j6_heterogeneous_dt_list_in_strict_mode_synthesizes_per_entry() {
2478 let images = vec![img(1, 100, 100)];
2484 let cats = vec![cat(1, "thing")];
2485 let anns = vec![
2486 ann_with_segm(
2487 1,
2488 1,
2489 1,
2490 (0.0, 0.0, 10.0, 10.0),
2491 square_polygon(0.0, 0.0, 10.0),
2492 ),
2493 ann_with_segm(
2494 2,
2495 1,
2496 1,
2497 (50.0, 50.0, 10.0, 10.0),
2498 square_polygon(50.0, 50.0, 10.0),
2499 ),
2500 ];
2501 let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
2502 let dts = CocoDetections::from_inputs(vec![
2504 dt_input_with_segm(
2505 1,
2506 1,
2507 0.9,
2508 (0.0, 0.0, 10.0, 10.0),
2509 square_polygon(0.0, 0.0, 10.0),
2510 ),
2511 dt_input(1, 1, 0.8, (50.0, 50.0, 10.0, 10.0)),
2512 ])
2513 .unwrap();
2514 let area = AreaRange::coco_default();
2515 let params = EvaluateParams {
2516 iou_thresholds: iou_thresholds(),
2517 area_ranges: &area,
2518 max_dets_per_image: 100,
2519 use_cats: true,
2520 retain_iou: false,
2521 };
2522 let grid = evaluate_segm(>, &dts, params, ParityMode::Strict).unwrap();
2523 let all = grid.cell(0, 0, 0).unwrap();
2524 assert_eq!(all.dt_matched.shape(), &[iou_thresholds().len(), 2]);
2527 assert!(all.dt_matched.iter().all(|&m| m));
2528 }
2529
2530 #[test]
2531 fn boundary_perfect_overlap_summarizes_to_one() {
2532 let images = vec![img(1, 100, 100)];
2535 let cats = vec![cat(1, "thing")];
2536 let anns = vec![ann_with_segm(
2537 1,
2538 1,
2539 1,
2540 (10.0, 10.0, 20.0, 20.0),
2541 square_polygon(10.0, 10.0, 20.0),
2542 )];
2543 let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
2544 let dts = CocoDetections::from_inputs(vec![dt_input_with_segm(
2545 1,
2546 1,
2547 0.9,
2548 (10.0, 10.0, 20.0, 20.0),
2549 square_polygon(10.0, 10.0, 20.0),
2550 )])
2551 .unwrap();
2552 let area = AreaRange::coco_default();
2553 let params = EvaluateParams {
2554 iou_thresholds: iou_thresholds(),
2555 area_ranges: &area,
2556 max_dets_per_image: 100,
2557 use_cats: true,
2558 retain_iou: false,
2559 };
2560 let grid = evaluate_boundary(>, &dts, params, ParityMode::Strict, 0.02).unwrap();
2561 let max_dets = vec![1usize, 10, 100];
2562 let acc = accumulate(
2563 &grid.eval_imgs,
2564 AccumulateParams {
2565 iou_thresholds: iou_thresholds(),
2566 recall_thresholds: recall_thresholds(),
2567 max_dets: &max_dets,
2568 n_categories: grid.n_categories,
2569 n_area_ranges: grid.n_area_ranges,
2570 n_images: grid.n_images,
2571 },
2572 ParityMode::Strict,
2573 )
2574 .unwrap();
2575 let summary = summarize_detection(&acc, iou_thresholds(), &max_dets).unwrap();
2576 let stats = summary.stats();
2577 assert!((stats[0] - 1.0).abs() < 1e-12, "AP={}", stats[0]);
2578 }
2579
2580 #[test]
2581 fn boundary_disjoint_masks_summarize_to_zero() {
2582 let images = vec![img(1, 100, 100)];
2585 let cats = vec![cat(1, "thing")];
2586 let anns = vec![ann_with_segm(
2587 1,
2588 1,
2589 1,
2590 (0.0, 0.0, 10.0, 10.0),
2591 square_polygon(0.0, 0.0, 10.0),
2592 )];
2593 let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
2594 let dts = CocoDetections::from_inputs(vec![dt_input_with_segm(
2595 1,
2596 1,
2597 0.9,
2598 (50.0, 50.0, 10.0, 10.0),
2599 square_polygon(50.0, 50.0, 10.0),
2600 )])
2601 .unwrap();
2602 let area = AreaRange::coco_default();
2603 let params = EvaluateParams {
2604 iou_thresholds: iou_thresholds(),
2605 area_ranges: &area,
2606 max_dets_per_image: 100,
2607 use_cats: true,
2608 retain_iou: false,
2609 };
2610 let grid = evaluate_boundary(>, &dts, params, ParityMode::Strict, 0.02).unwrap();
2611 let all = grid.cell(0, 0, 0).unwrap();
2612 assert!(all.dt_matched.iter().all(|&m| !m));
2613 }
2614
2615 fn boundary_cache_fixture() -> (
2620 CocoDataset,
2621 CocoDetections,
2622 CocoDetections,
2623 OwnedEvaluateParams,
2624 ) {
2625 let images = vec![img(1, 100, 100), img(2, 100, 100)];
2626 let cats = vec![cat(1, "thing"), cat(2, "other")];
2627 let anns = vec![
2628 ann_with_segm(
2629 10,
2630 1,
2631 1,
2632 (10.0, 10.0, 20.0, 20.0),
2633 square_polygon(10.0, 10.0, 20.0),
2634 ),
2635 ann_with_segm(
2636 11,
2637 1,
2638 2,
2639 (50.0, 50.0, 15.0, 15.0),
2640 square_polygon(50.0, 50.0, 15.0),
2641 ),
2642 ann_with_segm(
2643 12,
2644 2,
2645 1,
2646 (5.0, 5.0, 25.0, 25.0),
2647 square_polygon(5.0, 5.0, 25.0),
2648 ),
2649 ];
2650 let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
2651 let dts_a = CocoDetections::from_inputs(vec![
2652 dt_input_with_segm(
2653 1,
2654 1,
2655 0.9,
2656 (10.0, 10.0, 20.0, 20.0),
2657 square_polygon(10.0, 10.0, 20.0),
2658 ),
2659 dt_input_with_segm(
2660 2,
2661 1,
2662 0.8,
2663 (5.0, 5.0, 25.0, 25.0),
2664 square_polygon(5.0, 5.0, 25.0),
2665 ),
2666 ])
2667 .unwrap();
2668 let dts_b = CocoDetections::from_inputs(vec![
2671 dt_input_with_segm(
2672 1,
2673 1,
2674 0.7,
2675 (12.0, 12.0, 20.0, 20.0),
2676 square_polygon(12.0, 12.0, 20.0),
2677 ),
2678 dt_input_with_segm(
2679 2,
2680 1,
2681 0.6,
2682 (8.0, 8.0, 25.0, 25.0),
2683 square_polygon(8.0, 8.0, 25.0),
2684 ),
2685 ])
2686 .unwrap();
2687 let params = OwnedEvaluateParams {
2688 iou_thresholds: iou_thresholds().to_vec(),
2689 area_ranges: AreaRange::coco_default().to_vec(),
2690 max_dets_per_image: 100,
2691 use_cats: true,
2692 retain_iou: false,
2693 };
2694 (gt, dts_a, dts_b, params)
2695 }
2696
2697 fn boundary_grid_cells(grid: &EvalGrid) -> Vec<f64> {
2698 grid.eval_imgs
2699 .iter()
2700 .filter_map(|c| c.as_ref())
2701 .flat_map(|c| c.dt_scores.iter().copied())
2702 .collect()
2703 }
2704
2705 #[test]
2706 fn boundary_cached_matches_uncached_bit_exact() {
2707 let (gt, dts, _, params) = boundary_cache_fixture();
2711 let p = params.borrow();
2712 let baseline = evaluate_boundary(>, &dts, p, ParityMode::Strict, 0.02).unwrap();
2713 let cache = BoundaryGtCache::new();
2714 let cached_first =
2715 evaluate_boundary_cached(>, &dts, p, ParityMode::Strict, 0.02, &cache).unwrap();
2716 let cached_second =
2717 evaluate_boundary_cached(>, &dts, p, ParityMode::Strict, 0.02, &cache).unwrap();
2718
2719 let baseline_scores = boundary_grid_cells(&baseline);
2720 let first_scores = boundary_grid_cells(&cached_first);
2721 let second_scores = boundary_grid_cells(&cached_second);
2722 assert_eq!(baseline_scores.len(), first_scores.len());
2723 for (b, c) in baseline_scores.iter().zip(first_scores.iter()) {
2724 assert_eq!(b.to_bits(), c.to_bits());
2725 }
2726 for (b, c) in baseline_scores.iter().zip(second_scores.iter()) {
2727 assert_eq!(b.to_bits(), c.to_bits());
2728 }
2729 }
2730
2731 #[test]
2732 fn boundary_cache_populates_lazily_per_evaluated_cell() {
2733 let (gt, dts, _, params) = boundary_cache_fixture();
2741 let cache = BoundaryGtCache::new();
2742 assert!(cache.is_empty());
2743 evaluate_boundary_cached(>, &dts, params.borrow(), ParityMode::Strict, 0.02, &cache)
2744 .unwrap();
2745 assert_eq!(cache.len(), 2);
2746 }
2747
2748 #[test]
2749 fn boundary_cache_invalidates_on_ratio_change() {
2750 let (gt, dts, _, params) = boundary_cache_fixture();
2754 let cache = BoundaryGtCache::new();
2755 evaluate_boundary_cached(>, &dts, params.borrow(), ParityMode::Strict, 0.02, &cache)
2756 .unwrap();
2757 let after_first = cache.len();
2758 evaluate_boundary_cached(>, &dts, params.borrow(), ParityMode::Strict, 0.05, &cache)
2759 .unwrap();
2760 assert_eq!(cache.len(), after_first);
2764 let fresh =
2765 evaluate_boundary(>, &dts, params.borrow(), ParityMode::Strict, 0.05).unwrap();
2766 let cached =
2767 evaluate_boundary_cached(>, &dts, params.borrow(), ParityMode::Strict, 0.05, &cache)
2768 .unwrap();
2769 let fresh_scores = boundary_grid_cells(&fresh);
2770 let cached_scores = boundary_grid_cells(&cached);
2771 for (f, c) in fresh_scores.iter().zip(cached_scores.iter()) {
2772 assert_eq!(f.to_bits(), c.to_bits());
2773 }
2774 }
2775
2776 #[test]
2777 fn boundary_cache_clear_resets_state() {
2778 let (gt, dts, _, params) = boundary_cache_fixture();
2779 let cache = BoundaryGtCache::new();
2780 evaluate_boundary_cached(>, &dts, params.borrow(), ParityMode::Strict, 0.02, &cache)
2781 .unwrap();
2782 assert!(!cache.is_empty());
2783 cache.clear();
2784 assert!(cache.is_empty());
2785 let after =
2788 evaluate_boundary_cached(>, &dts, params.borrow(), ParityMode::Strict, 0.02, &cache)
2789 .unwrap();
2790 let baseline =
2791 evaluate_boundary(>, &dts, params.borrow(), ParityMode::Strict, 0.02).unwrap();
2792 let after_scores = boundary_grid_cells(&after);
2793 let baseline_scores = boundary_grid_cells(&baseline);
2794 for (a, b) in after_scores.iter().zip(baseline_scores.iter()) {
2795 assert_eq!(a.to_bits(), b.to_bits());
2796 }
2797 }
2798
2799 #[test]
2800 fn boundary_cache_survives_changing_dt() {
2801 let (gt, dts_a, dts_b, params) = boundary_cache_fixture();
2806 let cache = BoundaryGtCache::new();
2807 let cached_a = evaluate_boundary_cached(
2808 >,
2809 &dts_a,
2810 params.borrow(),
2811 ParityMode::Strict,
2812 0.02,
2813 &cache,
2814 )
2815 .unwrap();
2816 let len_after_a = cache.len();
2817 let cached_b = evaluate_boundary_cached(
2818 >,
2819 &dts_b,
2820 params.borrow(),
2821 ParityMode::Strict,
2822 0.02,
2823 &cache,
2824 )
2825 .unwrap();
2826 assert_eq!(cache.len(), len_after_a);
2827
2828 let baseline_a =
2829 evaluate_boundary(>, &dts_a, params.borrow(), ParityMode::Strict, 0.02).unwrap();
2830 let baseline_b =
2831 evaluate_boundary(>, &dts_b, params.borrow(), ParityMode::Strict, 0.02).unwrap();
2832 for (lhs, rhs) in boundary_grid_cells(&cached_a)
2833 .iter()
2834 .zip(boundary_grid_cells(&baseline_a).iter())
2835 {
2836 assert_eq!(lhs.to_bits(), rhs.to_bits());
2837 }
2838 for (lhs, rhs) in boundary_grid_cells(&cached_b)
2839 .iter()
2840 .zip(boundary_grid_cells(&baseline_b).iter())
2841 {
2842 assert_eq!(lhs.to_bits(), rhs.to_bits());
2843 }
2844 }
2845
2846 #[test]
2853 fn segm_cached_matches_uncached_bit_exact() {
2854 let (gt, dts, _, params) = boundary_cache_fixture();
2855 let p = params.borrow();
2856 let baseline = evaluate_segm(>, &dts, p, ParityMode::Strict).unwrap();
2857 let cache = SegmGtCache::new();
2858 let cached_first = evaluate_segm_cached(>, &dts, p, ParityMode::Strict, &cache).unwrap();
2859 let cached_second = evaluate_segm_cached(>, &dts, p, ParityMode::Strict, &cache).unwrap();
2860
2861 let baseline_scores = boundary_grid_cells(&baseline);
2862 let first_scores = boundary_grid_cells(&cached_first);
2863 let second_scores = boundary_grid_cells(&cached_second);
2864 assert_eq!(baseline_scores.len(), first_scores.len());
2865 for (b, c) in baseline_scores.iter().zip(first_scores.iter()) {
2866 assert_eq!(b.to_bits(), c.to_bits());
2867 }
2868 for (b, c) in baseline_scores.iter().zip(second_scores.iter()) {
2869 assert_eq!(b.to_bits(), c.to_bits());
2870 }
2871 }
2872
2873 #[test]
2874 fn segm_cache_populates_lazily_per_evaluated_cell() {
2875 let (gt, dts, _, params) = boundary_cache_fixture();
2881 let cache = SegmGtCache::new();
2882 assert!(cache.is_empty());
2883 evaluate_segm_cached(>, &dts, params.borrow(), ParityMode::Strict, &cache).unwrap();
2884 assert_eq!(cache.len(), 2);
2885 }
2886
2887 #[test]
2888 fn segm_cache_clear_resets_state() {
2889 let (gt, dts, _, params) = boundary_cache_fixture();
2890 let cache = SegmGtCache::new();
2891 evaluate_segm_cached(>, &dts, params.borrow(), ParityMode::Strict, &cache).unwrap();
2892 assert!(!cache.is_empty());
2893 cache.clear();
2894 assert!(cache.is_empty());
2895 let after =
2896 evaluate_segm_cached(>, &dts, params.borrow(), ParityMode::Strict, &cache).unwrap();
2897 let baseline = evaluate_segm(>, &dts, params.borrow(), ParityMode::Strict).unwrap();
2898 for (a, b) in boundary_grid_cells(&after)
2899 .iter()
2900 .zip(boundary_grid_cells(&baseline).iter())
2901 {
2902 assert_eq!(a.to_bits(), b.to_bits());
2903 }
2904 }
2905
2906 #[test]
2907 fn segm_cache_survives_changing_dt() {
2908 let (gt, dts_a, dts_b, params) = boundary_cache_fixture();
2913 let cache = SegmGtCache::new();
2914 let cached_a =
2915 evaluate_segm_cached(>, &dts_a, params.borrow(), ParityMode::Strict, &cache).unwrap();
2916 let len_after_a = cache.len();
2917 let cached_b =
2918 evaluate_segm_cached(>, &dts_b, params.borrow(), ParityMode::Strict, &cache).unwrap();
2919 assert_eq!(cache.len(), len_after_a);
2920
2921 let baseline_a = evaluate_segm(>, &dts_a, params.borrow(), ParityMode::Strict).unwrap();
2922 let baseline_b = evaluate_segm(>, &dts_b, params.borrow(), ParityMode::Strict).unwrap();
2923 for (lhs, rhs) in boundary_grid_cells(&cached_a)
2924 .iter()
2925 .zip(boundary_grid_cells(&baseline_a).iter())
2926 {
2927 assert_eq!(lhs.to_bits(), rhs.to_bits());
2928 }
2929 for (lhs, rhs) in boundary_grid_cells(&cached_b)
2930 .iter()
2931 .zip(boundary_grid_cells(&baseline_b).iter())
2932 {
2933 assert_eq!(lhs.to_bits(), rhs.to_bits());
2934 }
2935 }
2936
2937 fn const_kps_vec(x: f64, y: f64, v: u32, len: usize) -> Vec<f64> {
2945 let mut out = Vec::with_capacity(3 * len);
2946 for _ in 0..len {
2947 out.push(x);
2948 out.push(y);
2949 out.push(f64::from(v));
2950 }
2951 out
2952 }
2953
2954 fn ann_with_kps(
2955 id: i64,
2956 image: i64,
2957 cat: i64,
2958 bbox: (f64, f64, f64, f64),
2959 keypoints: Vec<f64>,
2960 num_keypoints: Option<u32>,
2961 ) -> CocoAnnotation {
2962 CocoAnnotation {
2963 id: AnnId(id),
2964 image_id: ImageId(image),
2965 category_id: CategoryId(cat),
2966 area: bbox.2 * bbox.3,
2967 is_crowd: false,
2968 ignore_flag: None,
2969 bbox: Bbox {
2970 x: bbox.0,
2971 y: bbox.1,
2972 w: bbox.2,
2973 h: bbox.3,
2974 },
2975 segmentation: None,
2976 keypoints: Some(keypoints),
2977 num_keypoints,
2978 }
2979 }
2980
2981 fn dt_input_with_kps(
2982 image: i64,
2983 cat: i64,
2984 score: f64,
2985 bbox: (f64, f64, f64, f64),
2986 keypoints: Vec<f64>,
2987 ) -> DetectionInput {
2988 DetectionInput {
2989 id: None,
2990 image_id: ImageId(image),
2991 category_id: CategoryId(cat),
2992 score,
2993 bbox: Bbox {
2994 x: bbox.0,
2995 y: bbox.1,
2996 w: bbox.2,
2997 h: bbox.3,
2998 },
2999 segmentation: None,
3000 keypoints: Some(keypoints),
3001 num_keypoints: None,
3002 }
3003 }
3004
3005 #[test]
3006 fn test_evaluate_keypoints_perfect_match() {
3007 let images = vec![img(1, 100, 100)];
3011 let cats = vec![cat(1, "person")];
3012 let kps = const_kps_vec(50.0, 50.0, 2, 17);
3013 let anns = vec![ann_with_kps(
3014 1,
3015 1,
3016 1,
3017 (40.0, 40.0, 20.0, 20.0),
3018 kps.clone(),
3019 None,
3020 )];
3021 let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
3022 let dts = CocoDetections::from_inputs(vec![dt_input_with_kps(
3023 1,
3024 1,
3025 0.9,
3026 (40.0, 40.0, 20.0, 20.0),
3027 kps,
3028 )])
3029 .unwrap();
3030 let area = AreaRange::coco_default();
3031 let params = EvaluateParams {
3032 iou_thresholds: iou_thresholds(),
3033 area_ranges: &area,
3034 max_dets_per_image: 100,
3035 use_cats: true,
3036 retain_iou: false,
3037 };
3038 let grid =
3039 evaluate_keypoints(>, &dts, params, ParityMode::Strict, HashMap::new()).unwrap();
3040 let cell = grid.cell(0, 0, 0).unwrap();
3041 assert_eq!(cell.gt_ignore, vec![false]);
3043 assert!(cell.dt_matched.iter().all(|&m| m));
3045 let meta = grid.cell_meta(0, 0, 0).unwrap();
3047 assert!(
3048 meta.gt_matches.iter().all(|&id| id > 0),
3049 "every threshold should match the DT id (>0)",
3050 );
3051 }
3052
3053 #[test]
3054 fn test_evaluate_keypoints_zero_overlap() {
3055 let images = vec![img(1, 2000, 2000)];
3059 let cats = vec![cat(1, "person")];
3060 let gt_kps = const_kps_vec(50.0, 50.0, 2, 17);
3061 let dt_kps = const_kps_vec(1500.0, 1500.0, 2, 17);
3062 let anns = vec![ann_with_kps(
3063 1,
3064 1,
3065 1,
3066 (40.0, 40.0, 20.0, 20.0),
3067 gt_kps,
3068 None,
3069 )];
3070 let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
3071 let dts = CocoDetections::from_inputs(vec![dt_input_with_kps(
3072 1,
3073 1,
3074 0.9,
3075 (1490.0, 1490.0, 20.0, 20.0),
3076 dt_kps,
3077 )])
3078 .unwrap();
3079 let area = AreaRange::coco_default();
3080 let params = EvaluateParams {
3081 iou_thresholds: iou_thresholds(),
3082 area_ranges: &area,
3083 max_dets_per_image: 100,
3084 use_cats: true,
3085 retain_iou: false,
3086 };
3087 let grid =
3088 evaluate_keypoints(>, &dts, params, ParityMode::Strict, HashMap::new()).unwrap();
3089 let cell = grid.cell(0, 0, 0).unwrap();
3090 assert!(
3091 cell.dt_matched.iter().all(|&m| !m),
3092 "DTs far from GT should not match at any IoU threshold",
3093 );
3094 }
3095
3096 #[test]
3097 fn test_evaluate_keypoints_d2_implicit_ignore() {
3098 let images = vec![img(1, 100, 100)];
3103 let cats = vec![cat(1, "person")];
3104 let gt_kps = const_kps_vec(50.0, 50.0, 0, 17);
3105 let dt_kps = const_kps_vec(50.0, 50.0, 2, 17);
3106 let anns = vec![ann_with_kps(
3107 1,
3108 1,
3109 1,
3110 (40.0, 40.0, 20.0, 20.0),
3111 gt_kps,
3112 Some(0),
3116 )];
3117 let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
3118 let dts = CocoDetections::from_inputs(vec![dt_input_with_kps(
3119 1,
3120 1,
3121 0.9,
3122 (40.0, 40.0, 20.0, 20.0),
3123 dt_kps,
3124 )])
3125 .unwrap();
3126 let area = AreaRange::coco_default();
3127 let params = EvaluateParams {
3128 iou_thresholds: iou_thresholds(),
3129 area_ranges: &area,
3130 max_dets_per_image: 100,
3131 use_cats: true,
3132 retain_iou: false,
3133 };
3134 let grid =
3135 evaluate_keypoints(>, &dts, params, ParityMode::Strict, HashMap::new()).unwrap();
3136 let cell = grid.cell(0, 0, 0).unwrap();
3137 assert_eq!(
3138 cell.gt_ignore,
3139 vec![true],
3140 "D2: zero-visible-keypoints GT must be ignored",
3141 );
3142 }
3143
3144 #[test]
3145 fn test_evaluate_keypoints_per_category_sigmas() {
3146 let images = vec![img(1, 200, 200)];
3153 let cats = vec![cat(1, "person"), cat(2, "dog")];
3154 let gt_kps = const_kps_vec(50.0, 50.0, 2, 17);
3155 let anns = vec![
3156 ann_with_kps(1, 1, 1, (40.0, 40.0, 20.0, 20.0), gt_kps, None),
3157 ann_with_kps(
3158 2,
3159 1,
3160 2,
3161 (140.0, 140.0, 20.0, 20.0),
3162 const_kps_vec(150.0, 150.0, 2, 17),
3163 None,
3164 ),
3165 ];
3166 let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
3167 let dts = CocoDetections::from_inputs(vec![
3170 dt_input_with_kps(
3171 1,
3172 1,
3173 0.9,
3174 (40.0, 40.0, 20.0, 20.0),
3175 const_kps_vec(51.0, 50.0, 2, 17),
3176 ),
3177 dt_input_with_kps(
3178 1,
3179 2,
3180 0.8,
3181 (140.0, 140.0, 20.0, 20.0),
3182 const_kps_vec(151.0, 150.0, 2, 17),
3183 ),
3184 ])
3185 .unwrap();
3186 let mut sigmas: HashMap<i64, Vec<f64>> = HashMap::new();
3187 sigmas.insert(1, vec![0.5_f64; 17]);
3188 sigmas.insert(2, vec![0.5_f64; 17]);
3189 let area = AreaRange::coco_default();
3190 let params = EvaluateParams {
3191 iou_thresholds: iou_thresholds(),
3192 area_ranges: &area,
3193 max_dets_per_image: 100,
3194 use_cats: true,
3195 retain_iou: false,
3196 };
3197 let grid = evaluate_keypoints(>, &dts, params, ParityMode::Strict, sigmas).unwrap();
3198 let cell_cat1 = grid.cell(0, 0, 0).unwrap();
3200 let cell_cat2 = grid.cell(1, 0, 0).unwrap();
3201 assert!(
3202 cell_cat1.dt_matched.iter().all(|&m| m),
3203 "cat-1 DT should match cat-1 GT under override sigmas",
3204 );
3205 assert!(
3206 cell_cat2.dt_matched.iter().all(|&m| m),
3207 "cat-2 DT should match cat-2 GT under override sigmas",
3208 );
3209 }
3210
3211 #[test]
3212 fn test_evaluate_keypoints_missing_dt_kps_rejected() {
3213 let images = vec![img(1, 100, 100)];
3217 let cats = vec![cat(1, "person")];
3218 let gt_kps = const_kps_vec(50.0, 50.0, 2, 17);
3219 let anns = vec![ann_with_kps(
3220 1,
3221 1,
3222 1,
3223 (40.0, 40.0, 20.0, 20.0),
3224 gt_kps,
3225 None,
3226 )];
3227 let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
3228 let dts = CocoDetections::from_inputs(vec![dt_input(1, 1, 0.9, (40.0, 40.0, 20.0, 20.0))])
3231 .unwrap();
3232 let area = AreaRange::coco_default();
3233 let params = EvaluateParams {
3234 iou_thresholds: iou_thresholds(),
3235 area_ranges: &area,
3236 max_dets_per_image: 100,
3237 use_cats: true,
3238 retain_iou: false,
3239 };
3240 let err =
3241 evaluate_keypoints(>, &dts, params, ParityMode::Strict, HashMap::new()).unwrap_err();
3242 match err {
3243 EvalError::InvalidAnnotation { detail } => {
3244 assert!(detail.contains("DT"), "expected DT in msg: {detail}");
3245 assert!(
3246 detail.contains("keypoints"),
3247 "expected keypoints in msg: {detail}",
3248 );
3249 }
3250 other => panic!("expected InvalidAnnotation, got {other:?}"),
3251 }
3252 }
3253
3254 #[test]
3255 fn test_keypoints_default_ignore_for_other_kernels() {
3256 let ann_zero_kps = ann_with_kps(
3261 1,
3262 1,
3263 1,
3264 (0.0, 0.0, 10.0, 10.0),
3265 const_kps_vec(0.0, 0.0, 0, 17),
3266 Some(0),
3267 );
3268 assert!(
3269 !BboxIou.extra_gt_ignore(&ann_zero_kps),
3270 "BboxIou must keep the default `false` ignore",
3271 );
3272 assert!(
3273 !SegmIou.extra_gt_ignore(&ann_zero_kps),
3274 "SegmIou must keep the default `false` ignore",
3275 );
3276 assert!(
3277 !BoundaryIou {
3278 dilation_ratio: 0.02,
3279 }
3280 .extra_gt_ignore(&ann_zero_kps),
3281 "BoundaryIou must keep the default `false` ignore",
3282 );
3283 assert!(
3285 OksSimilarity::default().extra_gt_ignore(&ann_zero_kps),
3286 "OksSimilarity must flip D2 to true on zero-visible-keypoints GT",
3287 );
3288 }
3289
3290 #[test]
3291 fn boundary_missing_gt_segmentation_surfaces_typed_error() {
3292 let images = vec![img(1, 100, 100)];
3296 let cats = vec![cat(1, "thing")];
3297 let anns = vec![ann(7, 1, 1, (0.0, 0.0, 10.0, 10.0))];
3298 let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
3299 let dts = CocoDetections::from_inputs(vec![dt_input_with_segm(
3300 1,
3301 1,
3302 0.9,
3303 (0.0, 0.0, 10.0, 10.0),
3304 square_polygon(0.0, 0.0, 10.0),
3305 )])
3306 .unwrap();
3307 let area = AreaRange::coco_default();
3308 let params = EvaluateParams {
3309 iou_thresholds: iou_thresholds(),
3310 area_ranges: &area,
3311 max_dets_per_image: 100,
3312 use_cats: true,
3313 retain_iou: false,
3314 };
3315 let err = evaluate_boundary(>, &dts, params, ParityMode::Strict, 0.02).unwrap_err();
3316 match err {
3317 EvalError::InvalidAnnotation { detail } => {
3318 assert!(detail.contains("GT id=7"), "msg: {detail}");
3319 }
3320 other => panic!("expected InvalidAnnotation, got {other:?}"),
3321 }
3322 }
3323
3324 fn lvis_dataset(
3331 images: &[ImageMeta],
3332 annotations: &[CocoAnnotation],
3333 categories: &[CategoryMeta],
3334 neg: &[(i64, Vec<i64>)],
3335 nel: &[(i64, Vec<i64>)],
3336 freq: &[(i64, crate::dataset::Frequency)],
3337 ) -> CocoDataset {
3338 let images_json: Vec<serde_json::Value> = images
3343 .iter()
3344 .map(|im| {
3345 let neg_for: Vec<i64> = neg
3346 .iter()
3347 .find(|(id, _)| *id == im.id.0)
3348 .map(|(_, v)| v.clone())
3349 .unwrap_or_default();
3350 let nel_for: Vec<i64> = nel
3351 .iter()
3352 .find(|(id, _)| *id == im.id.0)
3353 .map(|(_, v)| v.clone())
3354 .unwrap_or_default();
3355 serde_json::json!({
3356 "id": im.id.0,
3357 "width": im.width,
3358 "height": im.height,
3359 "neg_category_ids": neg_for,
3360 "not_exhaustive_category_ids": nel_for,
3361 })
3362 })
3363 .collect();
3364 let cats_json: Vec<serde_json::Value> = categories
3365 .iter()
3366 .map(|c| {
3367 let f = freq
3368 .iter()
3369 .find(|(id, _)| *id == c.id.0)
3370 .map(|(_, f)| match f {
3371 crate::dataset::Frequency::Rare => "r",
3372 crate::dataset::Frequency::Common => "c",
3373 crate::dataset::Frequency::Frequent => "f",
3374 })
3375 .expect("test fixture must include frequency for every category");
3376 serde_json::json!({
3377 "id": c.id.0,
3378 "name": c.name,
3379 "frequency": f,
3380 })
3381 })
3382 .collect();
3383 let anns_json = serde_json::to_value(annotations).unwrap();
3384 let payload = serde_json::json!({
3385 "images": images_json,
3386 "annotations": anns_json,
3387 "categories": cats_json,
3388 });
3389 let bytes = serde_json::to_vec(&payload).unwrap();
3390 CocoDataset::from_lvis_json_bytes(&bytes).unwrap()
3391 }
3392
3393 #[test]
3394 fn aa4_skips_cells_outside_pos_union_neg() {
3395 let images = vec![img(1, 100, 100), img(2, 100, 100)];
3403 let cats = vec![cat(1, "a"), cat(2, "b")];
3404 let anns = vec![
3405 ann(1, 1, 1, (0.0, 0.0, 10.0, 10.0)),
3406 ann(2, 2, 2, (0.0, 0.0, 10.0, 10.0)),
3407 ];
3408 let gt_lvis = lvis_dataset(
3409 &images,
3410 &anns,
3411 &cats,
3412 &[(1, vec![]), (2, vec![])],
3413 &[(1, vec![]), (2, vec![])],
3414 &[
3415 (1, crate::dataset::Frequency::Frequent),
3416 (2, crate::dataset::Frequency::Frequent),
3417 ],
3418 );
3419 let gt_coco = CocoDataset::from_parts(images, anns, cats).unwrap();
3420 let dts = CocoDetections::from_inputs(vec![
3423 dt_input(1, 1, 0.9, (0.0, 0.0, 10.0, 10.0)),
3424 dt_input(1, 2, 0.7, (50.0, 50.0, 10.0, 10.0)),
3425 dt_input(2, 2, 0.9, (0.0, 0.0, 10.0, 10.0)),
3426 ])
3427 .unwrap();
3428 let area = AreaRange::coco_default();
3429 let params = EvaluateParams {
3430 iou_thresholds: iou_thresholds(),
3431 area_ranges: &area,
3432 max_dets_per_image: 100,
3433 use_cats: true,
3434 retain_iou: false,
3435 };
3436 let grid_lvis = evaluate_bbox(>_lvis, &dts, params, ParityMode::Strict).unwrap();
3437 let grid_coco = evaluate_bbox(>_coco, &dts, params, ParityMode::Strict).unwrap();
3438
3439 let lvis_cell = grid_lvis.cell(1, 0, 0);
3443 let coco_cell = grid_coco.cell(1, 0, 0);
3444 assert!(lvis_cell.is_none(), "AA4: federated cell must be skipped");
3445 assert!(
3446 coco_cell.is_some(),
3447 "control: COCO dataset must evaluate the same cell"
3448 );
3449 assert_eq!(
3452 grid_lvis.cell(0, 0, 0).map(|c| c.dt_scores.len()),
3453 grid_coco.cell(0, 0, 0).map(|c| c.dt_scores.len()),
3454 );
3455 }
3456
3457 #[test]
3458 fn aa4_keeps_neg_cells_with_no_gts() {
3459 let images = vec![img(1, 100, 100)];
3463 let cats = vec![cat(1, "a"), cat(2, "b")];
3464 let anns = vec![ann(1, 1, 1, (0.0, 0.0, 10.0, 10.0))];
3465 let gt = lvis_dataset(
3466 &images,
3467 &anns,
3468 &cats,
3469 &[(1, vec![2])], &[(1, vec![])],
3471 &[
3472 (1, crate::dataset::Frequency::Frequent),
3473 (2, crate::dataset::Frequency::Frequent),
3474 ],
3475 );
3476 let dts = CocoDetections::from_inputs(vec![
3477 dt_input(1, 1, 0.9, (0.0, 0.0, 10.0, 10.0)),
3478 dt_input(1, 2, 0.7, (50.0, 50.0, 10.0, 10.0)),
3479 ])
3480 .unwrap();
3481 let area = AreaRange::coco_default();
3482 let params = EvaluateParams {
3483 iou_thresholds: iou_thresholds(),
3484 area_ranges: &area,
3485 max_dets_per_image: 100,
3486 use_cats: true,
3487 retain_iou: false,
3488 };
3489 let grid = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
3490 let cell = grid
3491 .cell(1, 0, 0)
3492 .expect("cat 2 ∈ neg[1] must produce an evaluated cell");
3493 assert_eq!(cell.dt_scores.len(), 1);
3496 assert!(cell.dt_ignore.iter().all(|&ig| !ig));
3497 }
3498
3499 #[test]
3500 fn aa3_dt_ignore_extension_in_not_exhaustive_cell() {
3501 let images = vec![img(1, 100, 100)];
3508 let cats = vec![cat(1, "a")];
3509 let anns = vec![ann(1, 1, 1, (0.0, 0.0, 10.0, 10.0))];
3510 let gt = lvis_dataset(
3511 &images,
3512 &anns,
3513 &cats,
3514 &[(1, vec![])],
3515 &[(1, vec![1])], &[(1, crate::dataset::Frequency::Frequent)],
3517 );
3518 let dts = CocoDetections::from_inputs(vec![
3519 dt_input(1, 1, 0.9, (0.0, 0.0, 10.0, 10.0)), dt_input(1, 1, 0.7, (50.0, 50.0, 10.0, 10.0)), ])
3522 .unwrap();
3523 let area = AreaRange::coco_default();
3524 let params = EvaluateParams {
3525 iou_thresholds: iou_thresholds(),
3526 area_ranges: &area,
3527 max_dets_per_image: 100,
3528 use_cats: true,
3529 retain_iou: false,
3530 };
3531 let grid = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
3532 let cell = grid.cell(0, 0, 0).expect("cell must evaluate");
3533 let n_t = cell.dt_ignore.shape()[0];
3534 for t in 0..n_t {
3539 assert!(!cell.dt_ignore[(t, 0)], "TP should not be dt_ignore");
3540 assert!(
3541 cell.dt_ignore[(t, 1)],
3542 "AA3: unmatched DT in not_exhaustive cell must be dt_ignore"
3543 );
3544 }
3545 }
3546
3547 #[test]
3548 fn aa3_dt_ignore_only_unmatched() {
3549 let images = vec![img(1, 100, 100)];
3553 let cats = vec![cat(1, "a")];
3554 let anns = vec![ann(1, 1, 1, (0.0, 0.0, 10.0, 10.0))];
3555 let gt = lvis_dataset(
3556 &images,
3557 &anns,
3558 &cats,
3559 &[(1, vec![])],
3560 &[(1, vec![])],
3561 &[(1, crate::dataset::Frequency::Frequent)],
3562 );
3563 let dts = CocoDetections::from_inputs(vec![
3564 dt_input(1, 1, 0.9, (0.0, 0.0, 10.0, 10.0)),
3565 dt_input(1, 1, 0.7, (50.0, 50.0, 10.0, 10.0)),
3566 ])
3567 .unwrap();
3568 let area = AreaRange::coco_default();
3569 let params = EvaluateParams {
3570 iou_thresholds: iou_thresholds(),
3571 area_ranges: &area,
3572 max_dets_per_image: 100,
3573 use_cats: true,
3574 retain_iou: false,
3575 };
3576 let grid = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
3577 let cell = grid.cell(0, 0, 0).expect("cell must evaluate");
3578 assert!(cell.dt_ignore.iter().all(|&ig| !ig));
3579 }
3580
3581 #[test]
3582 fn federated_dataset_with_use_cats_false_falls_back_to_coco() {
3583 let images = vec![img(1, 100, 100), img(2, 100, 100)];
3588 let cats = vec![cat(1, "a"), cat(2, "b")];
3589 let anns = vec![
3590 ann(1, 1, 1, (0.0, 0.0, 10.0, 10.0)),
3591 ann(2, 2, 2, (0.0, 0.0, 10.0, 10.0)),
3592 ];
3593 let gt = lvis_dataset(
3594 &images,
3595 &anns,
3596 &cats,
3597 &[(1, vec![]), (2, vec![])],
3598 &[(1, vec![]), (2, vec![])],
3599 &[
3600 (1, crate::dataset::Frequency::Frequent),
3601 (2, crate::dataset::Frequency::Frequent),
3602 ],
3603 );
3604 let dts = CocoDetections::from_inputs(vec![
3605 dt_input(1, 1, 0.9, (0.0, 0.0, 10.0, 10.0)),
3606 dt_input(1, 2, 0.7, (50.0, 50.0, 10.0, 10.0)),
3607 ])
3608 .unwrap();
3609 let area = AreaRange::coco_default();
3610 let params = EvaluateParams {
3611 iou_thresholds: iou_thresholds(),
3612 area_ranges: &area,
3613 max_dets_per_image: 100,
3614 use_cats: false,
3615 retain_iou: false,
3616 };
3617 let grid = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
3620 assert_eq!(grid.n_categories, 1);
3621 let cell = grid.cell(0, 0, 0).expect("collapsed cell must evaluate");
3624 assert_eq!(cell.dt_scores.len(), 2);
3625 }
3626
3627 #[test]
3628 fn coco_dataset_unaffected_by_federated_machinery() {
3629 let g = perfect_match_grid();
3635 let cell = g.cell(0, 0, 0).expect("perfect_match cell must exist");
3638 assert_eq!(cell.dt_scores.len(), 2);
3639 assert!(cell.dt_ignore.iter().all(|&ig| !ig));
3640 }
3641
3642 fn ann_with_area(
3649 id: i64,
3650 image: i64,
3651 cat: i64,
3652 bbox: (f64, f64, f64, f64),
3653 area: f64,
3654 ) -> CocoAnnotation {
3655 let mut a = ann(id, image, cat, bbox);
3656 a.area = area;
3657 a
3658 }
3659
3660 #[test]
3661 fn ag6_mixed_cell_drops_zero_area_gt_in_strict_mode() {
3662 let images = vec![img(1, 100, 100)];
3669 let cats = vec![cat(1, "a")];
3670 let anns = vec![
3671 ann(1, 1, 1, (10.0, 10.0, 20.0, 20.0)),
3672 ann_with_area(2, 1, 1, (50.0, 50.0, 0.1, 0.1), 0.0),
3673 ];
3674 let gt = lvis_dataset(
3675 &images,
3676 &anns,
3677 &cats,
3678 &[(1, vec![])],
3679 &[(1, vec![])],
3680 &[(1, crate::dataset::Frequency::Frequent)],
3681 );
3682 let dts = CocoDetections::from_inputs(vec![
3683 dt_input(1, 1, 0.9, (10.0, 10.0, 20.0, 20.0)),
3684 dt_input(1, 1, 0.8, (50.0, 50.0, 0.1, 0.1)),
3685 ])
3686 .unwrap();
3687 let area = AreaRange::coco_default();
3688 let params = EvaluateParams {
3689 iou_thresholds: iou_thresholds(),
3690 area_ranges: &area,
3691 max_dets_per_image: 100,
3692 use_cats: true,
3693 retain_iou: false,
3694 };
3695
3696 let strict = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
3697 let cell = strict
3698 .cell(0, 0, 0)
3699 .expect("mixed cell must still evaluate in strict mode");
3700 assert_eq!(cell.dt_scores.len(), 2);
3701 let strict_meta = strict.cell_meta(0, 0, 0).unwrap();
3705 assert_eq!(strict_meta.dt_matches[(0, 0)], 1, "DT_real → GT id=1");
3706 assert_eq!(
3707 strict_meta.dt_matches[(0, 1)],
3708 0,
3709 "DT_zero must be unmatched after strict filter drops GT id=2"
3710 );
3711
3712 let corrected = evaluate_bbox(>, &dts, params, ParityMode::Corrected).unwrap();
3713 let cor_meta = corrected.cell_meta(0, 0, 0).unwrap();
3714 assert_eq!(cor_meta.dt_matches[(0, 0)], 1, "DT_real → GT id=1");
3715 assert_eq!(
3716 cor_meta.dt_matches[(0, 1)],
3717 2,
3718 "Corrected mode keeps the zero-area GT and matches DT_zero → GT id=2"
3719 );
3720 }
3721
3722 #[test]
3723 fn ag6_all_zero_area_cell_skipped_via_aa4_in_strict_mode() {
3724 let images = vec![img(1, 100, 100)];
3731 let cats = vec![cat(1, "a")];
3732 let anns = vec![ann_with_area(1, 1, 1, (50.0, 50.0, 0.1, 0.1), 0.0)];
3733 let gt = lvis_dataset(
3734 &images,
3735 &anns,
3736 &cats,
3737 &[(1, vec![])],
3738 &[(1, vec![])],
3739 &[(1, crate::dataset::Frequency::Frequent)],
3740 );
3741 let dts =
3742 CocoDetections::from_inputs(vec![dt_input(1, 1, 0.9, (50.0, 50.0, 0.1, 0.1))]).unwrap();
3743 let area = AreaRange::coco_default();
3744 let params = EvaluateParams {
3745 iou_thresholds: iou_thresholds(),
3746 area_ranges: &area,
3747 max_dets_per_image: 100,
3748 use_cats: true,
3749 retain_iou: false,
3750 };
3751
3752 let strict = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
3753 assert!(
3754 strict.cell(0, 0, 0).is_none(),
3755 "AG6: all-zero-area cell must be skipped via AA4 in strict mode"
3756 );
3757
3758 let corrected = evaluate_bbox(>, &dts, params, ParityMode::Corrected).unwrap();
3759 let cell = corrected
3760 .cell(0, 0, 0)
3761 .expect("Corrected mode must keep the zero-area GT");
3762 assert_eq!(cell.dt_scores.len(), 1);
3763 }
3764
3765 #[test]
3766 fn ag6_strict_filter_is_noop_on_coco_dataset() {
3767 let images = vec![img(1, 100, 100)];
3772 let cats = vec![cat(1, "a")];
3773 let anns = vec![
3774 ann(1, 1, 1, (10.0, 10.0, 20.0, 20.0)),
3775 ann_with_area(2, 1, 1, (50.0, 50.0, 0.1, 0.1), 0.0),
3776 ];
3777 let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
3778 let dts = CocoDetections::from_inputs(vec![
3779 dt_input(1, 1, 0.9, (10.0, 10.0, 20.0, 20.0)),
3780 dt_input(1, 1, 0.8, (50.0, 50.0, 0.1, 0.1)),
3781 ])
3782 .unwrap();
3783 let area = AreaRange::coco_default();
3784 let params = EvaluateParams {
3785 iou_thresholds: iou_thresholds(),
3786 area_ranges: &area,
3787 max_dets_per_image: 100,
3788 use_cats: true,
3789 retain_iou: false,
3790 };
3791 let grid = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
3792 let meta = grid.cell_meta(0, 0, 0).unwrap();
3793 assert_eq!(meta.dt_matches[(0, 0)], 1);
3794 assert_eq!(
3795 meta.dt_matches[(0, 1)],
3796 2,
3797 "COCO strict mode must NOT drop the zero-area GT — AG6 is LVIS-only"
3798 );
3799 }
3800}