use ndarray::{Array2, ArrayView2, ArrayViewMut2};
use crate::accumulate::PerImageEval;
use crate::dataset::{
Bbox, CategoryId, CocoAnnotation, CocoDataset, CocoDetection, CocoDetections, EvalDataset,
ImageId, ImageMeta,
};
use crate::error::EvalError;
use crate::matching::{match_image, MatchResult};
use crate::parity::ParityMode;
use crate::segmentation::Segmentation;
use crate::similarity::{
boundary_iou_compute, segm_iou_compute, BboxAnn, BboxIou, BoundaryComputeScratch,
BoundaryGtCache, BoundaryIou, OksAnn, OksSimilarity, SegmAnn, SegmComputeScratch, SegmGtCache,
SegmIou, Similarity,
};
use std::collections::{HashMap, HashSet};
use std::sync::{Arc, Mutex};
use vernier_mask::Rle;
#[derive(Clone)]
pub enum GtCacheRef<'a, T: ?Sized> {
Borrowed(&'a T),
Owned(Arc<T>),
}
impl<T: ?Sized> GtCacheRef<'_, T> {
pub fn get(&self) -> &T {
match self {
GtCacheRef::Borrowed(r) => r,
GtCacheRef::Owned(a) => a.as_ref(),
}
}
}
pub const COLLAPSED_CATEGORY_SENTINEL: i64 = -1;
pub const AREA_UNBOUNDED: f64 = 1e10;
#[derive(Debug, Clone, Copy, PartialEq, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)]
#[rkyv(derive(Debug))]
pub struct AreaRange {
pub index: usize,
pub lo: f64,
pub hi: f64,
}
impl AreaRange {
pub fn coco_default() -> [Self; 4] {
[
Self {
index: 0,
lo: 0.0,
hi: AREA_UNBOUNDED,
},
Self {
index: 1,
lo: 0.0,
hi: 32.0 * 32.0,
},
Self {
index: 2,
lo: 32.0 * 32.0,
hi: 96.0 * 96.0,
},
Self {
index: 3,
lo: 96.0 * 96.0,
hi: AREA_UNBOUNDED,
},
]
}
pub fn keypoints_default() -> [Self; 3] {
[
Self {
index: 0,
lo: 0.0,
hi: AREA_UNBOUNDED,
},
Self {
index: 1,
lo: 32.0 * 32.0,
hi: 96.0 * 96.0,
},
Self {
index: 2,
lo: 96.0 * 96.0,
hi: AREA_UNBOUNDED,
},
]
}
fn contains(&self, area: f64) -> bool {
area >= self.lo && area <= self.hi
}
}
#[derive(Debug, Clone, Copy)]
pub struct EvaluateParams<'p> {
pub iou_thresholds: &'p [f64],
pub area_ranges: &'p [AreaRange],
pub max_dets_per_image: usize,
pub use_cats: bool,
pub retain_iou: bool,
}
#[derive(Debug, Clone, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)]
#[rkyv(derive(Debug))]
pub struct OwnedEvaluateParams {
pub iou_thresholds: Vec<f64>,
pub area_ranges: Vec<AreaRange>,
pub max_dets_per_image: usize,
pub use_cats: bool,
pub retain_iou: bool,
}
impl OwnedEvaluateParams {
pub fn borrow(&self) -> EvaluateParams<'_> {
EvaluateParams {
iou_thresholds: &self.iou_thresholds,
area_ranges: &self.area_ranges,
max_dets_per_image: self.max_dets_per_image,
use_cats: self.use_cats,
retain_iou: self.retain_iou,
}
}
pub fn params_hash(&self) -> Result<[u8; 32], EvalError> {
let bytes =
rkyv::to_bytes::<rkyv::rancor::Error>(self).map_err(|e| EvalError::InvalidConfig {
detail: format!("rkyv serialization of OwnedEvaluateParams failed: {e}"),
})?;
Ok(*blake3::hash(&bytes).as_bytes())
}
}
#[derive(
Debug, Clone, Copy, PartialEq, Eq, Hash, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize,
)]
#[rkyv(derive(Debug, PartialEq, Eq))]
pub enum KernelKind {
Bbox,
Segm,
Boundary,
Keypoints,
}
impl KernelKind {
pub const fn discriminator(self) -> u32 {
match self {
Self::Bbox => 0,
Self::Segm => 1,
Self::Boundary => 2,
Self::Keypoints => 3,
}
}
}
pub trait EvalKernel: Similarity {
fn kind(&self) -> KernelKind;
fn build_gt_anns(
&self,
gt_anns: &[CocoAnnotation],
indices: &[usize],
image: &ImageMeta,
) -> Result<Vec<Self::Annotation>, EvalError>;
fn build_dt_anns(
&self,
dt_anns: &[CocoDetection],
indices: &[usize],
image: &ImageMeta,
parity_mode: ParityMode,
) -> Result<Vec<Self::Annotation>, EvalError>;
fn extra_gt_ignore(&self, _ann: &CocoAnnotation) -> bool {
false
}
fn is_keypoints(&self) -> bool {
false
}
}
impl EvalKernel for BboxIou {
fn kind(&self) -> KernelKind {
KernelKind::Bbox
}
fn build_gt_anns(
&self,
gt_anns: &[CocoAnnotation],
indices: &[usize],
_image: &ImageMeta,
) -> Result<Vec<BboxAnn>, EvalError> {
Ok(indices
.iter()
.map(|&j| BboxAnn {
bbox: gt_anns[j].bbox,
is_crowd: gt_anns[j].is_crowd,
})
.collect())
}
fn build_dt_anns(
&self,
dt_anns: &[CocoDetection],
indices: &[usize],
_image: &ImageMeta,
_parity_mode: ParityMode,
) -> Result<Vec<BboxAnn>, EvalError> {
Ok(indices
.iter()
.map(|&j| BboxAnn {
bbox: dt_anns[j].bbox,
is_crowd: false,
})
.collect())
}
}
impl EvalKernel for SegmIou {
fn kind(&self) -> KernelKind {
KernelKind::Segm
}
fn build_gt_anns(
&self,
gt_anns: &[CocoAnnotation],
indices: &[usize],
image: &ImageMeta,
) -> Result<Vec<SegmAnn>, EvalError> {
build_segm_gt_anns(gt_anns, indices, image)
}
fn build_dt_anns(
&self,
dt_anns: &[CocoDetection],
indices: &[usize],
image: &ImageMeta,
parity_mode: ParityMode,
) -> Result<Vec<SegmAnn>, EvalError> {
build_segm_dt_anns(dt_anns, indices, image, parity_mode)
}
}
impl EvalKernel for BoundaryIou {
fn kind(&self) -> KernelKind {
KernelKind::Boundary
}
fn build_gt_anns(
&self,
gt_anns: &[CocoAnnotation],
indices: &[usize],
image: &ImageMeta,
) -> Result<Vec<SegmAnn>, EvalError> {
build_segm_gt_anns(gt_anns, indices, image)
}
fn build_dt_anns(
&self,
dt_anns: &[CocoDetection],
indices: &[usize],
image: &ImageMeta,
parity_mode: ParityMode,
) -> Result<Vec<SegmAnn>, EvalError> {
build_segm_dt_anns(dt_anns, indices, image, parity_mode)
}
}
impl EvalKernel for OksSimilarity {
fn kind(&self) -> KernelKind {
KernelKind::Keypoints
}
fn build_gt_anns(
&self,
gt_anns: &[CocoAnnotation],
indices: &[usize],
_image: &ImageMeta,
) -> Result<Vec<OksAnn>, EvalError> {
indices
.iter()
.map(|&j| {
let ann = >_anns[j];
let kps = ann
.keypoints
.as_deref()
.ok_or_else(|| missing_keypoints_err("GT", ann.id.0, ann.image_id.0))?;
let num_keypoints = ann
.num_keypoints
.unwrap_or_else(|| count_visible_keypoints(kps));
Ok(OksAnn {
category_id: ann.category_id.0,
keypoints: kps.to_vec(),
num_keypoints,
bbox: ann.bbox.into(),
area: ann.area,
})
})
.collect()
}
fn build_dt_anns(
&self,
dt_anns: &[CocoDetection],
indices: &[usize],
_image: &ImageMeta,
_parity_mode: ParityMode,
) -> Result<Vec<OksAnn>, EvalError> {
indices
.iter()
.map(|&j| {
let dt = &dt_anns[j];
let kps = dt
.keypoints
.as_deref()
.ok_or_else(|| missing_keypoints_err("DT", dt.id.0, dt.image_id.0))?;
let num_keypoints = dt
.num_keypoints
.unwrap_or_else(|| count_visible_keypoints(kps));
Ok(OksAnn {
category_id: dt.category_id.0,
keypoints: kps.to_vec(),
num_keypoints,
bbox: dt.bbox.into(),
area: dt.area,
})
})
.collect()
}
fn extra_gt_ignore(&self, ann: &CocoAnnotation) -> bool {
let visible = ann
.num_keypoints
.or_else(|| ann.keypoints.as_deref().map(count_visible_keypoints))
.unwrap_or(0);
visible == 0
}
fn is_keypoints(&self) -> bool {
true
}
}
fn count_visible_keypoints(kps: &[f64]) -> u32 {
kps.chunks_exact(3).filter(|t| t[2] > 0.0).count() as u32
}
fn missing_keypoints_err(kind: &str, ann_id: i64, image_id: i64) -> EvalError {
EvalError::InvalidAnnotation {
detail: format!(
"{kind} id={ann_id} on image {image_id} has no `keypoints` field; \
OKS eval requires keypoints on every entry. There is no \
pycocotools-equivalent bbox-synthesis fallback for keypoints \
(unlike segm quirk J2)."
),
}
}
fn build_segm_gt_anns(
gt_anns: &[CocoAnnotation],
indices: &[usize],
image: &ImageMeta,
) -> Result<Vec<SegmAnn>, EvalError> {
indices
.iter()
.map(|&j| {
let ann = >_anns[j];
let seg = ann
.segmentation
.as_ref()
.ok_or_else(|| missing_segmentation_err("GT", ann.id.0, image.id.0))?;
Ok(SegmAnn {
rle: seg.to_rle(image.height, image.width)?,
is_crowd: ann.is_crowd,
ann_id: ann.id.0,
})
})
.collect()
}
fn build_segm_dt_anns(
dt_anns: &[CocoDetection],
indices: &[usize],
image: &ImageMeta,
parity_mode: ParityMode,
) -> Result<Vec<SegmAnn>, EvalError> {
indices
.iter()
.map(|&j| {
let dt = &dt_anns[j];
let rle = match (&dt.segmentation, parity_mode) {
(Some(seg), _) => seg.to_rle(image.height, image.width)?,
(None, ParityMode::Strict) => {
synthesize_dt_segm_from_bbox(&dt.bbox, image.height, image.width)?
}
(None, ParityMode::Corrected) => {
return Err(missing_segmentation_err("DT", dt.id.0, image.id.0));
}
};
Ok(SegmAnn {
rle,
is_crowd: false,
ann_id: dt.id.0,
})
})
.collect()
}
fn synthesize_dt_segm_from_bbox(bbox: &Bbox, h: u32, w: u32) -> Result<Rle, EvalError> {
let x1 = bbox.x;
let y1 = bbox.y;
let x2 = bbox.x + bbox.w;
let y2 = bbox.y + bbox.h;
let polygon = vec![x1, y1, x1, y2, x2, y2, x2, y1];
let segm = Segmentation::Polygons(vec![polygon]);
segm.to_rle(h, w)
}
fn missing_segmentation_err(kind: &str, ann_id: i64, image_id: i64) -> EvalError {
EvalError::InvalidAnnotation {
detail: format!(
"{kind} id={ann_id} on image {image_id} has no `segmentation` field; \
segm eval in corrected mode requires one on every entry. \
pycocotools synthesizes a bbox-rectangle polygon here \
(quirks J2/J6); pass `ParityMode::Strict` to opt into that \
behavior."
),
}
}
#[derive(Debug, Clone)]
pub struct EvalImageMeta {
pub image_id: i64,
pub category_id: i64,
pub area_rng: [f64; 2],
pub max_det: usize,
pub dt_ids: Vec<i64>,
pub gt_ids: Vec<i64>,
pub dt_matches: Array2<i64>,
pub gt_matches: Array2<i64>,
}
#[derive(Debug, Clone)]
pub struct EvalGrid {
pub eval_imgs: Vec<Option<Box<PerImageEval>>>,
pub eval_imgs_meta: Vec<Option<Box<EvalImageMeta>>>,
pub n_categories: usize,
pub n_area_ranges: usize,
pub n_images: usize,
pub retained_ious: Option<crate::tables::RetainedIous>,
}
impl EvalGrid {
pub fn cell(&self, k: usize, a: usize, i: usize) -> Option<&PerImageEval> {
let idx = self.flat_index(k, a, i)?;
self.eval_imgs.get(idx).and_then(Option::as_deref)
}
pub fn cell_meta(&self, k: usize, a: usize, i: usize) -> Option<&EvalImageMeta> {
let idx = self.flat_index(k, a, i)?;
self.eval_imgs_meta.get(idx).and_then(Option::as_deref)
}
fn flat_index(&self, k: usize, a: usize, i: usize) -> Option<usize> {
if k >= self.n_categories || a >= self.n_area_ranges || i >= self.n_images {
return None;
}
Some(k * self.n_area_ranges * self.n_images + a * self.n_images + i)
}
}
pub fn evaluate_with<K: EvalKernel>(
gt: &CocoDataset,
dt: &CocoDetections,
params: EvaluateParams<'_>,
parity_mode: ParityMode,
kernel: &K,
) -> Result<EvalGrid, EvalError> {
let mut images: Vec<&ImageMeta> = gt.images().iter().collect();
images.sort_unstable_by_key(|im| im.id.0);
let n_i = images.len();
let n_a = params.area_ranges.len();
let category_buckets: Vec<Option<CategoryId>> = if params.use_cats {
let mut cats: Vec<_> = gt.categories().iter().map(|c| c.id).collect();
cats.sort_unstable_by_key(|id| id.0);
cats.into_iter().map(Some).collect()
} else {
vec![None]
};
let n_k = category_buckets.len();
let mut eval_imgs: Vec<Option<Box<PerImageEval>>> = vec![None; n_k * n_a * n_i];
let mut eval_imgs_meta: Vec<Option<Box<EvalImageMeta>>> = vec![None; n_k * n_a * n_i];
let mut retained_ious_map: Option<std::collections::HashMap<(usize, usize), Array2<f64>>> =
if params.retain_iou {
Some(std::collections::HashMap::new())
} else {
None
};
let federated_per_image: Vec<Option<(&HashSet<CategoryId>, &HashSet<CategoryId>)>> =
match (params.use_cats, gt.federated()) {
(true, Some(fed)) => images
.iter()
.map(|im| {
let neg = fed.neg_category_ids.get(&im.id)?;
let nel = fed.not_exhaustive_category_ids.get(&im.id)?;
Some((neg, nel))
})
.collect(),
_ => Vec::new(),
};
let mut scratch = CellScratch::new();
let gt_anns = gt.annotations();
let dt_anns = dt.detections();
let strict_lvis_zero_area_filter =
matches!(parity_mode, ParityMode::Strict) && gt.federated().is_some();
for (k, cat) in category_buckets.iter().enumerate() {
let nk = k * n_a * n_i;
let category_id = cat.map_or(COLLAPSED_CATEGORY_SENTINEL, |c| c.0);
for (i, image) in images.iter().enumerate() {
let image_id = image.id;
let gt_indices_raw = gt_indices_for_cell(gt, image_id, *cat);
let gt_indices_buf: Vec<usize>;
let gt_indices: &[usize] = if strict_lvis_zero_area_filter
&& gt_indices_raw.iter().any(|&j| gt_anns[j].area <= 0.0)
{
gt_indices_buf = gt_indices_raw
.iter()
.copied()
.filter(|&j| gt_anns[j].area > 0.0)
.collect();
>_indices_buf
} else {
gt_indices_raw
};
let raw_dt_indices = raw_dt_indices_for_cell(dt, image_id, *cat);
if gt_indices.is_empty() && raw_dt_indices.is_empty() {
continue;
}
let mut not_exhaustive_for_cell = false;
if let (Some(c), Some(Some((neg_set, nel_set)))) = (cat, federated_per_image.get(i)) {
if gt_indices.is_empty() && !neg_set.contains(c) {
continue;
}
not_exhaustive_for_cell = nel_set.contains(c);
}
dt_top_indices_for_cell_into(
&mut scratch.dt_indices,
&mut scratch.dt_score_buf,
&mut scratch.dt_perm_buf,
dt_anns,
raw_dt_indices,
params.max_dets_per_image,
);
scratch.gt_areas.clear();
scratch
.gt_areas
.extend(gt_indices.iter().map(|&j| gt_anns[j].area));
scratch.gt_iscrowd.clear();
scratch
.gt_iscrowd
.extend(gt_indices.iter().map(|&j| gt_anns[j].is_crowd));
scratch.gt_base_ignore.clear();
scratch.gt_base_ignore.extend(gt_indices.iter().map(|&j| {
gt_anns[j].effective_ignore(parity_mode) || kernel.extra_gt_ignore(>_anns[j])
}));
scratch.gt_ids.clear();
scratch
.gt_ids
.extend(gt_indices.iter().map(|&j| gt_anns[j].id.0));
scratch.dt_areas.clear();
scratch
.dt_areas
.extend(scratch.dt_indices.iter().map(|&j| dt_anns[j].area));
scratch.dt_scores.clear();
scratch
.dt_scores
.extend(scratch.dt_indices.iter().map(|&j| dt_anns[j].score));
scratch.dt_ids.clear();
scratch
.dt_ids
.extend(scratch.dt_indices.iter().map(|&j| dt_anns[j].id.0));
let gt_kernel = kernel.build_gt_anns(gt_anns, gt_indices, image)?;
let dt_kernel =
kernel.build_dt_anns(dt_anns, &scratch.dt_indices, image, parity_mode)?;
let g = gt_kernel.len();
let d = dt_kernel.len();
scratch.iou_buf.clear();
scratch.iou_buf.resize(g * d, 0.0);
if g > 0 && d > 0 {
let mut iou_view = ArrayViewMut2::from_shape((g, d), &mut scratch.iou_buf[..])
.map_err(|e| EvalError::DimensionMismatch {
detail: format!("iou scratch view: {e}"),
})?;
kernel.compute(>_kernel, &dt_kernel, &mut iou_view)?;
}
let iou_view = ArrayView2::from_shape((g, d), &scratch.iou_buf[..]).map_err(|e| {
EvalError::DimensionMismatch {
detail: format!("iou scratch view: {e}"),
}
})?;
let buffers = CellBuffers {
image_id: image_id.0,
category_id,
max_det: params.max_dets_per_image,
gt_areas: &scratch.gt_areas,
gt_iscrowd: &scratch.gt_iscrowd,
gt_base_ignore: &scratch.gt_base_ignore,
gt_ids: &scratch.gt_ids,
dt_areas: &scratch.dt_areas,
dt_scores: &scratch.dt_scores,
dt_ids: &scratch.dt_ids,
iou: iou_view,
not_exhaustive: not_exhaustive_for_cell,
};
for (a, area) in params.area_ranges.iter().enumerate() {
let (cell, meta) = evaluate_cell(
&mut scratch.gt_ignore_buf,
&buffers,
area,
params.iou_thresholds,
parity_mode,
)?;
let flat = nk + a * n_i + i;
eval_imgs[flat] = Some(Box::new(cell));
eval_imgs_meta[flat] = Some(Box::new(meta));
}
if let Some(map) = retained_ious_map.as_mut() {
let cloned =
Array2::from_shape_vec((g, d), scratch.iou_buf.clone()).map_err(|e| {
EvalError::DimensionMismatch {
detail: format!("retained iou clone: {e}"),
}
})?;
map.insert((k, i), cloned);
}
}
}
Ok(EvalGrid {
eval_imgs,
eval_imgs_meta,
n_categories: n_k,
n_area_ranges: n_a,
n_images: n_i,
retained_ious: retained_ious_map.map(crate::tables::RetainedIous::from_map),
})
}
pub(crate) fn evaluate_with_retention<K: EvalKernel>(
gt: &CocoDataset,
dt: &CocoDetections,
params: EvaluateParams<'_>,
parity_mode: ParityMode,
kernel: &K,
) -> Result<(EvalGrid, crate::tables::CrossClassIous), EvalError> {
let grid = evaluate_with(gt, dt, params, parity_mode, kernel)?;
let cross_class = crate::tide::compute_cross_class_ious(
gt,
dt,
kernel,
parity_mode,
params.max_dets_per_image,
)?;
Ok((grid, cross_class))
}
pub fn evaluate_bbox(
gt: &CocoDataset,
dt: &CocoDetections,
params: EvaluateParams<'_>,
parity_mode: ParityMode,
) -> Result<EvalGrid, EvalError> {
evaluate_with(gt, dt, params, parity_mode, &BboxIou)
}
pub fn evaluate_segm(
gt: &CocoDataset,
dt: &CocoDetections,
params: EvaluateParams<'_>,
parity_mode: ParityMode,
) -> Result<EvalGrid, EvalError> {
evaluate_with(gt, dt, params, parity_mode, &segm_kernel(None))
}
pub fn evaluate_segm_cached(
gt: &CocoDataset,
dt: &CocoDetections,
params: EvaluateParams<'_>,
parity_mode: ParityMode,
cache: &SegmGtCache,
) -> Result<EvalGrid, EvalError> {
evaluate_with(gt, dt, params, parity_mode, &segm_kernel(Some(cache)))
}
fn segm_kernel(gt_cache: Option<&SegmGtCache>) -> SegmIouCached<'_> {
SegmIouCached {
scratch: Mutex::new(SegmComputeScratch::new()),
gt_cache: gt_cache.map(GtCacheRef::Borrowed),
}
}
pub struct SegmIouCached<'a> {
scratch: Mutex<SegmComputeScratch>,
gt_cache: Option<GtCacheRef<'a, SegmGtCache>>,
}
impl SegmIouCached<'static> {
pub fn with_arc_cache(cache: Arc<SegmGtCache>) -> Self {
Self {
scratch: Mutex::new(SegmComputeScratch::new()),
gt_cache: Some(GtCacheRef::Owned(cache)),
}
}
}
impl Similarity for SegmIouCached<'_> {
type Annotation = SegmAnn;
fn compute(
&self,
gts: &[SegmAnn],
dts: &[SegmAnn],
out: &mut ArrayViewMut2<'_, f64>,
) -> Result<(), EvalError> {
let mut scratch = self
.scratch
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
segm_iou_compute(
gts,
dts,
out,
&mut scratch,
self.gt_cache.as_ref().map(GtCacheRef::get),
)
}
}
impl EvalKernel for SegmIouCached<'_> {
fn kind(&self) -> KernelKind {
KernelKind::Segm
}
fn build_gt_anns(
&self,
gt_anns: &[CocoAnnotation],
indices: &[usize],
image: &ImageMeta,
) -> Result<Vec<SegmAnn>, EvalError> {
build_segm_gt_anns(gt_anns, indices, image)
}
fn build_dt_anns(
&self,
dt_anns: &[CocoDetection],
indices: &[usize],
image: &ImageMeta,
parity_mode: ParityMode,
) -> Result<Vec<SegmAnn>, EvalError> {
build_segm_dt_anns(dt_anns, indices, image, parity_mode)
}
}
pub fn evaluate_boundary(
gt: &CocoDataset,
dt: &CocoDetections,
params: EvaluateParams<'_>,
parity_mode: ParityMode,
dilation_ratio: f64,
) -> Result<EvalGrid, EvalError> {
evaluate_with(gt, dt, params, parity_mode, &kernel(dilation_ratio, None))
}
pub fn evaluate_boundary_cached(
gt: &CocoDataset,
dt: &CocoDetections,
params: EvaluateParams<'_>,
parity_mode: ParityMode,
dilation_ratio: f64,
cache: &BoundaryGtCache,
) -> Result<EvalGrid, EvalError> {
cache.align_ratio(dilation_ratio);
evaluate_with(
gt,
dt,
params,
parity_mode,
&kernel(dilation_ratio, Some(cache)),
)
}
fn kernel(dilation_ratio: f64, gt_cache: Option<&BoundaryGtCache>) -> BoundaryIouCached<'_> {
BoundaryIouCached {
dilation_ratio,
scratch: Mutex::new(BoundaryComputeScratch::new()),
gt_cache: gt_cache.map(GtCacheRef::Borrowed),
}
}
pub struct BoundaryIouCached<'a> {
dilation_ratio: f64,
scratch: Mutex<BoundaryComputeScratch>,
gt_cache: Option<GtCacheRef<'a, BoundaryGtCache>>,
}
impl BoundaryIouCached<'static> {
pub fn with_arc_cache(dilation_ratio: f64, cache: Arc<BoundaryGtCache>) -> Self {
cache.align_ratio(dilation_ratio);
Self {
dilation_ratio,
scratch: Mutex::new(BoundaryComputeScratch::new()),
gt_cache: Some(GtCacheRef::Owned(cache)),
}
}
}
impl Similarity for BoundaryIouCached<'_> {
type Annotation = SegmAnn;
fn compute(
&self,
gts: &[SegmAnn],
dts: &[SegmAnn],
out: &mut ArrayViewMut2<'_, f64>,
) -> Result<(), EvalError> {
let mut scratch = self
.scratch
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
boundary_iou_compute(
self.dilation_ratio,
gts,
dts,
out,
&mut scratch,
self.gt_cache.as_ref().map(GtCacheRef::get),
)
}
}
impl EvalKernel for BoundaryIouCached<'_> {
fn kind(&self) -> KernelKind {
KernelKind::Boundary
}
fn build_gt_anns(
&self,
gt_anns: &[CocoAnnotation],
indices: &[usize],
image: &ImageMeta,
) -> Result<Vec<SegmAnn>, EvalError> {
build_segm_gt_anns(gt_anns, indices, image)
}
fn build_dt_anns(
&self,
dt_anns: &[CocoDetection],
indices: &[usize],
image: &ImageMeta,
parity_mode: ParityMode,
) -> Result<Vec<SegmAnn>, EvalError> {
build_segm_dt_anns(dt_anns, indices, image, parity_mode)
}
}
pub fn evaluate_keypoints(
gt: &CocoDataset,
dt: &CocoDetections,
params: EvaluateParams<'_>,
parity_mode: ParityMode,
sigmas: HashMap<i64, Vec<f64>>,
) -> Result<EvalGrid, EvalError> {
evaluate_with(gt, dt, params, parity_mode, &OksSimilarity::new(sigmas))
}
fn gt_indices_for_cell(gt: &CocoDataset, image: ImageId, cat: Option<CategoryId>) -> &[usize] {
match cat {
Some(c) => gt.ann_indices_for(image, c),
None => gt.ann_indices_for_image(image),
}
}
fn raw_dt_indices_for_cell(
dt: &CocoDetections,
image: ImageId,
cat: Option<CategoryId>,
) -> &[usize] {
match cat {
Some(c) => dt.indices_for(image, c),
None => dt.indices_for_image(image),
}
}
pub(crate) fn dt_top_indices_for_cell(
dt: &CocoDetections,
image: ImageId,
cat: Option<CategoryId>,
max_dets: usize,
) -> Vec<usize> {
let raw_indices = raw_dt_indices_for_cell(dt, image, cat);
let mut out = Vec::new();
let mut score_buf = Vec::new();
let mut perm_buf = Vec::new();
dt_top_indices_for_cell_into(
&mut out,
&mut score_buf,
&mut perm_buf,
dt.detections(),
raw_indices,
max_dets,
);
out
}
fn dt_top_indices_for_cell_into(
out: &mut Vec<usize>,
score_buf: &mut Vec<f64>,
perm_buf: &mut Vec<usize>,
dts: &[CocoDetection],
raw_indices: &[usize],
max_dets: usize,
) {
score_buf.clear();
score_buf.extend(raw_indices.iter().map(|&i| dts[i].score));
perm_buf.clear();
perm_buf.extend(0..score_buf.len());
perm_buf.sort_by(|&a, &b| {
score_buf[b]
.partial_cmp(&score_buf[a])
.unwrap_or(std::cmp::Ordering::Equal)
});
out.clear();
out.extend(perm_buf.iter().take(max_dets).map(|&k| raw_indices[k]));
}
#[derive(Default)]
struct CellScratch {
gt_areas: Vec<f64>,
gt_iscrowd: Vec<bool>,
gt_base_ignore: Vec<bool>,
gt_ids: Vec<i64>,
dt_indices: Vec<usize>,
dt_areas: Vec<f64>,
dt_scores: Vec<f64>,
dt_ids: Vec<i64>,
iou_buf: Vec<f64>,
dt_score_buf: Vec<f64>,
dt_perm_buf: Vec<usize>,
gt_ignore_buf: Vec<bool>,
}
impl CellScratch {
fn new() -> Self {
Self::default()
}
}
struct CellBuffers<'a> {
image_id: i64,
category_id: i64,
max_det: usize,
gt_areas: &'a [f64],
gt_iscrowd: &'a [bool],
gt_base_ignore: &'a [bool],
gt_ids: &'a [i64],
dt_areas: &'a [f64],
dt_scores: &'a [f64],
dt_ids: &'a [i64],
iou: ArrayView2<'a, f64>,
not_exhaustive: bool,
}
fn evaluate_cell(
gt_ignore_buf: &mut Vec<bool>,
buf: &CellBuffers<'_>,
area: &AreaRange,
iou_thresholds: &[f64],
parity_mode: ParityMode,
) -> Result<(PerImageEval, EvalImageMeta), EvalError> {
gt_ignore_buf.clear();
gt_ignore_buf.extend(
buf.gt_base_ignore
.iter()
.zip(buf.gt_areas)
.map(|(&base, &a)| base || !area.contains(a)),
);
let gt_ignore: &[bool] = gt_ignore_buf.as_slice();
let MatchResult {
dt_perm,
gt_perm,
dt_matches: dt_matches_pos,
gt_matches: gt_matches_pos,
mut dt_ignore,
} = match_image(
buf.iou,
gt_ignore,
buf.gt_iscrowd,
buf.dt_scores,
iou_thresholds,
parity_mode,
)?;
let n_t = iou_thresholds.len();
let n_d = buf.dt_scores.len();
let n_g = gt_ignore.len();
let dt_scores_sorted: Vec<f64> = dt_perm.iter().map(|&k| buf.dt_scores[k]).collect();
let gt_ignore_sorted: Vec<bool> = gt_perm.iter().map(|&k| gt_ignore[k]).collect();
let dt_ids_sorted: Vec<i64> = dt_perm.iter().map(|&k| buf.dt_ids[k]).collect();
let gt_ids_sorted: Vec<i64> = gt_perm.iter().map(|&k| buf.gt_ids[k]).collect();
let mut dt_matched = Array2::<bool>::default((n_t, n_d));
let mut dt_matches_id = Array2::<i64>::zeros((n_t, n_d));
let mut gt_matches_id = Array2::<i64>::zeros((n_t, n_g));
for d in 0..n_d {
let in_range = area.contains(buf.dt_areas[dt_perm[d]]);
for t in 0..n_t {
let m = dt_matches_pos[(t, d)];
let matched = m >= 0;
dt_matched[(t, d)] = matched;
if matched {
dt_matches_id[(t, d)] = gt_ids_sorted[m as usize];
}
if !matched && (!in_range || buf.not_exhaustive) {
dt_ignore[(t, d)] = true;
}
}
}
for t in 0..n_t {
for g in 0..n_g {
let p = gt_matches_pos[(t, g)];
if p >= 0 {
gt_matches_id[(t, g)] = dt_ids_sorted[p as usize];
}
}
}
let cell = PerImageEval {
dt_scores: dt_scores_sorted,
dt_matched,
dt_ignore,
gt_ignore: gt_ignore_sorted,
};
let meta = EvalImageMeta {
image_id: buf.image_id,
category_id: buf.category_id,
area_rng: [area.lo, area.hi],
max_det: buf.max_det,
dt_ids: dt_ids_sorted,
gt_ids: gt_ids_sorted,
dt_matches: dt_matches_id,
gt_matches: gt_matches_id,
};
Ok((cell, meta))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::accumulate::{accumulate, AccumulateParams};
use crate::dataset::{AnnId, Bbox, CategoryMeta, CocoAnnotation, DetectionInput, ImageMeta};
use crate::parity::{iou_thresholds, recall_thresholds};
use crate::summarize::summarize_detection;
fn img(id: i64, w: u32, h: u32) -> ImageMeta {
ImageMeta {
id: ImageId(id),
width: w,
height: h,
file_name: None,
}
}
fn cat(id: i64, name: &str) -> CategoryMeta {
CategoryMeta {
id: CategoryId(id),
name: name.into(),
supercategory: None,
}
}
fn ann(id: i64, image: i64, cat: i64, bbox: (f64, f64, f64, f64)) -> CocoAnnotation {
CocoAnnotation {
id: AnnId(id),
image_id: ImageId(image),
category_id: CategoryId(cat),
area: bbox.2 * bbox.3,
is_crowd: false,
ignore_flag: None,
bbox: Bbox {
x: bbox.0,
y: bbox.1,
w: bbox.2,
h: bbox.3,
},
segmentation: None,
keypoints: None,
num_keypoints: None,
}
}
fn dt_input(image: i64, cat: i64, score: f64, bbox: (f64, f64, f64, f64)) -> DetectionInput {
DetectionInput {
id: None,
image_id: ImageId(image),
category_id: CategoryId(cat),
score,
bbox: Bbox {
x: bbox.0,
y: bbox.1,
w: bbox.2,
h: bbox.3,
},
segmentation: None,
keypoints: None,
num_keypoints: None,
}
}
fn perfect_match_grid() -> EvalGrid {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "thing")];
let anns = vec![
ann(1, 1, 1, (0.0, 0.0, 10.0, 10.0)),
ann(2, 1, 1, (50.0, 50.0, 10.0, 10.0)),
];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![
dt_input(1, 1, 0.9, (0.0, 0.0, 10.0, 10.0)),
dt_input(1, 1, 0.8, (50.0, 50.0, 10.0, 10.0)),
])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
retain_iou: false,
};
evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap()
}
#[test]
fn d4_coco_default_area_ranges_pin_literal_values() {
let ranges = AreaRange::coco_default();
assert_eq!(ranges.len(), 4);
assert_eq!(
(ranges[0].lo, ranges[0].hi),
(0.0, 1e10),
"all bucket bounds"
);
assert_eq!(
(ranges[1].lo, ranges[1].hi),
(0.0, 1024.0),
"small bucket bounds"
);
assert_eq!(
(ranges[2].lo, ranges[2].hi),
(1024.0, 9216.0),
"medium bucket bounds"
);
assert_eq!(
(ranges[3].lo, ranges[3].hi),
(9216.0, 1e10),
"large bucket bounds"
);
use crate::summarize::AreaRng;
assert_eq!(ranges[0].index, AreaRng::ALL.index);
assert_eq!(AreaRng::ALL.label.as_ref(), "all");
assert_eq!(ranges[1].index, AreaRng::SMALL.index);
assert_eq!(AreaRng::SMALL.label.as_ref(), "small");
assert_eq!(ranges[2].index, AreaRng::MEDIUM.index);
assert_eq!(AreaRng::MEDIUM.label.as_ref(), "medium");
assert_eq!(ranges[3].index, AreaRng::LARGE.index);
assert_eq!(AreaRng::LARGE.label.as_ref(), "large");
let pyco_unbounded: f64 = 1e5_f64.powi(2);
assert_eq!(pyco_unbounded.to_bits(), 1e10_f64.to_bits());
assert_eq!(ranges[0].hi.to_bits(), 1e10_f64.to_bits());
assert_eq!(ranges[3].hi.to_bits(), 1e10_f64.to_bits());
}
#[test]
fn perfect_match_produces_one_cell_per_area_range() {
let grid = perfect_match_grid();
assert_eq!(grid.n_categories, 1);
assert_eq!(grid.n_area_ranges, 4);
assert_eq!(grid.n_images, 1);
let cells: Vec<_> = grid.eval_imgs.iter().filter(|c| c.is_some()).collect();
assert_eq!(cells.len(), 4);
let all_cell = grid.cell(0, 0, 0).unwrap();
assert_eq!(all_cell.dt_scores.len(), 2);
assert!(all_cell.dt_matched.iter().all(|&m| m));
assert!(all_cell.dt_ignore.iter().all(|&ig| !ig));
}
#[test]
fn perfect_match_summarizes_to_one() {
let grid = perfect_match_grid();
let max_dets = vec![1usize, 10, 100];
let acc = accumulate(
&grid.eval_imgs,
AccumulateParams {
iou_thresholds: iou_thresholds(),
recall_thresholds: recall_thresholds(),
max_dets: &max_dets,
n_categories: grid.n_categories,
n_area_ranges: grid.n_area_ranges,
n_images: grid.n_images,
},
ParityMode::Strict,
)
.unwrap();
let summary = summarize_detection(&acc, iou_thresholds(), &max_dets).unwrap();
let stats = summary.stats();
assert!((stats[0] - 1.0).abs() < 1e-12, "AP={}", stats[0]);
assert!((stats[3] - 1.0).abs() < 1e-12, "AP_S={}", stats[3]);
assert_eq!(stats[4], -1.0, "AP_M should be -1 with no medium GTs");
assert_eq!(stats[5], -1.0, "AP_L should be -1 with no large GTs");
assert!((stats[8] - 1.0).abs() < 1e-12, "AR@100={}", stats[8]);
}
#[test]
fn b7_unmatched_dt_outside_area_range_is_ignored() {
let images = vec![img(1, 300, 300)];
let cats = vec![cat(1, "thing")];
let anns = vec![ann(1, 1, 1, (0.0, 0.0, 200.0, 200.0))];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts =
CocoDetections::from_inputs(vec![dt_input(1, 1, 0.5, (200.0, 200.0, 50.0, 50.0))])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
retain_iou: false,
};
let grid = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
let small = grid.cell(0, 1, 0).unwrap();
assert_eq!(small.gt_ignore, vec![true]);
assert!(small.dt_ignore.iter().all(|&ig| ig));
assert!(small.dt_matched.iter().all(|&m| !m));
}
#[test]
fn d6_boundary_area_lands_in_both_buckets() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "thing")];
let anns = vec![ann(1, 1, 1, (0.0, 0.0, 32.0, 32.0))];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts =
CocoDetections::from_inputs(vec![dt_input(1, 1, 0.5, (0.0, 0.0, 32.0, 32.0))]).unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
retain_iou: false,
};
let grid = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
let small = grid.cell(0, 1, 0).unwrap();
assert_eq!(small.gt_ignore, vec![false]);
let medium = grid.cell(0, 2, 0).unwrap();
assert_eq!(medium.gt_ignore, vec![false]);
let all = grid.cell(0, 0, 0).unwrap();
assert_eq!(all.gt_ignore, vec![false]);
let large = grid.cell(0, 3, 0).unwrap();
assert_eq!(large.gt_ignore, vec![true]);
}
#[test]
fn l4_use_cats_false_collapses_categories() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "a"), cat(2, "b")];
let anns = vec![
ann(1, 1, 1, (0.0, 0.0, 10.0, 10.0)),
ann(2, 1, 2, (50.0, 50.0, 10.0, 10.0)),
];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![dt_input(1, 1, 0.9, (50.0, 50.0, 10.0, 10.0))])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: false,
retain_iou: false,
};
let grid = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
assert_eq!(grid.n_categories, 1);
let all = grid.cell(0, 0, 0).unwrap();
assert_eq!(all.gt_ignore.len(), 2);
assert_eq!(all.dt_scores.len(), 1);
assert!(all.dt_matched.iter().all(|&m| m));
}
#[test]
fn max_dets_per_image_caps_top_n_by_score() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "thing")];
let anns = vec![ann(1, 1, 1, (0.0, 0.0, 10.0, 10.0))];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![
dt_input(1, 1, 0.1, (50.0, 50.0, 5.0, 5.0)),
dt_input(1, 1, 0.9, (0.0, 0.0, 10.0, 10.0)),
dt_input(1, 1, 0.5, (50.0, 50.0, 5.0, 5.0)),
])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 2,
use_cats: true,
retain_iou: false,
};
let grid = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
let all = grid.cell(0, 0, 0).unwrap();
assert_eq!(all.dt_scores.len(), 2);
assert_eq!(all.dt_scores[0], 0.9);
assert_eq!(all.dt_scores[1], 0.5);
}
#[test]
fn d1_parity_mode_propagates_to_base_ignore() {
const ANN_JSON: &str = r#"{
"images": [{"id": 1, "width": 100, "height": 100}],
"annotations": [
{"id": 1, "image_id": 1, "category_id": 1,
"bbox": [0, 0, 10, 10], "area": 100,
"iscrowd": 0, "ignore": 1}
],
"categories": [{"id": 1, "name": "thing"}]
}"#;
let gt = CocoDataset::from_json_bytes(ANN_JSON.as_bytes()).unwrap();
let dts =
CocoDetections::from_inputs(vec![dt_input(1, 1, 0.9, (0.0, 0.0, 10.0, 10.0))]).unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
retain_iou: false,
};
let strict = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
let strict_all = strict.cell(0, 0, 0).unwrap();
assert_eq!(strict_all.gt_ignore, vec![false]);
assert!(strict_all.dt_ignore.iter().all(|&ig| !ig));
let corrected = evaluate_bbox(>, &dts, params, ParityMode::Corrected).unwrap();
let corrected_all = corrected.cell(0, 0, 0).unwrap();
assert_eq!(corrected_all.gt_ignore, vec![true]);
assert!(corrected_all.dt_ignore.iter().all(|&ig| ig));
}
#[test]
fn cell_meta_carries_pycocotools_shape() {
let grid = perfect_match_grid();
let meta = grid.cell_meta(0, 0, 0).unwrap();
assert_eq!(meta.image_id, 1);
assert_eq!(meta.category_id, 1);
assert_eq!(meta.area_rng, [0.0, AREA_UNBOUNDED]);
assert_eq!(meta.max_det, 100);
assert_eq!(meta.dt_ids, vec![1, 2]);
assert_eq!(meta.gt_ids, vec![1, 2]);
let n_t = iou_thresholds().len();
assert_eq!(meta.dt_matches.shape(), &[n_t, 2]);
assert_eq!(meta.gt_matches.shape(), &[n_t, 2]);
for t in 0..n_t {
assert_eq!(meta.dt_matches[(t, 0)], 1, "dt[0] -> gt[1] at t={t}");
assert_eq!(meta.dt_matches[(t, 1)], 2, "dt[1] -> gt[2] at t={t}");
assert_eq!(meta.gt_matches[(t, 0)], 1, "gt[1] -> dt[1] at t={t}");
assert_eq!(meta.gt_matches[(t, 1)], 2, "gt[2] -> dt[2] at t={t}");
}
}
#[test]
fn cell_meta_unmatched_dt_uses_zero_sentinel() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "thing")];
let anns = vec![ann(7, 1, 1, (0.0, 0.0, 10.0, 10.0))];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![dt_input(1, 1, 0.5, (50.0, 50.0, 10.0, 10.0))])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
retain_iou: false,
};
let grid = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
let meta = grid.cell_meta(0, 0, 0).unwrap();
assert_eq!(meta.gt_ids, vec![7]);
assert_eq!(meta.dt_ids.len(), 1);
assert!(meta.dt_matches.iter().all(|&x| x == 0));
assert!(meta.gt_matches.iter().all(|&x| x == 0));
}
#[test]
fn cell_meta_use_cats_false_emits_sentinel_category() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "a"), cat(2, "b")];
let anns = vec![ann(1, 1, 1, (0.0, 0.0, 10.0, 10.0))];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts =
CocoDetections::from_inputs(vec![dt_input(1, 1, 0.9, (0.0, 0.0, 10.0, 10.0))]).unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: false,
retain_iou: false,
};
let grid = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
let meta = grid.cell_meta(0, 0, 0).unwrap();
assert_eq!(meta.category_id, COLLAPSED_CATEGORY_SENTINEL);
}
#[test]
fn missing_dt_image_yields_none_cells() {
let images = vec![img(1, 100, 100), img(2, 100, 100)];
let cats = vec![cat(1, "thing")];
let anns = vec![ann(1, 1, 1, (0.0, 0.0, 10.0, 10.0))];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![]).unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
retain_iou: false,
};
let grid = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
for a in 0..4 {
assert!(grid.cell(0, a, 0).is_some(), "image 1 area {a}");
assert!(grid.cell(0, a, 1).is_none(), "image 2 area {a}");
}
}
fn square_polygon(x: f64, y: f64, side: f64) -> Segmentation {
Segmentation::Polygons(vec![vec![
x,
y,
x + side,
y,
x + side,
y + side,
x,
y + side,
]])
}
fn ann_with_segm(
id: i64,
image: i64,
cat: i64,
bbox: (f64, f64, f64, f64),
segm: Segmentation,
) -> CocoAnnotation {
CocoAnnotation {
id: AnnId(id),
image_id: ImageId(image),
category_id: CategoryId(cat),
area: bbox.2 * bbox.3,
is_crowd: false,
ignore_flag: None,
bbox: Bbox {
x: bbox.0,
y: bbox.1,
w: bbox.2,
h: bbox.3,
},
segmentation: Some(segm),
keypoints: None,
num_keypoints: None,
}
}
fn dt_input_with_segm(
image: i64,
cat: i64,
score: f64,
bbox: (f64, f64, f64, f64),
segm: Segmentation,
) -> DetectionInput {
DetectionInput {
id: None,
image_id: ImageId(image),
category_id: CategoryId(cat),
score,
bbox: Bbox {
x: bbox.0,
y: bbox.1,
w: bbox.2,
h: bbox.3,
},
segmentation: Some(segm),
keypoints: None,
num_keypoints: None,
}
}
#[test]
fn segm_perfect_overlap_summarizes_to_one() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "thing")];
let anns = vec![ann_with_segm(
1,
1,
1,
(10.0, 10.0, 20.0, 20.0),
square_polygon(10.0, 10.0, 20.0),
)];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![dt_input_with_segm(
1,
1,
0.9,
(10.0, 10.0, 20.0, 20.0),
square_polygon(10.0, 10.0, 20.0),
)])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
retain_iou: false,
};
let grid = evaluate_segm(>, &dts, params, ParityMode::Strict).unwrap();
let max_dets = vec![1usize, 10, 100];
let acc = accumulate(
&grid.eval_imgs,
AccumulateParams {
iou_thresholds: iou_thresholds(),
recall_thresholds: recall_thresholds(),
max_dets: &max_dets,
n_categories: grid.n_categories,
n_area_ranges: grid.n_area_ranges,
n_images: grid.n_images,
},
ParityMode::Strict,
)
.unwrap();
let summary = summarize_detection(&acc, iou_thresholds(), &max_dets).unwrap();
let stats = summary.stats();
assert!((stats[0] - 1.0).abs() < 1e-12, "AP={}", stats[0]);
}
#[test]
fn segm_disjoint_masks_summarize_to_zero() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "thing")];
let anns = vec![ann_with_segm(
1,
1,
1,
(0.0, 0.0, 10.0, 10.0),
square_polygon(0.0, 0.0, 10.0),
)];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![dt_input_with_segm(
1,
1,
0.9,
(50.0, 50.0, 10.0, 10.0),
square_polygon(50.0, 50.0, 10.0),
)])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
retain_iou: false,
};
let grid = evaluate_segm(>, &dts, params, ParityMode::Strict).unwrap();
let all = grid.cell(0, 0, 0).unwrap();
assert!(all.dt_matched.iter().all(|&m| !m));
}
#[test]
fn segm_missing_gt_segmentation_surfaces_typed_error() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "thing")];
let anns = vec![ann(7, 1, 1, (0.0, 0.0, 10.0, 10.0))];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![dt_input_with_segm(
1,
1,
0.9,
(0.0, 0.0, 10.0, 10.0),
square_polygon(0.0, 0.0, 10.0),
)])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
retain_iou: false,
};
let err = evaluate_segm(>, &dts, params, ParityMode::Strict).unwrap_err();
match err {
EvalError::InvalidAnnotation { detail } => {
assert!(detail.contains("GT id=7"), "msg: {detail}");
}
other => panic!("expected InvalidAnnotation, got {other:?}"),
}
}
#[test]
fn j2_bbox_only_dt_under_segm_iou_type_raises_in_corrected_mode() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "thing")];
let anns = vec![ann_with_segm(
1,
1,
1,
(0.0, 0.0, 10.0, 10.0),
square_polygon(0.0, 0.0, 10.0),
)];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts =
CocoDetections::from_inputs(vec![dt_input(1, 1, 0.9, (0.0, 0.0, 10.0, 10.0))]).unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
retain_iou: false,
};
let err = evaluate_segm(>, &dts, params, ParityMode::Corrected).unwrap_err();
match err {
EvalError::InvalidAnnotation { detail } => {
assert!(detail.contains("DT"), "expected DT in msg: {detail}");
assert!(detail.contains("J2"), "expected J2 cite in msg: {detail}");
}
other => panic!("expected InvalidAnnotation, got {other:?}"),
}
}
#[test]
fn j2_bbox_only_dt_under_segm_iou_type_synthesizes_in_strict_mode() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "thing")];
let anns = vec![ann_with_segm(
1,
1,
1,
(0.0, 0.0, 10.0, 10.0),
square_polygon(0.0, 0.0, 10.0),
)];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts =
CocoDetections::from_inputs(vec![dt_input(1, 1, 0.9, (0.0, 0.0, 10.0, 10.0))]).unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
retain_iou: false,
};
let grid = evaluate_segm(>, &dts, params, ParityMode::Strict).unwrap();
let all = grid.cell(0, 0, 0).unwrap();
assert!(all.dt_matched.iter().all(|&m| m), "expected matches");
}
#[test]
fn j6_heterogeneous_dt_list_first_with_segm_second_without_raises_in_corrected_mode() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "thing")];
let anns = vec![ann_with_segm(
1,
1,
1,
(0.0, 0.0, 10.0, 10.0),
square_polygon(0.0, 0.0, 10.0),
)];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![
dt_input_with_segm(
1,
1,
0.9,
(0.0, 0.0, 10.0, 10.0),
square_polygon(0.0, 0.0, 10.0),
),
dt_input(1, 1, 0.8, (50.0, 50.0, 10.0, 10.0)),
])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
retain_iou: false,
};
let err = evaluate_segm(>, &dts, params, ParityMode::Corrected).unwrap_err();
assert!(matches!(err, EvalError::InvalidAnnotation { .. }));
}
#[test]
fn j6_heterogeneous_dt_list_first_without_segm_second_with_raises_in_corrected_mode() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "thing")];
let anns = vec![ann_with_segm(
1,
1,
1,
(0.0, 0.0, 10.0, 10.0),
square_polygon(0.0, 0.0, 10.0),
)];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![
dt_input(1, 1, 0.9, (0.0, 0.0, 10.0, 10.0)),
dt_input_with_segm(
1,
1,
0.8,
(50.0, 50.0, 10.0, 10.0),
square_polygon(50.0, 50.0, 10.0),
),
])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
retain_iou: false,
};
let err = evaluate_segm(>, &dts, params, ParityMode::Corrected).unwrap_err();
assert!(matches!(err, EvalError::InvalidAnnotation { .. }));
}
#[test]
fn j6_heterogeneous_dt_list_in_strict_mode_synthesizes_per_entry() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "thing")];
let anns = vec![
ann_with_segm(
1,
1,
1,
(0.0, 0.0, 10.0, 10.0),
square_polygon(0.0, 0.0, 10.0),
),
ann_with_segm(
2,
1,
1,
(50.0, 50.0, 10.0, 10.0),
square_polygon(50.0, 50.0, 10.0),
),
];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![
dt_input_with_segm(
1,
1,
0.9,
(0.0, 0.0, 10.0, 10.0),
square_polygon(0.0, 0.0, 10.0),
),
dt_input(1, 1, 0.8, (50.0, 50.0, 10.0, 10.0)),
])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
retain_iou: false,
};
let grid = evaluate_segm(>, &dts, params, ParityMode::Strict).unwrap();
let all = grid.cell(0, 0, 0).unwrap();
assert_eq!(all.dt_matched.shape(), &[iou_thresholds().len(), 2]);
assert!(all.dt_matched.iter().all(|&m| m));
}
#[test]
fn boundary_perfect_overlap_summarizes_to_one() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "thing")];
let anns = vec![ann_with_segm(
1,
1,
1,
(10.0, 10.0, 20.0, 20.0),
square_polygon(10.0, 10.0, 20.0),
)];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![dt_input_with_segm(
1,
1,
0.9,
(10.0, 10.0, 20.0, 20.0),
square_polygon(10.0, 10.0, 20.0),
)])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
retain_iou: false,
};
let grid = evaluate_boundary(>, &dts, params, ParityMode::Strict, 0.02).unwrap();
let max_dets = vec![1usize, 10, 100];
let acc = accumulate(
&grid.eval_imgs,
AccumulateParams {
iou_thresholds: iou_thresholds(),
recall_thresholds: recall_thresholds(),
max_dets: &max_dets,
n_categories: grid.n_categories,
n_area_ranges: grid.n_area_ranges,
n_images: grid.n_images,
},
ParityMode::Strict,
)
.unwrap();
let summary = summarize_detection(&acc, iou_thresholds(), &max_dets).unwrap();
let stats = summary.stats();
assert!((stats[0] - 1.0).abs() < 1e-12, "AP={}", stats[0]);
}
#[test]
fn boundary_disjoint_masks_summarize_to_zero() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "thing")];
let anns = vec![ann_with_segm(
1,
1,
1,
(0.0, 0.0, 10.0, 10.0),
square_polygon(0.0, 0.0, 10.0),
)];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![dt_input_with_segm(
1,
1,
0.9,
(50.0, 50.0, 10.0, 10.0),
square_polygon(50.0, 50.0, 10.0),
)])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
retain_iou: false,
};
let grid = evaluate_boundary(>, &dts, params, ParityMode::Strict, 0.02).unwrap();
let all = grid.cell(0, 0, 0).unwrap();
assert!(all.dt_matched.iter().all(|&m| !m));
}
fn boundary_cache_fixture() -> (
CocoDataset,
CocoDetections,
CocoDetections,
OwnedEvaluateParams,
) {
let images = vec![img(1, 100, 100), img(2, 100, 100)];
let cats = vec![cat(1, "thing"), cat(2, "other")];
let anns = vec![
ann_with_segm(
10,
1,
1,
(10.0, 10.0, 20.0, 20.0),
square_polygon(10.0, 10.0, 20.0),
),
ann_with_segm(
11,
1,
2,
(50.0, 50.0, 15.0, 15.0),
square_polygon(50.0, 50.0, 15.0),
),
ann_with_segm(
12,
2,
1,
(5.0, 5.0, 25.0, 25.0),
square_polygon(5.0, 5.0, 25.0),
),
];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts_a = CocoDetections::from_inputs(vec![
dt_input_with_segm(
1,
1,
0.9,
(10.0, 10.0, 20.0, 20.0),
square_polygon(10.0, 10.0, 20.0),
),
dt_input_with_segm(
2,
1,
0.8,
(5.0, 5.0, 25.0, 25.0),
square_polygon(5.0, 5.0, 25.0),
),
])
.unwrap();
let dts_b = CocoDetections::from_inputs(vec![
dt_input_with_segm(
1,
1,
0.7,
(12.0, 12.0, 20.0, 20.0),
square_polygon(12.0, 12.0, 20.0),
),
dt_input_with_segm(
2,
1,
0.6,
(8.0, 8.0, 25.0, 25.0),
square_polygon(8.0, 8.0, 25.0),
),
])
.unwrap();
let params = OwnedEvaluateParams {
iou_thresholds: iou_thresholds().to_vec(),
area_ranges: AreaRange::coco_default().to_vec(),
max_dets_per_image: 100,
use_cats: true,
retain_iou: false,
};
(gt, dts_a, dts_b, params)
}
fn boundary_grid_cells(grid: &EvalGrid) -> Vec<f64> {
grid.eval_imgs
.iter()
.filter_map(|c| c.as_ref())
.flat_map(|c| c.dt_scores.iter().copied())
.collect()
}
#[test]
fn boundary_cached_matches_uncached_bit_exact() {
let (gt, dts, _, params) = boundary_cache_fixture();
let p = params.borrow();
let baseline = evaluate_boundary(>, &dts, p, ParityMode::Strict, 0.02).unwrap();
let cache = BoundaryGtCache::new();
let cached_first =
evaluate_boundary_cached(>, &dts, p, ParityMode::Strict, 0.02, &cache).unwrap();
let cached_second =
evaluate_boundary_cached(>, &dts, p, ParityMode::Strict, 0.02, &cache).unwrap();
let baseline_scores = boundary_grid_cells(&baseline);
let first_scores = boundary_grid_cells(&cached_first);
let second_scores = boundary_grid_cells(&cached_second);
assert_eq!(baseline_scores.len(), first_scores.len());
for (b, c) in baseline_scores.iter().zip(first_scores.iter()) {
assert_eq!(b.to_bits(), c.to_bits());
}
for (b, c) in baseline_scores.iter().zip(second_scores.iter()) {
assert_eq!(b.to_bits(), c.to_bits());
}
}
#[test]
fn boundary_cache_populates_lazily_per_evaluated_cell() {
let (gt, dts, _, params) = boundary_cache_fixture();
let cache = BoundaryGtCache::new();
assert!(cache.is_empty());
evaluate_boundary_cached(>, &dts, params.borrow(), ParityMode::Strict, 0.02, &cache)
.unwrap();
assert_eq!(cache.len(), 2);
}
#[test]
fn boundary_cache_invalidates_on_ratio_change() {
let (gt, dts, _, params) = boundary_cache_fixture();
let cache = BoundaryGtCache::new();
evaluate_boundary_cached(>, &dts, params.borrow(), ParityMode::Strict, 0.02, &cache)
.unwrap();
let after_first = cache.len();
evaluate_boundary_cached(>, &dts, params.borrow(), ParityMode::Strict, 0.05, &cache)
.unwrap();
assert_eq!(cache.len(), after_first);
let fresh =
evaluate_boundary(>, &dts, params.borrow(), ParityMode::Strict, 0.05).unwrap();
let cached =
evaluate_boundary_cached(>, &dts, params.borrow(), ParityMode::Strict, 0.05, &cache)
.unwrap();
let fresh_scores = boundary_grid_cells(&fresh);
let cached_scores = boundary_grid_cells(&cached);
for (f, c) in fresh_scores.iter().zip(cached_scores.iter()) {
assert_eq!(f.to_bits(), c.to_bits());
}
}
#[test]
fn boundary_cache_clear_resets_state() {
let (gt, dts, _, params) = boundary_cache_fixture();
let cache = BoundaryGtCache::new();
evaluate_boundary_cached(>, &dts, params.borrow(), ParityMode::Strict, 0.02, &cache)
.unwrap();
assert!(!cache.is_empty());
cache.clear();
assert!(cache.is_empty());
let after =
evaluate_boundary_cached(>, &dts, params.borrow(), ParityMode::Strict, 0.02, &cache)
.unwrap();
let baseline =
evaluate_boundary(>, &dts, params.borrow(), ParityMode::Strict, 0.02).unwrap();
let after_scores = boundary_grid_cells(&after);
let baseline_scores = boundary_grid_cells(&baseline);
for (a, b) in after_scores.iter().zip(baseline_scores.iter()) {
assert_eq!(a.to_bits(), b.to_bits());
}
}
#[test]
fn boundary_cache_survives_changing_dt() {
let (gt, dts_a, dts_b, params) = boundary_cache_fixture();
let cache = BoundaryGtCache::new();
let cached_a = evaluate_boundary_cached(
>,
&dts_a,
params.borrow(),
ParityMode::Strict,
0.02,
&cache,
)
.unwrap();
let len_after_a = cache.len();
let cached_b = evaluate_boundary_cached(
>,
&dts_b,
params.borrow(),
ParityMode::Strict,
0.02,
&cache,
)
.unwrap();
assert_eq!(cache.len(), len_after_a);
let baseline_a =
evaluate_boundary(>, &dts_a, params.borrow(), ParityMode::Strict, 0.02).unwrap();
let baseline_b =
evaluate_boundary(>, &dts_b, params.borrow(), ParityMode::Strict, 0.02).unwrap();
for (lhs, rhs) in boundary_grid_cells(&cached_a)
.iter()
.zip(boundary_grid_cells(&baseline_a).iter())
{
assert_eq!(lhs.to_bits(), rhs.to_bits());
}
for (lhs, rhs) in boundary_grid_cells(&cached_b)
.iter()
.zip(boundary_grid_cells(&baseline_b).iter())
{
assert_eq!(lhs.to_bits(), rhs.to_bits());
}
}
#[test]
fn segm_cached_matches_uncached_bit_exact() {
let (gt, dts, _, params) = boundary_cache_fixture();
let p = params.borrow();
let baseline = evaluate_segm(>, &dts, p, ParityMode::Strict).unwrap();
let cache = SegmGtCache::new();
let cached_first = evaluate_segm_cached(>, &dts, p, ParityMode::Strict, &cache).unwrap();
let cached_second = evaluate_segm_cached(>, &dts, p, ParityMode::Strict, &cache).unwrap();
let baseline_scores = boundary_grid_cells(&baseline);
let first_scores = boundary_grid_cells(&cached_first);
let second_scores = boundary_grid_cells(&cached_second);
assert_eq!(baseline_scores.len(), first_scores.len());
for (b, c) in baseline_scores.iter().zip(first_scores.iter()) {
assert_eq!(b.to_bits(), c.to_bits());
}
for (b, c) in baseline_scores.iter().zip(second_scores.iter()) {
assert_eq!(b.to_bits(), c.to_bits());
}
}
#[test]
fn segm_cache_populates_lazily_per_evaluated_cell() {
let (gt, dts, _, params) = boundary_cache_fixture();
let cache = SegmGtCache::new();
assert!(cache.is_empty());
evaluate_segm_cached(>, &dts, params.borrow(), ParityMode::Strict, &cache).unwrap();
assert_eq!(cache.len(), 2);
}
#[test]
fn segm_cache_clear_resets_state() {
let (gt, dts, _, params) = boundary_cache_fixture();
let cache = SegmGtCache::new();
evaluate_segm_cached(>, &dts, params.borrow(), ParityMode::Strict, &cache).unwrap();
assert!(!cache.is_empty());
cache.clear();
assert!(cache.is_empty());
let after =
evaluate_segm_cached(>, &dts, params.borrow(), ParityMode::Strict, &cache).unwrap();
let baseline = evaluate_segm(>, &dts, params.borrow(), ParityMode::Strict).unwrap();
for (a, b) in boundary_grid_cells(&after)
.iter()
.zip(boundary_grid_cells(&baseline).iter())
{
assert_eq!(a.to_bits(), b.to_bits());
}
}
#[test]
fn segm_cache_survives_changing_dt() {
let (gt, dts_a, dts_b, params) = boundary_cache_fixture();
let cache = SegmGtCache::new();
let cached_a =
evaluate_segm_cached(>, &dts_a, params.borrow(), ParityMode::Strict, &cache).unwrap();
let len_after_a = cache.len();
let cached_b =
evaluate_segm_cached(>, &dts_b, params.borrow(), ParityMode::Strict, &cache).unwrap();
assert_eq!(cache.len(), len_after_a);
let baseline_a = evaluate_segm(>, &dts_a, params.borrow(), ParityMode::Strict).unwrap();
let baseline_b = evaluate_segm(>, &dts_b, params.borrow(), ParityMode::Strict).unwrap();
for (lhs, rhs) in boundary_grid_cells(&cached_a)
.iter()
.zip(boundary_grid_cells(&baseline_a).iter())
{
assert_eq!(lhs.to_bits(), rhs.to_bits());
}
for (lhs, rhs) in boundary_grid_cells(&cached_b)
.iter()
.zip(boundary_grid_cells(&baseline_b).iter())
{
assert_eq!(lhs.to_bits(), rhs.to_bits());
}
}
fn const_kps_vec(x: f64, y: f64, v: u32, len: usize) -> Vec<f64> {
let mut out = Vec::with_capacity(3 * len);
for _ in 0..len {
out.push(x);
out.push(y);
out.push(f64::from(v));
}
out
}
fn ann_with_kps(
id: i64,
image: i64,
cat: i64,
bbox: (f64, f64, f64, f64),
keypoints: Vec<f64>,
num_keypoints: Option<u32>,
) -> CocoAnnotation {
CocoAnnotation {
id: AnnId(id),
image_id: ImageId(image),
category_id: CategoryId(cat),
area: bbox.2 * bbox.3,
is_crowd: false,
ignore_flag: None,
bbox: Bbox {
x: bbox.0,
y: bbox.1,
w: bbox.2,
h: bbox.3,
},
segmentation: None,
keypoints: Some(keypoints),
num_keypoints,
}
}
fn dt_input_with_kps(
image: i64,
cat: i64,
score: f64,
bbox: (f64, f64, f64, f64),
keypoints: Vec<f64>,
) -> DetectionInput {
DetectionInput {
id: None,
image_id: ImageId(image),
category_id: CategoryId(cat),
score,
bbox: Bbox {
x: bbox.0,
y: bbox.1,
w: bbox.2,
h: bbox.3,
},
segmentation: None,
keypoints: Some(keypoints),
num_keypoints: None,
}
}
#[test]
fn test_evaluate_keypoints_perfect_match() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "person")];
let kps = const_kps_vec(50.0, 50.0, 2, 17);
let anns = vec![ann_with_kps(
1,
1,
1,
(40.0, 40.0, 20.0, 20.0),
kps.clone(),
None,
)];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![dt_input_with_kps(
1,
1,
0.9,
(40.0, 40.0, 20.0, 20.0),
kps,
)])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
retain_iou: false,
};
let grid =
evaluate_keypoints(>, &dts, params, ParityMode::Strict, HashMap::new()).unwrap();
let cell = grid.cell(0, 0, 0).unwrap();
assert_eq!(cell.gt_ignore, vec![false]);
assert!(cell.dt_matched.iter().all(|&m| m));
let meta = grid.cell_meta(0, 0, 0).unwrap();
assert!(
meta.gt_matches.iter().all(|&id| id > 0),
"every threshold should match the DT id (>0)",
);
}
#[test]
fn test_evaluate_keypoints_zero_overlap() {
let images = vec![img(1, 2000, 2000)];
let cats = vec![cat(1, "person")];
let gt_kps = const_kps_vec(50.0, 50.0, 2, 17);
let dt_kps = const_kps_vec(1500.0, 1500.0, 2, 17);
let anns = vec![ann_with_kps(
1,
1,
1,
(40.0, 40.0, 20.0, 20.0),
gt_kps,
None,
)];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![dt_input_with_kps(
1,
1,
0.9,
(1490.0, 1490.0, 20.0, 20.0),
dt_kps,
)])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
retain_iou: false,
};
let grid =
evaluate_keypoints(>, &dts, params, ParityMode::Strict, HashMap::new()).unwrap();
let cell = grid.cell(0, 0, 0).unwrap();
assert!(
cell.dt_matched.iter().all(|&m| !m),
"DTs far from GT should not match at any IoU threshold",
);
}
#[test]
fn test_evaluate_keypoints_d2_implicit_ignore() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "person")];
let gt_kps = const_kps_vec(50.0, 50.0, 0, 17);
let dt_kps = const_kps_vec(50.0, 50.0, 2, 17);
let anns = vec![ann_with_kps(
1,
1,
1,
(40.0, 40.0, 20.0, 20.0),
gt_kps,
Some(0),
)];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![dt_input_with_kps(
1,
1,
0.9,
(40.0, 40.0, 20.0, 20.0),
dt_kps,
)])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
retain_iou: false,
};
let grid =
evaluate_keypoints(>, &dts, params, ParityMode::Strict, HashMap::new()).unwrap();
let cell = grid.cell(0, 0, 0).unwrap();
assert_eq!(
cell.gt_ignore,
vec![true],
"D2: zero-visible-keypoints GT must be ignored",
);
}
#[test]
fn test_evaluate_keypoints_per_category_sigmas() {
let images = vec![img(1, 200, 200)];
let cats = vec![cat(1, "person"), cat(2, "dog")];
let gt_kps = const_kps_vec(50.0, 50.0, 2, 17);
let anns = vec![
ann_with_kps(1, 1, 1, (40.0, 40.0, 20.0, 20.0), gt_kps, None),
ann_with_kps(
2,
1,
2,
(140.0, 140.0, 20.0, 20.0),
const_kps_vec(150.0, 150.0, 2, 17),
None,
),
];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![
dt_input_with_kps(
1,
1,
0.9,
(40.0, 40.0, 20.0, 20.0),
const_kps_vec(51.0, 50.0, 2, 17),
),
dt_input_with_kps(
1,
2,
0.8,
(140.0, 140.0, 20.0, 20.0),
const_kps_vec(151.0, 150.0, 2, 17),
),
])
.unwrap();
let mut sigmas: HashMap<i64, Vec<f64>> = HashMap::new();
sigmas.insert(1, vec![0.5_f64; 17]);
sigmas.insert(2, vec![0.5_f64; 17]);
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
retain_iou: false,
};
let grid = evaluate_keypoints(>, &dts, params, ParityMode::Strict, sigmas).unwrap();
let cell_cat1 = grid.cell(0, 0, 0).unwrap();
let cell_cat2 = grid.cell(1, 0, 0).unwrap();
assert!(
cell_cat1.dt_matched.iter().all(|&m| m),
"cat-1 DT should match cat-1 GT under override sigmas",
);
assert!(
cell_cat2.dt_matched.iter().all(|&m| m),
"cat-2 DT should match cat-2 GT under override sigmas",
);
}
#[test]
fn test_evaluate_keypoints_missing_dt_kps_rejected() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "person")];
let gt_kps = const_kps_vec(50.0, 50.0, 2, 17);
let anns = vec![ann_with_kps(
1,
1,
1,
(40.0, 40.0, 20.0, 20.0),
gt_kps,
None,
)];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![dt_input(1, 1, 0.9, (40.0, 40.0, 20.0, 20.0))])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
retain_iou: false,
};
let err =
evaluate_keypoints(>, &dts, params, ParityMode::Strict, HashMap::new()).unwrap_err();
match err {
EvalError::InvalidAnnotation { detail } => {
assert!(detail.contains("DT"), "expected DT in msg: {detail}");
assert!(
detail.contains("keypoints"),
"expected keypoints in msg: {detail}",
);
}
other => panic!("expected InvalidAnnotation, got {other:?}"),
}
}
#[test]
fn test_keypoints_default_ignore_for_other_kernels() {
let ann_zero_kps = ann_with_kps(
1,
1,
1,
(0.0, 0.0, 10.0, 10.0),
const_kps_vec(0.0, 0.0, 0, 17),
Some(0),
);
assert!(
!BboxIou.extra_gt_ignore(&ann_zero_kps),
"BboxIou must keep the default `false` ignore",
);
assert!(
!SegmIou.extra_gt_ignore(&ann_zero_kps),
"SegmIou must keep the default `false` ignore",
);
assert!(
!BoundaryIou {
dilation_ratio: 0.02,
}
.extra_gt_ignore(&ann_zero_kps),
"BoundaryIou must keep the default `false` ignore",
);
assert!(
OksSimilarity::default().extra_gt_ignore(&ann_zero_kps),
"OksSimilarity must flip D2 to true on zero-visible-keypoints GT",
);
}
#[test]
fn boundary_missing_gt_segmentation_surfaces_typed_error() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "thing")];
let anns = vec![ann(7, 1, 1, (0.0, 0.0, 10.0, 10.0))];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![dt_input_with_segm(
1,
1,
0.9,
(0.0, 0.0, 10.0, 10.0),
square_polygon(0.0, 0.0, 10.0),
)])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
retain_iou: false,
};
let err = evaluate_boundary(>, &dts, params, ParityMode::Strict, 0.02).unwrap_err();
match err {
EvalError::InvalidAnnotation { detail } => {
assert!(detail.contains("GT id=7"), "msg: {detail}");
}
other => panic!("expected InvalidAnnotation, got {other:?}"),
}
}
fn lvis_dataset(
images: &[ImageMeta],
annotations: &[CocoAnnotation],
categories: &[CategoryMeta],
neg: &[(i64, Vec<i64>)],
nel: &[(i64, Vec<i64>)],
freq: &[(i64, crate::dataset::Frequency)],
) -> CocoDataset {
let images_json: Vec<serde_json::Value> = images
.iter()
.map(|im| {
let neg_for: Vec<i64> = neg
.iter()
.find(|(id, _)| *id == im.id.0)
.map(|(_, v)| v.clone())
.unwrap_or_default();
let nel_for: Vec<i64> = nel
.iter()
.find(|(id, _)| *id == im.id.0)
.map(|(_, v)| v.clone())
.unwrap_or_default();
serde_json::json!({
"id": im.id.0,
"width": im.width,
"height": im.height,
"neg_category_ids": neg_for,
"not_exhaustive_category_ids": nel_for,
})
})
.collect();
let cats_json: Vec<serde_json::Value> = categories
.iter()
.map(|c| {
let f = freq
.iter()
.find(|(id, _)| *id == c.id.0)
.map(|(_, f)| match f {
crate::dataset::Frequency::Rare => "r",
crate::dataset::Frequency::Common => "c",
crate::dataset::Frequency::Frequent => "f",
})
.expect("test fixture must include frequency for every category");
serde_json::json!({
"id": c.id.0,
"name": c.name,
"frequency": f,
})
})
.collect();
let anns_json = serde_json::to_value(annotations).unwrap();
let payload = serde_json::json!({
"images": images_json,
"annotations": anns_json,
"categories": cats_json,
});
let bytes = serde_json::to_vec(&payload).unwrap();
CocoDataset::from_lvis_json_bytes(&bytes).unwrap()
}
#[test]
fn aa4_skips_cells_outside_pos_union_neg() {
let images = vec![img(1, 100, 100), img(2, 100, 100)];
let cats = vec![cat(1, "a"), cat(2, "b")];
let anns = vec![
ann(1, 1, 1, (0.0, 0.0, 10.0, 10.0)),
ann(2, 2, 2, (0.0, 0.0, 10.0, 10.0)),
];
let gt_lvis = lvis_dataset(
&images,
&anns,
&cats,
&[(1, vec![]), (2, vec![])],
&[(1, vec![]), (2, vec![])],
&[
(1, crate::dataset::Frequency::Frequent),
(2, crate::dataset::Frequency::Frequent),
],
);
let gt_coco = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![
dt_input(1, 1, 0.9, (0.0, 0.0, 10.0, 10.0)),
dt_input(1, 2, 0.7, (50.0, 50.0, 10.0, 10.0)),
dt_input(2, 2, 0.9, (0.0, 0.0, 10.0, 10.0)),
])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
retain_iou: false,
};
let grid_lvis = evaluate_bbox(>_lvis, &dts, params, ParityMode::Strict).unwrap();
let grid_coco = evaluate_bbox(>_coco, &dts, params, ParityMode::Strict).unwrap();
let lvis_cell = grid_lvis.cell(1, 0, 0);
let coco_cell = grid_coco.cell(1, 0, 0);
assert!(lvis_cell.is_none(), "AA4: federated cell must be skipped");
assert!(
coco_cell.is_some(),
"control: COCO dataset must evaluate the same cell"
);
assert_eq!(
grid_lvis.cell(0, 0, 0).map(|c| c.dt_scores.len()),
grid_coco.cell(0, 0, 0).map(|c| c.dt_scores.len()),
);
}
#[test]
fn aa4_keeps_neg_cells_with_no_gts() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "a"), cat(2, "b")];
let anns = vec![ann(1, 1, 1, (0.0, 0.0, 10.0, 10.0))];
let gt = lvis_dataset(
&images,
&anns,
&cats,
&[(1, vec![2])], &[(1, vec![])],
&[
(1, crate::dataset::Frequency::Frequent),
(2, crate::dataset::Frequency::Frequent),
],
);
let dts = CocoDetections::from_inputs(vec![
dt_input(1, 1, 0.9, (0.0, 0.0, 10.0, 10.0)),
dt_input(1, 2, 0.7, (50.0, 50.0, 10.0, 10.0)),
])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
retain_iou: false,
};
let grid = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
let cell = grid
.cell(1, 0, 0)
.expect("cat 2 ∈ neg[1] must produce an evaluated cell");
assert_eq!(cell.dt_scores.len(), 1);
assert!(cell.dt_ignore.iter().all(|&ig| !ig));
}
#[test]
fn aa3_dt_ignore_extension_in_not_exhaustive_cell() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "a")];
let anns = vec![ann(1, 1, 1, (0.0, 0.0, 10.0, 10.0))];
let gt = lvis_dataset(
&images,
&anns,
&cats,
&[(1, vec![])],
&[(1, vec![1])], &[(1, crate::dataset::Frequency::Frequent)],
);
let dts = CocoDetections::from_inputs(vec![
dt_input(1, 1, 0.9, (0.0, 0.0, 10.0, 10.0)), dt_input(1, 1, 0.7, (50.0, 50.0, 10.0, 10.0)), ])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
retain_iou: false,
};
let grid = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
let cell = grid.cell(0, 0, 0).expect("cell must evaluate");
let n_t = cell.dt_ignore.shape()[0];
for t in 0..n_t {
assert!(!cell.dt_ignore[(t, 0)], "TP should not be dt_ignore");
assert!(
cell.dt_ignore[(t, 1)],
"AA3: unmatched DT in not_exhaustive cell must be dt_ignore"
);
}
}
#[test]
fn aa3_dt_ignore_only_unmatched() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "a")];
let anns = vec![ann(1, 1, 1, (0.0, 0.0, 10.0, 10.0))];
let gt = lvis_dataset(
&images,
&anns,
&cats,
&[(1, vec![])],
&[(1, vec![])],
&[(1, crate::dataset::Frequency::Frequent)],
);
let dts = CocoDetections::from_inputs(vec![
dt_input(1, 1, 0.9, (0.0, 0.0, 10.0, 10.0)),
dt_input(1, 1, 0.7, (50.0, 50.0, 10.0, 10.0)),
])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
retain_iou: false,
};
let grid = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
let cell = grid.cell(0, 0, 0).expect("cell must evaluate");
assert!(cell.dt_ignore.iter().all(|&ig| !ig));
}
#[test]
fn federated_dataset_with_use_cats_false_falls_back_to_coco() {
let images = vec![img(1, 100, 100), img(2, 100, 100)];
let cats = vec![cat(1, "a"), cat(2, "b")];
let anns = vec![
ann(1, 1, 1, (0.0, 0.0, 10.0, 10.0)),
ann(2, 2, 2, (0.0, 0.0, 10.0, 10.0)),
];
let gt = lvis_dataset(
&images,
&anns,
&cats,
&[(1, vec![]), (2, vec![])],
&[(1, vec![]), (2, vec![])],
&[
(1, crate::dataset::Frequency::Frequent),
(2, crate::dataset::Frequency::Frequent),
],
);
let dts = CocoDetections::from_inputs(vec![
dt_input(1, 1, 0.9, (0.0, 0.0, 10.0, 10.0)),
dt_input(1, 2, 0.7, (50.0, 50.0, 10.0, 10.0)),
])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: false,
retain_iou: false,
};
let grid = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
assert_eq!(grid.n_categories, 1);
let cell = grid.cell(0, 0, 0).expect("collapsed cell must evaluate");
assert_eq!(cell.dt_scores.len(), 2);
}
#[test]
fn coco_dataset_unaffected_by_federated_machinery() {
let g = perfect_match_grid();
let cell = g.cell(0, 0, 0).expect("perfect_match cell must exist");
assert_eq!(cell.dt_scores.len(), 2);
assert!(cell.dt_ignore.iter().all(|&ig| !ig));
}
fn ann_with_area(
id: i64,
image: i64,
cat: i64,
bbox: (f64, f64, f64, f64),
area: f64,
) -> CocoAnnotation {
let mut a = ann(id, image, cat, bbox);
a.area = area;
a
}
#[test]
fn ag6_mixed_cell_drops_zero_area_gt_in_strict_mode() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "a")];
let anns = vec![
ann(1, 1, 1, (10.0, 10.0, 20.0, 20.0)),
ann_with_area(2, 1, 1, (50.0, 50.0, 0.1, 0.1), 0.0),
];
let gt = lvis_dataset(
&images,
&anns,
&cats,
&[(1, vec![])],
&[(1, vec![])],
&[(1, crate::dataset::Frequency::Frequent)],
);
let dts = CocoDetections::from_inputs(vec![
dt_input(1, 1, 0.9, (10.0, 10.0, 20.0, 20.0)),
dt_input(1, 1, 0.8, (50.0, 50.0, 0.1, 0.1)),
])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
retain_iou: false,
};
let strict = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
let cell = strict
.cell(0, 0, 0)
.expect("mixed cell must still evaluate in strict mode");
assert_eq!(cell.dt_scores.len(), 2);
let strict_meta = strict.cell_meta(0, 0, 0).unwrap();
assert_eq!(strict_meta.dt_matches[(0, 0)], 1, "DT_real → GT id=1");
assert_eq!(
strict_meta.dt_matches[(0, 1)],
0,
"DT_zero must be unmatched after strict filter drops GT id=2"
);
let corrected = evaluate_bbox(>, &dts, params, ParityMode::Corrected).unwrap();
let cor_meta = corrected.cell_meta(0, 0, 0).unwrap();
assert_eq!(cor_meta.dt_matches[(0, 0)], 1, "DT_real → GT id=1");
assert_eq!(
cor_meta.dt_matches[(0, 1)],
2,
"Corrected mode keeps the zero-area GT and matches DT_zero → GT id=2"
);
}
#[test]
fn ag6_all_zero_area_cell_skipped_via_aa4_in_strict_mode() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "a")];
let anns = vec![ann_with_area(1, 1, 1, (50.0, 50.0, 0.1, 0.1), 0.0)];
let gt = lvis_dataset(
&images,
&anns,
&cats,
&[(1, vec![])],
&[(1, vec![])],
&[(1, crate::dataset::Frequency::Frequent)],
);
let dts =
CocoDetections::from_inputs(vec![dt_input(1, 1, 0.9, (50.0, 50.0, 0.1, 0.1))]).unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
retain_iou: false,
};
let strict = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
assert!(
strict.cell(0, 0, 0).is_none(),
"AG6: all-zero-area cell must be skipped via AA4 in strict mode"
);
let corrected = evaluate_bbox(>, &dts, params, ParityMode::Corrected).unwrap();
let cell = corrected
.cell(0, 0, 0)
.expect("Corrected mode must keep the zero-area GT");
assert_eq!(cell.dt_scores.len(), 1);
}
#[test]
fn ag6_strict_filter_is_noop_on_coco_dataset() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "a")];
let anns = vec![
ann(1, 1, 1, (10.0, 10.0, 20.0, 20.0)),
ann_with_area(2, 1, 1, (50.0, 50.0, 0.1, 0.1), 0.0),
];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![
dt_input(1, 1, 0.9, (10.0, 10.0, 20.0, 20.0)),
dt_input(1, 1, 0.8, (50.0, 50.0, 0.1, 0.1)),
])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
retain_iou: false,
};
let grid = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
let meta = grid.cell_meta(0, 0, 0).unwrap();
assert_eq!(meta.dt_matches[(0, 0)], 1);
assert_eq!(
meta.dt_matches[(0, 1)],
2,
"COCO strict mode must NOT drop the zero-area GT — AG6 is LVIS-only"
);
}
}