use ndarray::{Array2, ArrayView2};
use crate::accumulate::PerImageEval;
use crate::dataset::{
Bbox, CategoryId, CocoAnnotation, CocoDataset, CocoDetection, CocoDetections, EvalDataset,
ImageId, ImageMeta,
};
use crate::error::EvalError;
use crate::matching::{match_image, MatchResult};
use crate::parity::{argsort_score_desc, ParityMode};
use crate::segmentation::Segmentation;
use crate::similarity::{
BboxAnn, BboxIou, BoundaryIou, OksAnn, OksSimilarity, SegmAnn, SegmIou, Similarity,
};
use std::collections::HashMap;
use vernier_mask::Rle;
pub const COLLAPSED_CATEGORY_SENTINEL: i64 = -1;
pub const AREA_UNBOUNDED: f64 = 1e10;
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct AreaRange {
pub index: usize,
pub lo: f64,
pub hi: f64,
}
impl AreaRange {
pub fn coco_default() -> [Self; 4] {
[
Self {
index: 0,
lo: 0.0,
hi: AREA_UNBOUNDED,
},
Self {
index: 1,
lo: 0.0,
hi: 32.0 * 32.0,
},
Self {
index: 2,
lo: 32.0 * 32.0,
hi: 96.0 * 96.0,
},
Self {
index: 3,
lo: 96.0 * 96.0,
hi: AREA_UNBOUNDED,
},
]
}
pub fn keypoints_default() -> [Self; 3] {
[
Self {
index: 0,
lo: 0.0,
hi: AREA_UNBOUNDED,
},
Self {
index: 1,
lo: 32.0 * 32.0,
hi: 96.0 * 96.0,
},
Self {
index: 2,
lo: 96.0 * 96.0,
hi: AREA_UNBOUNDED,
},
]
}
fn contains(&self, area: f64) -> bool {
area >= self.lo && area <= self.hi
}
}
#[derive(Debug, Clone, Copy)]
pub struct EvaluateParams<'p> {
pub iou_thresholds: &'p [f64],
pub area_ranges: &'p [AreaRange],
pub max_dets_per_image: usize,
pub use_cats: bool,
}
#[derive(Debug, Clone)]
pub struct OwnedEvaluateParams {
pub iou_thresholds: Vec<f64>,
pub area_ranges: Vec<AreaRange>,
pub max_dets_per_image: usize,
pub use_cats: bool,
}
impl OwnedEvaluateParams {
pub fn borrow(&self) -> EvaluateParams<'_> {
EvaluateParams {
iou_thresholds: &self.iou_thresholds,
area_ranges: &self.area_ranges,
max_dets_per_image: self.max_dets_per_image,
use_cats: self.use_cats,
}
}
}
pub trait EvalKernel: Similarity {
fn build_gt_anns(
&self,
gt_anns: &[CocoAnnotation],
indices: &[usize],
image: &ImageMeta,
) -> Result<Vec<Self::Annotation>, EvalError>;
fn build_dt_anns(
&self,
dt_anns: &[CocoDetection],
indices: &[usize],
image: &ImageMeta,
parity_mode: ParityMode,
) -> Result<Vec<Self::Annotation>, EvalError>;
fn extra_gt_ignore(&self, _ann: &CocoAnnotation) -> bool {
false
}
fn is_keypoints(&self) -> bool {
false
}
}
impl EvalKernel for BboxIou {
fn build_gt_anns(
&self,
gt_anns: &[CocoAnnotation],
indices: &[usize],
_image: &ImageMeta,
) -> Result<Vec<BboxAnn>, EvalError> {
Ok(indices
.iter()
.map(|&j| BboxAnn {
bbox: gt_anns[j].bbox,
is_crowd: gt_anns[j].is_crowd,
})
.collect())
}
fn build_dt_anns(
&self,
dt_anns: &[CocoDetection],
indices: &[usize],
_image: &ImageMeta,
_parity_mode: ParityMode,
) -> Result<Vec<BboxAnn>, EvalError> {
Ok(indices
.iter()
.map(|&j| BboxAnn {
bbox: dt_anns[j].bbox,
is_crowd: false,
})
.collect())
}
}
impl EvalKernel for SegmIou {
fn build_gt_anns(
&self,
gt_anns: &[CocoAnnotation],
indices: &[usize],
image: &ImageMeta,
) -> Result<Vec<SegmAnn>, EvalError> {
build_segm_gt_anns(gt_anns, indices, image)
}
fn build_dt_anns(
&self,
dt_anns: &[CocoDetection],
indices: &[usize],
image: &ImageMeta,
parity_mode: ParityMode,
) -> Result<Vec<SegmAnn>, EvalError> {
build_segm_dt_anns(dt_anns, indices, image, parity_mode)
}
}
impl EvalKernel for BoundaryIou {
fn build_gt_anns(
&self,
gt_anns: &[CocoAnnotation],
indices: &[usize],
image: &ImageMeta,
) -> Result<Vec<SegmAnn>, EvalError> {
build_segm_gt_anns(gt_anns, indices, image)
}
fn build_dt_anns(
&self,
dt_anns: &[CocoDetection],
indices: &[usize],
image: &ImageMeta,
parity_mode: ParityMode,
) -> Result<Vec<SegmAnn>, EvalError> {
build_segm_dt_anns(dt_anns, indices, image, parity_mode)
}
}
impl EvalKernel for OksSimilarity {
fn build_gt_anns(
&self,
gt_anns: &[CocoAnnotation],
indices: &[usize],
_image: &ImageMeta,
) -> Result<Vec<OksAnn>, EvalError> {
indices
.iter()
.map(|&j| {
let ann = >_anns[j];
let kps = ann
.keypoints
.as_deref()
.ok_or_else(|| missing_keypoints_err("GT", ann.id.0, ann.image_id.0))?;
let num_keypoints = ann
.num_keypoints
.unwrap_or_else(|| count_visible_keypoints(kps));
Ok(OksAnn {
category_id: ann.category_id.0,
keypoints: kps.to_vec(),
num_keypoints,
bbox: ann.bbox.into(),
area: ann.area,
})
})
.collect()
}
fn build_dt_anns(
&self,
dt_anns: &[CocoDetection],
indices: &[usize],
_image: &ImageMeta,
_parity_mode: ParityMode,
) -> Result<Vec<OksAnn>, EvalError> {
indices
.iter()
.map(|&j| {
let dt = &dt_anns[j];
let kps = dt
.keypoints
.as_deref()
.ok_or_else(|| missing_keypoints_err("DT", dt.id.0, dt.image_id.0))?;
let num_keypoints = dt
.num_keypoints
.unwrap_or_else(|| count_visible_keypoints(kps));
Ok(OksAnn {
category_id: dt.category_id.0,
keypoints: kps.to_vec(),
num_keypoints,
bbox: dt.bbox.into(),
area: dt.area,
})
})
.collect()
}
fn extra_gt_ignore(&self, ann: &CocoAnnotation) -> bool {
let visible = ann
.num_keypoints
.or_else(|| ann.keypoints.as_deref().map(count_visible_keypoints))
.unwrap_or(0);
visible == 0
}
fn is_keypoints(&self) -> bool {
true
}
}
fn count_visible_keypoints(kps: &[f64]) -> u32 {
kps.chunks_exact(3).filter(|t| t[2] > 0.0).count() as u32
}
fn missing_keypoints_err(kind: &str, ann_id: i64, image_id: i64) -> EvalError {
EvalError::InvalidAnnotation {
detail: format!(
"{kind} id={ann_id} on image {image_id} has no `keypoints` field; \
OKS eval requires keypoints on every entry. There is no \
pycocotools-equivalent bbox-synthesis fallback for keypoints \
(unlike segm quirk J2)."
),
}
}
fn build_segm_gt_anns(
gt_anns: &[CocoAnnotation],
indices: &[usize],
image: &ImageMeta,
) -> Result<Vec<SegmAnn>, EvalError> {
indices
.iter()
.map(|&j| {
let ann = >_anns[j];
let seg = ann
.segmentation
.as_ref()
.ok_or_else(|| missing_segmentation_err("GT", ann.id.0, image.id.0))?;
Ok(SegmAnn {
rle: seg.to_rle(image.height, image.width)?,
is_crowd: ann.is_crowd,
})
})
.collect()
}
fn build_segm_dt_anns(
dt_anns: &[CocoDetection],
indices: &[usize],
image: &ImageMeta,
parity_mode: ParityMode,
) -> Result<Vec<SegmAnn>, EvalError> {
indices
.iter()
.map(|&j| {
let dt = &dt_anns[j];
let rle = match (&dt.segmentation, parity_mode) {
(Some(seg), _) => seg.to_rle(image.height, image.width)?,
(None, ParityMode::Strict) => {
synthesize_dt_segm_from_bbox(&dt.bbox, image.height, image.width)?
}
(None, ParityMode::Corrected) => {
return Err(missing_segmentation_err("DT", dt.id.0, image.id.0));
}
};
Ok(SegmAnn {
rle,
is_crowd: false,
})
})
.collect()
}
fn synthesize_dt_segm_from_bbox(bbox: &Bbox, h: u32, w: u32) -> Result<Rle, EvalError> {
let x1 = bbox.x;
let y1 = bbox.y;
let x2 = bbox.x + bbox.w;
let y2 = bbox.y + bbox.h;
let polygon = vec![x1, y1, x1, y2, x2, y2, x2, y1];
let segm = Segmentation::Polygons(vec![polygon]);
segm.to_rle(h, w)
}
fn missing_segmentation_err(kind: &str, ann_id: i64, image_id: i64) -> EvalError {
EvalError::InvalidAnnotation {
detail: format!(
"{kind} id={ann_id} on image {image_id} has no `segmentation` field; \
segm eval in corrected mode requires one on every entry. \
pycocotools synthesizes a bbox-rectangle polygon here \
(quirks J2/J6); pass `ParityMode::Strict` to opt into that \
behavior."
),
}
}
#[derive(Debug, Clone)]
pub struct EvalImageMeta {
pub image_id: i64,
pub category_id: i64,
pub area_rng: [f64; 2],
pub max_det: usize,
pub dt_ids: Vec<i64>,
pub gt_ids: Vec<i64>,
pub dt_matches: Array2<i64>,
pub gt_matches: Array2<i64>,
}
#[derive(Debug, Clone)]
pub struct EvalGrid {
pub eval_imgs: Vec<Option<PerImageEval>>,
pub eval_imgs_meta: Vec<Option<EvalImageMeta>>,
pub n_categories: usize,
pub n_area_ranges: usize,
pub n_images: usize,
}
impl EvalGrid {
pub fn cell(&self, k: usize, a: usize, i: usize) -> Option<&PerImageEval> {
let idx = self.flat_index(k, a, i)?;
self.eval_imgs.get(idx).and_then(Option::as_ref)
}
pub fn cell_meta(&self, k: usize, a: usize, i: usize) -> Option<&EvalImageMeta> {
let idx = self.flat_index(k, a, i)?;
self.eval_imgs_meta.get(idx).and_then(Option::as_ref)
}
fn flat_index(&self, k: usize, a: usize, i: usize) -> Option<usize> {
if k >= self.n_categories || a >= self.n_area_ranges || i >= self.n_images {
return None;
}
Some(k * self.n_area_ranges * self.n_images + a * self.n_images + i)
}
}
pub fn evaluate_with<K: EvalKernel>(
gt: &CocoDataset,
dt: &CocoDetections,
params: EvaluateParams<'_>,
parity_mode: ParityMode,
kernel: &K,
) -> Result<EvalGrid, EvalError> {
let mut images: Vec<&ImageMeta> = gt.images().iter().collect();
images.sort_unstable_by_key(|im| im.id.0);
let n_i = images.len();
let n_a = params.area_ranges.len();
let category_buckets: Vec<Option<CategoryId>> = if params.use_cats {
let mut cats: Vec<_> = gt.categories().iter().map(|c| c.id).collect();
cats.sort_unstable_by_key(|id| id.0);
cats.into_iter().map(Some).collect()
} else {
vec![None]
};
let n_k = category_buckets.len();
let mut eval_imgs: Vec<Option<PerImageEval>> = vec![None; n_k * n_a * n_i];
let mut eval_imgs_meta: Vec<Option<EvalImageMeta>> = vec![None; n_k * n_a * n_i];
for (k, cat) in category_buckets.iter().enumerate() {
let nk = k * n_a * n_i;
let category_id = cat.map_or(COLLAPSED_CATEGORY_SENTINEL, |c| c.0);
for (i, image) in images.iter().enumerate() {
let image_id = image.id;
let gt_indices = gt_indices_for_cell(gt, image_id, *cat);
let dt_indices = dt_top_indices_for_cell(dt, image_id, *cat, params.max_dets_per_image);
if gt_indices.is_empty() && dt_indices.is_empty() {
continue;
}
let gt_anns = gt.annotations();
let dt_anns = dt.detections();
let gt_areas: Vec<f64> = gt_indices.iter().map(|&j| gt_anns[j].area).collect();
let gt_iscrowd: Vec<bool> = gt_indices.iter().map(|&j| gt_anns[j].is_crowd).collect();
let gt_base_ignore: Vec<bool> = gt_indices
.iter()
.map(|&j| {
gt_anns[j].effective_ignore(parity_mode) || kernel.extra_gt_ignore(>_anns[j])
})
.collect();
let gt_ids: Vec<i64> = gt_indices.iter().map(|&j| gt_anns[j].id.0).collect();
let dt_areas: Vec<f64> = dt_indices.iter().map(|&j| dt_anns[j].area).collect();
let dt_scores: Vec<f64> = dt_indices.iter().map(|&j| dt_anns[j].score).collect();
let dt_ids: Vec<i64> = dt_indices.iter().map(|&j| dt_anns[j].id.0).collect();
let gt_kernel = kernel.build_gt_anns(gt_anns, gt_indices, image)?;
let dt_kernel = kernel.build_dt_anns(dt_anns, &dt_indices, image, parity_mode)?;
let mut iou = Array2::<f64>::zeros((gt_kernel.len(), dt_kernel.len()));
if !gt_kernel.is_empty() && !dt_kernel.is_empty() {
kernel.compute(>_kernel, &dt_kernel, &mut iou.view_mut())?;
}
let buffers = CellBuffers {
image_id: image_id.0,
category_id,
max_det: params.max_dets_per_image,
gt_areas: >_areas,
gt_iscrowd: >_iscrowd,
gt_base_ignore: >_base_ignore,
gt_ids: >_ids,
dt_areas: &dt_areas,
dt_scores: &dt_scores,
dt_ids: &dt_ids,
iou: iou.view(),
};
for (a, area) in params.area_ranges.iter().enumerate() {
let (cell, meta) =
evaluate_cell(&buffers, area, params.iou_thresholds, parity_mode)?;
let flat = nk + a * n_i + i;
eval_imgs[flat] = Some(cell);
eval_imgs_meta[flat] = Some(meta);
}
}
}
Ok(EvalGrid {
eval_imgs,
eval_imgs_meta,
n_categories: n_k,
n_area_ranges: n_a,
n_images: n_i,
})
}
pub fn evaluate_bbox(
gt: &CocoDataset,
dt: &CocoDetections,
params: EvaluateParams<'_>,
parity_mode: ParityMode,
) -> Result<EvalGrid, EvalError> {
evaluate_with(gt, dt, params, parity_mode, &BboxIou)
}
pub fn evaluate_segm(
gt: &CocoDataset,
dt: &CocoDetections,
params: EvaluateParams<'_>,
parity_mode: ParityMode,
) -> Result<EvalGrid, EvalError> {
evaluate_with(gt, dt, params, parity_mode, &SegmIou)
}
pub fn evaluate_boundary(
gt: &CocoDataset,
dt: &CocoDetections,
params: EvaluateParams<'_>,
parity_mode: ParityMode,
dilation_ratio: f64,
) -> Result<EvalGrid, EvalError> {
evaluate_with(gt, dt, params, parity_mode, &BoundaryIou { dilation_ratio })
}
pub fn evaluate_keypoints(
gt: &CocoDataset,
dt: &CocoDetections,
params: EvaluateParams<'_>,
parity_mode: ParityMode,
sigmas: HashMap<i64, Vec<f64>>,
) -> Result<EvalGrid, EvalError> {
evaluate_with(gt, dt, params, parity_mode, &OksSimilarity::new(sigmas))
}
fn gt_indices_for_cell(gt: &CocoDataset, image: ImageId, cat: Option<CategoryId>) -> &[usize] {
match cat {
Some(c) => gt.ann_indices_for(image, c),
None => gt.ann_indices_for_image(image),
}
}
fn dt_top_indices_for_cell(
dt: &CocoDetections,
image: ImageId,
cat: Option<CategoryId>,
max_dets: usize,
) -> Vec<usize> {
let indices: &[usize] = match cat {
Some(c) => dt.indices_for(image, c),
None => dt.indices_for_image(image),
};
let dts = dt.detections();
let scores: Vec<f64> = indices.iter().map(|&i| dts[i].score).collect();
let perm = argsort_score_desc(&scores);
perm.into_iter()
.take(max_dets)
.map(|k| indices[k])
.collect()
}
struct CellBuffers<'a> {
image_id: i64,
category_id: i64,
max_det: usize,
gt_areas: &'a [f64],
gt_iscrowd: &'a [bool],
gt_base_ignore: &'a [bool],
gt_ids: &'a [i64],
dt_areas: &'a [f64],
dt_scores: &'a [f64],
dt_ids: &'a [i64],
iou: ArrayView2<'a, f64>,
}
fn evaluate_cell(
buf: &CellBuffers<'_>,
area: &AreaRange,
iou_thresholds: &[f64],
parity_mode: ParityMode,
) -> Result<(PerImageEval, EvalImageMeta), EvalError> {
let gt_ignore: Vec<bool> = buf
.gt_base_ignore
.iter()
.zip(buf.gt_areas)
.map(|(&base, &a)| base || !area.contains(a))
.collect();
let MatchResult {
dt_perm,
gt_perm,
dt_matches: dt_matches_pos,
gt_matches: gt_matches_pos,
mut dt_ignore,
} = match_image(
buf.iou,
>_ignore,
buf.gt_iscrowd,
buf.dt_scores,
iou_thresholds,
parity_mode,
)?;
let n_t = iou_thresholds.len();
let n_d = buf.dt_scores.len();
let n_g = gt_ignore.len();
let dt_scores_sorted: Vec<f64> = dt_perm.iter().map(|&k| buf.dt_scores[k]).collect();
let dt_in_range_sorted: Vec<bool> = dt_perm
.iter()
.map(|&k| area.contains(buf.dt_areas[k]))
.collect();
let gt_ignore_sorted: Vec<bool> = gt_perm.iter().map(|&k| gt_ignore[k]).collect();
let dt_ids_sorted: Vec<i64> = dt_perm.iter().map(|&k| buf.dt_ids[k]).collect();
let gt_ids_sorted: Vec<i64> = gt_perm.iter().map(|&k| buf.gt_ids[k]).collect();
let mut dt_matched = Array2::<bool>::default((n_t, n_d));
let mut dt_matches_id = Array2::<i64>::zeros((n_t, n_d));
let mut gt_matches_id = Array2::<i64>::zeros((n_t, n_g));
for t in 0..n_t {
for d in 0..n_d {
let m = dt_matches_pos[(t, d)];
let matched = m >= 0;
dt_matched[(t, d)] = matched;
if matched {
dt_matches_id[(t, d)] = gt_ids_sorted[m as usize];
}
if !matched && !dt_in_range_sorted[d] {
dt_ignore[(t, d)] = true;
}
}
for g in 0..n_g {
let p = gt_matches_pos[(t, g)];
if p >= 0 {
gt_matches_id[(t, g)] = dt_ids_sorted[p as usize];
}
}
}
let cell = PerImageEval {
dt_scores: dt_scores_sorted,
dt_matched,
dt_ignore,
gt_ignore: gt_ignore_sorted,
};
let meta = EvalImageMeta {
image_id: buf.image_id,
category_id: buf.category_id,
area_rng: [area.lo, area.hi],
max_det: buf.max_det,
dt_ids: dt_ids_sorted,
gt_ids: gt_ids_sorted,
dt_matches: dt_matches_id,
gt_matches: gt_matches_id,
};
Ok((cell, meta))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::accumulate::{accumulate, AccumulateParams};
use crate::dataset::{AnnId, Bbox, CategoryMeta, CocoAnnotation, DetectionInput, ImageMeta};
use crate::parity::{iou_thresholds, recall_thresholds};
use crate::summarize::summarize_detection;
fn img(id: i64, w: u32, h: u32) -> ImageMeta {
ImageMeta {
id: ImageId(id),
width: w,
height: h,
file_name: None,
}
}
fn cat(id: i64, name: &str) -> CategoryMeta {
CategoryMeta {
id: CategoryId(id),
name: name.into(),
supercategory: None,
}
}
fn ann(id: i64, image: i64, cat: i64, bbox: (f64, f64, f64, f64)) -> CocoAnnotation {
CocoAnnotation {
id: AnnId(id),
image_id: ImageId(image),
category_id: CategoryId(cat),
area: bbox.2 * bbox.3,
is_crowd: false,
ignore_flag: None,
bbox: Bbox {
x: bbox.0,
y: bbox.1,
w: bbox.2,
h: bbox.3,
},
segmentation: None,
keypoints: None,
num_keypoints: None,
}
}
fn dt_input(image: i64, cat: i64, score: f64, bbox: (f64, f64, f64, f64)) -> DetectionInput {
DetectionInput {
id: None,
image_id: ImageId(image),
category_id: CategoryId(cat),
score,
bbox: Bbox {
x: bbox.0,
y: bbox.1,
w: bbox.2,
h: bbox.3,
},
segmentation: None,
keypoints: None,
num_keypoints: None,
}
}
fn perfect_match_grid() -> EvalGrid {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "thing")];
let anns = vec![
ann(1, 1, 1, (0.0, 0.0, 10.0, 10.0)),
ann(2, 1, 1, (50.0, 50.0, 10.0, 10.0)),
];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![
dt_input(1, 1, 0.9, (0.0, 0.0, 10.0, 10.0)),
dt_input(1, 1, 0.8, (50.0, 50.0, 10.0, 10.0)),
])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
};
evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap()
}
#[test]
fn d4_coco_default_area_ranges_pin_literal_values() {
let ranges = AreaRange::coco_default();
assert_eq!(ranges.len(), 4);
assert_eq!(
(ranges[0].lo, ranges[0].hi),
(0.0, 1e10),
"all bucket bounds"
);
assert_eq!(
(ranges[1].lo, ranges[1].hi),
(0.0, 1024.0),
"small bucket bounds"
);
assert_eq!(
(ranges[2].lo, ranges[2].hi),
(1024.0, 9216.0),
"medium bucket bounds"
);
assert_eq!(
(ranges[3].lo, ranges[3].hi),
(9216.0, 1e10),
"large bucket bounds"
);
use crate::summarize::AreaRng;
assert_eq!(ranges[0].index, AreaRng::ALL.index);
assert_eq!(AreaRng::ALL.label.as_ref(), "all");
assert_eq!(ranges[1].index, AreaRng::SMALL.index);
assert_eq!(AreaRng::SMALL.label.as_ref(), "small");
assert_eq!(ranges[2].index, AreaRng::MEDIUM.index);
assert_eq!(AreaRng::MEDIUM.label.as_ref(), "medium");
assert_eq!(ranges[3].index, AreaRng::LARGE.index);
assert_eq!(AreaRng::LARGE.label.as_ref(), "large");
let pyco_unbounded: f64 = 1e5_f64.powi(2);
assert_eq!(pyco_unbounded.to_bits(), 1e10_f64.to_bits());
assert_eq!(ranges[0].hi.to_bits(), 1e10_f64.to_bits());
assert_eq!(ranges[3].hi.to_bits(), 1e10_f64.to_bits());
}
#[test]
fn perfect_match_produces_one_cell_per_area_range() {
let grid = perfect_match_grid();
assert_eq!(grid.n_categories, 1);
assert_eq!(grid.n_area_ranges, 4);
assert_eq!(grid.n_images, 1);
let cells: Vec<_> = grid.eval_imgs.iter().filter(|c| c.is_some()).collect();
assert_eq!(cells.len(), 4);
let all_cell = grid.cell(0, 0, 0).unwrap();
assert_eq!(all_cell.dt_scores.len(), 2);
assert!(all_cell.dt_matched.iter().all(|&m| m));
assert!(all_cell.dt_ignore.iter().all(|&ig| !ig));
}
#[test]
fn perfect_match_summarizes_to_one() {
let grid = perfect_match_grid();
let max_dets = vec![1usize, 10, 100];
let acc = accumulate(
&grid.eval_imgs,
AccumulateParams {
iou_thresholds: iou_thresholds(),
recall_thresholds: recall_thresholds(),
max_dets: &max_dets,
n_categories: grid.n_categories,
n_area_ranges: grid.n_area_ranges,
n_images: grid.n_images,
},
ParityMode::Strict,
)
.unwrap();
let summary = summarize_detection(&acc, iou_thresholds(), &max_dets).unwrap();
let stats = summary.stats();
assert!((stats[0] - 1.0).abs() < 1e-12, "AP={}", stats[0]);
assert!((stats[3] - 1.0).abs() < 1e-12, "AP_S={}", stats[3]);
assert_eq!(stats[4], -1.0, "AP_M should be -1 with no medium GTs");
assert_eq!(stats[5], -1.0, "AP_L should be -1 with no large GTs");
assert!((stats[8] - 1.0).abs() < 1e-12, "AR@100={}", stats[8]);
}
#[test]
fn b7_unmatched_dt_outside_area_range_is_ignored() {
let images = vec![img(1, 300, 300)];
let cats = vec![cat(1, "thing")];
let anns = vec![ann(1, 1, 1, (0.0, 0.0, 200.0, 200.0))];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts =
CocoDetections::from_inputs(vec![dt_input(1, 1, 0.5, (200.0, 200.0, 50.0, 50.0))])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
};
let grid = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
let small = grid.cell(0, 1, 0).unwrap();
assert_eq!(small.gt_ignore, vec![true]);
assert!(small.dt_ignore.iter().all(|&ig| ig));
assert!(small.dt_matched.iter().all(|&m| !m));
}
#[test]
fn d6_boundary_area_lands_in_both_buckets() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "thing")];
let anns = vec![ann(1, 1, 1, (0.0, 0.0, 32.0, 32.0))];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts =
CocoDetections::from_inputs(vec![dt_input(1, 1, 0.5, (0.0, 0.0, 32.0, 32.0))]).unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
};
let grid = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
let small = grid.cell(0, 1, 0).unwrap();
assert_eq!(small.gt_ignore, vec![false]);
let medium = grid.cell(0, 2, 0).unwrap();
assert_eq!(medium.gt_ignore, vec![false]);
let all = grid.cell(0, 0, 0).unwrap();
assert_eq!(all.gt_ignore, vec![false]);
let large = grid.cell(0, 3, 0).unwrap();
assert_eq!(large.gt_ignore, vec![true]);
}
#[test]
fn l4_use_cats_false_collapses_categories() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "a"), cat(2, "b")];
let anns = vec![
ann(1, 1, 1, (0.0, 0.0, 10.0, 10.0)),
ann(2, 1, 2, (50.0, 50.0, 10.0, 10.0)),
];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![dt_input(1, 1, 0.9, (50.0, 50.0, 10.0, 10.0))])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: false,
};
let grid = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
assert_eq!(grid.n_categories, 1);
let all = grid.cell(0, 0, 0).unwrap();
assert_eq!(all.gt_ignore.len(), 2);
assert_eq!(all.dt_scores.len(), 1);
assert!(all.dt_matched.iter().all(|&m| m));
}
#[test]
fn max_dets_per_image_caps_top_n_by_score() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "thing")];
let anns = vec![ann(1, 1, 1, (0.0, 0.0, 10.0, 10.0))];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![
dt_input(1, 1, 0.1, (50.0, 50.0, 5.0, 5.0)),
dt_input(1, 1, 0.9, (0.0, 0.0, 10.0, 10.0)),
dt_input(1, 1, 0.5, (50.0, 50.0, 5.0, 5.0)),
])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 2,
use_cats: true,
};
let grid = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
let all = grid.cell(0, 0, 0).unwrap();
assert_eq!(all.dt_scores.len(), 2);
assert_eq!(all.dt_scores[0], 0.9);
assert_eq!(all.dt_scores[1], 0.5);
}
#[test]
fn d1_parity_mode_propagates_to_base_ignore() {
const ANN_JSON: &str = r#"{
"images": [{"id": 1, "width": 100, "height": 100}],
"annotations": [
{"id": 1, "image_id": 1, "category_id": 1,
"bbox": [0, 0, 10, 10], "area": 100,
"iscrowd": 0, "ignore": 1}
],
"categories": [{"id": 1, "name": "thing"}]
}"#;
let gt = CocoDataset::from_json_bytes(ANN_JSON.as_bytes()).unwrap();
let dts =
CocoDetections::from_inputs(vec![dt_input(1, 1, 0.9, (0.0, 0.0, 10.0, 10.0))]).unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
};
let strict = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
let strict_all = strict.cell(0, 0, 0).unwrap();
assert_eq!(strict_all.gt_ignore, vec![false]);
assert!(strict_all.dt_ignore.iter().all(|&ig| !ig));
let corrected = evaluate_bbox(>, &dts, params, ParityMode::Corrected).unwrap();
let corrected_all = corrected.cell(0, 0, 0).unwrap();
assert_eq!(corrected_all.gt_ignore, vec![true]);
assert!(corrected_all.dt_ignore.iter().all(|&ig| ig));
}
#[test]
fn cell_meta_carries_pycocotools_shape() {
let grid = perfect_match_grid();
let meta = grid.cell_meta(0, 0, 0).unwrap();
assert_eq!(meta.image_id, 1);
assert_eq!(meta.category_id, 1);
assert_eq!(meta.area_rng, [0.0, AREA_UNBOUNDED]);
assert_eq!(meta.max_det, 100);
assert_eq!(meta.dt_ids, vec![1, 2]);
assert_eq!(meta.gt_ids, vec![1, 2]);
let n_t = iou_thresholds().len();
assert_eq!(meta.dt_matches.shape(), &[n_t, 2]);
assert_eq!(meta.gt_matches.shape(), &[n_t, 2]);
for t in 0..n_t {
assert_eq!(meta.dt_matches[(t, 0)], 1, "dt[0] -> gt[1] at t={t}");
assert_eq!(meta.dt_matches[(t, 1)], 2, "dt[1] -> gt[2] at t={t}");
assert_eq!(meta.gt_matches[(t, 0)], 1, "gt[1] -> dt[1] at t={t}");
assert_eq!(meta.gt_matches[(t, 1)], 2, "gt[2] -> dt[2] at t={t}");
}
}
#[test]
fn cell_meta_unmatched_dt_uses_zero_sentinel() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "thing")];
let anns = vec![ann(7, 1, 1, (0.0, 0.0, 10.0, 10.0))];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![dt_input(1, 1, 0.5, (50.0, 50.0, 10.0, 10.0))])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
};
let grid = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
let meta = grid.cell_meta(0, 0, 0).unwrap();
assert_eq!(meta.gt_ids, vec![7]);
assert_eq!(meta.dt_ids.len(), 1);
assert!(meta.dt_matches.iter().all(|&x| x == 0));
assert!(meta.gt_matches.iter().all(|&x| x == 0));
}
#[test]
fn cell_meta_use_cats_false_emits_sentinel_category() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "a"), cat(2, "b")];
let anns = vec![ann(1, 1, 1, (0.0, 0.0, 10.0, 10.0))];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts =
CocoDetections::from_inputs(vec![dt_input(1, 1, 0.9, (0.0, 0.0, 10.0, 10.0))]).unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: false,
};
let grid = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
let meta = grid.cell_meta(0, 0, 0).unwrap();
assert_eq!(meta.category_id, COLLAPSED_CATEGORY_SENTINEL);
}
#[test]
fn missing_dt_image_yields_none_cells() {
let images = vec![img(1, 100, 100), img(2, 100, 100)];
let cats = vec![cat(1, "thing")];
let anns = vec![ann(1, 1, 1, (0.0, 0.0, 10.0, 10.0))];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![]).unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
};
let grid = evaluate_bbox(>, &dts, params, ParityMode::Strict).unwrap();
for a in 0..4 {
assert!(grid.cell(0, a, 0).is_some(), "image 1 area {a}");
assert!(grid.cell(0, a, 1).is_none(), "image 2 area {a}");
}
}
fn square_polygon(x: f64, y: f64, side: f64) -> Segmentation {
Segmentation::Polygons(vec![vec![
x,
y,
x + side,
y,
x + side,
y + side,
x,
y + side,
]])
}
fn ann_with_segm(
id: i64,
image: i64,
cat: i64,
bbox: (f64, f64, f64, f64),
segm: Segmentation,
) -> CocoAnnotation {
CocoAnnotation {
id: AnnId(id),
image_id: ImageId(image),
category_id: CategoryId(cat),
area: bbox.2 * bbox.3,
is_crowd: false,
ignore_flag: None,
bbox: Bbox {
x: bbox.0,
y: bbox.1,
w: bbox.2,
h: bbox.3,
},
segmentation: Some(segm),
keypoints: None,
num_keypoints: None,
}
}
fn dt_input_with_segm(
image: i64,
cat: i64,
score: f64,
bbox: (f64, f64, f64, f64),
segm: Segmentation,
) -> DetectionInput {
DetectionInput {
id: None,
image_id: ImageId(image),
category_id: CategoryId(cat),
score,
bbox: Bbox {
x: bbox.0,
y: bbox.1,
w: bbox.2,
h: bbox.3,
},
segmentation: Some(segm),
keypoints: None,
num_keypoints: None,
}
}
#[test]
fn segm_perfect_overlap_summarizes_to_one() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "thing")];
let anns = vec![ann_with_segm(
1,
1,
1,
(10.0, 10.0, 20.0, 20.0),
square_polygon(10.0, 10.0, 20.0),
)];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![dt_input_with_segm(
1,
1,
0.9,
(10.0, 10.0, 20.0, 20.0),
square_polygon(10.0, 10.0, 20.0),
)])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
};
let grid = evaluate_segm(>, &dts, params, ParityMode::Strict).unwrap();
let max_dets = vec![1usize, 10, 100];
let acc = accumulate(
&grid.eval_imgs,
AccumulateParams {
iou_thresholds: iou_thresholds(),
recall_thresholds: recall_thresholds(),
max_dets: &max_dets,
n_categories: grid.n_categories,
n_area_ranges: grid.n_area_ranges,
n_images: grid.n_images,
},
ParityMode::Strict,
)
.unwrap();
let summary = summarize_detection(&acc, iou_thresholds(), &max_dets).unwrap();
let stats = summary.stats();
assert!((stats[0] - 1.0).abs() < 1e-12, "AP={}", stats[0]);
}
#[test]
fn segm_disjoint_masks_summarize_to_zero() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "thing")];
let anns = vec![ann_with_segm(
1,
1,
1,
(0.0, 0.0, 10.0, 10.0),
square_polygon(0.0, 0.0, 10.0),
)];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![dt_input_with_segm(
1,
1,
0.9,
(50.0, 50.0, 10.0, 10.0),
square_polygon(50.0, 50.0, 10.0),
)])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
};
let grid = evaluate_segm(>, &dts, params, ParityMode::Strict).unwrap();
let all = grid.cell(0, 0, 0).unwrap();
assert!(all.dt_matched.iter().all(|&m| !m));
}
#[test]
fn segm_missing_gt_segmentation_surfaces_typed_error() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "thing")];
let anns = vec![ann(7, 1, 1, (0.0, 0.0, 10.0, 10.0))];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![dt_input_with_segm(
1,
1,
0.9,
(0.0, 0.0, 10.0, 10.0),
square_polygon(0.0, 0.0, 10.0),
)])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
};
let err = evaluate_segm(>, &dts, params, ParityMode::Strict).unwrap_err();
match err {
EvalError::InvalidAnnotation { detail } => {
assert!(detail.contains("GT id=7"), "msg: {detail}");
}
other => panic!("expected InvalidAnnotation, got {other:?}"),
}
}
#[test]
fn j2_bbox_only_dt_under_segm_iou_type_raises_in_corrected_mode() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "thing")];
let anns = vec![ann_with_segm(
1,
1,
1,
(0.0, 0.0, 10.0, 10.0),
square_polygon(0.0, 0.0, 10.0),
)];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts =
CocoDetections::from_inputs(vec![dt_input(1, 1, 0.9, (0.0, 0.0, 10.0, 10.0))]).unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
};
let err = evaluate_segm(>, &dts, params, ParityMode::Corrected).unwrap_err();
match err {
EvalError::InvalidAnnotation { detail } => {
assert!(detail.contains("DT"), "expected DT in msg: {detail}");
assert!(detail.contains("J2"), "expected J2 cite in msg: {detail}");
}
other => panic!("expected InvalidAnnotation, got {other:?}"),
}
}
#[test]
fn j2_bbox_only_dt_under_segm_iou_type_synthesizes_in_strict_mode() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "thing")];
let anns = vec![ann_with_segm(
1,
1,
1,
(0.0, 0.0, 10.0, 10.0),
square_polygon(0.0, 0.0, 10.0),
)];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts =
CocoDetections::from_inputs(vec![dt_input(1, 1, 0.9, (0.0, 0.0, 10.0, 10.0))]).unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
};
let grid = evaluate_segm(>, &dts, params, ParityMode::Strict).unwrap();
let all = grid.cell(0, 0, 0).unwrap();
assert!(all.dt_matched.iter().all(|&m| m), "expected matches");
}
#[test]
fn j6_heterogeneous_dt_list_first_with_segm_second_without_raises_in_corrected_mode() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "thing")];
let anns = vec![ann_with_segm(
1,
1,
1,
(0.0, 0.0, 10.0, 10.0),
square_polygon(0.0, 0.0, 10.0),
)];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![
dt_input_with_segm(
1,
1,
0.9,
(0.0, 0.0, 10.0, 10.0),
square_polygon(0.0, 0.0, 10.0),
),
dt_input(1, 1, 0.8, (50.0, 50.0, 10.0, 10.0)),
])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
};
let err = evaluate_segm(>, &dts, params, ParityMode::Corrected).unwrap_err();
assert!(matches!(err, EvalError::InvalidAnnotation { .. }));
}
#[test]
fn j6_heterogeneous_dt_list_first_without_segm_second_with_raises_in_corrected_mode() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "thing")];
let anns = vec![ann_with_segm(
1,
1,
1,
(0.0, 0.0, 10.0, 10.0),
square_polygon(0.0, 0.0, 10.0),
)];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![
dt_input(1, 1, 0.9, (0.0, 0.0, 10.0, 10.0)),
dt_input_with_segm(
1,
1,
0.8,
(50.0, 50.0, 10.0, 10.0),
square_polygon(50.0, 50.0, 10.0),
),
])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
};
let err = evaluate_segm(>, &dts, params, ParityMode::Corrected).unwrap_err();
assert!(matches!(err, EvalError::InvalidAnnotation { .. }));
}
#[test]
fn j6_heterogeneous_dt_list_in_strict_mode_synthesizes_per_entry() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "thing")];
let anns = vec![
ann_with_segm(
1,
1,
1,
(0.0, 0.0, 10.0, 10.0),
square_polygon(0.0, 0.0, 10.0),
),
ann_with_segm(
2,
1,
1,
(50.0, 50.0, 10.0, 10.0),
square_polygon(50.0, 50.0, 10.0),
),
];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![
dt_input_with_segm(
1,
1,
0.9,
(0.0, 0.0, 10.0, 10.0),
square_polygon(0.0, 0.0, 10.0),
),
dt_input(1, 1, 0.8, (50.0, 50.0, 10.0, 10.0)),
])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
};
let grid = evaluate_segm(>, &dts, params, ParityMode::Strict).unwrap();
let all = grid.cell(0, 0, 0).unwrap();
assert_eq!(all.dt_matched.shape(), &[iou_thresholds().len(), 2]);
assert!(all.dt_matched.iter().all(|&m| m));
}
#[test]
fn boundary_perfect_overlap_summarizes_to_one() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "thing")];
let anns = vec![ann_with_segm(
1,
1,
1,
(10.0, 10.0, 20.0, 20.0),
square_polygon(10.0, 10.0, 20.0),
)];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![dt_input_with_segm(
1,
1,
0.9,
(10.0, 10.0, 20.0, 20.0),
square_polygon(10.0, 10.0, 20.0),
)])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
};
let grid = evaluate_boundary(>, &dts, params, ParityMode::Strict, 0.02).unwrap();
let max_dets = vec![1usize, 10, 100];
let acc = accumulate(
&grid.eval_imgs,
AccumulateParams {
iou_thresholds: iou_thresholds(),
recall_thresholds: recall_thresholds(),
max_dets: &max_dets,
n_categories: grid.n_categories,
n_area_ranges: grid.n_area_ranges,
n_images: grid.n_images,
},
ParityMode::Strict,
)
.unwrap();
let summary = summarize_detection(&acc, iou_thresholds(), &max_dets).unwrap();
let stats = summary.stats();
assert!((stats[0] - 1.0).abs() < 1e-12, "AP={}", stats[0]);
}
#[test]
fn boundary_disjoint_masks_summarize_to_zero() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "thing")];
let anns = vec![ann_with_segm(
1,
1,
1,
(0.0, 0.0, 10.0, 10.0),
square_polygon(0.0, 0.0, 10.0),
)];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![dt_input_with_segm(
1,
1,
0.9,
(50.0, 50.0, 10.0, 10.0),
square_polygon(50.0, 50.0, 10.0),
)])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
};
let grid = evaluate_boundary(>, &dts, params, ParityMode::Strict, 0.02).unwrap();
let all = grid.cell(0, 0, 0).unwrap();
assert!(all.dt_matched.iter().all(|&m| !m));
}
fn const_kps_vec(x: f64, y: f64, v: u32, len: usize) -> Vec<f64> {
let mut out = Vec::with_capacity(3 * len);
for _ in 0..len {
out.push(x);
out.push(y);
out.push(f64::from(v));
}
out
}
fn ann_with_kps(
id: i64,
image: i64,
cat: i64,
bbox: (f64, f64, f64, f64),
keypoints: Vec<f64>,
num_keypoints: Option<u32>,
) -> CocoAnnotation {
CocoAnnotation {
id: AnnId(id),
image_id: ImageId(image),
category_id: CategoryId(cat),
area: bbox.2 * bbox.3,
is_crowd: false,
ignore_flag: None,
bbox: Bbox {
x: bbox.0,
y: bbox.1,
w: bbox.2,
h: bbox.3,
},
segmentation: None,
keypoints: Some(keypoints),
num_keypoints,
}
}
fn dt_input_with_kps(
image: i64,
cat: i64,
score: f64,
bbox: (f64, f64, f64, f64),
keypoints: Vec<f64>,
) -> DetectionInput {
DetectionInput {
id: None,
image_id: ImageId(image),
category_id: CategoryId(cat),
score,
bbox: Bbox {
x: bbox.0,
y: bbox.1,
w: bbox.2,
h: bbox.3,
},
segmentation: None,
keypoints: Some(keypoints),
num_keypoints: None,
}
}
#[test]
fn test_evaluate_keypoints_perfect_match() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "person")];
let kps = const_kps_vec(50.0, 50.0, 2, 17);
let anns = vec![ann_with_kps(
1,
1,
1,
(40.0, 40.0, 20.0, 20.0),
kps.clone(),
None,
)];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![dt_input_with_kps(
1,
1,
0.9,
(40.0, 40.0, 20.0, 20.0),
kps,
)])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
};
let grid =
evaluate_keypoints(>, &dts, params, ParityMode::Strict, HashMap::new()).unwrap();
let cell = grid.cell(0, 0, 0).unwrap();
assert_eq!(cell.gt_ignore, vec![false]);
assert!(cell.dt_matched.iter().all(|&m| m));
let meta = grid.cell_meta(0, 0, 0).unwrap();
assert!(
meta.gt_matches.iter().all(|&id| id > 0),
"every threshold should match the DT id (>0)",
);
}
#[test]
fn test_evaluate_keypoints_zero_overlap() {
let images = vec![img(1, 2000, 2000)];
let cats = vec![cat(1, "person")];
let gt_kps = const_kps_vec(50.0, 50.0, 2, 17);
let dt_kps = const_kps_vec(1500.0, 1500.0, 2, 17);
let anns = vec![ann_with_kps(
1,
1,
1,
(40.0, 40.0, 20.0, 20.0),
gt_kps,
None,
)];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![dt_input_with_kps(
1,
1,
0.9,
(1490.0, 1490.0, 20.0, 20.0),
dt_kps,
)])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
};
let grid =
evaluate_keypoints(>, &dts, params, ParityMode::Strict, HashMap::new()).unwrap();
let cell = grid.cell(0, 0, 0).unwrap();
assert!(
cell.dt_matched.iter().all(|&m| !m),
"DTs far from GT should not match at any IoU threshold",
);
}
#[test]
fn test_evaluate_keypoints_d2_implicit_ignore() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "person")];
let gt_kps = const_kps_vec(50.0, 50.0, 0, 17);
let dt_kps = const_kps_vec(50.0, 50.0, 2, 17);
let anns = vec![ann_with_kps(
1,
1,
1,
(40.0, 40.0, 20.0, 20.0),
gt_kps,
Some(0),
)];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![dt_input_with_kps(
1,
1,
0.9,
(40.0, 40.0, 20.0, 20.0),
dt_kps,
)])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
};
let grid =
evaluate_keypoints(>, &dts, params, ParityMode::Strict, HashMap::new()).unwrap();
let cell = grid.cell(0, 0, 0).unwrap();
assert_eq!(
cell.gt_ignore,
vec![true],
"D2: zero-visible-keypoints GT must be ignored",
);
}
#[test]
fn test_evaluate_keypoints_per_category_sigmas() {
let images = vec![img(1, 200, 200)];
let cats = vec![cat(1, "person"), cat(2, "dog")];
let gt_kps = const_kps_vec(50.0, 50.0, 2, 17);
let anns = vec![
ann_with_kps(1, 1, 1, (40.0, 40.0, 20.0, 20.0), gt_kps, None),
ann_with_kps(
2,
1,
2,
(140.0, 140.0, 20.0, 20.0),
const_kps_vec(150.0, 150.0, 2, 17),
None,
),
];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![
dt_input_with_kps(
1,
1,
0.9,
(40.0, 40.0, 20.0, 20.0),
const_kps_vec(51.0, 50.0, 2, 17),
),
dt_input_with_kps(
1,
2,
0.8,
(140.0, 140.0, 20.0, 20.0),
const_kps_vec(151.0, 150.0, 2, 17),
),
])
.unwrap();
let mut sigmas: HashMap<i64, Vec<f64>> = HashMap::new();
sigmas.insert(1, vec![0.5_f64; 17]);
sigmas.insert(2, vec![0.5_f64; 17]);
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
};
let grid = evaluate_keypoints(>, &dts, params, ParityMode::Strict, sigmas).unwrap();
let cell_cat1 = grid.cell(0, 0, 0).unwrap();
let cell_cat2 = grid.cell(1, 0, 0).unwrap();
assert!(
cell_cat1.dt_matched.iter().all(|&m| m),
"cat-1 DT should match cat-1 GT under override sigmas",
);
assert!(
cell_cat2.dt_matched.iter().all(|&m| m),
"cat-2 DT should match cat-2 GT under override sigmas",
);
}
#[test]
fn test_evaluate_keypoints_missing_dt_kps_rejected() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "person")];
let gt_kps = const_kps_vec(50.0, 50.0, 2, 17);
let anns = vec![ann_with_kps(
1,
1,
1,
(40.0, 40.0, 20.0, 20.0),
gt_kps,
None,
)];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![dt_input(1, 1, 0.9, (40.0, 40.0, 20.0, 20.0))])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
};
let err =
evaluate_keypoints(>, &dts, params, ParityMode::Strict, HashMap::new()).unwrap_err();
match err {
EvalError::InvalidAnnotation { detail } => {
assert!(detail.contains("DT"), "expected DT in msg: {detail}");
assert!(
detail.contains("keypoints"),
"expected keypoints in msg: {detail}",
);
}
other => panic!("expected InvalidAnnotation, got {other:?}"),
}
}
#[test]
fn test_keypoints_default_ignore_for_other_kernels() {
let ann_zero_kps = ann_with_kps(
1,
1,
1,
(0.0, 0.0, 10.0, 10.0),
const_kps_vec(0.0, 0.0, 0, 17),
Some(0),
);
assert!(
!BboxIou.extra_gt_ignore(&ann_zero_kps),
"BboxIou must keep the default `false` ignore",
);
assert!(
!SegmIou.extra_gt_ignore(&ann_zero_kps),
"SegmIou must keep the default `false` ignore",
);
assert!(
!BoundaryIou {
dilation_ratio: 0.02,
}
.extra_gt_ignore(&ann_zero_kps),
"BoundaryIou must keep the default `false` ignore",
);
assert!(
OksSimilarity::default().extra_gt_ignore(&ann_zero_kps),
"OksSimilarity must flip D2 to true on zero-visible-keypoints GT",
);
}
#[test]
fn boundary_missing_gt_segmentation_surfaces_typed_error() {
let images = vec![img(1, 100, 100)];
let cats = vec![cat(1, "thing")];
let anns = vec![ann(7, 1, 1, (0.0, 0.0, 10.0, 10.0))];
let gt = CocoDataset::from_parts(images, anns, cats).unwrap();
let dts = CocoDetections::from_inputs(vec![dt_input_with_segm(
1,
1,
0.9,
(0.0, 0.0, 10.0, 10.0),
square_polygon(0.0, 0.0, 10.0),
)])
.unwrap();
let area = AreaRange::coco_default();
let params = EvaluateParams {
iou_thresholds: iou_thresholds(),
area_ranges: &area,
max_dets_per_image: 100,
use_cats: true,
};
let err = evaluate_boundary(>, &dts, params, ParityMode::Strict, 0.02).unwrap_err();
match err {
EvalError::InvalidAnnotation { detail } => {
assert!(detail.contains("GT id=7"), "msg: {detail}");
}
other => panic!("expected InvalidAnnotation, got {other:?}"),
}
}
}