use std::collections::HashMap;
use std::collections::hash_map::Entry;
use serde::Serialize;
use super::COCOeval;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
pub enum DtStatus {
Tp,
Fp,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
pub enum GtStatus {
Matched,
Fn,
}
#[derive(Debug, Clone, Serialize)]
pub struct AnnotationIndex {
pub dt_status: HashMap<u64, DtStatus>,
pub gt_status: HashMap<u64, GtStatus>,
pub dt_match: HashMap<u64, u64>,
pub gt_match: HashMap<u64, u64>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
pub enum ErrorProfile {
Perfect,
FpHeavy,
FnHeavy,
Mixed,
}
#[derive(Debug, Clone, Serialize)]
pub struct ImageSummary {
pub tp: u32,
pub fp: u32,
pub fn_count: u32,
pub f1: f64,
pub ap: f64,
pub error_profile: ErrorProfile,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
pub enum LabelErrorType {
WrongLabel,
MissingAnnotation,
}
#[derive(Debug, Clone, Serialize)]
pub struct LabelError {
pub image_id: u64,
pub dt_id: u64,
pub dt_score: f64,
pub dt_category_id: u64,
pub gt_id: Option<u64>,
pub gt_category_id: Option<u64>,
pub iou: f64,
pub error_type: LabelErrorType,
}
#[derive(Debug, Clone, Serialize)]
pub struct ImageDiagnostics {
pub annotations: AnnotationIndex,
pub images: HashMap<u64, ImageSummary>,
pub label_errors: Vec<LabelError>,
pub iou_thr: f64,
}
const WRONG_LABEL_IOU: f64 = 0.5;
const MISSING_ANNOTATION_IOU: f64 = 0.1;
const PROFILE_DOMINANCE_FACTOR: u32 = 2;
fn bbox_iou_plain(a: [f64; 4], b: [f64; 4]) -> f64 {
crate::primitives::sim::bbox_iou_pair(a, b, false)
}
fn compute_image_ap(scores: &[f64], matched: &[bool], n_gt: u32, rec_thrs: &[f64]) -> f64 {
if n_gt == 0 {
return if scores.is_empty() { 1.0 } else { 0.0 };
}
crate::metrics::counts::average_precision(scores, matched, None, n_gt as usize, rec_thrs)
}
#[derive(Default)]
struct ImageTally {
tp: u32,
fp: u32,
fn_count: u32,
scores: Vec<f64>,
matched: Vec<bool>,
}
struct FpDt {
dt_id: u64,
score: f64,
cat_id: u64,
bbox: [f64; 4],
}
struct FnGt {
gt_id: u64,
cat_id: u64,
bbox: [f64; 4],
}
#[derive(Default)]
struct ImageCandidates {
fps: Vec<FpDt>,
fn_gts: Vec<FnGt>,
all_gt_bboxes: Vec<[f64; 4]>,
}
impl COCOeval {
pub fn image_diagnostics(
&self,
iou_thr: f64,
score_thr: f64,
) -> crate::error::Result<ImageDiagnostics> {
if self.eval_imgs.is_empty() {
return Err("image_diagnostics() requires evaluate() to be called first".into());
}
let t_idx = self.params.nearest_iou_thr_idx(iou_thr);
let (annotations, tallies) = self.classify_annotations(t_idx);
let images = summarize_images(tallies, &self.params.rec_thrs);
let label_errors = self.find_label_errors(&annotations, score_thr);
Ok(ImageDiagnostics {
annotations,
images,
label_errors,
iou_thr: self.params.iou_thrs[t_idx],
})
}
fn classify_annotations(&self, t_idx: usize) -> (AnnotationIndex, HashMap<u64, ImageTally>) {
let mut index = AnnotationIndex {
dt_status: HashMap::new(),
gt_status: HashMap::new(),
dt_match: HashMap::new(),
gt_match: HashMap::new(),
};
let mut tallies: HashMap<u64, ImageTally> = HashMap::new();
for eval_img in self.default_cells() {
debug_assert!(t_idx < eval_img.dt_matched.num_rows());
let matched = eval_img.dt_matched.row(t_idx);
let ignored = eval_img.dt_ignore.row(t_idx);
let matches = eval_img.dt_matches.row(t_idx);
debug_assert_eq!(matched.len(), matches.len());
debug_assert_eq!(eval_img.dt_ids.len(), matched.len());
debug_assert_eq!(eval_img.dt_ids.len(), ignored.len());
let tally = tallies.entry(eval_img.image_id).or_default();
for (d, &did) in eval_img.dt_ids.iter().enumerate() {
if ignored[d] {
continue;
}
let Entry::Vacant(slot) = index.dt_status.entry(did) else {
continue;
};
let is_tp = matched[d];
if is_tp {
slot.insert(DtStatus::Tp);
index.dt_match.insert(did, matches[d]);
index.gt_match.insert(matches[d], did);
tally.tp += 1;
} else {
slot.insert(DtStatus::Fp);
tally.fp += 1;
}
tally.scores.push(eval_img.dt_scores[d]);
tally.matched.push(is_tp);
}
let gt_matched_at_t = eval_img.gt_matched.row(t_idx);
debug_assert_eq!(eval_img.gt_ids.len(), gt_matched_at_t.len());
for (g, &gid) in eval_img.gt_ids.iter().enumerate() {
let Entry::Vacant(slot) = index.gt_status.entry(gid) else {
continue;
};
if !eval_img.counts_as_miss(g) {
continue;
}
if gt_matched_at_t[g] {
slot.insert(GtStatus::Matched);
} else {
slot.insert(GtStatus::Fn);
tally.fn_count += 1;
}
}
}
(index, tallies)
}
fn find_label_errors(&self, index: &AnnotationIndex, score_thr: f64) -> Vec<LabelError> {
let mut by_image: HashMap<u64, ImageCandidates> = HashMap::new();
for (&dt_id, &status) in &index.dt_status {
if status != DtStatus::Fp {
continue;
}
let Some(ann) = self.coco_dt.get_ann(dt_id) else {
continue;
};
let (Some(score), Some(bbox)) = (ann.score, ann.bbox) else {
continue;
};
if score < score_thr {
continue;
}
by_image.entry(ann.image_id).or_default().fps.push(FpDt {
dt_id,
score,
cat_id: ann.category_id,
bbox,
});
}
for (>_id, &status) in &index.gt_status {
let Some(ann) = self.coco_gt.get_ann(gt_id) else {
continue;
};
let Some(bbox) = ann.bbox else {
continue;
};
let cand = by_image.entry(ann.image_id).or_default();
cand.all_gt_bboxes.push(bbox);
if status == GtStatus::Fn {
cand.fn_gts.push(FnGt {
gt_id,
cat_id: ann.category_id,
bbox,
});
}
}
let mut errors: Vec<LabelError> = by_image
.iter()
.flat_map(|(&img_id, cand)| {
cand.fps
.iter()
.filter_map(move |fp| classify_label_error(img_id, fp, cand))
})
.collect();
errors.sort_by(|a, b| {
b.dt_score
.partial_cmp(&a.dt_score)
.unwrap_or(std::cmp::Ordering::Equal)
.then(a.dt_id.cmp(&b.dt_id))
});
errors
}
}
fn summarize_images(
tallies: HashMap<u64, ImageTally>,
rec_thrs: &[f64],
) -> HashMap<u64, ImageSummary> {
tallies
.into_iter()
.map(|(img_id, t)| {
let denom = 2 * t.tp + t.fp + t.fn_count;
let f1 = if denom == 0 {
1.0
} else {
(2 * t.tp) as f64 / denom as f64
};
let n_gt = t.tp + t.fn_count; let ap = compute_image_ap(&t.scores, &t.matched, n_gt, rec_thrs);
let error_profile = match (t.fp, t.fn_count) {
(0, 0) => ErrorProfile::Perfect,
(f, n) if f > PROFILE_DOMINANCE_FACTOR * n => ErrorProfile::FpHeavy,
(f, n) if n > PROFILE_DOMINANCE_FACTOR * f => ErrorProfile::FnHeavy,
_ => ErrorProfile::Mixed,
};
(
img_id,
ImageSummary {
tp: t.tp,
fp: t.fp,
fn_count: t.fn_count,
f1,
ap,
error_profile,
},
)
})
.collect()
}
fn classify_label_error(img_id: u64, fp: &FpDt, cand: &ImageCandidates) -> Option<LabelError> {
let mut best_fn: Option<(f64, &FnGt)> = None;
for fg in cand.fn_gts.iter().filter(|fg| fg.cat_id != fp.cat_id) {
let overlap = bbox_iou_plain(fp.bbox, fg.bbox);
if overlap > best_fn.map_or(0.0, |(best, _)| best) {
best_fn = Some((overlap, fg));
}
}
if let Some((iou, fg)) = best_fn {
if iou >= WRONG_LABEL_IOU {
return Some(LabelError {
image_id: img_id,
dt_id: fp.dt_id,
dt_score: fp.score,
dt_category_id: fp.cat_id,
gt_id: Some(fg.gt_id),
gt_category_id: Some(fg.cat_id),
iou,
error_type: LabelErrorType::WrongLabel,
});
}
}
let max_iou_any_gt = cand
.all_gt_bboxes
.iter()
.map(|>_bbox| bbox_iou_plain(fp.bbox, gt_bbox))
.fold(0.0f64, f64::max);
(max_iou_any_gt < MISSING_ANNOTATION_IOU).then_some(LabelError {
image_id: img_id,
dt_id: fp.dt_id,
dt_score: fp.score,
dt_category_id: fp.cat_id,
gt_id: None,
gt_category_id: None,
iou: 0.0,
error_type: LabelErrorType::MissingAnnotation,
})
}
#[cfg(test)]
#[allow(clippy::unwrap_used)]
mod tests {
use super::*;
use crate::coco::COCO;
use crate::detection::COCOeval;
use crate::params::IouType;
use crate::types::{Annotation, Dataset};
fn make_gt(json: serde_json::Value) -> COCO {
let ds: Dataset = serde_json::from_value(json).unwrap();
COCO::from_dataset(ds)
}
fn make_dt(gt: &COCO, anns_json: serde_json::Value) -> COCO {
let anns: Vec<Annotation> = serde_json::from_value(anns_json).unwrap();
gt.load_res_anns(anns).unwrap()
}
fn make_gt_dt() -> (COCO, COCO) {
let gt = make_gt(serde_json::json!({
"images": [
{"id": 1, "width": 100, "height": 100},
{"id": 2, "width": 100, "height": 100}
],
"annotations": [
{"id": 1, "image_id": 1, "category_id": 1, "bbox": [10, 10, 20, 20], "area": 400, "iscrowd": 0},
{"id": 2, "image_id": 1, "category_id": 2, "bbox": [50, 50, 20, 20], "area": 400, "iscrowd": 0},
{"id": 3, "image_id": 2, "category_id": 1, "bbox": [10, 10, 30, 30], "area": 900, "iscrowd": 0}
],
"categories": [
{"id": 1, "name": "cat"},
{"id": 2, "name": "dog"}
]
}));
let dt = make_dt(
>,
serde_json::json!([
{"image_id": 1, "category_id": 1, "bbox": [10, 10, 20, 20], "score": 0.9},
{"image_id": 1, "category_id": 2, "bbox": [50, 50, 20, 20], "score": 0.8},
{"image_id": 2, "category_id": 1, "bbox": [10, 10, 30, 30], "score": 0.7}
]),
);
(gt, dt)
}
#[test]
fn test_diagnostics_perfect_detection() {
let (gt, dt) = make_gt_dt();
let mut ev = COCOeval::new(gt, dt, IouType::Bbox);
ev.evaluate();
let diag = ev.image_diagnostics(0.5, 0.5).unwrap();
assert_eq!(diag.annotations.dt_status.len(), 3);
for status in diag.annotations.dt_status.values() {
assert_eq!(*status, DtStatus::Tp);
}
assert_eq!(diag.annotations.gt_status.len(), 3);
for status in diag.annotations.gt_status.values() {
assert_eq!(*status, GtStatus::Matched);
}
let img1 = &diag.images[&1];
assert_eq!(img1.tp, 2);
assert_eq!(img1.fp, 0);
assert_eq!(img1.fn_count, 0);
assert!((img1.f1 - 1.0).abs() < 1e-9);
assert_eq!(img1.error_profile, ErrorProfile::Perfect);
let img2 = &diag.images[&2];
assert_eq!(img2.tp, 1);
assert!((img2.f1 - 1.0).abs() < 1e-9);
assert!(diag.label_errors.is_empty());
}
#[test]
fn test_diagnostics_with_fp_and_fn() {
let gt = make_gt(serde_json::json!({
"images": [{"id": 1, "width": 100, "height": 100}],
"annotations": [
{"id": 1, "image_id": 1, "category_id": 1, "bbox": [10, 10, 20, 20], "area": 400, "iscrowd": 0},
{"id": 2, "image_id": 1, "category_id": 1, "bbox": [60, 60, 20, 20], "area": 400, "iscrowd": 0}
],
"categories": [{"id": 1, "name": "cat"}]
}));
let dt = make_dt(
>,
serde_json::json!([
{"image_id": 1, "category_id": 1, "bbox": [10, 10, 20, 20], "score": 0.9},
{"image_id": 1, "category_id": 1, "bbox": [80, 80, 10, 10], "score": 0.6}
]),
);
let mut ev = COCOeval::new(gt, dt, IouType::Bbox);
ev.evaluate();
let diag = ev.image_diagnostics(0.5, 0.5).unwrap();
let img = &diag.images[&1];
assert_eq!(img.tp, 1);
assert_eq!(img.fp, 1);
assert_eq!(img.fn_count, 1);
assert!((img.f1 - 0.5).abs() < 1e-9);
assert_eq!(img.error_profile, ErrorProfile::Mixed);
}
#[test]
fn test_diagnostics_wrong_label() {
let gt = make_gt(serde_json::json!({
"images": [{"id": 1, "width": 100, "height": 100}],
"annotations": [
{"id": 1, "image_id": 1, "category_id": 1, "bbox": [10, 10, 20, 20], "area": 400, "iscrowd": 0}
],
"categories": [
{"id": 1, "name": "cat"},
{"id": 2, "name": "dog"}
]
}));
let dt = make_dt(
>,
serde_json::json!([
{"image_id": 1, "category_id": 2, "bbox": [10, 10, 20, 20], "score": 0.95}
]),
);
let mut ev = COCOeval::new(gt, dt, IouType::Bbox);
ev.evaluate();
let diag = ev.image_diagnostics(0.5, 0.5).unwrap();
assert_eq!(diag.images[&1].fp, 1);
assert_eq!(diag.images[&1].fn_count, 1);
assert_eq!(diag.label_errors.len(), 1);
let err = &diag.label_errors[0];
assert_eq!(err.error_type, LabelErrorType::WrongLabel);
assert_eq!(err.dt_category_id, 2); assert_eq!(err.gt_category_id, Some(1)); assert!(err.iou > 0.9); }
#[test]
fn test_diagnostics_missing_annotation() {
let gt = make_gt(serde_json::json!({
"images": [{"id": 1, "width": 200, "height": 200}],
"annotations": [
{"id": 1, "image_id": 1, "category_id": 1, "bbox": [10, 10, 20, 20], "area": 400, "iscrowd": 0}
],
"categories": [{"id": 1, "name": "cat"}]
}));
let dt = make_dt(
>,
serde_json::json!([
{"image_id": 1, "category_id": 1, "bbox": [10, 10, 20, 20], "score": 0.9},
{"image_id": 1, "category_id": 1, "bbox": [150, 150, 20, 20], "score": 0.85}
]),
);
let mut ev = COCOeval::new(gt, dt, IouType::Bbox);
ev.evaluate();
let diag = ev.image_diagnostics(0.5, 0.5).unwrap();
assert_eq!(diag.images[&1].tp, 1);
assert_eq!(diag.images[&1].fp, 1);
assert_eq!(diag.label_errors.len(), 1);
let err = &diag.label_errors[0];
assert_eq!(err.error_type, LabelErrorType::MissingAnnotation);
assert!(err.gt_id.is_none());
}
#[test]
fn test_diagnostics_requires_evaluate() {
let gt = make_gt(serde_json::json!({
"images": [{"id": 1, "width": 100, "height": 100}],
"annotations": [],
"categories": [{"id": 1, "name": "cat"}]
}));
let dt = make_dt(>, serde_json::json!([]));
let ev = COCOeval::new(gt, dt, IouType::Bbox);
assert!(ev.image_diagnostics(0.5, 0.5).is_err());
}
#[test]
fn test_bbox_iou_exact_overlap() {
let a = [10.0, 10.0, 20.0, 20.0];
assert!((bbox_iou_plain(a, a) - 1.0).abs() < 1e-9);
}
#[test]
fn test_bbox_iou_no_overlap() {
let a = [0.0, 0.0, 10.0, 10.0];
let b = [50.0, 50.0, 10.0, 10.0];
assert_eq!(bbox_iou_plain(a, b), 0.0);
}
}
#[cfg(test)]
#[allow(clippy::unwrap_used)]
mod image_ap_tests {
use super::compute_image_ap;
#[test]
fn image_ap_matches_closed_form() {
let rec_thrs = crate::params::default_rec_thrs();
let n_thr = rec_thrs.len() as f64;
assert_eq!(compute_image_ap(&[0.9], &[true], 1, &rec_thrs), 1.0);
assert_eq!(
compute_image_ap(&[0.9, 0.8], &[false, false], 1, &rec_thrs),
0.0
);
let ap = compute_image_ap(&[0.9, 0.8], &[true, false], 2, &rec_thrs);
let reachable = rec_thrs.iter().filter(|&&t| t <= 0.5 + 1e-12).count() as f64;
assert!(
(ap - reachable / n_thr).abs() < 1e-12,
"expected {}/{n_thr} = {}, got {ap}",
reachable,
reachable / n_thr
);
let ap_fp_first = compute_image_ap(&[0.9, 0.8], &[false, true], 2, &rec_thrs);
assert!(
(ap_fp_first - 0.5 * reachable / n_thr).abs() < 1e-12,
"expected half the previous AP, got {ap_fp_first}"
);
assert!(
ap_fp_first < ap,
"a false positive ranked above the true positive must not score higher"
);
}
#[test]
fn empty_image_is_perfect_only_when_nothing_was_predicted() {
let rec_thrs = crate::params::default_rec_thrs();
assert_eq!(compute_image_ap(&[], &[], 0, &rec_thrs), 1.0);
assert_eq!(compute_image_ap(&[0.9], &[false], 0, &rec_thrs), 0.0);
assert_eq!(compute_image_ap(&[], &[], 3, &rec_thrs), 0.0);
}
}