Skip to main content

sightloom_host/
detect_decode.rs

1//! YOLO-like detector-head decode + NMS (no model runtime).
2//!
3//! Accepts the common export layouts:
4//! - `N×6` already-decoded `x1,y1,x2,y2,score,class`
5//! - YOLOv8/v11 `[1, 4+C, N]` (transposed, no objectness)
6//! - `YOLOv5` `[1, N, 5+C]` (objectness × class)
7//! - the same shapes without a leading batch dim of 1
8
9use crate::error::HostError;
10use crate::preprocess::Letterbox;
11use sightloom::core::{ClassId, Detection, NmsConfig, NmsMode, OverlapMetric, Rect, nms_in_place};
12
13/// One decoded box in **network** pixel space (`x1,y1,x2,y2`).
14#[derive(Clone, Copy, Debug, PartialEq)]
15pub struct RawBox {
16    /// Left.
17    pub x1: f32,
18    /// Top.
19    pub y1: f32,
20    /// Right.
21    pub x2: f32,
22    /// Bottom.
23    pub y2: f32,
24    /// Confidence in `0..=1` (or model units, already thresholded).
25    pub score: f32,
26    /// Class index when the head provides one.
27    pub class: Option<u16>,
28}
29
30/// Thresholds for decode + NMS.
31#[derive(Clone, Copy, Debug, PartialEq)]
32pub struct DetectorDecodeConfig {
33    /// Minimum confidence to keep a raw box.
34    pub conf_thresh: f32,
35    /// `IoU` threshold for class-aware hard NMS.
36    pub nms_thresh: f32,
37    /// Maximum detections after NMS.
38    pub max_det: usize,
39    /// Class used when the head has no class channel.
40    pub default_class: u16,
41}
42
43impl Default for DetectorDecodeConfig {
44    fn default() -> Self {
45        Self {
46            conf_thresh: 0.25,
47            nms_thresh: 0.45,
48            max_det: 300,
49            default_class: 0,
50        }
51    }
52}
53
54/// Decodes a flattened detector tensor into network-space boxes.
55///
56/// `shape` is the ONNX output shape (may be empty — then layout is inferred
57/// from `raw.len()` the same way as the original Nx6 / stride probe).
58///
59/// # Errors
60///
61/// Unrecognized layout, or a tensor that is empty after a required parse.
62pub fn decode_detector_output(
63    raw: &[f32],
64    shape: &[usize],
65    net_w: f32,
66    net_h: f32,
67    conf_thresh: f32,
68) -> Result<Vec<RawBox>, HostError> {
69    if raw.is_empty() {
70        return Ok(Vec::new());
71    }
72
73    let squeezed = squeeze_ones(shape);
74    if squeezed.len() >= 2 {
75        let rows = squeezed[squeezed.len() - 2];
76        let cols = squeezed[squeezed.len() - 1];
77        if rows.checked_mul(cols) == Some(raw.len())
78            && let Some(out) = decode_matrix(raw, rows, cols, net_w, net_h, conf_thresh)
79        {
80            // Trust an explicit tensor shape even when nothing is above thresh.
81            return Ok(out);
82        }
83    }
84
85    if let Some(out) = decode_flat_xyxy6(raw, conf_thresh)
86        && !out.is_empty()
87    {
88        return Ok(out);
89    }
90
91    let mut recognized_empty = false;
92    let mut best: Option<Vec<RawBox>> = None;
93    for stride in [6_usize, 7, 84, 85, 5] {
94        if !raw.len().is_multiple_of(stride) {
95            continue;
96        }
97        let n = raw.len() / stride;
98        if n == 0 || n > 50_000 {
99            continue;
100        }
101        for (rows, cols) in [(n, stride), (stride, n)] {
102            if let Some(out) = decode_matrix(raw, rows, cols, net_w, net_h, conf_thresh) {
103                if out.is_empty() {
104                    recognized_empty = true;
105                    continue;
106                }
107                if best.as_ref().is_none_or(|b| out.len() > b.len()) {
108                    best = Some(out);
109                }
110            }
111        }
112    }
113    if let Some(out) = best {
114        return Ok(out);
115    }
116    if recognized_empty {
117        return Ok(Vec::new());
118    }
119
120    Err(HostError::Runtime(format!(
121        "unrecognized detector output length {} shape {shape:?} (expected Nx6 or YOLO-like)",
122        raw.len()
123    )))
124}
125
126/// Maps network-space boxes to source-frame [`Detection`]s and applies NMS.
127///
128/// # Errors
129///
130/// NMS configuration errors (should not happen with [`DetectorDecodeConfig::default`]).
131pub fn detections_from_raw_boxes(
132    boxes: &[RawBox],
133    letterbox: Letterbox,
134    cfg: DetectorDecodeConfig,
135) -> Result<Vec<Detection>, HostError> {
136    let src_w = letterbox.src_w as f32;
137    let src_h = letterbox.src_h as f32;
138    let mut dets = Vec::with_capacity(boxes.len());
139    for b in boxes {
140        let (left, top) = letterbox.to_source(b.x1, b.y1);
141        let (right, bottom) = letterbox.to_source(b.x2, b.y2);
142        let left = left.clamp(0.0, src_w);
143        let top = top.clamp(0.0, src_h);
144        let right = right.clamp(0.0, src_w).max(left + 1.0);
145        let bottom = bottom.clamp(0.0, src_h).max(top + 1.0);
146        let Ok(rect) = Rect::new(left, top, right, bottom) else {
147            continue;
148        };
149        let class = ClassId(b.class.unwrap_or(cfg.default_class));
150        if let Ok(det) = Detection::new(rect, b.score, Some(class), None) {
151            dets.push(det);
152        }
153    }
154    apply_class_aware_nms(&mut dets, cfg.nms_thresh, cfg.max_det)?;
155    Ok(dets)
156}
157
158fn apply_class_aware_nms(
159    dets: &mut Vec<Detection>,
160    nms_thresh: f32,
161    max_det: usize,
162) -> Result<(), HostError> {
163    if dets.len() <= 1 {
164        if dets.len() > max_det {
165            dets.truncate(max_det);
166        }
167        return Ok(());
168    }
169    let mut order = vec![0_usize; dets.len()];
170    let mut suppressed = vec![false; dets.len()];
171    let kept = nms_in_place(
172        dets,
173        &mut order,
174        &mut suppressed,
175        NmsConfig {
176            threshold: nms_thresh.clamp(0.0, 1.0),
177            mode: NmsMode::ClassAware,
178            metric: OverlapMetric::IoU,
179        },
180    )
181    .map_err(|e| HostError::Runtime(format!("nms: {e}")))?;
182    dets.truncate(kept);
183    dets.sort_by(|a, b| {
184        b.score()
185            .partial_cmp(&a.score())
186            .unwrap_or(core::cmp::Ordering::Equal)
187    });
188    if dets.len() > max_det {
189        dets.truncate(max_det);
190    }
191    Ok(())
192}
193
194fn squeeze_ones(shape: &[usize]) -> Vec<usize> {
195    let squeezed: Vec<usize> = shape.iter().copied().filter(|&d| d != 1).collect();
196    if squeezed.is_empty() && !shape.is_empty() {
197        vec![1]
198    } else {
199        squeezed
200    }
201}
202
203fn channel_preference(d: usize) -> i32 {
204    match d {
205        84 | 85 => 100,
206        6 | 7 => 80,
207        5 => 70,
208        80 | 81 | 90 | 91 => 60,
209        4 => 40,
210        8..=256 => 20,
211        _ => 0,
212    }
213}
214
215fn decode_matrix(
216    raw: &[f32],
217    rows: usize,
218    cols: usize,
219    net_w: f32,
220    net_h: f32,
221    conf_thresh: f32,
222) -> Option<Vec<RawBox>> {
223    if rows == 0 || cols == 0 {
224        return None;
225    }
226    let row_pref = channel_preference(rows);
227    let col_pref = channel_preference(cols);
228    let (n, c, transposed) = if row_pref > col_pref {
229        (cols, rows, true)
230    } else if col_pref > row_pref {
231        (rows, cols, false)
232    } else if rows <= 256 && cols > rows {
233        (cols, rows, true)
234    } else {
235        (rows, cols, false)
236    };
237    if c < 4 || n == 0 || n > 50_000 {
238        return None;
239    }
240    let at = |channel: usize, index: usize| -> f32 {
241        if transposed {
242            raw[channel * n + index]
243        } else {
244            raw[index * c + channel]
245        }
246    };
247    let mut out = Vec::new();
248    for i in 0..n {
249        if let Some(b) = decode_one(c, |ch| at(ch, i), net_w, net_h, conf_thresh) {
250            out.push(b);
251        }
252    }
253    Some(out)
254}
255
256fn decode_one(
257    channels: usize,
258    at: impl Fn(usize) -> f32,
259    net_w: f32,
260    net_h: f32,
261    conf_thresh: f32,
262) -> Option<RawBox> {
263    if channels == 6 {
264        let x1 = at(0);
265        let y1 = at(1);
266        let x2 = at(2);
267        let y2 = at(3);
268        let score = at(4);
269        // Prefer already-decoded xyxy when the box is ordered. Do not reject
270        // the row here: YOLOv8 2-class heads also have C=6 (cxcywh + 2 cls).
271        if x2 > x1 && y2 > y1 && score.is_finite() && score >= conf_thresh {
272            return Some(scale_xyxy(
273                x1,
274                y1,
275                x2,
276                y2,
277                score,
278                Some(at(5) as u16),
279                net_w,
280                net_h,
281            ));
282        }
283    }
284
285    let cx = at(0);
286    let cy = at(1);
287    let w = at(2);
288    let h = at(3);
289    if ![cx, cy, w, h].iter().all(|v| v.is_finite()) {
290        return None;
291    }
292
293    let (class, score) = class_and_score(channels, &at);
294    if !score.is_finite() || score < conf_thresh {
295        return None;
296    }
297    Some(cxcywh_to_box(cx, cy, w, h, score, class, net_w, net_h))
298}
299
300fn class_and_score(channels: usize, at: &impl Fn(usize) -> f32) -> (Option<u16>, f32) {
301    if channels == 4 {
302        return (None, 1.0);
303    }
304    if channels == 5 {
305        return (None, at(4));
306    }
307    // Prefer the modern no-objectness head (YOLOv8: 4 + C) when that class
308    // count looks plausible. YOLOv5 COCO (`C=85`) still takes objectness.
309    let yolo8_classes = channels.saturating_sub(4);
310    let yolo5_classes = channels.saturating_sub(5);
311    let use_objectness =
312        is_typical_class_count(yolo5_classes) && !is_typical_class_count(yolo8_classes);
313    if use_objectness {
314        let obj = at(4);
315        let (cls, cls_s) = best_class(channels, 5, at);
316        (Some(cls), obj * cls_s)
317    } else {
318        let (cls, cls_s) = best_class(channels, 4, at);
319        (Some(cls), cls_s)
320    }
321}
322
323fn is_typical_class_count(n: usize) -> bool {
324    (1..=10).contains(&n) || matches!(n, 20 | 80 | 90 | 91)
325}
326
327fn best_class(channels: usize, start: usize, at: &impl Fn(usize) -> f32) -> (u16, f32) {
328    let mut best_i = 0_u16;
329    let mut best_s = f32::NEG_INFINITY;
330    for j in start..channels {
331        let s = at(j);
332        if s > best_s {
333            best_s = s;
334            best_i = (j - start) as u16;
335        }
336    }
337    if best_s.is_finite() {
338        (best_i, best_s)
339    } else {
340        (0, 0.0)
341    }
342}
343
344#[allow(clippy::too_many_arguments)]
345fn cxcywh_to_box(
346    mut cx: f32,
347    mut cy: f32,
348    mut w: f32,
349    mut h: f32,
350    score: f32,
351    class: Option<u16>,
352    net_w: f32,
353    net_h: f32,
354) -> RawBox {
355    if cx <= 1.5 && cy <= 1.5 && w <= 1.5 && h <= 1.5 {
356        cx *= net_w;
357        cy *= net_h;
358        w *= net_w;
359        h *= net_h;
360    }
361    RawBox {
362        x1: cx - w * 0.5,
363        y1: cy - h * 0.5,
364        x2: cx + w * 0.5,
365        y2: cy + h * 0.5,
366        score,
367        class,
368    }
369}
370
371#[allow(clippy::too_many_arguments)]
372fn scale_xyxy(
373    mut x1: f32,
374    mut y1: f32,
375    mut x2: f32,
376    mut y2: f32,
377    score: f32,
378    class: Option<u16>,
379    net_w: f32,
380    net_h: f32,
381) -> RawBox {
382    if x1 >= 0.0 && y1 >= 0.0 && x2 <= 1.5 && y2 <= 1.5 {
383        x1 *= net_w;
384        y1 *= net_h;
385        x2 *= net_w;
386        y2 *= net_h;
387    }
388    RawBox {
389        x1,
390        y1,
391        x2,
392        y2,
393        score,
394        class,
395    }
396}
397
398fn decode_flat_xyxy6(raw: &[f32], conf_thresh: f32) -> Option<Vec<RawBox>> {
399    if !raw.len().is_multiple_of(6) || raw.len() < 6 {
400        return None;
401    }
402    let mut out = Vec::new();
403    for chunk in raw.chunks_exact(6) {
404        let score = chunk[4];
405        if score < conf_thresh {
406            continue;
407        }
408        if chunk[2] <= chunk[0] || chunk[3] <= chunk[1] {
409            return None;
410        }
411        out.push(RawBox {
412            x1: chunk[0],
413            y1: chunk[1],
414            x2: chunk[2],
415            y2: chunk[3],
416            score,
417            class: Some(chunk[5] as u16),
418        });
419    }
420    if out.is_empty() { None } else { Some(out) }
421}
422
423#[cfg(test)]
424mod tests {
425    use super::*;
426
427    #[test]
428    fn empty_shaped_head_is_ok() {
429        let raw = vec![0.0_f32; 6 * 4];
430        let boxes = decode_detector_output(&raw, &[1, 6, 4], 640.0, 640.0, 0.25).unwrap();
431        assert!(boxes.is_empty());
432    }
433
434    #[test]
435    fn parse_flat_boxes() {
436        let raw = [
437            10.0_f32, 10.0, 50.0, 80.0, 0.9, 0.0, 1.0, 1.0, 2.0, 2.0, 0.05, 1.0,
438        ];
439        let boxes = decode_detector_output(&raw, &[], 640.0, 640.0, 0.25).unwrap();
440        assert_eq!(boxes.len(), 1);
441        assert!((boxes[0].score - 0.9).abs() < 1e-5);
442        assert_eq!(boxes[0].class, Some(0));
443    }
444
445    #[test]
446    fn parse_yolov8_transposed() {
447        // shape [1, 6, 2] = 4 box + 2 classes, two predictions, channels-first
448        let n = 2_usize;
449        let c = 6_usize;
450        let mut raw = vec![0.0_f32; c * n];
451        // pred 0: cx=320, cy=320, w=40, h=80, cls0=0.1, cls1=0.9
452        raw[0] = 320.0;
453        raw[n] = 320.0;
454        raw[2 * n] = 40.0;
455        raw[3 * n] = 80.0;
456        raw[4 * n] = 0.1;
457        raw[5 * n] = 0.9;
458        // pred 1: low score
459        raw[1] = 10.0;
460        raw[n + 1] = 10.0;
461        raw[2 * n + 1] = 4.0;
462        raw[3 * n + 1] = 4.0;
463        raw[4 * n + 1] = 0.01;
464        raw[5 * n + 1] = 0.02;
465        let boxes = decode_detector_output(&raw, &[1, 6, 2], 640.0, 640.0, 0.25).unwrap();
466        assert_eq!(boxes.len(), 1);
467        assert_eq!(boxes[0].class, Some(1));
468        assert!((boxes[0].score - 0.9).abs() < 1e-5);
469        assert!((boxes[0].x1 - 300.0).abs() < 1e-3);
470        assert!((boxes[0].y2 - 360.0).abs() < 1e-3);
471    }
472
473    #[test]
474    fn parse_yolov5_rows() {
475        // [1, 2, 85] = two YOLOv5-COCO rows (obj × class).
476        let c = 85_usize;
477        let mut raw = vec![0.0_f32; 2 * c];
478        raw[0] = 100.0;
479        raw[1] = 50.0;
480        raw[2] = 20.0;
481        raw[3] = 10.0;
482        raw[4] = 0.8;
483        raw[5] = 0.5; // score 0.40
484        raw[c] = 200.0;
485        raw[c + 1] = 80.0;
486        raw[c + 2] = 10.0;
487        raw[c + 3] = 10.0;
488        raw[c + 4] = 0.9;
489        raw[c + 5] = 0.9; // score 0.81
490        let boxes = decode_detector_output(&raw, &[1, 2, 85], 640.0, 640.0, 0.5).unwrap();
491        assert_eq!(boxes.len(), 1);
492        assert!((boxes[0].score - 0.81).abs() < 1e-5);
493        assert_eq!(boxes[0].class, Some(0));
494    }
495
496    #[test]
497    fn nms_drops_overlap_keeps_class_split() {
498        let meta = Letterbox::stretch(640, 640, 640, 640);
499        let boxes = [
500            RawBox {
501                x1: 10.0,
502                y1: 10.0,
503                x2: 50.0,
504                y2: 50.0,
505                score: 0.9,
506                class: Some(0),
507            },
508            RawBox {
509                x1: 12.0,
510                y1: 12.0,
511                x2: 52.0,
512                y2: 52.0,
513                score: 0.8,
514                class: Some(0),
515            },
516            RawBox {
517                x1: 12.0,
518                y1: 12.0,
519                x2: 52.0,
520                y2: 52.0,
521                score: 0.85,
522                class: Some(1),
523            },
524        ];
525        let dets =
526            detections_from_raw_boxes(&boxes, meta, DetectorDecodeConfig::default()).unwrap();
527        assert_eq!(dets.len(), 2);
528        assert_eq!(dets[0].class_id(), Some(ClassId(0)));
529        assert_eq!(dets[1].class_id(), Some(ClassId(1)));
530    }
531
532    #[test]
533    fn letterbox_maps_boxes_back() {
534        let src = vec![0_u8; 40 * 20 * 3];
535        let (_, meta) = crate::preprocess::letterbox_rgb8(&src, 40, 20, 64, 64, 114).unwrap();
536        let boxes = [RawBox {
537            x1: meta.pad_x,
538            y1: meta.pad_y,
539            x2: meta.pad_x + 40.0 * meta.scale_x,
540            y2: meta.pad_y + 20.0 * meta.scale_y,
541            score: 0.99,
542            class: Some(0),
543        }];
544        let dets =
545            detections_from_raw_boxes(&boxes, meta, DetectorDecodeConfig::default()).unwrap();
546        assert_eq!(dets.len(), 1);
547        let b = dets[0].bbox();
548        assert!(b.left() < 2.0);
549        assert!(b.top() < 2.0);
550        assert!((b.right() - 40.0).abs() < 2.0);
551        assert!((b.bottom() - 20.0).abs() < 2.0);
552    }
553}