Skip to main content

edgefirst_decoder/
yolo.rs

1// SPDX-FileCopyrightText: Copyright 2025 Au-Zone Technologies
2// SPDX-License-Identifier: Apache-2.0
3
4//! Internal YOLO decoder kernels.
5//!
6//! All items in this module are `pub(crate)`. The public API surface for
7//! decoding is [`crate::Decoder`] + [`crate::DecoderBuilder`]; external
8//! callers must go through that entry point so we can evolve the kernels
9//! (split paths, dispatch tables, NEON tiers) without breaking semver.
10//! See `CHANGELOG.md` for the 0.20.0 narrowing.
11
12use std::fmt::Debug;
13
14use ndarray::{
15    parallel::prelude::{IntoParallelIterator, ParallelIterator},
16    s, Array2, Array3, ArrayView1, ArrayView2, ArrayView3,
17};
18use num_traits::{AsPrimitive, Float, PrimInt, Signed};
19
20use crate::{
21    byte::{
22        nms_class_aware_int, nms_extra_class_aware_int, nms_extra_int, nms_int,
23        postprocess_boxes_index_quant, postprocess_boxes_quant, quantize_score_threshold,
24    },
25    configs::Nms,
26    dequant_detect_box,
27    float::{
28        nms_class_aware_float, nms_extra_class_aware_float, nms_extra_float, nms_float,
29        postprocess_boxes_float, postprocess_boxes_index_float,
30        postprocess_boxes_multilabel_index_float,
31    },
32    BBoxTypeTrait, BoundingBox, DetectBox, DetectBoxQuantized, ProtoData, ProtoLayout,
33    Quantization, Segmentation, XYWH, XYXY,
34};
35
36/// Maximum number of above-threshold candidates fed to NMS.
37///
38/// At very low score thresholds (e.g., t=0.01 on YOLOv8 with
39/// 8400 anchors × 80 classes), the number of survivors approaches
40/// the full 672 000-entry score grid. NMS is O(n²) and the
41/// downstream mask matmul runs once per survivor, so an
42/// unbounded set produces minutes-per-frame decode times.
43///
44/// `MAX_NMS_CANDIDATES` matches the Ultralytics `max_nms` default
45/// and is applied as a top-K-by-score truncation immediately
46/// before NMS. Values above the cap are silently dropped — at the
47/// score thresholds where the cap activates the bottom of the
48/// candidate list is dominated by noise that NMS would discard
49/// anyway.
50///
51/// Production callers configure this via [`crate::DecoderBuilder::with_pre_nms_top_k`];
52/// only the test-only seg-det wrappers reach for this constant directly.
53#[cfg(test)]
54pub(crate) const MAX_NMS_CANDIDATES: usize = 30_000;
55
56/// Default post-NMS detection cap used by the public `decode_yolo_*`
57/// convenience wrappers when no explicit cap is plumbed in. Mirrors the
58/// `Decoder::max_det` default set by `DecoderBuilder` (also 300, matching
59/// the Ultralytics `max_det` default). Pre-EDGEAI-1302 these wrappers
60/// used `output_boxes.capacity()` as the cap, which silently dropped all
61/// detections when the caller passed `Vec::new()`.
62pub(crate) const DEFAULT_MAX_DETECTIONS: usize = 300;
63
64/// Truncate `boxes` to the highest-scoring `top_k` entries in-place when the
65/// input exceeds the cap. Uses partial sort (O(N)) via `select_nth_unstable_by`
66/// to avoid full O(N log N) sort. No-op when `top_k` is 0 (unbounded) or
67/// when the input length ≤ `top_k`.
68fn truncate_to_top_k_by_score<E: Send>(boxes: &mut Vec<(DetectBox, E)>, top_k: usize) {
69    if top_k > 0 && boxes.len() > top_k {
70        boxes.select_nth_unstable_by(top_k, |a, b| b.0.score.total_cmp(&a.0.score));
71        boxes.truncate(top_k);
72    }
73}
74
75/// Quantized counterpart of [`truncate_to_top_k_by_score`]. Sorts on
76/// the raw quantized score (which preserves order under monotonic
77/// dequantization). Uses partial sort (O(N)) via `select_nth_unstable_by`.
78/// No-op when `top_k` is 0 (unbounded) or when the input length ≤ `top_k`.
79fn truncate_to_top_k_by_score_quant<S: PrimInt + AsPrimitive<f32> + Send + Sync, E: Send>(
80    boxes: &mut Vec<(DetectBoxQuantized<S>, E)>,
81    top_k: usize,
82) {
83    if top_k > 0 && boxes.len() > top_k {
84        boxes.select_nth_unstable_by(top_k, |a, b| b.0.score.cmp(&a.0.score));
85        boxes.truncate(top_k);
86    }
87}
88
89/// Dispatches to the appropriate NMS function based on mode for float boxes.
90///
91/// `max_det` is the post-NMS detection cap; when present it lets the greedy
92/// inner loop break as soon as that many survivors are confirmed (the survivors
93/// are guaranteed to be the top-`max_det` by score because the input is sorted
94/// descending). Pass `None` to run the full O(N²) suppression.
95fn dispatch_nms_float(
96    nms: Option<Nms>,
97    iou: f32,
98    max_det: Option<usize>,
99    boxes: Vec<DetectBox>,
100) -> Vec<DetectBox> {
101    match nms {
102        Some(Nms::ClassAgnostic | Nms::Auto) => nms_float(iou, max_det, boxes),
103        Some(Nms::ClassAware) => nms_class_aware_float(iou, max_det, boxes),
104        None => boxes, // bypass NMS
105    }
106}
107
108/// Dispatches to the appropriate NMS function based on mode for float boxes
109/// with extra data.
110pub(super) fn dispatch_nms_extra_float<E: Send + Sync>(
111    nms: Option<Nms>,
112    iou: f32,
113    max_det: Option<usize>,
114    boxes: Vec<(DetectBox, E)>,
115) -> Vec<(DetectBox, E)> {
116    match nms {
117        Some(Nms::ClassAgnostic | Nms::Auto) => nms_extra_float(iou, max_det, boxes),
118        Some(Nms::ClassAware) => nms_extra_class_aware_float(iou, max_det, boxes),
119        None => boxes, // bypass NMS
120    }
121}
122
123/// Dispatches to the appropriate NMS function based on mode for quantized
124/// boxes.
125fn dispatch_nms_int<SCORE: PrimInt + AsPrimitive<f32> + Send + Sync>(
126    nms: Option<Nms>,
127    iou: f32,
128    max_det: Option<usize>,
129    boxes: Vec<DetectBoxQuantized<SCORE>>,
130) -> Vec<DetectBoxQuantized<SCORE>> {
131    match nms {
132        Some(Nms::ClassAgnostic | Nms::Auto) => nms_int(iou, max_det, boxes),
133        Some(Nms::ClassAware) => nms_class_aware_int(iou, max_det, boxes),
134        None => boxes, // bypass NMS
135    }
136}
137
138/// Dispatches to the appropriate NMS function based on mode for quantized boxes
139/// with extra data.
140fn dispatch_nms_extra_int<SCORE: PrimInt + AsPrimitive<f32> + Send + Sync, E: Send + Sync>(
141    nms: Option<Nms>,
142    iou: f32,
143    max_det: Option<usize>,
144    boxes: Vec<(DetectBoxQuantized<SCORE>, E)>,
145) -> Vec<(DetectBoxQuantized<SCORE>, E)> {
146    match nms {
147        Some(Nms::ClassAgnostic | Nms::Auto) => nms_extra_int(iou, max_det, boxes),
148        Some(Nms::ClassAware) => nms_extra_class_aware_int(iou, max_det, boxes),
149        None => boxes, // bypass NMS
150    }
151}
152
153/// Detection cap helper for the public free `decode_yolo_*` wrappers.
154///
155/// Encodes the convention documented above: if the caller passed a non-empty
156/// `Vec`, that capacity acts as the per-call cap; otherwise fall back to
157/// [`DEFAULT_MAX_DETECTIONS`] so freshly-constructed `Vec::new()` outputs
158/// don't silently drop every detection.
159#[inline]
160fn cap_or_default<T>(v: &Vec<T>) -> usize {
161    if v.capacity() > 0 {
162        v.capacity()
163    } else {
164        DEFAULT_MAX_DETECTIONS
165    }
166}
167
168// ─── Public free decode_yolo_* convenience wrappers ────────────────────────
169//
170// Detection cap convention (applies to every `decode_yolo_*` free function
171// below):
172//
173// These functions are the convenience layer for callers that don't go
174// through `Decoder::decode()` (benches, FFI shims, ad-hoc test harnesses).
175// They use **`output_boxes.capacity()` as a per-call detection cap**:
176//
177//   - When the caller passes `Vec::with_capacity(N)`, the post-NMS output
178//     is truncated to at most `N` detections.
179//   - When the caller passes `Vec::new()` (capacity 0), the implementation
180//     falls back to the [`DEFAULT_MAX_DETECTIONS`] constant (300) so a
181//     freshly-constructed `Vec` doesn't silently drop every detection.
182//
183// This is intentionally **different** from the `Decoder::decode()` /
184// `Decoder::decode_proto()` contract, which bounds output count solely
185// by [`Decoder::max_det`] (set via `DecoderBuilder::with_max_det`,
186// default 300) regardless of the caller's `Vec` capacity (EDGEAI-1302).
187//
188// Use the `Decoder` API when you need explicit control over `max_det`,
189// schema-driven decoding, or EDGEAI-1303 normalization. Use these free
190// functions when you have raw tensors in hand and want a one-shot decode.
191
192/// Decodes YOLO detection outputs from quantized tensors into detection boxes.
193///
194/// Boxes are expected to be in XYWH format.
195///
196/// Expected shapes of inputs:
197/// - output: (4 + num_classes, num_boxes)
198///
199/// See the "Detection cap convention" comment above for how
200/// `output_boxes.capacity()` bounds the result count.
201pub(crate) fn decode_yolo_det<BOX: PrimInt + AsPrimitive<f32> + Send + Sync>(
202    output: (ArrayView2<BOX>, Quantization),
203    score_threshold: f32,
204    iou_threshold: f32,
205    nms: Option<Nms>,
206    output_boxes: &mut Vec<DetectBox>,
207) where
208    f32: AsPrimitive<BOX>,
209{
210    impl_yolo_quant::<XYWH, _>(output, score_threshold, iou_threshold, nms, output_boxes);
211}
212
213/// Decodes YOLO detection outputs from float tensors into detection boxes.
214///
215/// Boxes are expected to be in XYWH format.
216///
217/// Expected shapes of inputs:
218/// - output: (4 + num_classes, num_boxes)
219pub(crate) fn decode_yolo_det_float<T>(
220    output: ArrayView2<T>,
221    score_threshold: f32,
222    iou_threshold: f32,
223    nms: Option<Nms>,
224    output_boxes: &mut Vec<DetectBox>,
225) where
226    T: Float + AsPrimitive<f32> + Send + Sync + 'static,
227    f32: AsPrimitive<T>,
228{
229    impl_yolo_float::<XYWH, _>(output, score_threshold, iou_threshold, nms, output_boxes);
230}
231
232/// Test-only seg-det quantized decode shim.
233///
234/// Production callers go through [`crate::Decoder::decode`]; this wrapper
235/// exists only so the parity tests in `lib.rs` and `yolo.rs` can compare the
236/// kernel output against the Decoder output without duplicating the
237/// `impl_yolo_segdet_quant` argument plumbing.
238///
239/// Boxes are expected to be in XYWH format. Expected shapes:
240/// - `boxes`: `(4 + num_classes + num_protos, num_boxes)`
241/// - `protos`: `(proto_height, proto_width, num_protos)`
242///
243/// # Errors
244/// Returns `DecoderError::InvalidShape` if bounding boxes are not normalized.
245#[cfg(test)]
246pub(crate) fn decode_yolo_segdet_quant<
247    BOX: PrimInt + AsPrimitive<i64> + AsPrimitive<i128> + AsPrimitive<f32> + Send + Sync,
248    PROTO: PrimInt + AsPrimitive<i64> + AsPrimitive<i128> + AsPrimitive<f32> + Send + Sync,
249>(
250    boxes: (ArrayView2<BOX>, Quantization),
251    protos: (ArrayView3<PROTO>, Quantization),
252    score_threshold: f32,
253    iou_threshold: f32,
254    nms: Option<Nms>,
255    output_boxes: &mut Vec<DetectBox>,
256    output_masks: &mut Vec<Segmentation>,
257) -> Result<(), crate::DecoderError>
258where
259    f32: AsPrimitive<BOX>,
260{
261    // Pre-Decoder convenience wrapper: no schema-derived `normalized`
262    // or `input_dims`, so the EDGEAI-1303 normalization is a no-op.
263    // Callers that need that path should go through `DecoderBuilder`
264    // with a schema.
265    let cap = cap_or_default(output_boxes);
266    impl_yolo_segdet_quant::<XYWH, _, _>(
267        boxes,
268        protos,
269        score_threshold,
270        iou_threshold,
271        nms,
272        MAX_NMS_CANDIDATES,
273        cap,
274        None,
275        None,
276        output_boxes,
277        output_masks,
278    )
279}
280
281/// Test-only seg-det float decode shim. See [`decode_yolo_segdet_quant`].
282#[cfg(test)]
283pub(crate) fn decode_yolo_segdet_float<T>(
284    boxes: ArrayView2<T>,
285    protos: ArrayView3<T>,
286    score_threshold: f32,
287    iou_threshold: f32,
288    nms: Option<Nms>,
289    output_boxes: &mut Vec<DetectBox>,
290    output_masks: &mut Vec<Segmentation>,
291) -> Result<(), crate::DecoderError>
292where
293    T: Float + AsPrimitive<f32> + Send + Sync + 'static,
294    f32: AsPrimitive<T>,
295{
296    // Pre-Decoder convenience wrapper: schema-derived normalization is
297    // not available here (see EDGEAI-1303 note in the quantized sibling).
298    let cap = cap_or_default(output_boxes);
299    impl_yolo_segdet_float::<XYWH, _, _>(
300        boxes,
301        protos,
302        score_threshold,
303        iou_threshold,
304        nms,
305        MAX_NMS_CANDIDATES,
306        cap,
307        None,
308        None,
309        false, // multi_label: test shim stays on argmax
310        output_boxes,
311        output_masks,
312    )
313}
314
315/// Decodes YOLO split detection outputs from quantized tensors into detection
316/// boxes.
317///
318/// Boxes are expected to be in XYWH format.
319///
320/// Expected shapes of inputs:
321/// - boxes: (4, num_boxes)
322/// - scores: (num_classes, num_boxes)
323///
324/// # Panics
325/// Panics if shapes don't match the expected dimensions.
326pub(crate) fn decode_yolo_split_det_quant<
327    BOX: PrimInt + AsPrimitive<i32> + AsPrimitive<f32> + Send + Sync,
328    SCORE: PrimInt + AsPrimitive<f32> + Send + Sync,
329>(
330    boxes: (ArrayView2<BOX>, Quantization),
331    scores: (ArrayView2<SCORE>, Quantization),
332    score_threshold: f32,
333    iou_threshold: f32,
334    nms: Option<Nms>,
335    output_boxes: &mut Vec<DetectBox>,
336) where
337    f32: AsPrimitive<SCORE>,
338{
339    impl_yolo_split_quant::<XYWH, _, _>(
340        boxes,
341        scores,
342        score_threshold,
343        iou_threshold,
344        nms,
345        output_boxes,
346    );
347}
348
349/// Decodes YOLO split detection outputs from float tensors into detection
350/// boxes.
351///
352/// Boxes are expected to be in XYWH format.
353///
354/// Expected shapes of inputs:
355/// - boxes: (4, num_boxes)
356/// - scores: (num_classes, num_boxes)
357///
358/// # Panics
359/// Panics if shapes don't match the expected dimensions.
360pub(crate) fn decode_yolo_split_det_float<T>(
361    boxes: ArrayView2<T>,
362    scores: ArrayView2<T>,
363    score_threshold: f32,
364    iou_threshold: f32,
365    nms: Option<Nms>,
366    output_boxes: &mut Vec<DetectBox>,
367) where
368    T: Float + AsPrimitive<f32> + Send + Sync + 'static,
369    f32: AsPrimitive<T>,
370{
371    impl_yolo_split_float::<XYWH, _, _>(
372        boxes,
373        scores,
374        score_threshold,
375        iou_threshold,
376        nms,
377        output_boxes,
378    );
379}
380
381/// Decodes end-to-end YOLO detection outputs (post-NMS from model).
382/// Expects an array of shape `(6, N)`, where the first dimension (rows)
383/// corresponds to the 6 per-detection features
384/// `[x1, y1, x2, y2, conf, class]` and the second dimension (columns)
385/// indexes the `N` detections.
386/// Boxes are output directly without NMS (the model already applied NMS).
387///
388/// Coordinates may be normalized `[0, 1]` or absolute pixel values depending
389/// on the model configuration. The caller should check
390/// `decoder.normalized_boxes()` to determine which.
391///
392/// # Errors
393///
394/// Returns `DecoderError::InvalidShape` if `output` has fewer than 6 rows.
395pub(crate) fn decode_yolo_end_to_end_det_float<T>(
396    output: ArrayView2<T>,
397    score_threshold: f32,
398    output_boxes: &mut Vec<DetectBox>,
399) -> Result<(), crate::DecoderError>
400where
401    T: Float + AsPrimitive<f32> + Send + Sync + 'static,
402    f32: AsPrimitive<T>,
403{
404    // Validate input shape: need at least 6 rows (x1, y1, x2, y2, conf, class)
405    if output.shape()[0] < 6 {
406        return Err(crate::DecoderError::InvalidShape(format!(
407            "End-to-end detection output requires at least 6 rows, got {}",
408            output.shape()[0]
409        )));
410    }
411
412    // Input shape: (6, N) -> transpose to (N, 4) for boxes and (N, 1) for scores
413    let boxes = output.slice(s![0..4, ..]).reversed_axes();
414    let scores = output.slice(s![4..5, ..]).reversed_axes();
415    let classes = output.slice(s![5, ..]);
416    let mut boxes =
417        postprocess_boxes_index_float::<XYXY, _, _>(score_threshold.as_(), boxes, scores);
418    boxes.truncate(cap_or_default(output_boxes));
419    output_boxes.clear();
420    for (mut b, i) in boxes.into_iter() {
421        b.label = classes[i].as_() as usize;
422        output_boxes.push(b);
423    }
424    // No NMS — model output is already post-NMS
425    Ok(())
426}
427
428/// Decodes end-to-end YOLO detection + segmentation outputs (post-NMS from
429/// model).
430///
431/// Input shapes:
432/// - detection: (6 + num_protos, N) where rows are [x1, y1, x2, y2, conf,
433///   class, mask_coeff_0, ..., mask_coeff_31]
434/// - protos: (proto_height, proto_width, num_protos)
435///
436/// Boxes are output directly without NMS (model already applied NMS).
437/// Coordinates may be normalized [0,1] or pixel values depending on model
438/// config.
439///
440/// # Errors
441///
442/// Returns `DecoderError::InvalidShape` if:
443/// - output has fewer than 7 rows (6 base + at least 1 mask coefficient)
444/// - protos shape doesn't match mask coefficients count
445pub(crate) fn decode_yolo_end_to_end_segdet_float<T>(
446    output: ArrayView2<T>,
447    protos: ArrayView3<T>,
448    score_threshold: f32,
449    output_boxes: &mut Vec<DetectBox>,
450    output_masks: &mut Vec<crate::Segmentation>,
451) -> Result<(), crate::DecoderError>
452where
453    T: Float + AsPrimitive<f32> + Send + Sync + 'static,
454    f32: AsPrimitive<T>,
455{
456    let (boxes, scores, classes, mask_coeff) =
457        postprocess_yolo_end_to_end_segdet(&output, protos.dim().2)?;
458    let cap = cap_or_default(output_boxes);
459    let boxes = impl_yolo_end_to_end_segdet_get_boxes::<XYXY, _, _, _>(
460        boxes,
461        scores,
462        classes,
463        score_threshold,
464        cap,
465    );
466
467    // No NMS — model output is already post-NMS
468
469    impl_yolo_split_segdet_process_masks(boxes, mask_coeff, protos, output_boxes, output_masks)
470}
471
472/// Decodes split end-to-end YOLO detection outputs (post-NMS from model).
473///
474/// Input shapes (after batch dim removed):
475/// - boxes: (4, N) — xyxy pixel coordinates
476/// - scores: (1, N) — confidence of the top class
477/// - classes: (1, N) — class index of the top class
478///
479/// Boxes are output directly without NMS (model already applied NMS).
480pub(crate) fn decode_yolo_split_end_to_end_det_float<T: Float + AsPrimitive<f32>>(
481    boxes: ArrayView2<T>,
482    scores: ArrayView2<T>,
483    classes: ArrayView2<T>,
484    score_threshold: f32,
485    output_boxes: &mut Vec<DetectBox>,
486) -> Result<(), crate::DecoderError> {
487    let n = boxes.shape()[1];
488
489    let cap = cap_or_default(output_boxes);
490    output_boxes.clear();
491
492    let (boxes, scores, classes) = postprocess_yolo_split_end_to_end_det(boxes, scores, &classes)?;
493
494    for i in 0..n {
495        let score: f32 = scores[[i, 0]].as_();
496        if score < score_threshold {
497            continue;
498        }
499        if output_boxes.len() >= cap {
500            break;
501        }
502        output_boxes.push(DetectBox {
503            bbox: BoundingBox {
504                xmin: boxes[[i, 0]].as_(),
505                ymin: boxes[[i, 1]].as_(),
506                xmax: boxes[[i, 2]].as_(),
507                ymax: boxes[[i, 3]].as_(),
508            },
509            score,
510            label: classes[i].as_() as usize,
511        });
512    }
513    Ok(())
514}
515
516/// Decodes split end-to-end YOLO detection + segmentation outputs.
517///
518/// Input shapes (after batch dim removed):
519/// - boxes: (4, N) — xyxy pixel coordinates
520/// - scores: (1, N) — confidence
521/// - classes: (1, N) — class index
522/// - mask_coeff: (num_protos, N) — mask coefficients per detection
523/// - protos: (proto_h, proto_w, num_protos) — prototype masks
524#[allow(clippy::too_many_arguments)]
525pub(crate) fn decode_yolo_split_end_to_end_segdet_float<T>(
526    boxes: ArrayView2<T>,
527    scores: ArrayView2<T>,
528    classes: ArrayView2<T>,
529    mask_coeff: ArrayView2<T>,
530    protos: ArrayView3<T>,
531    score_threshold: f32,
532    output_boxes: &mut Vec<DetectBox>,
533    output_masks: &mut Vec<crate::Segmentation>,
534) -> Result<(), crate::DecoderError>
535where
536    T: Float + AsPrimitive<f32> + Send + Sync + 'static,
537    f32: AsPrimitive<T>,
538{
539    let (boxes, scores, classes, mask_coeff) =
540        postprocess_yolo_split_end_to_end_segdet(boxes, scores, &classes, mask_coeff)?;
541    let cap = cap_or_default(output_boxes);
542    let boxes = impl_yolo_end_to_end_segdet_get_boxes::<XYXY, _, _, _>(
543        boxes,
544        scores,
545        classes,
546        score_threshold,
547        cap,
548    );
549
550    impl_yolo_split_segdet_process_masks(boxes, mask_coeff, protos, output_boxes, output_masks)
551}
552
553#[allow(clippy::type_complexity)]
554pub(crate) fn postprocess_yolo_end_to_end_segdet<'a, T>(
555    output: &'a ArrayView2<'_, T>,
556    num_protos: usize,
557) -> Result<
558    (
559        ArrayView2<'a, T>,
560        ArrayView2<'a, T>,
561        ArrayView1<'a, T>,
562        ArrayView2<'a, T>,
563    ),
564    crate::DecoderError,
565> {
566    // Validate input shape: need at least 7 rows (6 base + at least 1 mask coeff)
567    if output.shape()[0] < 7 {
568        return Err(crate::DecoderError::InvalidShape(format!(
569            "End-to-end segdet output requires at least 7 rows, got {}",
570            output.shape()[0]
571        )));
572    }
573
574    let num_mask_coeffs = output.shape()[0] - 6;
575    if num_mask_coeffs != num_protos {
576        return Err(crate::DecoderError::InvalidShape(format!(
577            "Mask coefficients count ({}) doesn't match protos count ({})",
578            num_mask_coeffs, num_protos
579        )));
580    }
581
582    // Input shape: (6+num_protos, N) -> transpose for postprocessing
583    let boxes = output.slice(s![0..4, ..]).reversed_axes();
584    let scores = output.slice(s![4..5, ..]).reversed_axes();
585    let classes = output.slice(s![5, ..]);
586    let mask_coeff = output.slice(s![6.., ..]).reversed_axes();
587    Ok((boxes, scores, classes, mask_coeff))
588}
589
590/// Postprocess yolo split end to end det by reversing axes of boxes,
591/// scores, and flattening the class tensor.
592/// Expected input shapes:
593/// - boxes: (4, N)
594/// - scores: (1, N)
595/// - classes: (1, N)
596#[allow(clippy::type_complexity)]
597pub(crate) fn postprocess_yolo_split_end_to_end_det<'a, 'b, 'c, BOXES, SCORES, CLASS>(
598    boxes: ArrayView2<'a, BOXES>,
599    scores: ArrayView2<'b, SCORES>,
600    classes: &'c ArrayView2<CLASS>,
601) -> Result<
602    (
603        ArrayView2<'a, BOXES>,
604        ArrayView2<'b, SCORES>,
605        ArrayView1<'c, CLASS>,
606    ),
607    crate::DecoderError,
608> {
609    let num_boxes = boxes.shape()[1];
610    if boxes.shape()[0] != 4 {
611        return Err(crate::DecoderError::InvalidShape(format!(
612            "Split end-to-end box_coords must be 4, got {}",
613            boxes.shape()[0]
614        )));
615    }
616
617    if scores.shape()[0] != 1 {
618        return Err(crate::DecoderError::InvalidShape(format!(
619            "Split end-to-end scores num_classes must be 1, got {}",
620            scores.shape()[0]
621        )));
622    }
623
624    if classes.shape()[0] != 1 {
625        return Err(crate::DecoderError::InvalidShape(format!(
626            "Split end-to-end classes num_classes must be 1, got {}",
627            classes.shape()[0]
628        )));
629    }
630
631    if scores.shape()[1] != num_boxes {
632        return Err(crate::DecoderError::InvalidShape(format!(
633            "Split end-to-end scores must have same num_boxes as boxes ({}), got {}",
634            num_boxes,
635            scores.shape()[1]
636        )));
637    }
638
639    if classes.shape()[1] != num_boxes {
640        return Err(crate::DecoderError::InvalidShape(format!(
641            "Split end-to-end classes must have same num_boxes as boxes ({}), got {}",
642            num_boxes,
643            classes.shape()[1]
644        )));
645    }
646
647    let boxes = boxes.reversed_axes();
648    let scores = scores.reversed_axes();
649    let classes = classes.slice(s![0, ..]);
650    Ok((boxes, scores, classes))
651}
652
653/// Postprocess yolo split end to end segdet by reversing axes of boxes,
654/// scores, mask tensors and flattening the class tensor.
655#[allow(clippy::type_complexity)]
656pub(crate) fn postprocess_yolo_split_end_to_end_segdet<
657    'a,
658    'b,
659    'c,
660    'd,
661    BOXES,
662    SCORES,
663    CLASS,
664    MASK,
665>(
666    boxes: ArrayView2<'a, BOXES>,
667    scores: ArrayView2<'b, SCORES>,
668    classes: &'c ArrayView2<CLASS>,
669    mask_coeff: ArrayView2<'d, MASK>,
670) -> Result<
671    (
672        ArrayView2<'a, BOXES>,
673        ArrayView2<'b, SCORES>,
674        ArrayView1<'c, CLASS>,
675        ArrayView2<'d, MASK>,
676    ),
677    crate::DecoderError,
678> {
679    let num_boxes = boxes.shape()[1];
680    if boxes.shape()[0] != 4 {
681        return Err(crate::DecoderError::InvalidShape(format!(
682            "Split end-to-end box_coords must be 4, got {}",
683            boxes.shape()[0]
684        )));
685    }
686
687    if scores.shape()[0] != 1 {
688        return Err(crate::DecoderError::InvalidShape(format!(
689            "Split end-to-end scores num_classes must be 1, got {}",
690            scores.shape()[0]
691        )));
692    }
693
694    if classes.shape()[0] != 1 {
695        return Err(crate::DecoderError::InvalidShape(format!(
696            "Split end-to-end classes num_classes must be 1, got {}",
697            classes.shape()[0]
698        )));
699    }
700
701    if scores.shape()[1] != num_boxes {
702        return Err(crate::DecoderError::InvalidShape(format!(
703            "Split end-to-end scores must have same num_boxes as boxes ({}), got {}",
704            num_boxes,
705            scores.shape()[1]
706        )));
707    }
708
709    if classes.shape()[1] != num_boxes {
710        return Err(crate::DecoderError::InvalidShape(format!(
711            "Split end-to-end classes must have same num_boxes as boxes ({}), got {}",
712            num_boxes,
713            classes.shape()[1]
714        )));
715    }
716
717    if mask_coeff.shape()[1] != num_boxes {
718        return Err(crate::DecoderError::InvalidShape(format!(
719            "Split end-to-end mask_coeff must have same num_boxes as boxes ({}), got {}",
720            num_boxes,
721            mask_coeff.shape()[1]
722        )));
723    }
724
725    let boxes = boxes.reversed_axes();
726    let scores = scores.reversed_axes();
727    let classes = classes.slice(s![0, ..]);
728    let mask_coeff = mask_coeff.reversed_axes();
729    Ok((boxes, scores, classes, mask_coeff))
730}
731/// Internal implementation of YOLO decoding for quantized tensors.
732///
733/// Expected shapes of inputs:
734/// - output: (4 + num_classes, num_boxes)
735pub(crate) fn impl_yolo_quant<B: BBoxTypeTrait, T: PrimInt + AsPrimitive<f32> + Send + Sync>(
736    output: (ArrayView2<T>, Quantization),
737    score_threshold: f32,
738    iou_threshold: f32,
739    nms: Option<Nms>,
740    output_boxes: &mut Vec<DetectBox>,
741) where
742    f32: AsPrimitive<T>,
743{
744    let _span = tracing::trace_span!("decoder.decode.yolo_quant_flat").entered();
745    let (boxes, quant_boxes) = output;
746    let (boxes_tensor, scores_tensor) = postprocess_yolo(&boxes);
747
748    let boxes = {
749        let score_threshold = quantize_score_threshold(score_threshold, quant_boxes);
750        postprocess_boxes_quant::<B, _, _>(
751            score_threshold,
752            boxes_tensor,
753            scores_tensor,
754            quant_boxes,
755        )
756    };
757
758    let cap = cap_or_default(output_boxes);
759    let boxes = dispatch_nms_int(nms, iou_threshold, Some(cap), boxes);
760    // Detection cap convention (see `cap_or_default`). NMS already capped to
761    // `cap`; the `min` here is a redundant guard for non-NMS bypass mode.
762    let len = cap.min(boxes.len());
763    output_boxes.clear();
764    for b in boxes.iter().take(len) {
765        output_boxes.push(dequant_detect_box(b, quant_boxes));
766    }
767}
768
769/// Internal implementation of YOLO decoding for float tensors.
770///
771/// Expected shapes of inputs:
772/// - output: (4 + num_classes, num_boxes)
773pub(crate) fn impl_yolo_float<B: BBoxTypeTrait, T: Float + AsPrimitive<f32> + Send + Sync>(
774    output: ArrayView2<T>,
775    score_threshold: f32,
776    iou_threshold: f32,
777    nms: Option<Nms>,
778    output_boxes: &mut Vec<DetectBox>,
779) where
780    f32: AsPrimitive<T>,
781{
782    let _span = tracing::trace_span!("decoder.decode.yolo_float_flat").entered();
783    let (boxes_tensor, scores_tensor) = postprocess_yolo(&output);
784    let boxes =
785        postprocess_boxes_float::<B, _, _>(score_threshold.as_(), boxes_tensor, scores_tensor);
786    let cap = cap_or_default(output_boxes);
787    let boxes = dispatch_nms_float(nms, iou_threshold, Some(cap), boxes);
788    // Detection cap convention (see `cap_or_default`). NMS already capped to
789    // `cap`; the `min` here is a redundant guard for non-NMS bypass mode.
790    let len = cap.min(boxes.len());
791    output_boxes.clear();
792    for b in boxes.into_iter().take(len) {
793        output_boxes.push(b);
794    }
795}
796
797/// Internal implementation of YOLO split detection decoding for quantized
798/// tensors.
799///
800/// Expected shapes of inputs:
801/// - boxes: (4, num_boxes)
802/// - scores: (num_classes, num_boxes)
803///
804/// # Panics
805/// Panics if shapes don't match the expected dimensions.
806pub(crate) fn impl_yolo_split_quant<
807    B: BBoxTypeTrait,
808    BOX: PrimInt + AsPrimitive<f32> + Send + Sync,
809    SCORE: PrimInt + AsPrimitive<f32> + Send + Sync,
810>(
811    boxes: (ArrayView2<BOX>, Quantization),
812    scores: (ArrayView2<SCORE>, Quantization),
813    score_threshold: f32,
814    iou_threshold: f32,
815    nms: Option<Nms>,
816    output_boxes: &mut Vec<DetectBox>,
817) where
818    f32: AsPrimitive<SCORE>,
819{
820    let _span = tracing::trace_span!("decoder.decode.yolo_quant_split").entered();
821    let (boxes_tensor, quant_boxes) = boxes;
822    let (scores_tensor, quant_scores) = scores;
823
824    let boxes_tensor = boxes_tensor.reversed_axes();
825    let scores_tensor = scores_tensor.reversed_axes();
826
827    let boxes = {
828        let score_threshold = quantize_score_threshold(score_threshold, quant_scores);
829        postprocess_boxes_quant::<B, _, _>(
830            score_threshold,
831            boxes_tensor,
832            scores_tensor,
833            quant_boxes,
834        )
835    };
836
837    let cap = cap_or_default(output_boxes);
838    let boxes = dispatch_nms_int(nms, iou_threshold, Some(cap), boxes);
839    // Detection cap convention (see `cap_or_default`). NMS already capped to
840    // `cap`; the `min` here is a redundant guard for non-NMS bypass mode.
841    let len = cap.min(boxes.len());
842    output_boxes.clear();
843    for b in boxes.iter().take(len) {
844        output_boxes.push(dequant_detect_box(b, quant_scores));
845    }
846}
847
848/// Internal implementation of YOLO split detection decoding for float tensors.
849///
850/// Expected shapes of inputs:
851/// - boxes: (4, num_boxes)
852/// - scores: (num_classes, num_boxes)
853///
854/// # Panics
855/// Panics if shapes don't match the expected dimensions.
856pub(crate) fn impl_yolo_split_float<
857    B: BBoxTypeTrait,
858    BOX: Float + AsPrimitive<f32> + Send + Sync,
859    SCORE: Float + AsPrimitive<f32> + Send + Sync,
860>(
861    boxes_tensor: ArrayView2<BOX>,
862    scores_tensor: ArrayView2<SCORE>,
863    score_threshold: f32,
864    iou_threshold: f32,
865    nms: Option<Nms>,
866    output_boxes: &mut Vec<DetectBox>,
867) where
868    f32: AsPrimitive<SCORE>,
869{
870    let _span = tracing::trace_span!("decoder.decode.yolo_float_split").entered();
871    let boxes_tensor = boxes_tensor.reversed_axes();
872    let scores_tensor = scores_tensor.reversed_axes();
873    let boxes =
874        postprocess_boxes_float::<B, _, _>(score_threshold.as_(), boxes_tensor, scores_tensor);
875    let cap = cap_or_default(output_boxes);
876    let boxes = dispatch_nms_float(nms, iou_threshold, Some(cap), boxes);
877    // Detection cap convention (see `cap_or_default`). NMS already capped to
878    // `cap`; the `min` here is a redundant guard for non-NMS bypass mode.
879    let len = cap.min(boxes.len());
880    output_boxes.clear();
881    for b in boxes.into_iter().take(len) {
882        output_boxes.push(b);
883    }
884}
885
886/// Divide each survivor's bbox by `(input_w, input_h)` when the schema
887/// declares `normalized: false`. Pixel-space box coords from the model
888/// are pulled into the canonical `[0, 1]` range expected by `protobox`
889/// and downstream callers — see EDGEAI-1303.
890///
891/// No-op when `normalized` is `Some(true)` / `None`, when `input_dims`
892/// is `None`, or when there are no survivors.
893#[inline]
894pub(crate) fn maybe_normalize_boxes_in_place(
895    boxes: &mut [(DetectBox, usize)],
896    normalized: Option<bool>,
897    input_dims: Option<(usize, usize)>,
898) {
899    if normalized != Some(false) {
900        return;
901    }
902    let Some((w, h)) = input_dims else {
903        return;
904    };
905    if w == 0 || h == 0 {
906        return;
907    }
908    let inv_w = 1.0 / w as f32;
909    let inv_h = 1.0 / h as f32;
910    for (b, _) in boxes.iter_mut() {
911        b.bbox.xmin *= inv_w;
912        b.bbox.ymin *= inv_h;
913        b.bbox.xmax *= inv_w;
914        b.bbox.ymax *= inv_h;
915    }
916}
917
918/// Internal implementation of YOLO detection segmentation decoding for
919/// quantized tensors.
920///
921/// Expected shapes of inputs:
922/// - boxes: (4 + num_classes + num_protos, num_boxes)
923/// - protos: (proto_height, proto_width, num_protos)
924///
925/// # Errors
926/// Returns `DecoderError::InvalidShape` if bounding boxes are not normalized.
927#[allow(clippy::too_many_arguments)]
928pub(crate) fn impl_yolo_segdet_quant<
929    B: BBoxTypeTrait,
930    BOX: PrimInt + AsPrimitive<i64> + AsPrimitive<i128> + AsPrimitive<f32> + Send + Sync,
931    PROTO: PrimInt + AsPrimitive<i64> + AsPrimitive<i128> + AsPrimitive<f32> + Send + Sync,
932>(
933    boxes: (ArrayView2<BOX>, Quantization),
934    protos: (ArrayView3<PROTO>, Quantization),
935    score_threshold: f32,
936    iou_threshold: f32,
937    nms: Option<Nms>,
938    pre_nms_top_k: usize,
939    max_det: usize,
940    normalized: Option<bool>,
941    input_dims: Option<(usize, usize)>,
942    output_boxes: &mut Vec<DetectBox>,
943    output_masks: &mut Vec<Segmentation>,
944) -> Result<(), crate::DecoderError>
945where
946    f32: AsPrimitive<BOX>,
947{
948    let (boxes, quant_boxes) = boxes;
949    let num_protos = protos.0.dim().2;
950
951    let (boxes_tensor, scores_tensor, mask_tensor) = postprocess_yolo_seg(&boxes, num_protos);
952    let mut boxes = impl_yolo_split_segdet_quant_get_boxes::<B, _, _>(
953        (boxes_tensor, quant_boxes),
954        (scores_tensor, quant_boxes),
955        score_threshold,
956        iou_threshold,
957        nms,
958        pre_nms_top_k,
959        max_det,
960    );
961    maybe_normalize_boxes_in_place(&mut boxes, normalized, input_dims);
962
963    impl_yolo_split_segdet_quant_process_masks::<_, _>(
964        boxes,
965        (mask_tensor, quant_boxes),
966        protos,
967        output_boxes,
968        output_masks,
969    )
970}
971
972/// Internal implementation of YOLO detection segmentation decoding for
973/// float tensors.
974///
975/// Expected shapes of inputs:
976/// - boxes: (4 + num_classes + num_protos, num_boxes)
977/// - protos: (proto_height, proto_width, num_protos)
978///
979/// # Panics
980/// Panics if shapes don't match the expected dimensions.
981#[allow(clippy::too_many_arguments)]
982pub(crate) fn impl_yolo_segdet_float<
983    B: BBoxTypeTrait,
984    BOX: Float + AsPrimitive<f32> + Send + Sync,
985    PROTO: Float + AsPrimitive<f32> + Send + Sync,
986>(
987    boxes: ArrayView2<BOX>,
988    protos: ArrayView3<PROTO>,
989    score_threshold: f32,
990    iou_threshold: f32,
991    nms: Option<Nms>,
992    pre_nms_top_k: usize,
993    max_det: usize,
994    normalized: Option<bool>,
995    input_dims: Option<(usize, usize)>,
996    multi_label: bool,
997    output_boxes: &mut Vec<DetectBox>,
998    output_masks: &mut Vec<Segmentation>,
999) -> Result<(), crate::DecoderError>
1000where
1001    f32: AsPrimitive<BOX>,
1002{
1003    let num_protos = protos.dim().2;
1004    let (boxes_tensor, scores_tensor, mask_tensor) = postprocess_yolo_seg(&boxes, num_protos);
1005    let mut boxes = impl_yolo_segdet_get_boxes::<B, _, _>(
1006        boxes_tensor,
1007        scores_tensor,
1008        score_threshold,
1009        iou_threshold,
1010        nms,
1011        pre_nms_top_k,
1012        max_det,
1013        multi_label,
1014    );
1015    maybe_normalize_boxes_in_place(&mut boxes, normalized, input_dims);
1016    impl_yolo_split_segdet_process_masks(boxes, mask_tensor, protos, output_boxes, output_masks)
1017}
1018
1019#[allow(clippy::too_many_arguments)]
1020pub(crate) fn impl_yolo_segdet_get_boxes<
1021    B: BBoxTypeTrait,
1022    BOX: Float + AsPrimitive<f32> + Send + Sync,
1023    SCORE: Float + AsPrimitive<f32> + Send + Sync,
1024>(
1025    boxes_tensor: ArrayView2<BOX>,
1026    scores_tensor: ArrayView2<SCORE>,
1027    score_threshold: f32,
1028    iou_threshold: f32,
1029    nms: Option<Nms>,
1030    pre_nms_top_k: usize,
1031    max_det: usize,
1032    multi_label: bool,
1033) -> Vec<(DetectBox, usize)>
1034where
1035    f32: AsPrimitive<SCORE>,
1036{
1037    let span = tracing::trace_span!(
1038        "decoder.nms_get_boxes",
1039        n_candidates = tracing::field::Empty,
1040        n_after_topk = tracing::field::Empty,
1041        n_after_nms = tracing::field::Empty,
1042        n_detections = tracing::field::Empty,
1043    );
1044    let _guard = span.enter();
1045
1046    // Multi-label emits one candidate per (anchor, class) for every class
1047    // above threshold; argmax emits one candidate per anchor (highest class).
1048    // Multi-label requires class-aware NMS so per-class duplicates don't
1049    // cross-suppress boxes from different classes on the same anchor.
1050    let (mut boxes, effective_nms) = {
1051        let _s = tracing::trace_span!("decoder.nms_get_boxes.score_filter").entered();
1052        if multi_label {
1053            let candidates = postprocess_boxes_multilabel_index_float::<B, _, _>(
1054                score_threshold.as_(),
1055                boxes_tensor,
1056                scores_tensor,
1057            );
1058            // Class-agnostic NMS would suppress boxes that share the same
1059            // anchor region but have different class labels — defeating the
1060            // purpose of multi-label decode.  Force class-aware here.
1061            let nms_override = Some(Nms::ClassAware);
1062            (candidates, nms_override)
1063        } else {
1064            let candidates = postprocess_boxes_index_float::<B, _, _>(
1065                score_threshold.as_(),
1066                boxes_tensor,
1067                scores_tensor,
1068            );
1069            (candidates, nms)
1070        }
1071    };
1072    span.record("n_candidates", boxes.len());
1073
1074    if effective_nms.is_some() {
1075        let _s = tracing::trace_span!("decoder.nms_get_boxes.top_k", k = pre_nms_top_k).entered();
1076        truncate_to_top_k_by_score(&mut boxes, pre_nms_top_k);
1077    }
1078    span.record("n_after_topk", boxes.len());
1079
1080    let mut boxes = {
1081        let _s = tracing::trace_span!("decoder.nms_get_boxes.suppress").entered();
1082        dispatch_nms_extra_float(effective_nms, iou_threshold, Some(max_det), boxes)
1083    };
1084    span.record("n_after_nms", boxes.len());
1085
1086    // NMS already capped to `max_det`; the trailing sort+truncate is a
1087    // redundant guard for the bypass-NMS path (`nms = None`).
1088    boxes.sort_unstable_by(|a, b| b.0.score.total_cmp(&a.0.score));
1089    boxes.truncate(max_det);
1090    span.record("n_detections", boxes.len());
1091
1092    boxes
1093}
1094
1095pub(crate) fn impl_yolo_end_to_end_segdet_get_boxes<
1096    B: BBoxTypeTrait,
1097    BOX: Float + AsPrimitive<f32> + Send + Sync,
1098    SCORE: Float + AsPrimitive<f32> + Send + Sync,
1099    CLASS: AsPrimitive<f32> + Send + Sync,
1100>(
1101    boxes: ArrayView2<BOX>,
1102    scores: ArrayView2<SCORE>,
1103    classes: ArrayView1<CLASS>,
1104    score_threshold: f32,
1105    max_boxes: usize,
1106) -> Vec<(DetectBox, usize)>
1107where
1108    f32: AsPrimitive<SCORE>,
1109{
1110    let mut boxes = postprocess_boxes_index_float::<B, _, _>(score_threshold.as_(), boxes, scores);
1111    boxes.truncate(max_boxes);
1112    for (b, ind) in &mut boxes {
1113        b.label = classes[*ind].as_().round() as usize;
1114    }
1115    boxes
1116}
1117
1118pub(crate) fn impl_yolo_split_segdet_process_masks<
1119    MASK: Float + AsPrimitive<f32> + Send + Sync,
1120    PROTO: Float + AsPrimitive<f32> + Send + Sync,
1121>(
1122    boxes: Vec<(DetectBox, usize)>,
1123    masks_tensor: ArrayView2<MASK>,
1124    protos_tensor: ArrayView3<PROTO>,
1125    output_boxes: &mut Vec<DetectBox>,
1126    output_masks: &mut Vec<Segmentation>,
1127) -> Result<(), crate::DecoderError> {
1128    let _span = tracing::trace_span!(
1129        "decoder.decode.process_masks",
1130        n = boxes.len(),
1131        mode = "float"
1132    )
1133    .entered();
1134    // Boxes are already bounded by the upstream `max_det` cap from
1135    // `_get_boxes`; no second cap is needed here (EDGEAI-1302).
1136
1137    let boxes = decode_segdet_f32(boxes, masks_tensor, protos_tensor)?;
1138    output_boxes.clear();
1139    output_masks.clear();
1140    for (b, roi, m) in boxes.into_iter() {
1141        output_boxes.push(b);
1142        output_masks.push(Segmentation {
1143            xmin: roi.xmin,
1144            ymin: roi.ymin,
1145            xmax: roi.xmax,
1146            ymax: roi.ymax,
1147            segmentation: m,
1148        });
1149    }
1150    Ok(())
1151}
1152/// Expected input shapes:
1153/// - boxes_tensor: (num_boxes, 4)
1154/// - scores_tensor: (num_boxes, num_classes)
1155pub(crate) fn impl_yolo_split_segdet_quant_get_boxes<
1156    B: BBoxTypeTrait,
1157    BOX: PrimInt + AsPrimitive<f32> + Send + Sync,
1158    SCORE: PrimInt + AsPrimitive<f32> + Send + Sync,
1159>(
1160    boxes: (ArrayView2<BOX>, Quantization),
1161    scores: (ArrayView2<SCORE>, Quantization),
1162    score_threshold: f32,
1163    iou_threshold: f32,
1164    nms: Option<Nms>,
1165    pre_nms_top_k: usize,
1166    max_det: usize,
1167) -> Vec<(DetectBox, usize)>
1168where
1169    f32: AsPrimitive<SCORE>,
1170{
1171    let (boxes_tensor, quant_boxes) = boxes;
1172    let (scores_tensor, quant_scores) = scores;
1173
1174    let span = tracing::trace_span!(
1175        "decoder.nms_get_boxes",
1176        n_candidates = tracing::field::Empty,
1177        n_after_topk = tracing::field::Empty,
1178        n_after_nms = tracing::field::Empty,
1179        n_detections = tracing::field::Empty,
1180    );
1181    let _guard = span.enter();
1182
1183    let mut boxes = {
1184        let _s = tracing::trace_span!("decoder.nms_get_boxes.score_filter").entered();
1185        let score_threshold = quantize_score_threshold(score_threshold, quant_scores);
1186        postprocess_boxes_index_quant::<B, _, _>(
1187            score_threshold,
1188            boxes_tensor,
1189            scores_tensor,
1190            quant_boxes,
1191        )
1192    };
1193    span.record("n_candidates", boxes.len());
1194
1195    if nms.is_some() {
1196        let _s = tracing::trace_span!("decoder.nms_get_boxes.top_k", k = pre_nms_top_k).entered();
1197        truncate_to_top_k_by_score_quant(&mut boxes, pre_nms_top_k);
1198    }
1199    span.record("n_after_topk", boxes.len());
1200
1201    let mut boxes = {
1202        let _s = tracing::trace_span!("decoder.nms_get_boxes.suppress").entered();
1203        dispatch_nms_extra_int(nms, iou_threshold, Some(max_det), boxes)
1204    };
1205    span.record("n_after_nms", boxes.len());
1206
1207    // NMS already capped to `max_det`; the trailing sort+truncate is a
1208    // redundant guard for the bypass-NMS path (`nms = None`).
1209    boxes.sort_unstable_by_key(|b| std::cmp::Reverse(b.0.score));
1210    boxes.truncate(max_det);
1211    let result: Vec<_> = {
1212        let _s =
1213            tracing::trace_span!("decoder.nms_get_boxes.dequant_boxes", n = boxes.len()).entered();
1214        boxes
1215            .into_iter()
1216            .map(|(b, i)| (dequant_detect_box(&b, quant_scores), i))
1217            .collect()
1218    };
1219    span.record("n_detections", result.len());
1220
1221    result
1222}
1223
1224pub(crate) fn impl_yolo_split_segdet_quant_process_masks<
1225    MASK: PrimInt + AsPrimitive<i64> + AsPrimitive<i128> + AsPrimitive<f32> + Send + Sync,
1226    PROTO: PrimInt + AsPrimitive<i64> + AsPrimitive<i128> + AsPrimitive<f32> + Send + Sync,
1227>(
1228    boxes: Vec<(DetectBox, usize)>,
1229    mask_coeff: (ArrayView2<MASK>, Quantization),
1230    protos: (ArrayView3<PROTO>, Quantization),
1231    output_boxes: &mut Vec<DetectBox>,
1232    output_masks: &mut Vec<Segmentation>,
1233) -> Result<(), crate::DecoderError> {
1234    let _span = tracing::trace_span!(
1235        "decoder.decode.process_masks",
1236        n = boxes.len(),
1237        mode = "quant"
1238    )
1239    .entered();
1240    let (masks, quant_masks) = mask_coeff;
1241    let (protos, quant_protos) = protos;
1242
1243    // Boxes are already bounded by the upstream `max_det` cap from
1244    // `_get_boxes`; no second cap is needed here (EDGEAI-1302).
1245
1246    let boxes = decode_segdet_quant(boxes, masks, protos, quant_masks, quant_protos)?;
1247    output_boxes.clear();
1248    output_masks.clear();
1249    for (b, roi, m) in boxes.into_iter() {
1250        output_boxes.push(b);
1251        output_masks.push(Segmentation {
1252            xmin: roi.xmin,
1253            ymin: roi.ymin,
1254            xmax: roi.xmax,
1255            ymax: roi.ymax,
1256            segmentation: m,
1257        });
1258    }
1259    Ok(())
1260}
1261
1262/// Internal implementation of YOLO split detection segmentation decoding for
1263/// float tensors.
1264///
1265/// Expected shapes of inputs:
1266/// - boxes_tensor: (4, num_boxes)
1267/// - scores_tensor: (num_classes, num_boxes)
1268/// - mask_tensor: (num_protos, num_boxes)
1269/// - protos: (proto_height, proto_width, num_protos)
1270///
1271/// # Errors
1272/// Returns `DecoderError::InvalidShape` if bounding boxes are not normalized.
1273#[allow(clippy::too_many_arguments)]
1274pub(crate) fn impl_yolo_split_segdet_float<
1275    B: BBoxTypeTrait,
1276    BOX: Float + AsPrimitive<f32> + Send + Sync,
1277    SCORE: Float + AsPrimitive<f32> + Send + Sync,
1278    MASK: Float + AsPrimitive<f32> + Send + Sync,
1279    PROTO: Float + AsPrimitive<f32> + Send + Sync,
1280>(
1281    boxes_tensor: ArrayView2<BOX>,
1282    scores_tensor: ArrayView2<SCORE>,
1283    mask_tensor: ArrayView2<MASK>,
1284    protos: ArrayView3<PROTO>,
1285    score_threshold: f32,
1286    iou_threshold: f32,
1287    nms: Option<Nms>,
1288    pre_nms_top_k: usize,
1289    max_det: usize,
1290    normalized: Option<bool>,
1291    input_dims: Option<(usize, usize)>,
1292    output_boxes: &mut Vec<DetectBox>,
1293    output_masks: &mut Vec<Segmentation>,
1294) -> Result<(), crate::DecoderError>
1295where
1296    f32: AsPrimitive<SCORE>,
1297{
1298    let (boxes_tensor, scores_tensor, mask_tensor) =
1299        postprocess_yolo_split_segdet(boxes_tensor, scores_tensor, mask_tensor);
1300
1301    let mut boxes = impl_yolo_segdet_get_boxes::<B, _, _>(
1302        boxes_tensor,
1303        scores_tensor,
1304        score_threshold,
1305        iou_threshold,
1306        nms,
1307        pre_nms_top_k,
1308        max_det,
1309        false, // multi_label: split path is argmax-only (no Decoder context here)
1310    );
1311    maybe_normalize_boxes_in_place(&mut boxes, normalized, input_dims);
1312    impl_yolo_split_segdet_process_masks(boxes, mask_tensor, protos, output_boxes, output_masks)
1313}
1314
1315// ---------------------------------------------------------------------------
1316// Proto-extraction variants: return ProtoData instead of materialized masks
1317// ---------------------------------------------------------------------------
1318
1319/// Proto-extraction variant of `impl_yolo_segdet_quant`.
1320/// Runs NMS but returns raw `ProtoData` instead of materialized masks.
1321#[allow(clippy::too_many_arguments)]
1322pub(crate) fn impl_yolo_segdet_quant_proto<
1323    B: BBoxTypeTrait,
1324    BOX: PrimInt
1325        + AsPrimitive<i64>
1326        + AsPrimitive<i128>
1327        + AsPrimitive<f32>
1328        + AsPrimitive<i8>
1329        + Send
1330        + Sync,
1331    PROTO: PrimInt
1332        + AsPrimitive<i64>
1333        + AsPrimitive<i128>
1334        + AsPrimitive<f32>
1335        + AsPrimitive<i8>
1336        + Send
1337        + Sync,
1338>(
1339    boxes: (ArrayView2<BOX>, Quantization),
1340    protos: (ArrayView3<PROTO>, Quantization),
1341    score_threshold: f32,
1342    iou_threshold: f32,
1343    nms: Option<Nms>,
1344    pre_nms_top_k: usize,
1345    max_det: usize,
1346    normalized: Option<bool>,
1347    input_dims: Option<(usize, usize)>,
1348    output_boxes: &mut Vec<DetectBox>,
1349) -> ProtoData
1350where
1351    f32: AsPrimitive<BOX>,
1352{
1353    let (boxes_arr, quant_boxes) = boxes;
1354    let (protos_arr, quant_protos) = protos;
1355    let num_protos = protos_arr.dim().2;
1356
1357    let (boxes_tensor, scores_tensor, mask_tensor) = postprocess_yolo_seg(&boxes_arr, num_protos);
1358
1359    let mut det_indices = impl_yolo_split_segdet_quant_get_boxes::<B, _, _>(
1360        (boxes_tensor, quant_boxes),
1361        (scores_tensor, quant_boxes),
1362        score_threshold,
1363        iou_threshold,
1364        nms,
1365        pre_nms_top_k,
1366        max_det,
1367    );
1368    maybe_normalize_boxes_in_place(&mut det_indices, normalized, input_dims);
1369
1370    extract_proto_data_quant(
1371        det_indices,
1372        mask_tensor,
1373        quant_boxes,
1374        protos_arr,
1375        quant_protos,
1376        output_boxes,
1377    )
1378}
1379
1380/// Proto-extraction variant of `impl_yolo_segdet_float`.
1381/// Runs NMS but returns raw `ProtoData` instead of materialized masks.
1382#[allow(clippy::too_many_arguments)]
1383pub(crate) fn impl_yolo_segdet_float_proto<
1384    B: BBoxTypeTrait,
1385    BOX: Float + AsPrimitive<f32> + Copy + Send + Sync + FloatProtoElem,
1386    PROTO: Float + AsPrimitive<f32> + Copy + Send + Sync + FloatProtoElem,
1387>(
1388    boxes: ArrayView2<BOX>,
1389    protos: ArrayView3<PROTO>,
1390    score_threshold: f32,
1391    iou_threshold: f32,
1392    nms: Option<Nms>,
1393    pre_nms_top_k: usize,
1394    max_det: usize,
1395    normalized: Option<bool>,
1396    input_dims: Option<(usize, usize)>,
1397    multi_label: bool,
1398    output_boxes: &mut Vec<DetectBox>,
1399) -> ProtoData
1400where
1401    f32: AsPrimitive<BOX>,
1402{
1403    let num_protos = protos.dim().2;
1404    let (boxes_tensor, scores_tensor, mask_tensor) = postprocess_yolo_seg(&boxes, num_protos);
1405
1406    let mut boxes = impl_yolo_segdet_get_boxes::<B, _, _>(
1407        boxes_tensor,
1408        scores_tensor,
1409        score_threshold,
1410        iou_threshold,
1411        nms,
1412        pre_nms_top_k,
1413        max_det,
1414        multi_label,
1415    );
1416    maybe_normalize_boxes_in_place(&mut boxes, normalized, input_dims);
1417
1418    extract_proto_data_float(boxes, mask_tensor, protos, output_boxes)
1419}
1420
1421/// Proto-extraction variant of `impl_yolo_split_segdet_float`.
1422/// Runs NMS but returns raw `ProtoData` instead of materialized masks.
1423#[allow(clippy::too_many_arguments)]
1424pub(crate) fn impl_yolo_split_segdet_float_proto<
1425    B: BBoxTypeTrait,
1426    BOX: Float + AsPrimitive<f32> + Send + Sync,
1427    SCORE: Float + AsPrimitive<f32> + Send + Sync,
1428    MASK: Float + AsPrimitive<f32> + Copy + Send + Sync + FloatProtoElem,
1429    PROTO: Float + AsPrimitive<f32> + Copy + Send + Sync + FloatProtoElem,
1430>(
1431    boxes_tensor: ArrayView2<BOX>,
1432    scores_tensor: ArrayView2<SCORE>,
1433    mask_tensor: ArrayView2<MASK>,
1434    protos: ArrayView3<PROTO>,
1435    score_threshold: f32,
1436    iou_threshold: f32,
1437    nms: Option<Nms>,
1438    pre_nms_top_k: usize,
1439    max_det: usize,
1440    normalized: Option<bool>,
1441    input_dims: Option<(usize, usize)>,
1442    output_boxes: &mut Vec<DetectBox>,
1443) -> ProtoData
1444where
1445    f32: AsPrimitive<SCORE>,
1446{
1447    let (boxes_tensor, scores_tensor, mask_tensor) =
1448        postprocess_yolo_split_segdet(boxes_tensor, scores_tensor, mask_tensor);
1449    let mut det_indices = impl_yolo_segdet_get_boxes::<B, _, _>(
1450        boxes_tensor,
1451        scores_tensor,
1452        score_threshold,
1453        iou_threshold,
1454        nms,
1455        pre_nms_top_k,
1456        max_det,
1457        false, // multi_label: split proto-extraction variant is argmax-only
1458    );
1459    maybe_normalize_boxes_in_place(&mut det_indices, normalized, input_dims);
1460
1461    extract_proto_data_float(det_indices, mask_tensor, protos, output_boxes)
1462}
1463
1464/// Proto-extraction variant of `decode_yolo_end_to_end_segdet_float`.
1465pub(crate) fn decode_yolo_end_to_end_segdet_float_proto<T>(
1466    output: ArrayView2<T>,
1467    protos: ArrayView3<T>,
1468    score_threshold: f32,
1469    output_boxes: &mut Vec<DetectBox>,
1470) -> Result<ProtoData, crate::DecoderError>
1471where
1472    T: Float + AsPrimitive<f32> + Copy + Send + Sync + FloatProtoElem,
1473    f32: AsPrimitive<T>,
1474{
1475    let (boxes, scores, classes, mask_coeff) =
1476        postprocess_yolo_end_to_end_segdet(&output, protos.dim().2)?;
1477    let cap = cap_or_default(output_boxes);
1478    let boxes = impl_yolo_end_to_end_segdet_get_boxes::<XYXY, _, _, _>(
1479        boxes,
1480        scores,
1481        classes,
1482        score_threshold,
1483        cap,
1484    );
1485
1486    Ok(extract_proto_data_float(
1487        boxes,
1488        mask_coeff,
1489        protos,
1490        output_boxes,
1491    ))
1492}
1493
1494/// Proto-extraction variant of `decode_yolo_split_end_to_end_segdet_float`.
1495#[allow(clippy::too_many_arguments)]
1496pub(crate) fn decode_yolo_split_end_to_end_segdet_float_proto<T>(
1497    boxes: ArrayView2<T>,
1498    scores: ArrayView2<T>,
1499    classes: ArrayView2<T>,
1500    mask_coeff: ArrayView2<T>,
1501    protos: ArrayView3<T>,
1502    score_threshold: f32,
1503    output_boxes: &mut Vec<DetectBox>,
1504) -> Result<ProtoData, crate::DecoderError>
1505where
1506    T: Float + AsPrimitive<f32> + Copy + Send + Sync + FloatProtoElem,
1507    f32: AsPrimitive<T>,
1508{
1509    let (boxes, scores, classes, mask_coeff) =
1510        postprocess_yolo_split_end_to_end_segdet(boxes, scores, &classes, mask_coeff)?;
1511    let cap = cap_or_default(output_boxes);
1512    let boxes = impl_yolo_end_to_end_segdet_get_boxes::<XYXY, _, _, _>(
1513        boxes,
1514        scores,
1515        classes,
1516        score_threshold,
1517        cap,
1518    );
1519
1520    Ok(extract_proto_data_float(
1521        boxes,
1522        mask_coeff,
1523        protos,
1524        output_boxes,
1525    ))
1526}
1527
1528/// Helper: extract ProtoData from float mask coefficients + protos.
1529///
1530/// Builds [`ProtoData`] with both `protos` and `mask_coefficients` as
1531/// [`edgefirst_tensor::TensorDyn`]. Preserves the native element type for
1532/// `f16` and `f32`; narrows `f64` to `f32` (there is no native f64 kernel
1533/// path). `mask_coefficients` shape is `[num_detections, num_protos]`.
1534pub(super) fn extract_proto_data_float<
1535    MASK: Float + AsPrimitive<f32> + Copy + Send + Sync + FloatProtoElem,
1536    PROTO: Float + AsPrimitive<f32> + Copy + Send + Sync + FloatProtoElem,
1537>(
1538    det_indices: Vec<(DetectBox, usize)>,
1539    mask_tensor: ArrayView2<MASK>,
1540    protos: ArrayView3<PROTO>,
1541    output_boxes: &mut Vec<DetectBox>,
1542) -> ProtoData {
1543    let _span = tracing::trace_span!(
1544        "decoder.decode_proto.extract_proto_data",
1545        mode = "float",
1546        n = det_indices.len(),
1547        num_protos = mask_tensor.ncols(),
1548        layout = "nhwc",
1549    )
1550    .entered();
1551
1552    let num_protos = mask_tensor.ncols();
1553    let n = det_indices.len();
1554
1555    // Per-detection coefficients packed row-major into a contiguous buffer,
1556    // preserving the source dtype. Shape: [N, num_protos] — N=0 is permitted
1557    // (tracker path emits no detections this frame) since Mem-backed tensors
1558    // accept zero-element shapes as "empty collection" sentinels.
1559    let mut coeff_rows: Vec<MASK> = Vec::with_capacity(n * num_protos);
1560    output_boxes.clear();
1561    for (det, idx) in det_indices {
1562        output_boxes.push(det);
1563        let row = mask_tensor.row(idx);
1564        coeff_rows.extend(row.iter().copied());
1565    }
1566
1567    let mask_coefficients = MASK::slice_into_tensor_dyn(&coeff_rows, &[n, num_protos])
1568        .expect("allocating mask_coefficients TensorDyn");
1569    let protos_tensor =
1570        PROTO::arrayview3_into_tensor_dyn(protos).expect("allocating protos TensorDyn");
1571
1572    ProtoData {
1573        mask_coefficients,
1574        protos: protos_tensor,
1575        layout: ProtoLayout::Nhwc,
1576    }
1577}
1578
1579/// Helper: extract ProtoData from quantized mask coefficients + protos.
1580///
1581/// Dequantizes mask coefficients to f32 at extraction (one-time cost on a
1582/// `num_detections * num_protos` slice) and keeps protos in raw i8,
1583/// attaching the dequantization params as
1584/// [`edgefirst_tensor::Quantization::per_tensor`] metadata on the proto
1585/// tensor. The GPU shader / CPU kernel reads `protos.quantization()` and
1586/// dequantizes per-texel.
1587pub(crate) fn extract_proto_data_quant<
1588    MASK: PrimInt + AsPrimitive<f32> + AsPrimitive<i8> + Send + Sync + 'static,
1589    PROTO: PrimInt + AsPrimitive<f32> + AsPrimitive<i8> + Send + Sync + 'static,
1590>(
1591    det_indices: Vec<(DetectBox, usize)>,
1592    mask_tensor: ArrayView2<MASK>,
1593    quant_masks: Quantization,
1594    protos: ArrayView3<PROTO>,
1595    quant_protos: Quantization,
1596    output_boxes: &mut Vec<DetectBox>,
1597) -> ProtoData {
1598    use edgefirst_tensor::{Tensor, TensorDyn, TensorMapTrait, TensorMemory, TensorTrait};
1599
1600    let span = tracing::trace_span!(
1601        "decoder.decode_proto.extract_proto_data",
1602        mode = "quant",
1603        n = det_indices.len(),
1604        num_protos = tracing::field::Empty,
1605        layout = tracing::field::Empty,
1606    );
1607    let _guard = span.enter();
1608
1609    let num_protos = mask_tensor.ncols();
1610    let n = det_indices.len();
1611    span.record("num_protos", num_protos);
1612
1613    // Fast path: when no detections survive NMS, skip the expensive proto
1614    // tensor copy (819KB for 160×160×32). Allocate with the correct shape
1615    // (preserving the documented ProtoData.protos shape contract) but skip
1616    // copying from the source tensor — the zeroed allocation is sufficient
1617    // since materialize_masks early-returns on empty detect slices.
1618    if n == 0 {
1619        output_boxes.clear();
1620        let (h, w, k) = protos.dim();
1621
1622        // Detect physical layout (same logic as the normal path).
1623        let (proto_shape, proto_layout) = if std::any::TypeId::of::<PROTO>()
1624            == std::any::TypeId::of::<i8>()
1625        {
1626            if protos.is_standard_layout() {
1627                (&[h, w, k][..], ProtoLayout::Nhwc)
1628            } else if protos.ndim() == 3 && protos.strides() == [w as isize, 1, (h * w) as isize] {
1629                (&[k, h, w][..], ProtoLayout::Nchw)
1630            } else {
1631                (&[h, w, k][..], ProtoLayout::Nhwc)
1632            }
1633        } else {
1634            (&[h, w, k][..], ProtoLayout::Nhwc)
1635        };
1636
1637        let coeff_tensor = Tensor::<i8>::new(&[0, num_protos], Some(TensorMemory::Mem), None)
1638            .expect("allocating empty mask_coefficients tensor");
1639        let coeff_quant =
1640            edgefirst_tensor::Quantization::per_tensor(quant_masks.scale, quant_masks.zero_point);
1641        let coeff_tensor = coeff_tensor
1642            .with_quantization(coeff_quant)
1643            .expect("per-tensor quantization on mask coefficients");
1644        let protos_tensor = Tensor::<i8>::new(proto_shape, Some(TensorMemory::Mem), None)
1645            .expect("allocating protos tensor");
1646        let tensor_quant =
1647            edgefirst_tensor::Quantization::per_tensor(quant_protos.scale, quant_protos.zero_point);
1648        let protos_tensor = protos_tensor
1649            .with_quantization(tensor_quant)
1650            .expect("per-tensor quantization on protos tensor");
1651        return ProtoData {
1652            mask_coefficients: TensorDyn::I8(coeff_tensor),
1653            protos: TensorDyn::I8(protos_tensor),
1654            layout: proto_layout,
1655        };
1656    }
1657
1658    // Mask coefficients: keep i8 losslessly when MASK == i8 (preserves
1659    // the fast i8×i8→i32 integer kernel in materialize_masks). Preserve
1660    // i16 natively so the downstream i16×i8 integer path can avoid lossy
1661    // truncation. Other wider types (u16, …) dequantize to f32 at
1662    // extraction because the downstream mask kernels accept F32 natively.
1663    let mask_coefficients: TensorDyn = if std::any::TypeId::of::<MASK>()
1664        == std::any::TypeId::of::<i8>()
1665    {
1666        let mut coeff_i8 = Vec::<i8>::with_capacity(n * num_protos);
1667        output_boxes.clear();
1668        for (det, idx) in det_indices {
1669            output_boxes.push(det);
1670            let row = mask_tensor.row(idx);
1671            coeff_i8.extend(row.iter().map(|v| {
1672                let v_i8: i8 = v.as_();
1673                v_i8
1674            }));
1675        }
1676        let coeff_tensor = Tensor::<i8>::new(&[n, num_protos], Some(TensorMemory::Mem), None)
1677            .expect("allocating mask_coefficients tensor");
1678        if n > 0 {
1679            let mut m = coeff_tensor
1680                .map()
1681                .expect("mapping mask_coefficients tensor");
1682            m.as_mut_slice().copy_from_slice(&coeff_i8);
1683        }
1684        let coeff_quant =
1685            edgefirst_tensor::Quantization::per_tensor(quant_masks.scale, quant_masks.zero_point);
1686        let coeff_tensor = coeff_tensor
1687            .with_quantization(coeff_quant)
1688            .expect("per-tensor quantization on mask coefficients");
1689        TensorDyn::I8(coeff_tensor)
1690    } else if std::any::TypeId::of::<MASK>() == std::any::TypeId::of::<i16>() {
1691        // i16 path: preserve natively for the fast i16×i8→i32 integer kernel.
1692        // f32 has 24-bit mantissa, so all i16 values are exactly representable.
1693        let mut coeff_i16 = Vec::<i16>::with_capacity(n * num_protos);
1694        output_boxes.clear();
1695        for (det, idx) in det_indices {
1696            output_boxes.push(det);
1697            let row = mask_tensor.row(idx);
1698            coeff_i16.extend(row.iter().map(|v| {
1699                let v_f32: f32 = v.as_();
1700                v_f32 as i16
1701            }));
1702        }
1703        let coeff_tensor = Tensor::<i16>::new(&[n, num_protos], Some(TensorMemory::Mem), None)
1704            .expect("allocating mask_coefficients tensor");
1705        if n > 0 {
1706            let mut m = coeff_tensor
1707                .map()
1708                .expect("mapping mask_coefficients tensor");
1709            m.as_mut_slice().copy_from_slice(&coeff_i16);
1710        }
1711        let coeff_quant =
1712            edgefirst_tensor::Quantization::per_tensor(quant_masks.scale, quant_masks.zero_point);
1713        let coeff_tensor = coeff_tensor
1714            .with_quantization(coeff_quant)
1715            .expect("per-tensor quantization on mask coefficients");
1716        TensorDyn::I16(coeff_tensor)
1717    } else {
1718        // Other types (u8, u16, etc.): dequantize to f32 to avoid lossy truncation.
1719        let scale = quant_masks.scale;
1720        let zp = quant_masks.zero_point as f32;
1721        let mut coeff_f32 = Vec::<f32>::with_capacity(n * num_protos);
1722        output_boxes.clear();
1723        for (det, idx) in det_indices {
1724            output_boxes.push(det);
1725            let row = mask_tensor.row(idx);
1726            coeff_f32.extend(row.iter().map(|v| {
1727                let v_f32: f32 = v.as_();
1728                (v_f32 - zp) * scale
1729            }));
1730        }
1731        let coeff_tensor = Tensor::<f32>::new(&[n, num_protos], Some(TensorMemory::Mem), None)
1732            .expect("allocating mask_coefficients tensor");
1733        if n > 0 {
1734            let mut m = coeff_tensor
1735                .map()
1736                .expect("mapping mask_coefficients tensor");
1737            m.as_mut_slice().copy_from_slice(&coeff_f32);
1738        }
1739        TensorDyn::F32(coeff_tensor)
1740    };
1741
1742    // Keep protos in raw i8 — consumers dequantize via protos.quantization().
1743    // When PROTO is already i8, detect layout and copy efficiently without
1744    // transposing. The mask materialisation kernels dispatch on the layout.
1745    let (h, w, k) = protos.dim();
1746
1747    // Determine physical layout and copy strategy.
1748    let (proto_shape, proto_layout) =
1749        if std::any::TypeId::of::<PROTO>() == std::any::TypeId::of::<i8>() {
1750            if protos.is_standard_layout() {
1751                // Already NHWC [H, W, K] in contiguous memory.
1752                (&[h, w, k][..], ProtoLayout::Nhwc)
1753            } else if protos.ndim() == 3 && protos.strides() == [w as isize, 1, (h * w) as isize] {
1754                // NCHW reinterpreted as NHWC via stride swap. Physical storage
1755                // is [K, H, W] contiguous. Keep in NCHW — eliminates the costly
1756                // 3.1ms transpose entirely.
1757                (&[k, h, w][..], ProtoLayout::Nchw)
1758            } else {
1759                // Unknown layout — fall back to iter copy as NHWC.
1760                (&[h, w, k][..], ProtoLayout::Nhwc)
1761            }
1762        } else {
1763            (&[h, w, k][..], ProtoLayout::Nhwc)
1764        };
1765
1766    let protos_tensor = Tensor::<i8>::new(proto_shape, Some(TensorMemory::Mem), None)
1767        .expect("allocating protos tensor");
1768    {
1769        let mut m = protos_tensor.map().expect("mapping protos tensor");
1770        let dst = m.as_mut_slice();
1771        if std::any::TypeId::of::<PROTO>() == std::any::TypeId::of::<i8>() {
1772            // SAFETY: PROTO == i8 checked via TypeId; cast slice view is
1773            // size/alignment-compatible by construction.
1774            if protos.is_standard_layout() {
1775                let src: &[i8] = unsafe {
1776                    std::slice::from_raw_parts(protos.as_ptr() as *const i8, protos.len())
1777                };
1778                dst.copy_from_slice(src);
1779            } else if protos.ndim() == 3 && protos.strides() == [w as isize, 1, (h * w) as isize] {
1780                // NCHW physical layout — sequential copy WITHOUT transpose.
1781                // This saves ~3.1ms on A53/A55 by avoiding the tiled
1782                // NCHW→NHWC transpose of the 819KB proto buffer.
1783                let total = h * w * k;
1784                // SAFETY: ArrayView was constructed from a contiguous slice of
1785                // `total` elements. as_ptr() points to the base of that slice.
1786                let src: &[i8] =
1787                    unsafe { std::slice::from_raw_parts(protos.as_ptr() as *const i8, total) };
1788                dst.copy_from_slice(src);
1789            } else {
1790                for (d, s) in dst.iter_mut().zip(protos.iter()) {
1791                    let v_i8: i8 = s.as_();
1792                    *d = v_i8;
1793                }
1794            }
1795        } else {
1796            for (d, s) in dst.iter_mut().zip(protos.iter()) {
1797                let v_i8: i8 = s.as_();
1798                *d = v_i8;
1799            }
1800        }
1801    }
1802    let tensor_quant =
1803        edgefirst_tensor::Quantization::per_tensor(quant_protos.scale, quant_protos.zero_point);
1804    let protos_tensor = protos_tensor
1805        .with_quantization(tensor_quant)
1806        .expect("per-tensor quantization on new Tensor<i8>");
1807
1808    span.record("layout", tracing::field::debug(&proto_layout));
1809
1810    ProtoData {
1811        mask_coefficients,
1812        protos: TensorDyn::I8(protos_tensor),
1813        layout: proto_layout,
1814    }
1815}
1816
1817/// Per-float-dtype construction of a [`TensorDyn`] from a flat slice / 3-D
1818/// `ArrayView`. Replaces the old `IntoProtoTensor` trait. Each implementor
1819/// either passes its element type straight to `Tensor::from_slice` /
1820/// `Tensor::from_arrayview3`, or narrows `f64` to `f32` (no native f64 kernel
1821/// path exists).
1822pub trait FloatProtoElem: Copy + 'static {
1823    fn slice_into_tensor_dyn(
1824        values: &[Self],
1825        shape: &[usize],
1826    ) -> edgefirst_tensor::Result<edgefirst_tensor::TensorDyn>;
1827
1828    fn arrayview3_into_tensor_dyn(
1829        view: ArrayView3<'_, Self>,
1830    ) -> edgefirst_tensor::Result<edgefirst_tensor::TensorDyn>;
1831}
1832
1833impl FloatProtoElem for f32 {
1834    fn slice_into_tensor_dyn(
1835        values: &[f32],
1836        shape: &[usize],
1837    ) -> edgefirst_tensor::Result<edgefirst_tensor::TensorDyn> {
1838        edgefirst_tensor::Tensor::<f32>::from_slice(values, shape)
1839            .map(edgefirst_tensor::TensorDyn::F32)
1840    }
1841    fn arrayview3_into_tensor_dyn(
1842        view: ArrayView3<'_, f32>,
1843    ) -> edgefirst_tensor::Result<edgefirst_tensor::TensorDyn> {
1844        edgefirst_tensor::Tensor::<f32>::from_arrayview3(view).map(edgefirst_tensor::TensorDyn::F32)
1845    }
1846}
1847
1848impl FloatProtoElem for half::f16 {
1849    fn slice_into_tensor_dyn(
1850        values: &[half::f16],
1851        shape: &[usize],
1852    ) -> edgefirst_tensor::Result<edgefirst_tensor::TensorDyn> {
1853        edgefirst_tensor::Tensor::<half::f16>::from_slice(values, shape)
1854            .map(edgefirst_tensor::TensorDyn::F16)
1855    }
1856    fn arrayview3_into_tensor_dyn(
1857        view: ArrayView3<'_, half::f16>,
1858    ) -> edgefirst_tensor::Result<edgefirst_tensor::TensorDyn> {
1859        edgefirst_tensor::Tensor::<half::f16>::from_arrayview3(view)
1860            .map(edgefirst_tensor::TensorDyn::F16)
1861    }
1862}
1863
1864impl FloatProtoElem for f64 {
1865    fn slice_into_tensor_dyn(
1866        values: &[f64],
1867        shape: &[usize],
1868    ) -> edgefirst_tensor::Result<edgefirst_tensor::TensorDyn> {
1869        // Narrow to f32 — no native f64 kernel path.
1870        let narrowed: Vec<f32> = values.iter().map(|&v| v as f32).collect();
1871        edgefirst_tensor::Tensor::<f32>::from_slice(&narrowed, shape)
1872            .map(edgefirst_tensor::TensorDyn::F32)
1873    }
1874    fn arrayview3_into_tensor_dyn(
1875        view: ArrayView3<'_, f64>,
1876    ) -> edgefirst_tensor::Result<edgefirst_tensor::TensorDyn> {
1877        let narrowed: ndarray::Array3<f32> = view.mapv(|v| v as f32);
1878        edgefirst_tensor::Tensor::<f32>::from_arrayview3(narrowed.view())
1879            .map(edgefirst_tensor::TensorDyn::F32)
1880    }
1881}
1882
1883fn postprocess_yolo<'a, T>(
1884    output: &'a ArrayView2<'_, T>,
1885) -> (ArrayView2<'a, T>, ArrayView2<'a, T>) {
1886    let boxes_tensor = output.slice(s![..4, ..,]).reversed_axes();
1887    let scores_tensor = output.slice(s![4.., ..,]).reversed_axes();
1888    (boxes_tensor, scores_tensor)
1889}
1890
1891pub(crate) fn postprocess_yolo_seg<'a, T>(
1892    output: &'a ArrayView2<'_, T>,
1893    num_protos: usize,
1894) -> (ArrayView2<'a, T>, ArrayView2<'a, T>, ArrayView2<'a, T>) {
1895    assert!(
1896        output.shape()[0] > num_protos + 4,
1897        "Output shape is too short: {} <= {} + 4",
1898        output.shape()[0],
1899        num_protos
1900    );
1901    let num_classes = output.shape()[0] - 4 - num_protos;
1902    let boxes_tensor = output.slice(s![..4, ..,]).reversed_axes();
1903    let scores_tensor = output.slice(s![4..(num_classes + 4), ..,]).reversed_axes();
1904    let mask_tensor = output.slice(s![(num_classes + 4).., ..,]).reversed_axes();
1905    (boxes_tensor, scores_tensor, mask_tensor)
1906}
1907
1908pub(crate) fn postprocess_yolo_split_segdet<'a, 'b, 'c, BOX, SCORE, MASK>(
1909    boxes_tensor: ArrayView2<'a, BOX>,
1910    scores_tensor: ArrayView2<'b, SCORE>,
1911    mask_tensor: ArrayView2<'c, MASK>,
1912) -> (
1913    ArrayView2<'a, BOX>,
1914    ArrayView2<'b, SCORE>,
1915    ArrayView2<'c, MASK>,
1916) {
1917    let boxes_tensor = boxes_tensor.reversed_axes();
1918    let scores_tensor = scores_tensor.reversed_axes();
1919    let mask_tensor = mask_tensor.reversed_axes();
1920    (boxes_tensor, scores_tensor, mask_tensor)
1921}
1922
1923fn decode_segdet_f32<
1924    MASK: Float + AsPrimitive<f32> + Send + Sync,
1925    PROTO: Float + AsPrimitive<f32> + Send + Sync,
1926>(
1927    boxes: Vec<(DetectBox, usize)>,
1928    masks: ArrayView2<MASK>,
1929    protos: ArrayView3<PROTO>,
1930) -> Result<Vec<(DetectBox, BoundingBox, Array3<u8>)>, crate::DecoderError> {
1931    if boxes.is_empty() {
1932        return Ok(Vec::new());
1933    }
1934    if masks.shape()[1] != protos.shape()[2] {
1935        return Err(crate::DecoderError::InvalidShape(format!(
1936            "Mask coefficients count ({}) doesn't match protos channel count ({})",
1937            masks.shape()[1],
1938            protos.shape()[2],
1939        )));
1940    }
1941    boxes
1942        .into_par_iter()
1943        .map(|b| {
1944            let ind = b.1;
1945            // `protobox` returns the cropped proto slice for `make_segmentation`
1946            // and a `roi` snapped to the 1/proto-grid step. The detection bbox
1947            // stays untouched (EDGEAI-1304); the snapped roi is reported back
1948            // separately so callers can describe where the cropped mask lives.
1949            let (protos, roi) = protobox(&protos, &b.0.bbox)?;
1950            Ok((b.0, roi, make_segmentation(masks.row(ind), protos.view())))
1951        })
1952        .collect()
1953}
1954
1955pub(crate) fn decode_segdet_quant<
1956    MASK: PrimInt + AsPrimitive<i64> + AsPrimitive<i128> + Send + Sync,
1957    PROTO: PrimInt + AsPrimitive<i64> + AsPrimitive<i128> + Send + Sync,
1958>(
1959    boxes: Vec<(DetectBox, usize)>,
1960    masks: ArrayView2<MASK>,
1961    protos: ArrayView3<PROTO>,
1962    quant_masks: Quantization,
1963    quant_protos: Quantization,
1964) -> Result<Vec<(DetectBox, BoundingBox, Array3<u8>)>, crate::DecoderError> {
1965    if boxes.is_empty() {
1966        return Ok(Vec::new());
1967    }
1968    if masks.shape()[1] != protos.shape()[2] {
1969        return Err(crate::DecoderError::InvalidShape(format!(
1970            "Mask coefficients count ({}) doesn't match protos channel count ({})",
1971            masks.shape()[1],
1972            protos.shape()[2],
1973        )));
1974    }
1975
1976    let total_bits = MASK::zero().count_zeros() + PROTO::zero().count_zeros() + 5; // 32 protos is 2^5
1977    boxes
1978        .into_iter()
1979        .map(|b| {
1980            let i = b.1;
1981            // See EDGEAI-1304: the caller's bbox stays untouched; the
1982            // proto-grid-snapped `roi` is reported back so the Segmentation's
1983            // bounds can describe the actual cropped mask region.
1984            let (protos, roi) = protobox(&protos, &b.0.bbox.to_canonical())?;
1985            let seg = match total_bits {
1986                0..=64 => make_segmentation_quant::<MASK, PROTO, i64>(
1987                    masks.row(i),
1988                    protos.view(),
1989                    quant_masks,
1990                    quant_protos,
1991                ),
1992                65..=128 => make_segmentation_quant::<MASK, PROTO, i128>(
1993                    masks.row(i),
1994                    protos.view(),
1995                    quant_masks,
1996                    quant_protos,
1997                ),
1998                _ => {
1999                    return Err(crate::DecoderError::NotSupported(format!(
2000                        "Unsupported bit width ({total_bits}) for segmentation computation"
2001                    )));
2002                }
2003            };
2004            Ok((b.0, roi, seg))
2005        })
2006        .collect()
2007}
2008
2009fn protobox<'a, T>(
2010    protos: &'a ArrayView3<T>,
2011    roi: &BoundingBox,
2012) -> Result<(ArrayView3<'a, T>, BoundingBox), crate::DecoderError> {
2013    let width = protos.dim().1 as f32;
2014    let height = protos.dim().0 as f32;
2015
2016    // Detect un-normalized bounding boxes (pixel-space coordinates).
2017    // protobox expects normalized coordinates in [0, 1]. The decoder will
2018    // normalize pixel-space coords automatically when the schema declares
2019    // `Detection::normalized = false` AND model input dimensions are known
2020    // (EDGEAI-1303); reaching this guard means at least one of those is
2021    // missing.
2022    //
2023    // The limit is set to 2.0 (not 1.01) because YOLO models legitimately
2024    // predict coordinates slightly > 1.0 for objects near frame edges.
2025    // Any value > 2.0 is clearly pixel-space (even the smallest practical
2026    // model input of 32×32 would produce coordinates >> 2.0).
2027    const NORM_LIMIT: f32 = 2.0;
2028    if roi.xmin > NORM_LIMIT
2029        || roi.ymin > NORM_LIMIT
2030        || roi.xmax > NORM_LIMIT
2031        || roi.ymax > NORM_LIMIT
2032    {
2033        return Err(crate::DecoderError::InvalidShape(format!(
2034            "Bounding box coordinates appear un-normalized (pixel-space). \
2035             Got bbox=({:.2}, {:.2}, {:.2}, {:.2}) but expected values in [0, 1]. \
2036             Two ways to fix this: \
2037             (1) declare `Detection::normalized = false` in the model schema \
2038             AND make sure the schema's `input.shape` / `input.dshape` carries \
2039             the model input dims so the decoder can divide by (W, H) before NMS \
2040             (EDGEAI-1303 — verify with `Decoder::input_dims().is_some()`); or \
2041             (2) normalize the boxes in-graph before decode().",
2042            roi.xmin, roi.ymin, roi.xmax, roi.ymax,
2043        )));
2044    }
2045
2046    let roi = [
2047        (roi.xmin * width).clamp(0.0, width) as usize,
2048        (roi.ymin * height).clamp(0.0, height) as usize,
2049        (roi.xmax * width).clamp(0.0, width).ceil() as usize,
2050        (roi.ymax * height).clamp(0.0, height).ceil() as usize,
2051    ];
2052
2053    let roi_norm = [
2054        roi[0] as f32 / width,
2055        roi[1] as f32 / height,
2056        roi[2] as f32 / width,
2057        roi[3] as f32 / height,
2058    ]
2059    .into();
2060
2061    let cropped = protos.slice(s![roi[1]..roi[3], roi[0]..roi[2], ..]);
2062
2063    Ok((cropped, roi_norm))
2064}
2065
2066/// Compute a single instance segmentation mask from mask coefficients and
2067/// proto maps (float path).
2068///
2069/// Computes `sigmoid(coefficients · protos)` and maps to `[0, 255]`.
2070/// Returns an `(H, W, 1)` u8 array.
2071fn make_segmentation<
2072    MASK: Float + AsPrimitive<f32> + Send + Sync,
2073    PROTO: Float + AsPrimitive<f32> + Send + Sync,
2074>(
2075    mask: ArrayView1<MASK>,
2076    protos: ArrayView3<PROTO>,
2077) -> Array3<u8> {
2078    let shape = protos.shape();
2079
2080    // Safe to unwrap since the shapes will always be compatible
2081    let mask = mask.to_shape((1, mask.len())).unwrap();
2082    let protos = protos.to_shape([shape[0] * shape[1], shape[2]]).unwrap();
2083    let protos = protos.reversed_axes();
2084    let mask = mask.map(|x| x.as_());
2085    let protos = protos.map(|x| x.as_());
2086
2087    // Safe to unwrap since the shapes will always be compatible
2088    let mask = mask
2089        .dot(&protos)
2090        .into_shape_with_order((shape[0], shape[1], 1))
2091        .unwrap();
2092
2093    mask.map(|x| {
2094        let sigmoid = 1.0 / (1.0 + (-*x).exp());
2095        (sigmoid * 255.0).round() as u8
2096    })
2097}
2098
2099/// Compute a single instance segmentation mask from quantized mask
2100/// coefficients and proto maps.
2101///
2102/// Dequantizes both inputs (subtracting zero-points), computes the dot
2103/// product, applies sigmoid, and maps to `[0, 255]`.
2104/// Returns an `(H, W, 1)` u8 array.
2105fn make_segmentation_quant<
2106    MASK: PrimInt + AsPrimitive<DEST> + Send + Sync,
2107    PROTO: PrimInt + AsPrimitive<DEST> + Send + Sync,
2108    DEST: PrimInt + 'static + Signed + AsPrimitive<f32> + Debug,
2109>(
2110    mask: ArrayView1<MASK>,
2111    protos: ArrayView3<PROTO>,
2112    quant_masks: Quantization,
2113    quant_protos: Quantization,
2114) -> Array3<u8>
2115where
2116    i32: AsPrimitive<DEST>,
2117    f32: AsPrimitive<DEST>,
2118{
2119    let shape = protos.shape();
2120
2121    // Safe to unwrap since the shapes will always be compatible
2122    let mask = mask.to_shape((1, mask.len())).unwrap();
2123
2124    let protos = protos.to_shape([shape[0] * shape[1], shape[2]]).unwrap();
2125    let protos = protos.reversed_axes();
2126
2127    let zp = quant_masks.zero_point.as_();
2128
2129    let mask = mask.mapv(|x| x.as_() - zp);
2130
2131    let zp = quant_protos.zero_point.as_();
2132    let protos = protos.mapv(|x| x.as_() - zp);
2133
2134    // Safe to unwrap since the shapes will always be compatible
2135    let segmentation = mask
2136        .dot(&protos)
2137        .into_shape_with_order((shape[0], shape[1], 1))
2138        .unwrap();
2139
2140    let combined_scale = quant_masks.scale * quant_protos.scale;
2141    segmentation.map(|x| {
2142        let val: f32 = (*x).as_() * combined_scale;
2143        let sigmoid = 1.0 / (1.0 + (-val).exp());
2144        (sigmoid * 255.0).round() as u8
2145    })
2146}
2147
2148/// Converts Yolo Instance Segmentation into a 2D mask.
2149///
2150/// The input segmentation is expected to have shape (H, W, 1).
2151///
2152/// The output mask will have shape (H, W), with values 0 or 1 based on the
2153/// threshold.
2154///
2155/// # Errors
2156///
2157/// Returns `DecoderError::InvalidShape` if the input segmentation does not
2158/// have shape (H, W, 1).
2159pub(crate) fn yolo_segmentation_to_mask(
2160    segmentation: ArrayView3<u8>,
2161    threshold: u8,
2162) -> Result<Array2<u8>, crate::DecoderError> {
2163    if segmentation.shape()[2] != 1 {
2164        return Err(crate::DecoderError::InvalidShape(format!(
2165            "Yolo Instance Segmentation should have shape (H, W, 1), got (H, W, {})",
2166            segmentation.shape()[2]
2167        )));
2168    }
2169    Ok(segmentation
2170        .slice(s![.., .., 0])
2171        .map(|x| if *x >= threshold { 1 } else { 0 }))
2172}
2173
2174#[cfg(test)]
2175#[cfg_attr(coverage_nightly, coverage(off))]
2176mod tests {
2177    use super::*;
2178    use ndarray::Array2;
2179
2180    // ========================================================================
2181    // Tests for decode_yolo_end_to_end_det_float
2182    // ========================================================================
2183
2184    #[test]
2185    fn test_end_to_end_det_basic_filtering() {
2186        // Create synthetic end-to-end detection output: (6, N) where rows are
2187        // [x1, y1, x2, y2, conf, class]
2188        // 3 detections: one above threshold, two below
2189        let data: Vec<f32> = vec![
2190            // Detection 0: high score (0.9)
2191            0.1, 0.2, 0.3, // x1 values
2192            0.1, 0.2, 0.3, // y1 values
2193            0.5, 0.6, 0.7, // x2 values
2194            0.5, 0.6, 0.7, // y2 values
2195            0.9, 0.1, 0.2, // confidence scores
2196            0.0, 1.0, 2.0, // class indices
2197        ];
2198        let output = Array2::from_shape_vec((6, 3), data).unwrap();
2199
2200        let mut boxes = Vec::with_capacity(10);
2201        decode_yolo_end_to_end_det_float(output.view(), 0.5, &mut boxes).unwrap();
2202
2203        // Only 1 detection should pass threshold of 0.5
2204        assert_eq!(boxes.len(), 1);
2205        assert_eq!(boxes[0].label, 0);
2206        assert!((boxes[0].score - 0.9).abs() < 0.01);
2207        assert!((boxes[0].bbox.xmin - 0.1).abs() < 0.01);
2208        assert!((boxes[0].bbox.ymin - 0.1).abs() < 0.01);
2209        assert!((boxes[0].bbox.xmax - 0.5).abs() < 0.01);
2210        assert!((boxes[0].bbox.ymax - 0.5).abs() < 0.01);
2211    }
2212
2213    #[test]
2214    fn test_end_to_end_det_all_pass_threshold() {
2215        // All detections above threshold
2216        let data: Vec<f32> = vec![
2217            10.0, 20.0, // x1
2218            10.0, 20.0, // y1
2219            50.0, 60.0, // x2
2220            50.0, 60.0, // y2
2221            0.8, 0.7, // conf (both above 0.5)
2222            1.0, 2.0, // class
2223        ];
2224        let output = Array2::from_shape_vec((6, 2), data).unwrap();
2225
2226        let mut boxes = Vec::with_capacity(10);
2227        decode_yolo_end_to_end_det_float(output.view(), 0.5, &mut boxes).unwrap();
2228
2229        assert_eq!(boxes.len(), 2);
2230        assert_eq!(boxes[0].label, 1);
2231        assert_eq!(boxes[1].label, 2);
2232    }
2233
2234    #[test]
2235    fn test_end_to_end_det_none_pass_threshold() {
2236        // All detections below threshold
2237        let data: Vec<f32> = vec![
2238            10.0, 20.0, // x1
2239            10.0, 20.0, // y1
2240            50.0, 60.0, // x2
2241            50.0, 60.0, // y2
2242            0.1, 0.2, // conf (both below 0.5)
2243            1.0, 2.0, // class
2244        ];
2245        let output = Array2::from_shape_vec((6, 2), data).unwrap();
2246
2247        let mut boxes = Vec::with_capacity(10);
2248        decode_yolo_end_to_end_det_float(output.view(), 0.5, &mut boxes).unwrap();
2249
2250        assert_eq!(boxes.len(), 0);
2251    }
2252
2253    #[test]
2254    fn test_end_to_end_det_capacity_limit() {
2255        // Test that output is truncated to capacity
2256        let data: Vec<f32> = vec![
2257            0.1, 0.2, 0.3, 0.4, 0.5, // x1
2258            0.1, 0.2, 0.3, 0.4, 0.5, // y1
2259            0.5, 0.6, 0.7, 0.8, 0.9, // x2
2260            0.5, 0.6, 0.7, 0.8, 0.9, // y2
2261            0.9, 0.9, 0.9, 0.9, 0.9, // conf (all pass)
2262            0.0, 1.0, 2.0, 3.0, 4.0, // class
2263        ];
2264        let output = Array2::from_shape_vec((6, 5), data).unwrap();
2265
2266        let mut boxes = Vec::with_capacity(2); // Only allow 2 boxes
2267        decode_yolo_end_to_end_det_float(output.view(), 0.5, &mut boxes).unwrap();
2268
2269        assert_eq!(boxes.len(), 2);
2270    }
2271
2272    #[test]
2273    fn test_end_to_end_det_empty_output() {
2274        // Test with zero detections
2275        let output = Array2::<f32>::zeros((6, 0));
2276
2277        let mut boxes = Vec::with_capacity(10);
2278        decode_yolo_end_to_end_det_float(output.view(), 0.5, &mut boxes).unwrap();
2279
2280        assert_eq!(boxes.len(), 0);
2281    }
2282
2283    #[test]
2284    fn test_end_to_end_det_pixel_coordinates() {
2285        // Test with pixel coordinates (non-normalized)
2286        let data: Vec<f32> = vec![
2287            100.0, // x1
2288            200.0, // y1
2289            300.0, // x2
2290            400.0, // y2
2291            0.95,  // conf
2292            5.0,   // class
2293        ];
2294        let output = Array2::from_shape_vec((6, 1), data).unwrap();
2295
2296        let mut boxes = Vec::with_capacity(10);
2297        decode_yolo_end_to_end_det_float(output.view(), 0.5, &mut boxes).unwrap();
2298
2299        assert_eq!(boxes.len(), 1);
2300        assert_eq!(boxes[0].label, 5);
2301        assert!((boxes[0].bbox.xmin - 100.0).abs() < 0.01);
2302        assert!((boxes[0].bbox.ymin - 200.0).abs() < 0.01);
2303        assert!((boxes[0].bbox.xmax - 300.0).abs() < 0.01);
2304        assert!((boxes[0].bbox.ymax - 400.0).abs() < 0.01);
2305    }
2306
2307    #[test]
2308    fn test_end_to_end_det_invalid_shape() {
2309        // Test with too few rows (needs at least 6)
2310        let output = Array2::<f32>::zeros((5, 3));
2311
2312        let mut boxes = Vec::with_capacity(10);
2313        let result = decode_yolo_end_to_end_det_float(output.view(), 0.5, &mut boxes);
2314
2315        assert!(result.is_err());
2316        assert!(matches!(
2317            result,
2318            Err(crate::DecoderError::InvalidShape(s)) if s.contains("at least 6 rows")
2319        ));
2320    }
2321
2322    // ========================================================================
2323    // Tests for decode_yolo_end_to_end_segdet_float
2324    // ========================================================================
2325
2326    #[test]
2327    fn test_end_to_end_segdet_basic() {
2328        // Create synthetic segdet output: (6 + num_protos, N)
2329        // Detection format: [x1, y1, x2, y2, conf, class, mask_coeff_0..31]
2330        let num_protos = 32;
2331        let num_detections = 2;
2332        let num_features = 6 + num_protos;
2333
2334        // Build detection tensor
2335        let mut data = vec![0.0f32; num_features * num_detections];
2336        // Detection 0: passes threshold
2337        data[0] = 0.1; // x1[0]
2338        data[1] = 0.5; // x1[1]
2339        data[num_detections] = 0.1; // y1[0]
2340        data[num_detections + 1] = 0.5; // y1[1]
2341        data[2 * num_detections] = 0.4; // x2[0]
2342        data[2 * num_detections + 1] = 0.9; // x2[1]
2343        data[3 * num_detections] = 0.4; // y2[0]
2344        data[3 * num_detections + 1] = 0.9; // y2[1]
2345        data[4 * num_detections] = 0.9; // conf[0] - passes
2346        data[4 * num_detections + 1] = 0.3; // conf[1] - fails
2347        data[5 * num_detections] = 1.0; // class[0]
2348        data[5 * num_detections + 1] = 2.0; // class[1]
2349                                            // Fill mask coefficients with small values
2350        for i in 6..num_features {
2351            data[i * num_detections] = 0.1;
2352            data[i * num_detections + 1] = 0.1;
2353        }
2354
2355        let output = Array2::from_shape_vec((num_features, num_detections), data).unwrap();
2356
2357        // Create protos tensor: (proto_height, proto_width, num_protos)
2358        let protos = Array3::<f32>::zeros((16, 16, num_protos));
2359
2360        let mut boxes = Vec::with_capacity(10);
2361        let mut masks = Vec::with_capacity(10);
2362        decode_yolo_end_to_end_segdet_float(
2363            output.view(),
2364            protos.view(),
2365            0.5,
2366            &mut boxes,
2367            &mut masks,
2368        )
2369        .unwrap();
2370
2371        // Only detection 0 should pass
2372        assert_eq!(boxes.len(), 1);
2373        assert_eq!(masks.len(), 1);
2374        assert_eq!(boxes[0].label, 1);
2375        assert!((boxes[0].score - 0.9).abs() < 0.01);
2376    }
2377
2378    #[test]
2379    fn test_end_to_end_segdet_mask_coordinates() {
2380        // Test that mask coordinates match box coordinates
2381        let num_protos = 32;
2382        let num_features = 6 + num_protos;
2383
2384        let mut data = vec![0.0f32; num_features];
2385        data[0] = 0.2; // x1
2386        data[1] = 0.2; // y1
2387        data[2] = 0.8; // x2
2388        data[3] = 0.8; // y2
2389        data[4] = 0.95; // conf
2390        data[5] = 3.0; // class
2391
2392        let output = Array2::from_shape_vec((num_features, 1), data).unwrap();
2393        let protos = Array3::<f32>::zeros((16, 16, num_protos));
2394
2395        let mut boxes = Vec::with_capacity(10);
2396        let mut masks = Vec::with_capacity(10);
2397        decode_yolo_end_to_end_segdet_float(
2398            output.view(),
2399            protos.view(),
2400            0.5,
2401            &mut boxes,
2402            &mut masks,
2403        )
2404        .unwrap();
2405
2406        assert_eq!(boxes.len(), 1);
2407        assert_eq!(masks.len(), 1);
2408
2409        // Mask region is the proto-grid-aligned crop and encloses the
2410        // post-NMS bbox (EDGEAI-1304); on a 16x16 grid each side may snap
2411        // by up to 1/16 = 0.0625.
2412        let step = 1.0 / 16.0;
2413        assert!(masks[0].xmin <= boxes[0].bbox.xmin);
2414        assert!(masks[0].ymin <= boxes[0].bbox.ymin);
2415        assert!(masks[0].xmax >= boxes[0].bbox.xmax);
2416        assert!(masks[0].ymax >= boxes[0].bbox.ymax);
2417        assert!((boxes[0].bbox.xmin - masks[0].xmin) < step);
2418        assert!((boxes[0].bbox.ymin - masks[0].ymin) < step);
2419        assert!((masks[0].xmax - boxes[0].bbox.xmax) < step);
2420        assert!((masks[0].ymax - boxes[0].bbox.ymax) < step);
2421    }
2422
2423    #[test]
2424    fn test_end_to_end_segdet_empty_output() {
2425        let num_protos = 32;
2426        let output = Array2::<f32>::zeros((6 + num_protos, 0));
2427        let protos = Array3::<f32>::zeros((16, 16, num_protos));
2428
2429        let mut boxes = Vec::with_capacity(10);
2430        let mut masks = Vec::with_capacity(10);
2431        decode_yolo_end_to_end_segdet_float(
2432            output.view(),
2433            protos.view(),
2434            0.5,
2435            &mut boxes,
2436            &mut masks,
2437        )
2438        .unwrap();
2439
2440        assert_eq!(boxes.len(), 0);
2441        assert_eq!(masks.len(), 0);
2442    }
2443
2444    #[test]
2445    fn test_end_to_end_segdet_capacity_limit() {
2446        let num_protos = 32;
2447        let num_detections = 5;
2448        let num_features = 6 + num_protos;
2449
2450        let mut data = vec![0.0f32; num_features * num_detections];
2451        // All detections pass threshold
2452        for i in 0..num_detections {
2453            data[i] = 0.1 * (i as f32); // x1
2454            data[num_detections + i] = 0.1 * (i as f32); // y1
2455            data[2 * num_detections + i] = 0.1 * (i as f32) + 0.2; // x2
2456            data[3 * num_detections + i] = 0.1 * (i as f32) + 0.2; // y2
2457            data[4 * num_detections + i] = 0.9; // conf
2458            data[5 * num_detections + i] = i as f32; // class
2459        }
2460
2461        let output = Array2::from_shape_vec((num_features, num_detections), data).unwrap();
2462        let protos = Array3::<f32>::zeros((16, 16, num_protos));
2463
2464        let mut boxes = Vec::with_capacity(2); // Limit to 2
2465        let mut masks = Vec::with_capacity(2);
2466        decode_yolo_end_to_end_segdet_float(
2467            output.view(),
2468            protos.view(),
2469            0.5,
2470            &mut boxes,
2471            &mut masks,
2472        )
2473        .unwrap();
2474
2475        assert_eq!(boxes.len(), 2);
2476        assert_eq!(masks.len(), 2);
2477    }
2478
2479    #[test]
2480    fn test_end_to_end_segdet_invalid_shape_too_few_rows() {
2481        // Test with too few rows (needs at least 7: 6 base + 1 mask coeff)
2482        let output = Array2::<f32>::zeros((6, 3));
2483        let protos = Array3::<f32>::zeros((16, 16, 32));
2484
2485        let mut boxes = Vec::with_capacity(10);
2486        let mut masks = Vec::with_capacity(10);
2487        let result = decode_yolo_end_to_end_segdet_float(
2488            output.view(),
2489            protos.view(),
2490            0.5,
2491            &mut boxes,
2492            &mut masks,
2493        );
2494
2495        assert!(result.is_err());
2496        assert!(matches!(
2497            result,
2498            Err(crate::DecoderError::InvalidShape(s)) if s.contains("at least 7 rows")
2499        ));
2500    }
2501
2502    #[test]
2503    fn test_end_to_end_segdet_invalid_shape_protos_mismatch() {
2504        // Test with mismatched mask coefficients and protos count
2505        let num_protos = 32;
2506        let output = Array2::<f32>::zeros((6 + 16, 3)); // 16 mask coeffs
2507        let protos = Array3::<f32>::zeros((16, 16, num_protos)); // 32 protos
2508
2509        let mut boxes = Vec::with_capacity(10);
2510        let mut masks = Vec::with_capacity(10);
2511        let result = decode_yolo_end_to_end_segdet_float(
2512            output.view(),
2513            protos.view(),
2514            0.5,
2515            &mut boxes,
2516            &mut masks,
2517        );
2518
2519        assert!(result.is_err());
2520        assert!(matches!(
2521            result,
2522            Err(crate::DecoderError::InvalidShape(s)) if s.contains("doesn't match protos count")
2523        ));
2524    }
2525
2526    // ========================================================================
2527    // Tests for decode_yolo_split_end_to_end_segdet_float
2528    // ========================================================================
2529
2530    #[test]
2531    fn test_split_end_to_end_segdet_basic() {
2532        // Create synthetic segdet output: (6 + num_protos, N)
2533        // Detection format: [x1, y1, x2, y2, conf, class, mask_coeff_0..31]
2534        let num_protos = 32;
2535        let num_detections = 2;
2536        let num_features = 6 + num_protos;
2537
2538        // Build detection tensor
2539        let mut data = vec![0.0f32; num_features * num_detections];
2540        // Detection 0: passes threshold
2541        data[0] = 0.1; // x1[0]
2542        data[1] = 0.5; // x1[1]
2543        data[num_detections] = 0.1; // y1[0]
2544        data[num_detections + 1] = 0.5; // y1[1]
2545        data[2 * num_detections] = 0.4; // x2[0]
2546        data[2 * num_detections + 1] = 0.9; // x2[1]
2547        data[3 * num_detections] = 0.4; // y2[0]
2548        data[3 * num_detections + 1] = 0.9; // y2[1]
2549        data[4 * num_detections] = 0.9; // conf[0] - passes
2550        data[4 * num_detections + 1] = 0.3; // conf[1] - fails
2551        data[5 * num_detections] = 1.0; // class[0]
2552        data[5 * num_detections + 1] = 2.0; // class[1]
2553                                            // Fill mask coefficients with small values
2554        for i in 6..num_features {
2555            data[i * num_detections] = 0.1;
2556            data[i * num_detections + 1] = 0.1;
2557        }
2558
2559        let output = Array2::from_shape_vec((num_features, num_detections), data).unwrap();
2560        let box_coords = output.slice(s![..4, ..]);
2561        let scores = output.slice(s![4..5, ..]);
2562        let classes = output.slice(s![5..6, ..]);
2563        let mask_coeff = output.slice(s![6.., ..]);
2564        // Create protos tensor: (proto_height, proto_width, num_protos)
2565        let protos = Array3::<f32>::zeros((16, 16, num_protos));
2566
2567        let mut boxes = Vec::with_capacity(10);
2568        let mut masks = Vec::with_capacity(10);
2569        decode_yolo_split_end_to_end_segdet_float(
2570            box_coords,
2571            scores,
2572            classes,
2573            mask_coeff,
2574            protos.view(),
2575            0.5,
2576            &mut boxes,
2577            &mut masks,
2578        )
2579        .unwrap();
2580
2581        // Only detection 0 should pass
2582        assert_eq!(boxes.len(), 1);
2583        assert_eq!(masks.len(), 1);
2584        assert_eq!(boxes[0].label, 1);
2585        assert!((boxes[0].score - 0.9).abs() < 0.01);
2586    }
2587
2588    // ========================================================================
2589    // Tests for yolo_segmentation_to_mask
2590    // ========================================================================
2591
2592    #[test]
2593    fn test_segmentation_to_mask_basic() {
2594        // Create a 4x4x1 segmentation with values above and below threshold
2595        let data: Vec<u8> = vec![
2596            100, 200, 50, 150, // row 0
2597            10, 255, 128, 64, // row 1
2598            0, 127, 128, 255, // row 2
2599            64, 64, 192, 192, // row 3
2600        ];
2601        let segmentation = Array3::from_shape_vec((4, 4, 1), data).unwrap();
2602
2603        let mask = yolo_segmentation_to_mask(segmentation.view(), 128).unwrap();
2604
2605        // Values >= 128 should be 1, others 0
2606        assert_eq!(mask[[0, 0]], 0); // 100 < 128
2607        assert_eq!(mask[[0, 1]], 1); // 200 >= 128
2608        assert_eq!(mask[[0, 2]], 0); // 50 < 128
2609        assert_eq!(mask[[0, 3]], 1); // 150 >= 128
2610        assert_eq!(mask[[1, 1]], 1); // 255 >= 128
2611        assert_eq!(mask[[1, 2]], 1); // 128 >= 128
2612        assert_eq!(mask[[2, 0]], 0); // 0 < 128
2613        assert_eq!(mask[[2, 1]], 0); // 127 < 128
2614    }
2615
2616    #[test]
2617    fn test_segmentation_to_mask_all_above() {
2618        let segmentation = Array3::from_elem((4, 4, 1), 255u8);
2619        let mask = yolo_segmentation_to_mask(segmentation.view(), 128).unwrap();
2620        assert!(mask.iter().all(|&x| x == 1));
2621    }
2622
2623    #[test]
2624    fn test_segmentation_to_mask_all_below() {
2625        let segmentation = Array3::from_elem((4, 4, 1), 64u8);
2626        let mask = yolo_segmentation_to_mask(segmentation.view(), 128).unwrap();
2627        assert!(mask.iter().all(|&x| x == 0));
2628    }
2629
2630    #[test]
2631    fn test_segmentation_to_mask_invalid_shape() {
2632        let segmentation = Array3::from_elem((4, 4, 3), 128u8);
2633        let result = yolo_segmentation_to_mask(segmentation.view(), 128);
2634
2635        assert!(result.is_err());
2636        assert!(matches!(
2637            result,
2638            Err(crate::DecoderError::InvalidShape(s)) if s.contains("(H, W, 1)")
2639        ));
2640    }
2641
2642    // ========================================================================
2643    // Tests for protobox / NORM_LIMIT regression
2644    // ========================================================================
2645
2646    #[test]
2647    fn test_protobox_clamps_edge_coordinates() {
2648        // bbox with xmax=1.0 should not panic (OOB guard)
2649        let protos = Array3::<f32>::zeros((16, 16, 4));
2650        let view = protos.view();
2651        let roi = BoundingBox {
2652            xmin: 0.5,
2653            ymin: 0.5,
2654            xmax: 1.0,
2655            ymax: 1.0,
2656        };
2657        let result = protobox(&view, &roi);
2658        assert!(result.is_ok(), "protobox should accept xmax=1.0");
2659        let (cropped, _roi_norm) = result.unwrap();
2660        // Cropped region must have non-zero spatial dimensions
2661        assert!(cropped.shape()[0] > 0);
2662        assert!(cropped.shape()[1] > 0);
2663        assert_eq!(cropped.shape()[2], 4);
2664    }
2665
2666    #[test]
2667    fn test_protobox_rejects_wildly_out_of_range() {
2668        // bbox with coords > NORM_LIMIT (e.g. 3.0) returns error
2669        let protos = Array3::<f32>::zeros((16, 16, 4));
2670        let view = protos.view();
2671        let roi = BoundingBox {
2672            xmin: 0.0,
2673            ymin: 0.0,
2674            xmax: 3.0,
2675            ymax: 3.0,
2676        };
2677        let result = protobox(&view, &roi);
2678        assert!(
2679            matches!(result, Err(crate::DecoderError::InvalidShape(s)) if s.contains("un-normalized")),
2680            "protobox should reject coords > NORM_LIMIT"
2681        );
2682    }
2683
2684    #[test]
2685    fn test_protobox_accepts_slightly_over_one() {
2686        // bbox with coords at 1.5 (within NORM_LIMIT=2.0) succeeds
2687        let protos = Array3::<f32>::zeros((16, 16, 4));
2688        let view = protos.view();
2689        let roi = BoundingBox {
2690            xmin: 0.0,
2691            ymin: 0.0,
2692            xmax: 1.5,
2693            ymax: 1.5,
2694        };
2695        let result = protobox(&view, &roi);
2696        assert!(
2697            result.is_ok(),
2698            "protobox should accept coords <= NORM_LIMIT (2.0)"
2699        );
2700        let (cropped, _roi_norm) = result.unwrap();
2701        // Entire proto map should be selected when coords > 1.0 (clamped to boundary)
2702        assert_eq!(cropped.shape()[0], 16);
2703        assert_eq!(cropped.shape()[1], 16);
2704    }
2705
2706    #[test]
2707    fn test_segdet_float_proto_no_panic() {
2708        // Simulates YOLOv8n-seg: output0 = [116, 8400] (4 box + 80 class + 32 mask coeff)
2709        // output1 (protos) = [32, 160, 160]
2710        let num_proposals = 100; // enough to produce idx >= 32
2711        let num_classes = 80;
2712        let num_mask_coeffs = 32;
2713        let rows = 4 + num_classes + num_mask_coeffs; // 116
2714
2715        // Fill boxes with valid xywh data so some detections pass the threshold.
2716        // Layout is [116, num_proposals] row-major: row 0=cx, 1=cy, 2=w, 3=h,
2717        // rows 4..84=class scores, rows 84..116=mask coefficients.
2718        let mut data = vec![0.0f32; rows * num_proposals];
2719        for i in 0..num_proposals {
2720            let row = |r: usize| r * num_proposals + i;
2721            data[row(0)] = 320.0; // cx
2722            data[row(1)] = 320.0; // cy
2723            data[row(2)] = 50.0; // w
2724            data[row(3)] = 50.0; // h
2725            data[row(4)] = 0.9; // class-0 score
2726        }
2727        let boxes = ndarray::Array2::from_shape_vec((rows, num_proposals), data).unwrap();
2728
2729        // Protos must be in HWC order. Under the HAL physical-order
2730        // contract, callers declare shape+dshape matching producer memory
2731        // and swap_axes_if_needed permutes the stride tuple into canonical
2732        // [batch, height, width, num_protos] before this function sees it.
2733        let protos = ndarray::Array3::<f32>::zeros((160, 160, num_mask_coeffs));
2734
2735        let mut output_boxes = Vec::with_capacity(300);
2736
2737        // This panicked before fix: mask_tensor.row(idx) with idx >= 32
2738        let proto_data = impl_yolo_segdet_float_proto::<XYWH, _, _>(
2739            boxes.view(),
2740            protos.view(),
2741            0.5,
2742            0.7,
2743            Some(Nms::default()),
2744            MAX_NMS_CANDIDATES,
2745            300,
2746            None,
2747            None,
2748            false, // multi_label: argmax for this test
2749            &mut output_boxes,
2750        );
2751
2752        // Should produce detections (NMS will collapse many overlapping boxes)
2753        assert!(!output_boxes.is_empty());
2754        let coeffs_shape = proto_data.mask_coefficients.shape();
2755        assert_eq!(coeffs_shape[0], output_boxes.len());
2756        // Each mask coefficient vector should have 32 elements
2757        assert_eq!(coeffs_shape[1], num_mask_coeffs);
2758    }
2759
2760    // ========================================================================
2761    // Pre-NMS top-K cap (MAX_NMS_CANDIDATES)
2762    // ========================================================================
2763
2764    /// At very low score thresholds (e.g., t=0.01 on YOLOv8 with 8400×80
2765    /// candidates) almost every score passes the filter, feeding O(n²)
2766    /// NMS and a per-survivor mask matmul. The decoder caps the
2767    /// candidate set fed to NMS at `MAX_NMS_CANDIDATES` (Ultralytics
2768    /// default 30 000) to bound worst-case decode time.
2769    ///
2770    /// This regression test pumps 50 000 above-threshold candidates
2771    /// into `impl_yolo_segdet_get_boxes` with NMS bypassed (Nms=None)
2772    /// and a generous post-NMS cap. Before the fix, the function
2773    /// returned all 50 000; after the fix, exactly 30 000.
2774    #[test]
2775    fn test_pre_nms_cap_truncates_excess_candidates() {
2776        let n: usize = 50_000;
2777        let num_classes = 1;
2778
2779        // Identical valid boxes. Distinct scores (descending) so the
2780        // top-K cap keeps the highest-scoring ones in deterministic
2781        // order — letting us assert *which* ones survived.
2782        let mut boxes_data = Vec::with_capacity(n * 4);
2783        let mut scores_data = Vec::with_capacity(n * num_classes);
2784        for i in 0..n {
2785            boxes_data.extend_from_slice(&[0.1f32, 0.1, 0.5, 0.5]);
2786            // score_i = 0.99 - i * 1e-7 keeps everything well above 0.1
2787            // threshold but strictly decreasing.
2788            scores_data.push(0.99 - (i as f32) * 1e-7);
2789        }
2790        let boxes = Array2::from_shape_vec((n, 4), boxes_data).unwrap();
2791        let scores = Array2::from_shape_vec((n, num_classes), scores_data).unwrap();
2792
2793        let result = impl_yolo_segdet_get_boxes::<XYXY, _, _>(
2794            boxes.view(),
2795            scores.view(),
2796            0.1,
2797            1.0,                             // IoU 1.0 → NMS suppresses nothing
2798            Some(Nms::ClassAgnostic),        // NMS enabled so pre_nms_top_k applies
2799            crate::yolo::MAX_NMS_CANDIDATES, // pre_nms_top_k
2800            usize::MAX,                      // no post-NMS truncation
2801            false,                           // multi_label: argmax for this test
2802        );
2803
2804        assert_eq!(
2805            result.len(),
2806            crate::yolo::MAX_NMS_CANDIDATES,
2807            "pre-NMS cap should truncate to MAX_NMS_CANDIDATES; got {}",
2808            result.len()
2809        );
2810        // Top-K survivors: highest scores were the first n indices,
2811        // so survivor 0 must have score ~0.99.
2812        let top_score = result[0].0.score;
2813        assert!(
2814            top_score > 0.98,
2815            "highest-ranked survivor should have the largest score, got {top_score}"
2816        );
2817    }
2818
2819    /// Counterpart for the quantized split path. Same contract: feed
2820    /// more than `MAX_NMS_CANDIDATES` survivors above the quantized
2821    /// threshold, confirm `impl_yolo_split_segdet_quant_get_boxes`
2822    /// truncates before NMS.
2823    #[test]
2824    fn test_pre_nms_cap_truncates_excess_candidates_quant() {
2825        use crate::Quantization;
2826        let n: usize = 50_000;
2827        let num_classes = 1;
2828
2829        // i8 boxes with simple scale/zp; the box value 50 dequantizes
2830        // to 0.5 with scale=0.01, zp=0 — fine for a flat box set.
2831        let boxes_data = (0..n).flat_map(|_| [10i8, 10, 50, 50]).collect::<Vec<_>>();
2832        let boxes = Array2::from_shape_vec((n, 4), boxes_data).unwrap();
2833        let quant_boxes = Quantization {
2834            scale: 0.01,
2835            zero_point: 0,
2836        };
2837
2838        // u8 scores: distinct descending values, all well above threshold.
2839        // value 250 → 0.98 with scale 0.00392, zp 0.
2840        // value (250 - i % 200) keeps a wide spread above the dequant
2841        // threshold of 0.5.
2842        let scores_data: Vec<u8> = (0..n)
2843            .map(|i| 250u8.saturating_sub((i % 200) as u8))
2844            .collect();
2845        let scores = Array2::from_shape_vec((n, num_classes), scores_data).unwrap();
2846        let quant_scores = Quantization {
2847            scale: 0.00392,
2848            zero_point: 0,
2849        };
2850
2851        let result = impl_yolo_split_segdet_quant_get_boxes::<XYXY, _, _>(
2852            (boxes.view(), quant_boxes),
2853            (scores.view(), quant_scores),
2854            0.1,
2855            1.0,                             // IoU 1.0 → NMS suppresses nothing
2856            Some(Nms::ClassAgnostic),        // NMS enabled so pre_nms_top_k applies
2857            crate::yolo::MAX_NMS_CANDIDATES, // pre_nms_top_k
2858            usize::MAX,                      // no post-NMS truncation
2859        );
2860
2861        assert_eq!(
2862            result.len(),
2863            crate::yolo::MAX_NMS_CANDIDATES,
2864            "quant path pre-NMS cap should truncate to MAX_NMS_CANDIDATES; got {}",
2865            result.len()
2866        );
2867    }
2868
2869    /// Regression test for HAILORT_BUG.md — the YoloSegDet path
2870    /// (combined `(4 + nc + nm, N)` detection tensor + separate protos)
2871    /// must pair each surviving detection with the mask coefficient
2872    /// row at the SAME anchor index the box came from. The validator
2873    /// sees this path miss the pairing under schema-v2 Hailo inputs
2874    /// (mAP collapse from 46.8 → 3.65 while mask IoU stays at 66.9,
2875    /// the fingerprint of mask-to-detection misalignment).
2876    ///
2877    /// Construction: three anchors with distinct mask-coef signatures
2878    /// that, after dot(coefs, protos) + sigmoid, produce HIGH vs LOW
2879    /// mask pixel values. Two anchors survive (one high, one low); if
2880    /// the mask row is looked up at the wrong index, the per-detection
2881    /// mean mask value would cross the threshold and we catch it.
2882    #[test]
2883    fn segdet_combined_tensor_pairs_detection_with_matching_mask_row() {
2884        let nc = 2; // num_classes
2885        let nm = 2; // num_protos
2886        let n = 3; // num_anchors
2887        let feat = 4 + nc + nm; // 8
2888
2889        // Tensor layout: (8, 3) rows=features, cols=anchors.
2890        // Row indices:  0..4 = xywh, 4..6 = scores, 6..8 = mask_coefs.
2891        //
2892        //         anchor 0 | anchor 1 | anchor 2
2893        // xc       0.2      | 0.5      | 0.8
2894        // yc       0.2      | 0.5      | 0.8
2895        // w        0.1      | 0.1      | 0.1
2896        // h        0.1      | 0.1      | 0.1
2897        // s[0]     0.9      | 0.0      | 0.8   (class 0)
2898        // s[1]     0.0      | 0.0      | 0.0   (class 1 — always loses)
2899        // m[0]     3.0      | 0.0      | -3.0  (high for a0, low for a2)
2900        // m[1]     3.0      | 0.0      | -3.0
2901        //
2902        // Proto[0] = Proto[1] = all-ones (8x8), so
2903        //   mask(a0) = sigmoid(3 + 3) ≈ 0.9975 → 254
2904        //   mask(a2) = sigmoid(-3 + -3) ≈ 0.0025 → 1
2905        // 250-point gap makes any misalignment trivially detectable.
2906        let mut data = vec![0.0f32; feat * n];
2907        let set = |d: &mut [f32], r: usize, c: usize, v: f32| d[r * n + c] = v;
2908        set(&mut data, 0, 0, 0.2);
2909        set(&mut data, 1, 0, 0.2);
2910        set(&mut data, 2, 0, 0.1);
2911        set(&mut data, 3, 0, 0.1);
2912        set(&mut data, 0, 1, 0.5);
2913        set(&mut data, 1, 1, 0.5);
2914        set(&mut data, 2, 1, 0.1);
2915        set(&mut data, 3, 1, 0.1);
2916        set(&mut data, 0, 2, 0.8);
2917        set(&mut data, 1, 2, 0.8);
2918        set(&mut data, 2, 2, 0.1);
2919        set(&mut data, 3, 2, 0.1);
2920        set(&mut data, 4, 0, 0.9);
2921        set(&mut data, 4, 2, 0.8);
2922        set(&mut data, 6, 0, 3.0);
2923        set(&mut data, 7, 0, 3.0);
2924        set(&mut data, 6, 2, -3.0);
2925        set(&mut data, 7, 2, -3.0);
2926
2927        let output = Array2::from_shape_vec((feat, n), data).unwrap();
2928        let protos = Array3::<f32>::from_elem((8, 8, nm), 1.0);
2929
2930        let mut boxes: Vec<DetectBox> = Vec::with_capacity(10);
2931        let mut masks: Vec<Segmentation> = Vec::with_capacity(10);
2932        decode_yolo_segdet_float(
2933            output.view(),
2934            protos.view(),
2935            0.5,
2936            0.5,
2937            Some(Nms::ClassAgnostic),
2938            &mut boxes,
2939            &mut masks,
2940        )
2941        .unwrap();
2942
2943        assert_eq!(
2944            boxes.len(),
2945            2,
2946            "two anchors above threshold should survive (a0 score=0.9, a2 score=0.8); got {}",
2947            boxes.len()
2948        );
2949
2950        // Build a (anchor_index → mask_mean) mapping from the results.
2951        // Anchor 0 has centre (0.2, 0.2), anchor 2 has centre (0.8,
2952        // 0.8). The DetectBox bbox is the post-XYWH-to-XYXY conversion
2953        // of the original xywh; cropping inside protobox may shrink it,
2954        // so match by centre (0.2 vs 0.8) rather than exact bbox.
2955        for (b, m) in boxes.iter().zip(masks.iter()) {
2956            let cx = (b.bbox.xmin + b.bbox.xmax) * 0.5;
2957            let mean = {
2958                let s = &m.segmentation;
2959                let total: u32 = s.iter().map(|&v| v as u32).sum();
2960                total as f32 / s.len() as f32
2961            };
2962            if cx < 0.3 {
2963                // anchor 0 — expect HIGH mask values ≈ 254
2964                assert!(
2965                    mean > 200.0,
2966                    "anchor 0 detection (centre {cx:.2}) should have high-value mask; got mean {mean}"
2967                );
2968            } else if cx > 0.7 {
2969                // anchor 2 — expect LOW mask values ≈ 1
2970                assert!(
2971                    mean < 50.0,
2972                    "anchor 2 detection (centre {cx:.2}) should have low-value mask; got mean {mean}"
2973                );
2974            } else {
2975                panic!("unexpected detection centre {cx:.2}");
2976            }
2977        }
2978    }
2979
2980    // ========================================================================
2981    // Tests for truncate_to_top_k_by_score / truncate_to_top_k_by_score_quant
2982    // ========================================================================
2983
2984    /// Helper: build a Vec of (DetectBox, ()) with the given scores.
2985    fn make_float_boxes(scores: &[f32]) -> Vec<(DetectBox, ())> {
2986        scores
2987            .iter()
2988            .enumerate()
2989            .map(|(i, &s)| {
2990                (
2991                    DetectBox {
2992                        bbox: BoundingBox {
2993                            xmin: 0.0,
2994                            ymin: 0.0,
2995                            xmax: 1.0,
2996                            ymax: 1.0,
2997                        },
2998                        score: s,
2999                        label: i,
3000                    },
3001                    (),
3002                )
3003            })
3004            .collect()
3005    }
3006
3007    /// Helper: build a Vec of (DetectBoxQuantized<i8>, ()) with the given scores.
3008    fn make_quant_boxes(scores: &[i8]) -> Vec<(DetectBoxQuantized<i8>, ())> {
3009        scores
3010            .iter()
3011            .enumerate()
3012            .map(|(i, &s)| {
3013                (
3014                    DetectBoxQuantized {
3015                        bbox: BoundingBox {
3016                            xmin: 0.0,
3017                            ymin: 0.0,
3018                            xmax: 1.0,
3019                            ymax: 1.0,
3020                        },
3021                        score: s,
3022                        label: i,
3023                    },
3024                    (),
3025                )
3026            })
3027            .collect()
3028    }
3029
3030    #[test]
3031    fn truncate_float_top_k_zero_is_unbounded() {
3032        let mut boxes = make_float_boxes(&[0.9, 0.1, 0.5, 0.3, 0.7]);
3033        let original_len = boxes.len();
3034        truncate_to_top_k_by_score(&mut boxes, 0);
3035        assert_eq!(
3036            boxes.len(),
3037            original_len,
3038            "top_k=0 should keep all candidates (no-limit semantics)"
3039        );
3040    }
3041
3042    #[test]
3043    fn truncate_float_top_k_normal() {
3044        let mut boxes = make_float_boxes(&[0.9, 0.1, 0.5, 0.3, 0.7]);
3045        truncate_to_top_k_by_score(&mut boxes, 3);
3046        assert_eq!(boxes.len(), 3);
3047        // The top-3 scores should be 0.9, 0.7, 0.5 (order within top-K is unspecified)
3048        let mut retained: Vec<f32> = boxes.iter().map(|(b, _)| b.score).collect();
3049        retained.sort_by(|a, b| b.total_cmp(a));
3050        assert_eq!(retained, vec![0.9, 0.7, 0.5]);
3051    }
3052
3053    #[test]
3054    fn truncate_float_top_k_noop_when_under_cap() {
3055        let mut boxes = make_float_boxes(&[0.9, 0.5]);
3056        truncate_to_top_k_by_score(&mut boxes, 10);
3057        assert_eq!(boxes.len(), 2, "should be no-op when len <= top_k");
3058    }
3059
3060    #[test]
3061    fn truncate_quant_top_k_zero_is_unbounded() {
3062        let mut boxes = make_quant_boxes(&[120, -50, 30, -10, 80]);
3063        let original_len = boxes.len();
3064        truncate_to_top_k_by_score_quant(&mut boxes, 0);
3065        assert_eq!(
3066            boxes.len(),
3067            original_len,
3068            "top_k=0 should keep all candidates (no-limit semantics)"
3069        );
3070    }
3071
3072    #[test]
3073    fn truncate_quant_top_k_normal() {
3074        let mut boxes = make_quant_boxes(&[120, -50, 30, -10, 80]);
3075        truncate_to_top_k_by_score_quant(&mut boxes, 3);
3076        assert_eq!(boxes.len(), 3);
3077        let mut retained: Vec<i8> = boxes.iter().map(|(b, _)| b.score).collect();
3078        retained.sort_by(|a, b| b.cmp(a));
3079        assert_eq!(retained, vec![120, 80, 30]);
3080    }
3081
3082    #[test]
3083    fn truncate_quant_top_k_noop_when_under_cap() {
3084        let mut boxes = make_quant_boxes(&[120, 80]);
3085        truncate_to_top_k_by_score_quant(&mut boxes, 10);
3086        assert_eq!(boxes.len(), 2, "should be no-op when len <= top_k");
3087    }
3088}