Skip to main content

astrs_node_api/message/
vision.rs

1//! `std/vision/v1` — perception outputs (§24.3).
2//!
3//! | Type | Layout |
4//! |---|---|
5//! | [`Detections`] | `{boxes: List<FixedSizeList<Float32, 4>>, scores: List<Float32>, labels: List<UInt32>}` |
6//! | [`Keypoints`] | `{points: List<FixedSizeList<Float32, 2>>, scores: List<Float32>, labels: List<UInt32>}` |
7//! | [`Mask`] | `{width: UInt32, height: UInt32, data: List<UInt16>}` |
8//!
9//! The three `List` columns of [`Detections`] and [`Keypoints`] are
10//! *index-aligned*: entry *i* of `scores` is the confidence of entry *i* of
11//! `boxes`. Nothing in the columnar layout enforces that — three lists of
12//! different lengths are a well-formed `Struct` — so both types check it on
13//! encode and on decode, which is the difference between "the payload is
14//! malformed" and "the labels are silently off by one".
15//!
16//! # Examples
17//!
18//! ```
19//! use astrs_node_api::message::{AstrsMessage, BoundingBox, Detections};
20//!
21//! let detections = Detections::new(
22//!     vec![BoundingBox::new(0.0, 0.0, 10.0, 20.0)],
23//!     vec![0.93],
24//!     vec![7],
25//! )?;
26//! let batch = detections.to_record_batch()?;
27//! assert_eq!(Detections::from_record_batch(&batch)?.len(), 1);
28//! # Ok::<(), astrs_data::DataError>(())
29//! ```
30
31use astrs_data::{ArrayRef, AstrsMessage, DataError, DataType, Field, RecordBatch, Result};
32
33use super::{build, read, single_row_column};
34
35/// One axis-aligned detection box, in image pixel coordinates.
36#[derive(Debug, Clone, Copy, Default, PartialEq, PartialOrd)]
37pub struct BoundingBox {
38    /// The left edge.
39    pub x: f32,
40    /// The top edge.
41    pub y: f32,
42    /// The width.
43    pub width: f32,
44    /// The height.
45    pub height: f32,
46}
47
48impl BoundingBox {
49    /// A box from its `xywh` components.
50    #[must_use]
51    pub const fn new(x: f32, y: f32, width: f32, height: f32) -> Self {
52        Self {
53            x,
54            y,
55            width,
56            height,
57        }
58    }
59
60    /// The components in `xywh` order — the layout's own order.
61    #[must_use]
62    pub const fn to_array(self) -> [f32; 4] {
63        [self.x, self.y, self.width, self.height]
64    }
65
66    /// A box from an `xywh` array.
67    #[must_use]
68    pub const fn from_array(values: [f32; 4]) -> Self {
69        Self::new(values[0], values[1], values[2], values[3])
70    }
71
72    /// The box's area, clamped at zero for a degenerate box.
73    #[must_use]
74    pub fn area(self) -> f32 {
75        self.width.max(0.0) * self.height.max(0.0)
76    }
77}
78
79impl From<[f32; 4]> for BoundingBox {
80    fn from(values: [f32; 4]) -> Self {
81        Self::from_array(values)
82    }
83}
84
85/// One landmark, in image pixel coordinates.
86#[derive(Debug, Clone, Copy, Default, PartialEq, PartialOrd)]
87pub struct Keypoint {
88    /// The horizontal coordinate.
89    pub x: f32,
90    /// The vertical coordinate.
91    pub y: f32,
92}
93
94impl Keypoint {
95    /// A keypoint from its coordinates.
96    #[must_use]
97    pub const fn new(x: f32, y: f32) -> Self {
98        Self { x, y }
99    }
100
101    /// The coordinates in `xy` order.
102    #[must_use]
103    pub const fn to_array(self) -> [f32; 2] {
104        [self.x, self.y]
105    }
106}
107
108impl From<[f32; 2]> for Keypoint {
109    fn from(values: [f32; 2]) -> Self {
110        Self::new(values[0], values[1])
111    }
112}
113
114/// Detection boxes with per-box confidence and label —
115/// `std/vision/v1/Detections`.
116#[derive(Debug, Clone, Default, PartialEq)]
117pub struct Detections {
118    /// The boxes.
119    pub boxes: Vec<BoundingBox>,
120    /// The per-box confidences, index-aligned with `boxes`.
121    pub scores: Vec<f32>,
122    /// The per-box class labels, index-aligned with `boxes`.
123    pub labels: Vec<u32>,
124}
125
126impl Detections {
127    /// Detections whose three lists are the same length.
128    ///
129    /// # Errors
130    ///
131    /// [`DataError::ColumnLengthMismatch`] when they are not.
132    pub fn new(boxes: Vec<BoundingBox>, scores: Vec<f32>, labels: Vec<u32>) -> Result<Self> {
133        let value = Self {
134            boxes,
135            scores,
136            labels,
137        };
138        value.check_alignment()?;
139        Ok(value)
140    }
141
142    /// An empty detection set.
143    #[must_use]
144    pub const fn empty() -> Self {
145        Self {
146            boxes: Vec::new(),
147            scores: Vec::new(),
148            labels: Vec::new(),
149        }
150    }
151
152    /// How many detections the message carries.
153    #[must_use]
154    pub fn len(&self) -> usize {
155        self.boxes.len()
156    }
157
158    /// Whether the message carries no detections.
159    #[must_use]
160    pub fn is_empty(&self) -> bool {
161        self.boxes.is_empty()
162    }
163
164    /// Checks that the three index-aligned lists agree on length.
165    ///
166    /// # Errors
167    ///
168    /// [`DataError::ColumnLengthMismatch`] naming the first list that
169    /// disagrees.
170    pub fn check_alignment(&self) -> Result<()> {
171        check_three(
172            ("boxes", self.boxes.len()),
173            ("scores", self.scores.len()),
174            ("labels", self.labels.len()),
175        )
176    }
177
178    /// The columnar layout of this type.
179    #[must_use]
180    pub fn layout() -> DataType {
181        DataType::strukt([
182            Field::required(
183                "boxes",
184                DataType::list(Field::required(
185                    "xywh",
186                    DataType::fixed_size_list(Field::required("v", DataType::Float32), 4),
187                )),
188            ),
189            Field::required(
190                "scores",
191                DataType::list(Field::required("score", DataType::Float32)),
192            ),
193            Field::required(
194                "labels",
195                DataType::list(Field::required("label", DataType::UInt32)),
196            ),
197        ])
198    }
199}
200
201impl AstrsMessage for Detections {
202    const URN: &'static str = "std/vision/v1/Detections";
203
204    fn data_type() -> DataType {
205        Self::layout()
206    }
207
208    fn to_record_batch(&self) -> Result<RecordBatch> {
209        self.check_alignment()?;
210        let flat: Vec<f32> = self
211            .boxes
212            .iter()
213            .flat_map(|value| value.to_array())
214            .collect();
215        let column = build::structure(vec![
216            (
217                "boxes",
218                build::fixed_tuple_lists::<f32>("xywh", "v", 4, &[&flat])?,
219            ),
220            (
221                "scores",
222                build::primitive_lists::<f32>("score", &[&self.scores])?,
223            ),
224            (
225                "labels",
226                build::primitive_lists::<u32>("label", &[&self.labels])?,
227            ),
228        ])?;
229        Ok(RecordBatch::from_payload(column))
230    }
231
232    fn from_record_batch(batch: &RecordBatch) -> Result<Self> {
233        let column = single_row_column(batch)?;
234        let (flat, width) = read::fixed_tuple_list_at::<f32>(read::child(column, "boxes")?, 0)?;
235        if width != 4 {
236            return Err(DataError::ChildLengthMismatch {
237                expected: 4,
238                actual: width,
239            });
240        }
241        // The width check above is what makes `as_chunks::<4>` exact: its
242        // remainder is empty by construction, and each chunk arrives as the
243        // `[f32; 4]` `BoundingBox`'s own `From` takes.
244        let boxes = flat
245            .as_chunks::<4>()
246            .0
247            .iter()
248            .copied()
249            .map(BoundingBox::from)
250            .collect();
251        let scores = read::primitive_list_at::<f32>(read::child(column, "scores")?, 0)?;
252        let labels = read::primitive_list_at::<u32>(read::child(column, "labels")?, 0)?;
253        Self::new(boxes, scores, labels)
254    }
255}
256
257super::impl_from_payload!(Detections);
258
259/// Landmark sets with per-point confidence — `std/vision/v1/Keypoints`.
260#[derive(Debug, Clone, Default, PartialEq)]
261pub struct Keypoints {
262    /// The points.
263    pub points: Vec<Keypoint>,
264    /// The per-point confidences, index-aligned with `points`.
265    pub scores: Vec<f32>,
266    /// The per-point landmark ids, index-aligned with `points`.
267    pub labels: Vec<u32>,
268}
269
270impl Keypoints {
271    /// Keypoints whose three lists are the same length.
272    ///
273    /// # Errors
274    ///
275    /// [`DataError::ColumnLengthMismatch`] when they are not.
276    pub fn new(points: Vec<Keypoint>, scores: Vec<f32>, labels: Vec<u32>) -> Result<Self> {
277        let value = Self {
278            points,
279            scores,
280            labels,
281        };
282        value.check_alignment()?;
283        Ok(value)
284    }
285
286    /// How many points the message carries.
287    #[must_use]
288    pub fn len(&self) -> usize {
289        self.points.len()
290    }
291
292    /// Whether the message carries no points.
293    #[must_use]
294    pub fn is_empty(&self) -> bool {
295        self.points.is_empty()
296    }
297
298    /// Checks that the three index-aligned lists agree on length.
299    ///
300    /// # Errors
301    ///
302    /// [`DataError::ColumnLengthMismatch`].
303    pub fn check_alignment(&self) -> Result<()> {
304        check_three(
305            ("points", self.points.len()),
306            ("scores", self.scores.len()),
307            ("labels", self.labels.len()),
308        )
309    }
310
311    /// The columnar layout of this type.
312    #[must_use]
313    pub fn layout() -> DataType {
314        DataType::strukt([
315            Field::required(
316                "points",
317                DataType::list(Field::required(
318                    "xy",
319                    DataType::fixed_size_list(Field::required("v", DataType::Float32), 2),
320                )),
321            ),
322            Field::required(
323                "scores",
324                DataType::list(Field::required("score", DataType::Float32)),
325            ),
326            Field::required(
327                "labels",
328                DataType::list(Field::required("label", DataType::UInt32)),
329            ),
330        ])
331    }
332}
333
334impl AstrsMessage for Keypoints {
335    const URN: &'static str = "std/vision/v1/Keypoints";
336
337    fn data_type() -> DataType {
338        Self::layout()
339    }
340
341    fn to_record_batch(&self) -> Result<RecordBatch> {
342        self.check_alignment()?;
343        let flat: Vec<f32> = self
344            .points
345            .iter()
346            .flat_map(|value| value.to_array())
347            .collect();
348        let column = build::structure(vec![
349            (
350                "points",
351                build::fixed_tuple_lists::<f32>("xy", "v", 2, &[&flat])?,
352            ),
353            (
354                "scores",
355                build::primitive_lists::<f32>("score", &[&self.scores])?,
356            ),
357            (
358                "labels",
359                build::primitive_lists::<u32>("label", &[&self.labels])?,
360            ),
361        ])?;
362        Ok(RecordBatch::from_payload(column))
363    }
364
365    fn from_record_batch(batch: &RecordBatch) -> Result<Self> {
366        let column = single_row_column(batch)?;
367        let (flat, width) = read::fixed_tuple_list_at::<f32>(read::child(column, "points")?, 0)?;
368        if width != 2 {
369            return Err(DataError::ChildLengthMismatch {
370                expected: 2,
371                actual: width,
372            });
373        }
374        // As `Detections` above: the width check makes `as_chunks::<2>`
375        // exact, and each chunk is the `[f32; 2]` `Keypoint`'s `From` takes.
376        let points = flat
377            .as_chunks::<2>()
378            .0
379            .iter()
380            .copied()
381            .map(Keypoint::from)
382            .collect();
383        let scores = read::primitive_list_at::<f32>(read::child(column, "scores")?, 0)?;
384        let labels = read::primitive_list_at::<u32>(read::child(column, "labels")?, 0)?;
385        Self::new(points, scores, labels)
386    }
387}
388
389super::impl_from_payload!(Keypoints);
390
391/// A per-pixel label plane — `std/vision/v1/Mask`, row-major.
392#[derive(Debug, Clone, Default, PartialEq, Eq)]
393pub struct Mask {
394    /// The plane's width in pixels.
395    pub width: u32,
396    /// The plane's height in pixels.
397    pub height: u32,
398    /// `width * height` labels, row-major.
399    pub data: Vec<u16>,
400}
401
402impl Mask {
403    /// A mask whose data length matches its dimensions.
404    ///
405    /// # Errors
406    ///
407    /// [`DataError::ChildLengthMismatch`] when `data.len() != width * height`.
408    pub fn new(width: u32, height: u32, data: Vec<u16>) -> Result<Self> {
409        let value = Self {
410            width,
411            height,
412            data,
413        };
414        value.check_dimensions()?;
415        Ok(value)
416    }
417
418    /// The label at `(x, y)`, or `None` outside the plane.
419    #[must_use]
420    pub fn label_at(&self, x: u32, y: u32) -> Option<u16> {
421        if x >= self.width || y >= self.height {
422            return None;
423        }
424        let index = usize::try_from(y)
425            .ok()?
426            .checked_mul(usize::try_from(self.width).ok()?)?
427            + usize::try_from(x).ok()?;
428        self.data.get(index).copied()
429    }
430
431    /// Checks that the data length matches the declared dimensions.
432    ///
433    /// # Errors
434    ///
435    /// [`DataError::ChildLengthMismatch`].
436    pub fn check_dimensions(&self) -> Result<()> {
437        let expected = expected_cells(self.width, self.height);
438        if self.data.len() == expected {
439            Ok(())
440        } else {
441            Err(DataError::ChildLengthMismatch {
442                expected,
443                actual: self.data.len(),
444            })
445        }
446    }
447
448    /// The columnar layout of this type.
449    #[must_use]
450    pub fn layout() -> DataType {
451        DataType::strukt([
452            Field::required("width", DataType::UInt32),
453            Field::required("height", DataType::UInt32),
454            Field::required(
455                "data",
456                DataType::list(Field::required("label", DataType::UInt16)),
457            ),
458        ])
459    }
460}
461
462impl AstrsMessage for Mask {
463    const URN: &'static str = "std/vision/v1/Mask";
464
465    fn data_type() -> DataType {
466        Self::layout()
467    }
468
469    fn to_record_batch(&self) -> Result<RecordBatch> {
470        self.check_dimensions()?;
471        let column = build::structure(vec![
472            ("width", build::primitive::<u32>(&[self.width])),
473            ("height", build::primitive::<u32>(&[self.height])),
474            (
475                "data",
476                build::primitive_lists::<u16>("label", &[&self.data])?,
477            ),
478        ])?;
479        Ok(RecordBatch::from_payload(column))
480    }
481
482    fn from_record_batch(batch: &RecordBatch) -> Result<Self> {
483        let column = single_row_column(batch)?;
484        let width = read::primitive_at::<u32>(read::child(column, "width")?, 0)?;
485        let height = read::primitive_at::<u32>(read::child(column, "height")?, 0)?;
486        let data = read::primitive_list_at::<u16>(read::child(column, "data")?, 0)?;
487        Self::new(width, height, data)
488    }
489}
490
491super::impl_from_payload!(Mask);
492
493/// The cell count `width * height` implies, saturating rather than wrapping.
494fn expected_cells(width: u32, height: u32) -> usize {
495    usize::try_from(u64::from(width) * u64::from(height)).unwrap_or(usize::MAX)
496}
497
498/// Checks that three index-aligned lists agree on length.
499///
500/// Each argument is the list's `(name, length)`, so the error names the
501/// column that disagreed rather than only its position.
502fn check_three(first: (&str, usize), second: (&str, usize), third: (&str, usize)) -> Result<()> {
503    if first.1 != second.1 {
504        return Err(DataError::ColumnLengthMismatch {
505            index: 1,
506            name: second.0.to_owned(),
507            expected: first.1,
508            actual: second.1,
509        });
510    }
511    if first.1 != third.1 {
512        return Err(DataError::ColumnLengthMismatch {
513            index: 2,
514            name: third.0.to_owned(),
515            expected: first.1,
516            actual: third.1,
517        });
518    }
519    Ok(())
520}
521
522/// The `boxes` column of a `Detections` payload, without decoding the rest.
523///
524/// The shape a tracker wants: it needs the geometry every frame and the
525/// labels only when a track is born.
526///
527/// # Errors
528///
529/// [`DataError`] when the payload is not a `Detections` layout.
530pub fn detection_boxes(batch: &RecordBatch) -> Result<Vec<BoundingBox>> {
531    let column = single_row_column(batch)?;
532    let (flat, width) = read::fixed_tuple_list_at::<f32>(read::child(column, "boxes")?, 0)?;
533    if width != 4 {
534        return Err(DataError::ChildLengthMismatch {
535            expected: 4,
536            actual: width,
537        });
538    }
539    Ok(flat
540        .as_chunks::<4>()
541        .0
542        .iter()
543        .copied()
544        .map(BoundingBox::from)
545        .collect())
546}
547
548/// The raw `data` column of a `Mask` payload, without copying the dimensions.
549///
550/// # Errors
551///
552/// [`DataError`] when the payload is not a `Mask` layout.
553pub fn mask_labels(batch: &RecordBatch) -> Result<ArrayRef> {
554    let column = single_row_column(batch)?;
555    Ok(std::sync::Arc::clone(read::child(column, "data")?))
556}
557
558#[cfg(test)]
559mod tests {
560    #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
561
562    use super::*;
563    use crate::message::assert_registry_layout;
564
565    fn detections() -> Detections {
566        Detections::new(
567            vec![
568                BoundingBox::new(0.0, 0.0, 10.0, 20.0),
569                BoundingBox::new(5.0, 5.0, 1.0, 2.0),
570            ],
571            vec![0.93, 0.41],
572            vec![7, 12],
573        )
574        .unwrap()
575    }
576
577    #[test]
578    fn every_vision_type_conforms_to_the_registry() {
579        assert_registry_layout::<Detections>().unwrap();
580        assert_registry_layout::<Keypoints>().unwrap();
581        assert_registry_layout::<Mask>().unwrap();
582    }
583
584    #[test]
585    fn detections_round_trip() {
586        let value = detections();
587        let batch = value.to_record_batch().unwrap();
588        assert_eq!(batch.num_rows(), 1);
589        let decoded = Detections::from_record_batch(&batch).unwrap();
590        assert_eq!(decoded, value);
591        assert_eq!(decoded.len(), 2);
592        assert!(!decoded.is_empty());
593        assert_eq!(decoded.boxes[0].area(), 200.0);
594        assert_eq!(detection_boxes(&batch).unwrap(), value.boxes);
595    }
596
597    #[test]
598    fn empty_detections_round_trip() {
599        let value = Detections::empty();
600        assert!(value.is_empty());
601        let batch = value.to_record_batch().unwrap();
602        assert_eq!(Detections::from_record_batch(&batch).unwrap(), value);
603    }
604
605    #[test]
606    fn misaligned_detection_lists_are_refused() {
607        let error =
608            Detections::new(vec![BoundingBox::default()], vec![0.5, 0.6], vec![1]).unwrap_err();
609        assert!(matches!(error, DataError::ColumnLengthMismatch { .. }));
610
611        assert!(
612            Detections::new(vec![BoundingBox::default()], vec![0.5], vec![1, 2]).is_err(),
613            "labels must align too"
614        );
615    }
616
617    #[test]
618    fn keypoints_round_trip() {
619        let value = Keypoints::new(
620            vec![Keypoint::new(1.0, 2.0), Keypoint::new(3.0, 4.0)],
621            vec![0.5, 0.25],
622            vec![0, 1],
623        )
624        .unwrap();
625        let batch = value.to_record_batch().unwrap();
626        assert_eq!(Keypoints::from_record_batch(&batch).unwrap(), value);
627        assert_eq!(value.len(), 2);
628        assert!(!value.is_empty());
629        assert_eq!(Keypoint::from([9.0, 8.0]).to_array(), [9.0, 8.0]);
630        assert!(Keypoints::new(vec![Keypoint::default()], vec![], vec![]).is_err());
631    }
632
633    #[test]
634    fn masks_round_trip_and_check_their_dimensions() {
635        let value = Mask::new(3, 2, vec![1, 2, 3, 4, 5, 6]).unwrap();
636        let batch = value.to_record_batch().unwrap();
637        assert_eq!(Mask::from_record_batch(&batch).unwrap(), value);
638        assert_eq!(value.label_at(0, 0), Some(1));
639        assert_eq!(value.label_at(2, 1), Some(6));
640        assert_eq!(value.label_at(3, 0), None);
641        assert_eq!(value.label_at(0, 2), None);
642        assert_eq!(mask_labels(&batch).unwrap().len(), 1);
643
644        let error = Mask::new(3, 2, vec![1, 2, 3]).unwrap_err();
645        assert!(matches!(error, DataError::ChildLengthMismatch { .. }));
646        assert!(Mask::new(0, 0, Vec::new()).is_ok());
647    }
648
649    #[test]
650    fn bounding_boxes_convert_both_ways() {
651        let value = BoundingBox::from([1.0, 2.0, 3.0, 4.0]);
652        assert_eq!(value.to_array(), [1.0, 2.0, 3.0, 4.0]);
653        assert_eq!(value.area(), 12.0);
654        assert_eq!(BoundingBox::new(0.0, 0.0, -1.0, 5.0).area(), 0.0);
655        assert_eq!(BoundingBox::default().area(), 0.0);
656    }
657
658    #[test]
659    fn a_wrong_layout_is_refused() {
660        let mask = Mask::new(1, 1, vec![0]).unwrap().to_record_batch().unwrap();
661        assert!(Detections::from_record_batch(&mask).is_err());
662        assert!(Keypoints::from_record_batch(&mask).is_err());
663        assert!(detection_boxes(&mask).is_err());
664
665        let detections = detections().to_record_batch().unwrap();
666        assert!(Mask::from_record_batch(&detections).is_err());
667        assert!(
668            Keypoints::from_record_batch(&detections).is_err(),
669            "four-wide boxes are not two-wide points"
670        );
671    }
672}