Skip to main content

astrs_node_api/message/
build.rs

1//! Column constructors shared by every [`AstrsMessage`](super::AstrsMessage)
2//! implementation.
3//!
4//! The std layouts of §24.3 are built from four shapes — a primitive column, a
5//! `List` of a primitive, a `FixedSizeList` of a primitive, and a `Struct`
6//! composing those — so four constructors cover all of them. Writing the
7//! implementations against these rather than against the array types directly
8//! keeps every one of them short enough to read against the normative layout
9//! it is supposed to produce.
10//!
11//! Everything here builds **non-nullable** columns. The std types are total:
12//! a `Pose` has a position, and a null one would be a different type
13//! (`Option<Pose>` is expressed as an absent message, not a null row). Nothing
14//! in this module can therefore produce a validity bitmap, which is also why
15//! [`astrs_data::RecordBatch::from_payload`] marks the field non-nullable for
16//! every payload built here.
17//!
18//! # Examples
19//!
20//! ```
21//! use astrs_data::{DataType, Field};
22//! use astrs_node_api::message::build;
23//!
24//! let column = build::primitive::<f64>(&[1.0, 2.0, 3.0]);
25//! assert_eq!(column.len(), 3);
26//!
27//! let lists = build::primitive_lists::<f32>("range", &[&[1.0, 2.0][..], &[3.0][..]])?;
28//! assert_eq!(lists.len(), 2);
29//! assert_eq!(
30//!     lists.data_type(),
31//!     &DataType::list(Field::required("range", DataType::Float32))
32//! );
33//! # Ok::<(), astrs_data::DataError>(())
34//! ```
35
36use astrs_data::array::{
37    ArrayRef, BinaryArray, BooleanArray, FixedSizeListArray, IntoArrayRef, ListArray,
38    PrimitiveArray, StringArray, StructArray, TimestampArray,
39};
40use astrs_data::buffer::ScalarBuffer;
41use astrs_data::{ArrowNativeType, DataError, DataType, Field, Result};
42
43/// A non-nullable primitive column.
44///
45/// # Examples
46///
47/// ```
48/// use astrs_node_api::message::build;
49///
50/// assert_eq!(build::primitive::<u32>(&[1, 2]).len(), 2);
51/// ```
52#[must_use]
53pub fn primitive<T: ArrowNativeType>(values: &[T]) -> ArrayRef {
54    PrimitiveArray::<T>::try_new(ScalarBuffer::from_slice(values), None)
55        .map_or_else(|_| empty_primitive::<T>(), IntoArrayRef::into_array_ref)
56}
57
58/// An empty primitive column, used as the total fallback above.
59///
60/// `PrimitiveArray::try_new` with no validity bitmap cannot fail, so this is
61/// unreachable; it exists so [`primitive`] is infallible at its call sites
62/// (the layouts it feeds are all fixed, and a `Result` there would propagate
63/// an error that cannot happen through every message implementation).
64fn empty_primitive<T: ArrowNativeType>() -> ArrayRef {
65    PrimitiveArray::<T>::from_values(core::iter::empty()).into_array_ref()
66}
67
68/// A non-nullable boolean column.
69#[must_use]
70pub fn boolean(values: &[bool]) -> ArrayRef {
71    BooleanArray::from_values(values.iter().copied()).into_array_ref()
72}
73
74/// A non-nullable `Utf8` column.
75#[must_use]
76pub fn strings<S: AsRef<str>>(values: &[S]) -> ArrayRef {
77    StringArray::from_values(values.iter().map(AsRef::as_ref)).into_array_ref()
78}
79
80/// A non-nullable `Binary` column.
81#[must_use]
82pub fn binaries<S: AsRef<[u8]>>(values: &[S]) -> ArrayRef {
83    BinaryArray::from_values(values.iter().map(AsRef::as_ref)).into_array_ref()
84}
85
86/// A non-nullable `Timestamp(ns)` column.
87#[must_use]
88pub fn timestamps(nanos: &[i64]) -> ArrayRef {
89    TimestampArray::from_nanos(nanos.iter().copied()).into_array_ref()
90}
91
92/// A `List<item_name: T>` column, one list per entry of `rows`.
93///
94/// # Errors
95///
96/// [`DataError`] if the offsets cannot be laid out, which
97/// requires a total element count beyond `i32::MAX`.
98///
99/// # Examples
100///
101/// ```
102/// use astrs_node_api::message::build;
103///
104/// let column = build::primitive_lists::<i8>("cell", &[&[1, 2, 3][..], &[][..]])?;
105/// assert_eq!(column.len(), 2);
106/// # Ok::<(), astrs_data::DataError>(())
107/// ```
108pub fn primitive_lists<T: ArrowNativeType>(item_name: &str, rows: &[&[T]]) -> Result<ArrayRef> {
109    let total: usize = rows.iter().map(|row| row.len()).sum();
110    let mut flat = Vec::with_capacity(total);
111    let mut offsets = Vec::with_capacity(rows.len() + 1);
112    let mut cursor = 0i32;
113    offsets.push(cursor);
114    for row in rows {
115        flat.extend_from_slice(row);
116        cursor = cursor.saturating_add(i32::try_from(row.len()).unwrap_or(i32::MAX));
117        offsets.push(cursor);
118    }
119    let field = Field::required(item_name, T::DATA_TYPE);
120    let values = primitive::<T>(&flat);
121    Ok(
122        ListArray::try_new(field, ScalarBuffer::from_slice(&offsets), values, None)?
123            .into_array_ref(),
124    )
125}
126
127/// A `List<FixedSizeList<item_name: T, width>>` column.
128///
129/// `rows` holds, per row, the flattened tuples of that row: a row of two
130/// four-wide boxes is eight values. This is the shape `Detections.boxes` and
131/// `Keypoints.points` need.
132///
133/// # Errors
134///
135/// [`DataError`] when a row's length is not a multiple of
136/// `width`, or when the offsets cannot be laid out.
137pub fn fixed_tuple_lists<T: ArrowNativeType>(
138    outer_name: &str,
139    item_name: &str,
140    width: i32,
141    rows: &[&[T]],
142) -> Result<ArrayRef> {
143    let stride = usize::try_from(width.max(1)).unwrap_or(1);
144    let total: usize = rows.iter().map(|row| row.len()).sum();
145    let mut flat = Vec::with_capacity(total);
146    let mut offsets = Vec::with_capacity(rows.len() + 1);
147    let mut cursor = 0i32;
148    offsets.push(cursor);
149    for row in rows {
150        if !row.len().is_multiple_of(stride) {
151            return Err(DataError::ChildLengthMismatch {
152                expected: row.len().next_multiple_of(stride),
153                actual: row.len(),
154            });
155        }
156        flat.extend_from_slice(row);
157        let tuples = row.len() / stride;
158        cursor = cursor.saturating_add(i32::try_from(tuples).unwrap_or(i32::MAX));
159        offsets.push(cursor);
160    }
161    let inner = FixedSizeListArray::try_new(
162        Field::required(item_name, T::DATA_TYPE),
163        width,
164        primitive::<T>(&flat),
165        None,
166    )?;
167    let outer_field = Field::required(
168        outer_name,
169        DataType::fixed_size_list(Field::required(item_name, T::DATA_TYPE), width),
170    );
171    Ok(ListArray::try_new(
172        outer_field,
173        ScalarBuffer::from_slice(&offsets),
174        inner.into_array_ref(),
175        None,
176    )?
177    .into_array_ref())
178}
179
180/// A `FixedSizeList<item_name: T, width>` column, one tuple per row.
181///
182/// `flat` holds `rows * width` values in row-major order — the shape every
183/// covariance matrix in §24.3 uses.
184///
185/// # Errors
186///
187/// [`DataError`] when `flat.len()` is not a multiple of `width`.
188pub fn fixed_tuples<T: ArrowNativeType>(
189    item_name: &str,
190    width: i32,
191    flat: &[T],
192) -> Result<ArrayRef> {
193    Ok(FixedSizeListArray::try_new(
194        Field::required(item_name, T::DATA_TYPE),
195        width,
196        primitive::<T>(flat),
197        None,
198    )?
199    .into_array_ref())
200}
201
202/// A non-nullable `Struct` column from named child columns.
203///
204/// The field types are taken from the columns themselves, so a caller cannot
205/// declare one shape and pass another.
206///
207/// # Errors
208///
209/// [`DataError`] when the children disagree on length, or when
210/// there are no children at all (a struct's row count would be unknowable).
211///
212/// # Examples
213///
214/// ```
215/// use astrs_node_api::message::build;
216///
217/// let column = build::structure(vec![
218///     ("x", build::primitive::<f64>(&[1.0])),
219///     ("y", build::primitive::<f64>(&[2.0])),
220/// ])?;
221/// assert_eq!(column.len(), 1);
222/// # Ok::<(), astrs_data::DataError>(())
223/// ```
224pub fn structure(children: Vec<(&str, ArrayRef)>) -> Result<ArrayRef> {
225    let mut fields = Vec::with_capacity(children.len());
226    let mut columns = Vec::with_capacity(children.len());
227    for (name, column) in children {
228        fields.push(Field::required(name, column.data_type().clone()));
229        columns.push(column);
230    }
231    Ok(StructArray::try_new(fields, columns, None)?.into_array_ref())
232}
233
234/// A `List<item_name: item_type>` column over already-built per-row children.
235///
236/// Used where the list's element is itself a struct — `Path`'s stamped poses
237/// — so the child column is built once for every row concatenated and the
238/// offsets slice it back apart.
239///
240/// # Errors
241///
242/// [`DataError`] when the lengths do not sum to the child's
243/// length, or when the child's type differs from `item_type`.
244pub fn nested_lists(
245    item_name: &str,
246    item_type: DataType,
247    lengths: &[usize],
248    values: ArrayRef,
249) -> Result<ArrayRef> {
250    let field = Field::required(item_name, item_type);
251    Ok(ListArray::try_from_lengths(field, lengths.iter().copied(), values)?.into_array_ref())
252}
253
254#[cfg(test)]
255mod tests {
256    #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
257
258    use super::*;
259    use astrs_data::array::{ArrayExt, Float32Array, Float64Array};
260
261    #[test]
262    fn primitive_columns_carry_their_values_and_type() {
263        let column = primitive::<f64>(&[1.0, 2.0]);
264        assert_eq!(column.data_type(), &DataType::Float64);
265        let values = column.downcast::<Float64Array>().unwrap();
266        assert_eq!(values.values(), &[1.0, 2.0]);
267        assert_eq!(column.null_count(), 0);
268        assert_eq!(primitive::<u8>(&[]).len(), 0);
269    }
270
271    #[test]
272    fn boolean_string_binary_and_timestamp_columns() {
273        assert_eq!(boolean(&[true, false]).len(), 2);
274        assert_eq!(boolean(&[]).data_type(), &DataType::Bool);
275        assert_eq!(strings(&["a", "bb"]).len(), 2);
276        assert_eq!(strings::<&str>(&[]).data_type(), &DataType::Utf8);
277        assert_eq!(binaries(&[&b"x"[..]]).data_type(), &DataType::Binary);
278        assert_eq!(timestamps(&[1, 2, 3]).data_type(), &DataType::Timestamp);
279    }
280
281    #[test]
282    fn primitive_lists_slice_their_rows_apart() {
283        let column =
284            primitive_lists::<f32>("range", &[&[1.0, 2.0][..], &[][..], &[3.0][..]]).unwrap();
285        assert_eq!(column.len(), 3);
286        let lists = column.downcast::<ListArray>().unwrap();
287        assert_eq!(lists.value_length(0), Some(2));
288        assert_eq!(lists.value_length(1), Some(0));
289        assert_eq!(lists.value_length(2), Some(1));
290        let first = lists.value(0).unwrap();
291        assert_eq!(
292            first.downcast::<Float32Array>().unwrap().values(),
293            &[1.0, 2.0]
294        );
295    }
296
297    #[test]
298    fn fixed_tuple_lists_group_by_width() {
299        let column = fixed_tuple_lists::<f32>(
300            "xywh",
301            "v",
302            4,
303            &[&[0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0][..]],
304        )
305        .unwrap();
306        assert_eq!(column.len(), 1, "one row");
307        let lists = column.downcast::<ListArray>().unwrap();
308        assert_eq!(lists.value_length(0), Some(2), "two boxes in the row");
309    }
310
311    #[test]
312    fn a_row_that_is_not_a_whole_number_of_tuples_is_refused() {
313        let error = fixed_tuple_lists::<f32>("xywh", "v", 4, &[&[0.0, 1.0][..]]).unwrap_err();
314        assert!(
315            matches!(error, DataError::ChildLengthMismatch { .. }),
316            "{error}"
317        );
318    }
319
320    #[test]
321    fn fixed_tuples_lay_out_covariances() {
322        let flat: Vec<f64> = (0..18).map(f64::from).collect();
323        let column = fixed_tuples::<f64>("v", 9, &flat).unwrap();
324        assert_eq!(column.len(), 2, "two rows of nine");
325        assert_eq!(
326            column.data_type(),
327            &DataType::fixed_size_list(Field::required("v", DataType::Float64), 9)
328        );
329        assert!(fixed_tuples::<f64>("v", 9, &flat[..10]).is_err());
330    }
331
332    #[test]
333    fn structures_take_their_types_from_their_columns() {
334        let column = structure(vec![
335            ("x", primitive::<f64>(&[1.0, 4.0])),
336            ("y", primitive::<f64>(&[2.0, 5.0])),
337            ("z", primitive::<f64>(&[3.0, 6.0])),
338        ])
339        .unwrap();
340        assert_eq!(column.len(), 2);
341        assert_eq!(
342            column.data_type(),
343            &DataType::strukt([
344                Field::required("x", DataType::Float64),
345                Field::required("y", DataType::Float64),
346                Field::required("z", DataType::Float64),
347            ])
348        );
349    }
350
351    #[test]
352    fn a_structure_with_mismatched_children_is_refused() {
353        let error = structure(vec![
354            ("x", primitive::<f64>(&[1.0, 2.0])),
355            ("y", primitive::<f64>(&[3.0])),
356        ])
357        .unwrap_err();
358        assert!(!error.to_string().is_empty());
359        assert!(structure(Vec::new()).is_err(), "no children, no row count");
360    }
361
362    #[test]
363    fn nested_lists_slice_a_shared_child() {
364        let child = structure(vec![
365            ("stamp", timestamps(&[1, 2, 3])),
366            ("value", primitive::<f64>(&[1.0, 2.0, 3.0])),
367        ])
368        .unwrap();
369        let item_type = child.data_type().clone();
370        let column = nested_lists("pose", item_type, &[2, 1], child).unwrap();
371        assert_eq!(column.len(), 2);
372        let lists = column.downcast::<ListArray>().unwrap();
373        assert_eq!(lists.value_length(0), Some(2));
374        assert_eq!(lists.value_length(1), Some(1));
375    }
376
377    #[test]
378    fn empty_inputs_produce_empty_columns_not_errors() {
379        assert_eq!(primitive_lists::<f32>("v", &[]).unwrap().len(), 0);
380        assert_eq!(
381            fixed_tuple_lists::<f32>("xy", "v", 2, &[]).unwrap().len(),
382            0
383        );
384        assert_eq!(fixed_tuples::<f64>("v", 4, &[]).unwrap().len(), 0);
385    }
386}