use astrs_data::array::{
ArrayRef, BinaryArray, BooleanArray, FixedSizeListArray, IntoArrayRef, ListArray,
PrimitiveArray, StringArray, StructArray, TimestampArray,
};
use astrs_data::buffer::ScalarBuffer;
use astrs_data::{ArrowNativeType, DataError, DataType, Field, Result};
#[must_use]
pub fn primitive<T: ArrowNativeType>(values: &[T]) -> ArrayRef {
PrimitiveArray::<T>::try_new(ScalarBuffer::from_slice(values), None)
.map_or_else(|_| empty_primitive::<T>(), IntoArrayRef::into_array_ref)
}
fn empty_primitive<T: ArrowNativeType>() -> ArrayRef {
PrimitiveArray::<T>::from_values(core::iter::empty()).into_array_ref()
}
#[must_use]
pub fn boolean(values: &[bool]) -> ArrayRef {
BooleanArray::from_values(values.iter().copied()).into_array_ref()
}
#[must_use]
pub fn strings<S: AsRef<str>>(values: &[S]) -> ArrayRef {
StringArray::from_values(values.iter().map(AsRef::as_ref)).into_array_ref()
}
#[must_use]
pub fn binaries<S: AsRef<[u8]>>(values: &[S]) -> ArrayRef {
BinaryArray::from_values(values.iter().map(AsRef::as_ref)).into_array_ref()
}
#[must_use]
pub fn timestamps(nanos: &[i64]) -> ArrayRef {
TimestampArray::from_nanos(nanos.iter().copied()).into_array_ref()
}
pub fn primitive_lists<T: ArrowNativeType>(item_name: &str, rows: &[&[T]]) -> Result<ArrayRef> {
let total: usize = rows.iter().map(|row| row.len()).sum();
let mut flat = Vec::with_capacity(total);
let mut offsets = Vec::with_capacity(rows.len() + 1);
let mut cursor = 0i32;
offsets.push(cursor);
for row in rows {
flat.extend_from_slice(row);
cursor = cursor.saturating_add(i32::try_from(row.len()).unwrap_or(i32::MAX));
offsets.push(cursor);
}
let field = Field::required(item_name, T::DATA_TYPE);
let values = primitive::<T>(&flat);
Ok(
ListArray::try_new(field, ScalarBuffer::from_slice(&offsets), values, None)?
.into_array_ref(),
)
}
pub fn fixed_tuple_lists<T: ArrowNativeType>(
outer_name: &str,
item_name: &str,
width: i32,
rows: &[&[T]],
) -> Result<ArrayRef> {
let stride = usize::try_from(width.max(1)).unwrap_or(1);
let total: usize = rows.iter().map(|row| row.len()).sum();
let mut flat = Vec::with_capacity(total);
let mut offsets = Vec::with_capacity(rows.len() + 1);
let mut cursor = 0i32;
offsets.push(cursor);
for row in rows {
if !row.len().is_multiple_of(stride) {
return Err(DataError::ChildLengthMismatch {
expected: row.len().next_multiple_of(stride),
actual: row.len(),
});
}
flat.extend_from_slice(row);
let tuples = row.len() / stride;
cursor = cursor.saturating_add(i32::try_from(tuples).unwrap_or(i32::MAX));
offsets.push(cursor);
}
let inner = FixedSizeListArray::try_new(
Field::required(item_name, T::DATA_TYPE),
width,
primitive::<T>(&flat),
None,
)?;
let outer_field = Field::required(
outer_name,
DataType::fixed_size_list(Field::required(item_name, T::DATA_TYPE), width),
);
Ok(ListArray::try_new(
outer_field,
ScalarBuffer::from_slice(&offsets),
inner.into_array_ref(),
None,
)?
.into_array_ref())
}
pub fn fixed_tuples<T: ArrowNativeType>(
item_name: &str,
width: i32,
flat: &[T],
) -> Result<ArrayRef> {
Ok(FixedSizeListArray::try_new(
Field::required(item_name, T::DATA_TYPE),
width,
primitive::<T>(flat),
None,
)?
.into_array_ref())
}
pub fn structure(children: Vec<(&str, ArrayRef)>) -> Result<ArrayRef> {
let mut fields = Vec::with_capacity(children.len());
let mut columns = Vec::with_capacity(children.len());
for (name, column) in children {
fields.push(Field::required(name, column.data_type().clone()));
columns.push(column);
}
Ok(StructArray::try_new(fields, columns, None)?.into_array_ref())
}
pub fn nested_lists(
item_name: &str,
item_type: DataType,
lengths: &[usize],
values: ArrayRef,
) -> Result<ArrayRef> {
let field = Field::required(item_name, item_type);
Ok(ListArray::try_from_lengths(field, lengths.iter().copied(), values)?.into_array_ref())
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
use astrs_data::array::{ArrayExt, Float32Array, Float64Array};
#[test]
fn primitive_columns_carry_their_values_and_type() {
let column = primitive::<f64>(&[1.0, 2.0]);
assert_eq!(column.data_type(), &DataType::Float64);
let values = column.downcast::<Float64Array>().unwrap();
assert_eq!(values.values(), &[1.0, 2.0]);
assert_eq!(column.null_count(), 0);
assert_eq!(primitive::<u8>(&[]).len(), 0);
}
#[test]
fn boolean_string_binary_and_timestamp_columns() {
assert_eq!(boolean(&[true, false]).len(), 2);
assert_eq!(boolean(&[]).data_type(), &DataType::Bool);
assert_eq!(strings(&["a", "bb"]).len(), 2);
assert_eq!(strings::<&str>(&[]).data_type(), &DataType::Utf8);
assert_eq!(binaries(&[&b"x"[..]]).data_type(), &DataType::Binary);
assert_eq!(timestamps(&[1, 2, 3]).data_type(), &DataType::Timestamp);
}
#[test]
fn primitive_lists_slice_their_rows_apart() {
let column =
primitive_lists::<f32>("range", &[&[1.0, 2.0][..], &[][..], &[3.0][..]]).unwrap();
assert_eq!(column.len(), 3);
let lists = column.downcast::<ListArray>().unwrap();
assert_eq!(lists.value_length(0), Some(2));
assert_eq!(lists.value_length(1), Some(0));
assert_eq!(lists.value_length(2), Some(1));
let first = lists.value(0).unwrap();
assert_eq!(
first.downcast::<Float32Array>().unwrap().values(),
&[1.0, 2.0]
);
}
#[test]
fn fixed_tuple_lists_group_by_width() {
let column = fixed_tuple_lists::<f32>(
"xywh",
"v",
4,
&[&[0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0][..]],
)
.unwrap();
assert_eq!(column.len(), 1, "one row");
let lists = column.downcast::<ListArray>().unwrap();
assert_eq!(lists.value_length(0), Some(2), "two boxes in the row");
}
#[test]
fn a_row_that_is_not_a_whole_number_of_tuples_is_refused() {
let error = fixed_tuple_lists::<f32>("xywh", "v", 4, &[&[0.0, 1.0][..]]).unwrap_err();
assert!(
matches!(error, DataError::ChildLengthMismatch { .. }),
"{error}"
);
}
#[test]
fn fixed_tuples_lay_out_covariances() {
let flat: Vec<f64> = (0..18).map(f64::from).collect();
let column = fixed_tuples::<f64>("v", 9, &flat).unwrap();
assert_eq!(column.len(), 2, "two rows of nine");
assert_eq!(
column.data_type(),
&DataType::fixed_size_list(Field::required("v", DataType::Float64), 9)
);
assert!(fixed_tuples::<f64>("v", 9, &flat[..10]).is_err());
}
#[test]
fn structures_take_their_types_from_their_columns() {
let column = structure(vec![
("x", primitive::<f64>(&[1.0, 4.0])),
("y", primitive::<f64>(&[2.0, 5.0])),
("z", primitive::<f64>(&[3.0, 6.0])),
])
.unwrap();
assert_eq!(column.len(), 2);
assert_eq!(
column.data_type(),
&DataType::strukt([
Field::required("x", DataType::Float64),
Field::required("y", DataType::Float64),
Field::required("z", DataType::Float64),
])
);
}
#[test]
fn a_structure_with_mismatched_children_is_refused() {
let error = structure(vec![
("x", primitive::<f64>(&[1.0, 2.0])),
("y", primitive::<f64>(&[3.0])),
])
.unwrap_err();
assert!(!error.to_string().is_empty());
assert!(structure(Vec::new()).is_err(), "no children, no row count");
}
#[test]
fn nested_lists_slice_a_shared_child() {
let child = structure(vec![
("stamp", timestamps(&[1, 2, 3])),
("value", primitive::<f64>(&[1.0, 2.0, 3.0])),
])
.unwrap();
let item_type = child.data_type().clone();
let column = nested_lists("pose", item_type, &[2, 1], child).unwrap();
assert_eq!(column.len(), 2);
let lists = column.downcast::<ListArray>().unwrap();
assert_eq!(lists.value_length(0), Some(2));
assert_eq!(lists.value_length(1), Some(1));
}
#[test]
fn empty_inputs_produce_empty_columns_not_errors() {
assert_eq!(primitive_lists::<f32>("v", &[]).unwrap().len(), 0);
assert_eq!(
fixed_tuple_lists::<f32>("xy", "v", 2, &[]).unwrap().len(),
0
);
assert_eq!(fixed_tuples::<f64>("v", 4, &[]).unwrap().len(), 0);
}
}