Skip to main content

astrs_data/tensor/
image.rs

1//! [`ImageView`] — the `std/media/v1/Image` accessor.
2//!
3//! [`crate::urn::layouts::media::image_layout`] fixes the columnar shape:
4//! `{width, height, stride: UInt32, channels: UInt8, data: List<sample>}`,
5//! one row per image, `sample` being `UInt8`, `UInt16` or `Float32`
6//! depending on the port's `pixel` parameter. `ImageView` is what turns one
7//! such row back into the `(height, width, channels)` [`TensorView`] a
8//! vision node actually wants to index — reading `width`/`height`/`channels`
9//! from the row itself, so it never needs the URN or a caller-supplied
10//! shape at all.
11//!
12//! ```
13//! use astrs_data::array::{Array, IntoArrayRef, ListArray, StructArray, UInt32Array, UInt8Array};
14//! use astrs_data::tensor::ImageView;
15//! use astrs_data::urn::layouts::media::image_layout;
16//! use astrs_data::{DataType, Field};
17//!
18//! // One 2x2 mono8 image, packed row-major.
19//! let DataType::Struct(fields) = image_layout("mono8")? else { unreachable!() };
20//! let data = ListArray::try_from_lengths(
21//!     Field::required("sample", DataType::UInt8),
22//!     [4],
23//!     UInt8Array::from_values([10, 20, 30, 40]).into_array_ref(),
24//! )?
25//! .into_array_ref();
26//! let row = StructArray::try_new(
27//!     fields,
28//!     vec![
29//!         UInt32Array::from_values([2u32]).into_array_ref(), // width
30//!         UInt32Array::from_values([2u32]).into_array_ref(), // height
31//!         UInt32Array::from_values([2u32]).into_array_ref(), // stride
32//!         UInt8Array::from_values([1u8]).into_array_ref(),   // channels
33//!         data,
34//!     ],
35//!     None,
36//! )?;
37//!
38//! let view = ImageView::from_struct_row(&row, 0)?;
39//! assert_eq!((view.height(), view.width(), view.channels()), (2, 2, 1));
40//! assert_eq!(view.get_f64(&[1, 0, 0])?, 30.0);
41//! # Ok::<(), astrs_data::DataError>(())
42//! ```
43
44use crate::array::structs::StructArray;
45use crate::array::{
46    Array, ArrayExt, Float32Array, ListArray, UInt8Array, UInt16Array, UInt32Array,
47};
48use crate::datatype::DataType;
49use crate::error::{DataError, Result};
50use crate::tensor::view::TensorView;
51
52/// Reads the `UInt32` value of column `field` at `row`.
53///
54/// # Errors
55///
56/// [`DataError::FieldNotFound`], [`DataError::DowncastFailed`],
57/// [`DataError::RequiredFieldIsNull`], or [`DataError::IndexOutOfBounds`].
58fn read_u32_field(array: &StructArray, field: &str, row: usize) -> Result<u32> {
59    let column = array.try_column_by_name(field)?;
60    let typed = column.try_downcast::<UInt32Array>()?;
61    typed.get(row).ok_or_else(|| {
62        if row < typed.len() {
63            DataError::RequiredFieldIsNull {
64                field: field.to_owned(),
65                row,
66            }
67        } else {
68            DataError::IndexOutOfBounds {
69                index: row,
70                len: typed.len(),
71            }
72        }
73    })
74}
75
76/// Reads the `UInt8` value of column `field` at `row`. See [`read_u32_field`].
77fn read_u8_field(array: &StructArray, field: &str, row: usize) -> Result<u8> {
78    let column = array.try_column_by_name(field)?;
79    let typed = column.try_downcast::<UInt8Array>()?;
80    typed.get(row).ok_or_else(|| {
81        if row < typed.len() {
82            DataError::RequiredFieldIsNull {
83                field: field.to_owned(),
84                row,
85            }
86        } else {
87            DataError::IndexOutOfBounds {
88                index: row,
89                len: typed.len(),
90            }
91        }
92    })
93}
94
95/// A `(height, width, channels)` view over one `std/media/v1/Image` row.
96///
97/// The sample type is exactly the row's `data` column's element type —
98/// `UInt8`, `UInt16` or `Float32`, per [`image_layout`](crate::urn::layouts::media::image_layout).
99/// Match on the variant to work with the concrete [`TensorView`] directly,
100/// or use [`ImageView::get_f64`] for a type-erased read.
101#[derive(Debug, Clone)]
102#[non_exhaustive]
103pub enum ImageView {
104    /// `sample` is `UInt8` (`mono8`, `rgb8`, `rgba8`, `bgr8`, `bgra8`, the
105    /// Bayer mosaics).
106    U8(TensorView<u8>),
107    /// `sample` is `UInt16` (`mono16`, `rgb16`, `rgba16`, `bgr16`, `bgra16`).
108    U16(TensorView<u16>),
109    /// `sample` is `Float32` (`mono32f`, `rgb32f`, `rgba32f`).
110    F32(TensorView<f32>),
111}
112
113impl ImageView {
114    /// Builds a view over row `row` of a `std/media/v1/Image`-shaped struct
115    /// array (or of a payload's own top-level column downcast to one).
116    ///
117    /// # Errors
118    ///
119    /// * [`DataError::FieldNotFound`] — `array` is missing one of
120    ///   `width`/`height`/`channels`/`data`.
121    /// * [`DataError::DowncastFailed`] — a field's column is not the type
122    ///   [`image_layout`](crate::urn::layouts::media::image_layout) declares.
123    /// * [`DataError::RequiredFieldIsNull`] — `width`/`height`/`channels` or
124    ///   the whole row is null.
125    /// * [`DataError::IndexOutOfBounds`] — `row >= array.len()`.
126    /// * [`DataError::TypeMismatch`] — `data`'s element type is not
127    ///   `UInt8`/`UInt16`/`Float32`.
128    /// * [`DataError::TensorShapeMismatch`] — `data`'s row does not hold
129    ///   exactly `width * height * channels` samples.
130    pub fn from_struct_row(array: &StructArray, row: usize) -> Result<Self> {
131        let width = read_u32_field(array, "width", row)? as usize;
132        let height = read_u32_field(array, "height", row)? as usize;
133        let channels = read_u8_field(array, "channels", row)? as usize;
134        let shape = [height, width, channels];
135
136        let data_column = array.try_column_by_name("data")?;
137        let list = data_column.try_downcast::<ListArray>()?;
138        let Some(samples) = list.get(row) else {
139            return Err(if row < list.len() {
140                DataError::RequiredFieldIsNull {
141                    field: "data".to_owned(),
142                    row,
143                }
144            } else {
145                DataError::IndexOutOfBounds {
146                    index: row,
147                    len: list.len(),
148                }
149            });
150        };
151
152        match samples.data_type() {
153            DataType::UInt8 => {
154                let typed = samples.try_downcast::<UInt8Array>()?;
155                Ok(Self::U8(TensorView::from_primitive(typed, shape)?))
156            }
157            DataType::UInt16 => {
158                let typed = samples.try_downcast::<UInt16Array>()?;
159                Ok(Self::U16(TensorView::from_primitive(typed, shape)?))
160            }
161            DataType::Float32 => {
162                let typed = samples.try_downcast::<Float32Array>()?;
163                Ok(Self::F32(TensorView::from_primitive(typed, shape)?))
164            }
165            other => Err(DataError::type_mismatch(DataType::UInt8, other.clone())),
166        }
167    }
168
169    /// The `(height, width, channels)` shape, whatever the sample type.
170    #[must_use]
171    pub fn shape(&self) -> &[usize] {
172        match self {
173            Self::U8(tensor) => tensor.shape(),
174            Self::U16(tensor) => tensor.shape(),
175            Self::F32(tensor) => tensor.shape(),
176        }
177    }
178
179    /// Row count.
180    #[inline]
181    #[must_use]
182    pub fn height(&self) -> usize {
183        self.shape()[0]
184    }
185
186    /// Column count.
187    #[inline]
188    #[must_use]
189    pub fn width(&self) -> usize {
190        self.shape()[1]
191    }
192
193    /// Samples per pixel.
194    #[inline]
195    #[must_use]
196    pub fn channels(&self) -> usize {
197        self.shape()[2]
198    }
199
200    /// The `UInt8`-sample tensor, if this view holds one.
201    #[must_use]
202    pub fn as_u8(&self) -> Option<&TensorView<u8>> {
203        match self {
204            Self::U8(tensor) => Some(tensor),
205            _ => None,
206        }
207    }
208
209    /// The `UInt16`-sample tensor, if this view holds one.
210    #[must_use]
211    pub fn as_u16(&self) -> Option<&TensorView<u16>> {
212        match self {
213            Self::U16(tensor) => Some(tensor),
214            _ => None,
215        }
216    }
217
218    /// The `Float32`-sample tensor, if this view holds one.
219    #[must_use]
220    pub fn as_f32(&self) -> Option<&TensorView<f32>> {
221        match self {
222            Self::F32(tensor) => Some(tensor),
223            _ => None,
224        }
225    }
226
227    /// The sample at `[row, column, channel]`, widened to `f64` — every
228    /// sample type this view can hold widens losslessly.
229    ///
230    /// # Errors
231    ///
232    /// Whatever the underlying [`TensorView::get`] reports.
233    pub fn get_f64(&self, index: &[usize]) -> Result<f64> {
234        match self {
235            Self::U8(tensor) => tensor.get(index).map(f64::from),
236            Self::U16(tensor) => tensor.get(index).map(f64::from),
237            Self::F32(tensor) => tensor.get(index).map(f64::from),
238        }
239    }
240}
241
242#[cfg(test)]
243mod tests {
244    #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
245
246    use super::*;
247    use crate::Field;
248    use crate::array::{Float32Array, IntoArrayRef, UInt16Array};
249    use crate::urn::layouts::media::image_layout;
250
251    fn struct_fields(pixel: &str) -> Vec<Field> {
252        match image_layout(pixel).unwrap() {
253            DataType::Struct(fields) => fields,
254            _ => panic!("image_layout always returns a Struct"),
255        }
256    }
257
258    fn mono8_row(width: u32, height: u32, samples: Vec<u8>) -> StructArray {
259        let data = ListArray::try_from_lengths(
260            Field::required("sample", DataType::UInt8),
261            [samples.len()],
262            UInt8Array::from_values(samples).into_array_ref(),
263        )
264        .unwrap()
265        .into_array_ref();
266        StructArray::try_new(
267            struct_fields("mono8"),
268            vec![
269                UInt32Array::from_values([width]).into_array_ref(),
270                UInt32Array::from_values([height]).into_array_ref(),
271                UInt32Array::from_values([width]).into_array_ref(),
272                UInt8Array::from_values([1u8]).into_array_ref(),
273                data,
274            ],
275            None,
276        )
277        .unwrap()
278    }
279
280    #[test]
281    fn a_mono8_row_round_trips() {
282        let row = mono8_row(3, 2, vec![0, 1, 2, 3, 4, 5]);
283        let view = ImageView::from_struct_row(&row, 0).unwrap();
284        assert_eq!((view.height(), view.width(), view.channels()), (2, 3, 1));
285        assert!(view.as_u8().is_some());
286        assert!(view.as_u16().is_none());
287        assert_eq!(view.get_f64(&[0, 0, 0]).unwrap(), 0.0);
288        assert_eq!(view.get_f64(&[1, 2, 0]).unwrap(), 5.0);
289    }
290
291    #[test]
292    fn a_multi_channel_row_shapes_the_channel_axis() {
293        // 2x2 rgb8: 2*2*3 = 12 samples.
294        let data = ListArray::try_from_lengths(
295            Field::required("sample", DataType::UInt8),
296            [12],
297            UInt8Array::from_values(0..12u8).into_array_ref(),
298        )
299        .unwrap()
300        .into_array_ref();
301        let row = StructArray::try_new(
302            struct_fields("rgb8"),
303            vec![
304                UInt32Array::from_values([2u32]).into_array_ref(),
305                UInt32Array::from_values([2u32]).into_array_ref(),
306                UInt32Array::from_values([6u32]).into_array_ref(),
307                UInt8Array::from_values([3u8]).into_array_ref(),
308                data,
309            ],
310            None,
311        )
312        .unwrap();
313
314        let view = ImageView::from_struct_row(&row, 0).unwrap();
315        assert_eq!(view.shape(), &[2, 2, 3]);
316        // Row-major over (height, width, channels): pixel (row=1, col=0)
317        // starts at flat index 1*2*3 + 0*3 = 6, so its channels are 6,7,8.
318        assert_eq!(view.get_f64(&[1, 0, 0]).unwrap(), 6.0);
319        assert_eq!(view.get_f64(&[1, 0, 2]).unwrap(), 8.0);
320        // Pixel (row=1, col=1) is the last one: samples 9,10,11.
321        assert_eq!(view.get_f64(&[1, 1, 0]).unwrap(), 9.0);
322        assert_eq!(view.get_f64(&[1, 1, 2]).unwrap(), 11.0);
323    }
324
325    #[test]
326    fn a_u16_row_selects_the_u16_variant() {
327        let data = ListArray::try_from_lengths(
328            Field::required("sample", DataType::UInt16),
329            [4],
330            UInt16Array::from_values([100u16, 200, 300, 400]).into_array_ref(),
331        )
332        .unwrap()
333        .into_array_ref();
334        let row = StructArray::try_new(
335            struct_fields("mono16"),
336            vec![
337                UInt32Array::from_values([2u32]).into_array_ref(),
338                UInt32Array::from_values([2u32]).into_array_ref(),
339                UInt32Array::from_values([2u32]).into_array_ref(),
340                UInt8Array::from_values([1u8]).into_array_ref(),
341                data,
342            ],
343            None,
344        )
345        .unwrap();
346
347        let view = ImageView::from_struct_row(&row, 0).unwrap();
348        assert!(view.as_u16().is_some());
349        assert_eq!(view.get_f64(&[0, 1, 0]).unwrap(), 200.0);
350    }
351
352    #[test]
353    fn an_f32_row_selects_the_f32_variant() {
354        let data = ListArray::try_from_lengths(
355            Field::required("sample", DataType::Float32),
356            [4],
357            Float32Array::from_values([1.5f32, 2.5, 3.5, 4.5]).into_array_ref(),
358        )
359        .unwrap()
360        .into_array_ref();
361        let row = StructArray::try_new(
362            struct_fields("mono32f"),
363            vec![
364                UInt32Array::from_values([2u32]).into_array_ref(),
365                UInt32Array::from_values([2u32]).into_array_ref(),
366                UInt32Array::from_values([2u32]).into_array_ref(),
367                UInt8Array::from_values([1u8]).into_array_ref(),
368                data,
369            ],
370            None,
371        )
372        .unwrap();
373
374        let view = ImageView::from_struct_row(&row, 0).unwrap();
375        assert!(view.as_f32().is_some());
376        assert_eq!(view.get_f64(&[1, 1, 0]).unwrap(), 4.5);
377    }
378
379    #[test]
380    fn a_second_row_in_a_multi_row_batch_is_independent() {
381        let data = ListArray::try_from_lengths(
382            Field::required("sample", DataType::UInt8),
383            [4, 4],
384            UInt8Array::from_values([0, 1, 2, 3, 10, 11, 12, 13]).into_array_ref(),
385        )
386        .unwrap()
387        .into_array_ref();
388        let row = StructArray::try_new(
389            struct_fields("mono8"),
390            vec![
391                UInt32Array::from_values([2u32, 2]).into_array_ref(),
392                UInt32Array::from_values([2u32, 2]).into_array_ref(),
393                UInt32Array::from_values([2u32, 2]).into_array_ref(),
394                UInt8Array::from_values([1u8, 1]).into_array_ref(),
395                data,
396            ],
397            None,
398        )
399        .unwrap();
400
401        let first = ImageView::from_struct_row(&row, 0).unwrap();
402        let second = ImageView::from_struct_row(&row, 1).unwrap();
403        assert_eq!(first.get_f64(&[0, 0, 0]).unwrap(), 0.0);
404        assert_eq!(second.get_f64(&[0, 0, 0]).unwrap(), 10.0);
405    }
406
407    #[test]
408    fn a_missing_field_is_reported() {
409        let fields = vec![Field::required("width", DataType::UInt32)];
410        let row = StructArray::try_new(
411            fields,
412            vec![UInt32Array::from_values([1u32]).into_array_ref()],
413            None,
414        )
415        .unwrap();
416        assert!(matches!(
417            ImageView::from_struct_row(&row, 0),
418            Err(DataError::FieldNotFound { .. })
419        ));
420    }
421
422    #[test]
423    fn a_null_required_field_is_reported_precisely() {
424        // A struct's own row-level validity (tested elsewhere on
425        // `StructArray` itself) does not null out its children — Arrow
426        // leaves their values at a null parent slot unspecified, not
427        // necessarily null — so this nulls the `width` *column* directly,
428        // the only way `read_u32_field` can actually observe a null here.
429        let width = UInt32Array::from_opt_iter([None]).into_array_ref();
430        let height = UInt32Array::from_values([2u32]).into_array_ref();
431        let stride = UInt32Array::from_values([2u32]).into_array_ref();
432        let channels = UInt8Array::from_values([1u8]).into_array_ref();
433        let data = ListArray::try_from_lengths(
434            Field::required("sample", DataType::UInt8),
435            [4],
436            UInt8Array::from_values([0u8, 1, 2, 3]).into_array_ref(),
437        )
438        .unwrap()
439        .into_array_ref();
440        let row = StructArray::try_new(
441            struct_fields("mono8"),
442            vec![width, height, stride, channels, data],
443            None,
444        )
445        .unwrap();
446
447        assert!(matches!(
448            ImageView::from_struct_row(&row, 0),
449            Err(DataError::RequiredFieldIsNull { field, row: 0 }) if field == "width"
450        ));
451    }
452
453    #[test]
454    fn an_out_of_range_row_is_index_out_of_bounds() {
455        let row = mono8_row(2, 2, vec![0, 1, 2, 3]);
456        assert!(matches!(
457            ImageView::from_struct_row(&row, 5),
458            Err(DataError::IndexOutOfBounds { index: 5, .. })
459        ));
460    }
461
462    #[test]
463    fn a_shape_mismatch_is_reported() {
464        // Declares 2x2x1 = 4 samples but the data column only holds 3.
465        let data = ListArray::try_from_lengths(
466            Field::required("sample", DataType::UInt8),
467            [3],
468            UInt8Array::from_values([0u8, 1, 2]).into_array_ref(),
469        )
470        .unwrap()
471        .into_array_ref();
472        let row = StructArray::try_new(
473            struct_fields("mono8"),
474            vec![
475                UInt32Array::from_values([2u32]).into_array_ref(),
476                UInt32Array::from_values([2u32]).into_array_ref(),
477                UInt32Array::from_values([2u32]).into_array_ref(),
478                UInt8Array::from_values([1u8]).into_array_ref(),
479                data,
480            ],
481            None,
482        )
483        .unwrap();
484        assert!(matches!(
485            ImageView::from_struct_row(&row, 0),
486            Err(DataError::TensorShapeMismatch {
487                expected: 4,
488                actual: 3
489            })
490        ));
491    }
492}