1use serde_core::de::{self, MapAccess, Visitor};
10use serde_core::ser::SerializeStruct;
11use serde_core::{Deserialize, Deserializer, Serialize, Serializer};
12use std::fmt;
13
14use crate::{ArrowError, DataType, extension::ExtensionType};
15
16#[derive(Debug, Clone, PartialEq)]
63pub struct FixedShapeTensor {
64 value_type: DataType,
66
67 metadata: FixedShapeTensorMetadata,
69}
70
71impl FixedShapeTensor {
72 pub fn try_new(
79 value_type: DataType,
80 shape: impl IntoIterator<Item = usize>,
81 dimension_names: Option<Vec<String>>,
82 permutations: Option<Vec<usize>>,
83 ) -> Result<Self, ArrowError> {
84 FixedShapeTensorMetadata::try_new(shape, dimension_names, permutations).map(|metadata| {
86 Self {
87 value_type,
88 metadata,
89 }
90 })
91 }
92
93 pub fn value_type(&self) -> &DataType {
95 &self.value_type
96 }
97
98 pub fn list_size(&self) -> usize {
100 self.metadata.list_size()
101 }
102
103 pub fn shape(&self) -> &[usize] {
117 self.metadata.shape()
118 }
119
120 pub fn dimensions(&self) -> usize {
122 self.metadata.dimensions()
123 }
124
125 pub fn dimension_names(&self) -> Option<&[String]> {
128 self.metadata.dimension_names()
129 }
130
131 pub fn permutations(&self) -> Option<&[usize]> {
134 self.metadata.permutations()
135 }
136
137 pub fn logical_shape(&self) -> Vec<usize> {
158 self.metadata.logical_shape()
159 }
160}
161
162#[derive(Debug, Clone, PartialEq)]
164pub struct FixedShapeTensorMetadata {
165 shape: Vec<usize>,
167
168 dim_names: Option<Vec<String>>,
170
171 permutations: Option<Vec<usize>>,
173}
174
175impl Serialize for FixedShapeTensorMetadata {
176 fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
177 where
178 S: Serializer,
179 {
180 let mut state = serializer.serialize_struct("FixedShapeTensorMetadata", 3)?;
181 state.serialize_field("shape", &self.shape)?;
182 state.serialize_field("dim_names", &self.dim_names)?;
183 state.serialize_field("permutations", &self.permutations)?;
184 state.end()
185 }
186}
187
188#[derive(Debug)]
189enum MetadataField {
190 Shape,
191 DimNames,
192 Permutations,
193}
194
195struct MetadataFieldVisitor;
196
197impl Visitor<'_> for MetadataFieldVisitor {
198 type Value = MetadataField;
199
200 fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
201 formatter.write_str("`shape`, `dim_names`, or `permutations`")
202 }
203
204 fn visit_str<E>(self, value: &str) -> Result<MetadataField, E>
205 where
206 E: de::Error,
207 {
208 match value {
209 "shape" => Ok(MetadataField::Shape),
210 "dim_names" => Ok(MetadataField::DimNames),
211 "permutations" => Ok(MetadataField::Permutations),
212 _ => Err(de::Error::unknown_field(
213 value,
214 &["shape", "dim_names", "permutations"],
215 )),
216 }
217 }
218}
219
220impl<'de> Deserialize<'de> for MetadataField {
221 fn deserialize<D>(deserializer: D) -> Result<MetadataField, D::Error>
222 where
223 D: Deserializer<'de>,
224 {
225 deserializer.deserialize_identifier(MetadataFieldVisitor)
226 }
227}
228
229struct FixedShapeTensorMetadataVisitor;
230
231impl<'de> Visitor<'de> for FixedShapeTensorMetadataVisitor {
232 type Value = FixedShapeTensorMetadata;
233
234 fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
235 formatter.write_str("struct FixedShapeTensorMetadata")
236 }
237
238 fn visit_seq<V>(self, mut seq: V) -> Result<FixedShapeTensorMetadata, V::Error>
239 where
240 V: de::SeqAccess<'de>,
241 {
242 let shape = seq
243 .next_element()?
244 .ok_or_else(|| de::Error::invalid_length(0, &self))?;
245 let dim_names = seq
246 .next_element()?
247 .ok_or_else(|| de::Error::invalid_length(1, &self))?;
248 let permutations = seq
249 .next_element()?
250 .ok_or_else(|| de::Error::invalid_length(2, &self))?;
251 Ok(FixedShapeTensorMetadata {
252 shape,
253 dim_names,
254 permutations,
255 })
256 }
257
258 fn visit_map<V>(self, mut map: V) -> Result<FixedShapeTensorMetadata, V::Error>
259 where
260 V: MapAccess<'de>,
261 {
262 let mut shape = None;
263 let mut dim_names = None;
264 let mut permutations = None;
265
266 while let Some(key) = map.next_key()? {
267 match key {
268 MetadataField::Shape => {
269 if shape.is_some() {
270 return Err(de::Error::duplicate_field("shape"));
271 }
272 shape = Some(map.next_value()?);
273 }
274 MetadataField::DimNames => {
275 if dim_names.is_some() {
276 return Err(de::Error::duplicate_field("dim_names"));
277 }
278 dim_names = Some(map.next_value()?);
279 }
280 MetadataField::Permutations => {
281 if permutations.is_some() {
282 return Err(de::Error::duplicate_field("permutations"));
283 }
284 permutations = Some(map.next_value()?);
285 }
286 }
287 }
288
289 let shape = shape.ok_or_else(|| de::Error::missing_field("shape"))?;
290
291 Ok(FixedShapeTensorMetadata {
292 shape,
293 dim_names,
294 permutations,
295 })
296 }
297}
298
299impl<'de> Deserialize<'de> for FixedShapeTensorMetadata {
300 fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
301 where
302 D: Deserializer<'de>,
303 {
304 deserializer.deserialize_struct(
305 "FixedShapeTensorMetadata",
306 &["shape", "dim_names", "permutations"],
307 FixedShapeTensorMetadataVisitor,
308 )
309 }
310}
311
312impl FixedShapeTensorMetadata {
313 pub fn try_new(
320 shape: impl IntoIterator<Item = usize>,
321 dimension_names: Option<Vec<String>>,
322 permutations: Option<Vec<usize>>,
323 ) -> Result<Self, ArrowError> {
324 let shape = shape.into_iter().collect::<Vec<_>>();
325 let dimensions = shape.len();
326
327 let dim_names = dimension_names.map(|dimension_names| {
328 if dimension_names.len() != dimensions {
329 Err(ArrowError::InvalidArgumentError(format!(
330 "FixedShapeTensor dimension names size mismatch, expected {dimensions}, found {}", dimension_names.len()
331 )))
332 } else {
333 Ok(dimension_names)
334 }
335 }).transpose()?;
336
337 let permutations = permutations
338 .map(|permutations| {
339 if permutations.len() != dimensions {
340 Err(ArrowError::InvalidArgumentError(format!(
341 "FixedShapeTensor permutations size mismatch, expected {dimensions}, found {}",
342 permutations.len()
343 )))
344 } else {
345 let mut sorted_permutations = permutations.clone();
346 sorted_permutations.sort_unstable();
347 if (0..dimensions).zip(sorted_permutations).any(|(a, b)| a != b) {
348 Err(ArrowError::InvalidArgumentError(format!(
349 "FixedShapeTensor permutations invalid, expected a permutation of [0, 1, .., N-1], where N is the number of dimensions: {dimensions}"
350 )))
351 } else {
352 Ok(permutations)
353 }
354 }
355 })
356 .transpose()?;
357
358 Ok(Self {
359 shape,
360 dim_names,
361 permutations,
362 })
363 }
364
365 pub fn shape(&self) -> &[usize] {
367 &self.shape
368 }
369
370 pub fn list_size(&self) -> usize {
372 self.shape.iter().product()
373 }
374
375 pub fn dimensions(&self) -> usize {
377 self.shape.len()
378 }
379
380 pub fn dimension_names(&self) -> Option<&[String]> {
383 self.dim_names.as_ref().map(AsRef::as_ref)
384 }
385
386 pub fn permutations(&self) -> Option<&[usize]> {
389 self.permutations.as_ref().map(AsRef::as_ref)
390 }
391
392 pub fn logical_shape(&self) -> Vec<usize> {
396 match &self.permutations {
397 Some(permutation) => permutation.iter().map(|&index| self.shape[index]).collect(),
398 None => self.shape.clone(),
399 }
400 }
401}
402
403impl ExtensionType for FixedShapeTensor {
404 const NAME: &'static str = "arrow.fixed_shape_tensor";
405
406 type Metadata = FixedShapeTensorMetadata;
407
408 fn metadata(&self) -> &Self::Metadata {
409 &self.metadata
410 }
411
412 fn serialize_metadata(&self) -> Option<String> {
413 Some(serde_json::to_string(&self.metadata).expect("metadata serialization"))
414 }
415
416 fn deserialize_metadata(metadata: Option<&str>) -> Result<Self::Metadata, ArrowError> {
417 metadata.map_or_else(
418 || {
419 Err(ArrowError::InvalidArgumentError(
420 "FixedShapeTensor extension types requires metadata".to_owned(),
421 ))
422 },
423 |value| {
424 serde_json::from_str(value).map_err(|e| {
425 ArrowError::InvalidArgumentError(format!(
426 "FixedShapeTensor metadata deserialization failed: {e}"
427 ))
428 })
429 },
430 )
431 }
432
433 fn supports_data_type(&self, data_type: &DataType) -> Result<(), ArrowError> {
434 let expected = DataType::new_fixed_size_list(
435 self.value_type.clone(),
436 i32::try_from(self.list_size()).expect("overflow"),
437 false,
438 );
439 data_type
440 .equals_datatype(&expected)
441 .then_some(())
442 .ok_or_else(|| {
443 ArrowError::InvalidArgumentError(format!(
444 "FixedShapeTensor data type mismatch, expected {expected}, found {data_type}"
445 ))
446 })
447 }
448
449 fn try_new(data_type: &DataType, metadata: Self::Metadata) -> Result<Self, ArrowError> {
450 match data_type {
451 DataType::FixedSizeList(field, list_size) if !field.is_nullable() => {
452 let metadata = FixedShapeTensorMetadata::try_new(
454 metadata.shape,
455 metadata.dim_names,
456 metadata.permutations,
457 )?;
458 let expected_size = i32::try_from(metadata.list_size()).expect("overflow");
460 if *list_size != expected_size {
461 Err(ArrowError::InvalidArgumentError(format!(
462 "FixedShapeTensor list size mismatch, expected {expected_size} (metadata), found {list_size} (data type)"
463 )))
464 } else {
465 Ok(Self {
466 value_type: field.data_type().clone(),
467 metadata,
468 })
469 }
470 }
471 data_type => Err(ArrowError::InvalidArgumentError(format!(
472 "FixedShapeTensor data type mismatch, expected FixedSizeList with non-nullable field, found {data_type}"
473 ))),
474 }
475 }
476}
477
478#[cfg(test)]
479mod tests {
480 #[cfg(feature = "canonical_extension_types")]
481 use crate::extension::CanonicalExtensionType;
482 use crate::{
483 Field,
484 extension::{EXTENSION_TYPE_METADATA_KEY, EXTENSION_TYPE_NAME_KEY, ExtensionType},
485 };
486
487 use super::*;
488
489 #[test]
490 fn valid() -> Result<(), ArrowError> {
491 let fixed_shape_tensor = FixedShapeTensor::try_new(
492 DataType::Float32,
493 [100, 200, 500],
494 Some(vec!["C".to_owned(), "H".to_owned(), "W".to_owned()]),
495 Some(vec![2, 0, 1]),
496 )?;
497 let mut field = Field::new_fixed_size_list(
498 "",
499 Field::new("", DataType::Float32, false),
500 i32::try_from(fixed_shape_tensor.list_size()).expect("overflow"),
501 false,
502 );
503 field.try_with_extension_type(fixed_shape_tensor.clone())?;
504 assert_eq!(
505 field.try_extension_type::<FixedShapeTensor>()?,
506 fixed_shape_tensor
507 );
508 assert_eq!(fixed_shape_tensor.shape(), &[100, 200, 500]);
509 assert_eq!(fixed_shape_tensor.metadata().shape(), &[100, 200, 500]);
510 #[cfg(feature = "canonical_extension_types")]
511 assert_eq!(
512 field.try_canonical_extension_type()?,
513 CanonicalExtensionType::FixedShapeTensor(fixed_shape_tensor)
514 );
515 Ok(())
516 }
517
518 #[test]
519 fn logical_shape_follows_permutation() -> Result<(), ArrowError> {
520 let physical = FixedShapeTensor::try_new(DataType::Float32, [100, 200, 500], None, None)?;
521 assert_eq!(physical.logical_shape(), vec![100, 200, 500]);
522 assert_eq!(physical.metadata().logical_shape(), vec![100, 200, 500]);
523
524 let permuted = FixedShapeTensor::try_new(
525 DataType::Float32,
526 [100, 200, 500],
527 Some(vec!["C".to_owned(), "H".to_owned(), "W".to_owned()]),
528 Some(vec![2, 0, 1]),
529 )?;
530 assert_eq!(permuted.logical_shape(), vec![500, 100, 200]);
531 assert_eq!(permuted.metadata().logical_shape(), vec![500, 100, 200]);
532
533 let identity =
534 FixedShapeTensor::try_new(DataType::Float32, [2, 6], None, Some(vec![0, 1]))?;
535 assert_eq!(identity.logical_shape(), vec![2, 6]);
536 Ok(())
537 }
538
539 #[test]
540 fn shape_distinguishes_equal_sizes() -> Result<(), ArrowError> {
541 let wide = FixedShapeTensor::try_new(DataType::Float32, [2, 6], None, None)?;
542 let tall = FixedShapeTensor::try_new(DataType::Float32, [3, 4], None, None)?;
543 assert_eq!(wide.dimensions(), tall.dimensions());
544 assert_eq!(wide.list_size(), tall.list_size());
545 assert_eq!(wide.shape(), &[2, 6]);
546 assert_eq!(tall.shape(), &[3, 4]);
547 assert_ne!(wide.shape(), tall.shape());
548 Ok(())
549 }
550
551 #[test]
552 #[should_panic(expected = "Extension type name missing")]
553 fn missing_name() {
554 let field =
555 Field::new_fixed_size_list("", Field::new("", DataType::Float32, false), 3, false)
556 .with_metadata([(
557 EXTENSION_TYPE_METADATA_KEY,
558 r#"{ "shape": [100, 200, 500], }"#,
559 )]);
560 field.extension_type::<FixedShapeTensor>();
561 }
562
563 #[test]
564 #[should_panic(expected = "FixedShapeTensor data type mismatch, expected FixedSizeList")]
565 fn invalid_type() {
566 let fixed_shape_tensor =
567 FixedShapeTensor::try_new(DataType::Int32, [100, 200, 500], None, None).unwrap();
568 let field = Field::new_fixed_size_list(
569 "",
570 Field::new("", DataType::Float32, false),
571 i32::try_from(fixed_shape_tensor.list_size()).expect("overflow"),
572 false,
573 );
574 field.with_extension_type(fixed_shape_tensor);
575 }
576
577 #[test]
578 #[should_panic(expected = "FixedShapeTensor extension types requires metadata")]
579 fn missing_metadata() {
580 let field =
581 Field::new_fixed_size_list("", Field::new("", DataType::Float32, false), 3, false)
582 .with_metadata([(EXTENSION_TYPE_NAME_KEY, FixedShapeTensor::NAME)]);
583 field.extension_type::<FixedShapeTensor>();
584 }
585
586 #[test]
587 #[should_panic(expected = "FixedShapeTensor metadata deserialization failed: \
588 unknown field `not-shape`, expected one of `shape`, `dim_names`, `permutations`")]
589 fn invalid_metadata() {
590 let fixed_shape_tensor =
591 FixedShapeTensor::try_new(DataType::Float32, [100, 200, 500], None, None).unwrap();
592 let field = Field::new_fixed_size_list(
593 "",
594 Field::new("", DataType::Float32, false),
595 i32::try_from(fixed_shape_tensor.list_size()).expect("overflow"),
596 false,
597 )
598 .with_metadata([
599 (EXTENSION_TYPE_NAME_KEY, FixedShapeTensor::NAME),
600 (EXTENSION_TYPE_METADATA_KEY, r#"{ "not-shape": [] }"#),
601 ]);
602 field.extension_type::<FixedShapeTensor>();
603 }
604
605 #[test]
606 #[should_panic(
607 expected = "FixedShapeTensor dimension names size mismatch, expected 3, found 2"
608 )]
609 fn invalid_metadata_dimension_names() {
610 FixedShapeTensor::try_new(
611 DataType::Float32,
612 [100, 200, 500],
613 Some(vec!["a".to_owned(), "b".to_owned()]),
614 None,
615 )
616 .unwrap();
617 }
618
619 #[test]
620 #[should_panic(expected = "FixedShapeTensor permutations size mismatch, expected 3, found 2")]
621 fn invalid_metadata_permutations_len() {
622 FixedShapeTensor::try_new(DataType::Float32, [100, 200, 500], None, Some(vec![1, 0]))
623 .unwrap();
624 }
625
626 #[test]
627 #[should_panic(
628 expected = "FixedShapeTensor permutations invalid, expected a permutation of [0, 1, .., N-1], where N is the number of dimensions: 3"
629 )]
630 fn invalid_metadata_permutations_values() {
631 FixedShapeTensor::try_new(
632 DataType::Float32,
633 [100, 200, 500],
634 None,
635 Some(vec![4, 3, 2]),
636 )
637 .unwrap();
638 }
639}