datafusion_common/types/
extension.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use crate::error::Result;
19use arrow::array::Array;
20use arrow::util::display::{ArrayFormatter, FormatOptions};
21use arrow_schema::DataType;
22use std::fmt::Debug;
23use std::sync::Arc;
24
25/// A cheaply cloneable pointer to a [`DFExtensionType`].
26pub type DFExtensionTypeRef = Arc<dyn DFExtensionType>;
27
28/// Represents an implementation of a DataFusion extension type, including the storage [`DataType`].
29/// While, in general, an extension type can support several different storage types, a specific
30/// instance of it is always locked into just one exact storage type and metadata pairing.
31///
32/// This trait allows users to customize the behavior of DataFusion for certain types. Having this
33/// ability is necessary because extension types affect how columns should be treated by the query
34/// engine. This effect includes which operations are possible on a column and what are the expected
35/// results from these operations. The extension type mechanism allows users to define how these
36/// operations apply to a particular extension type.
37///
38/// For example, adding two values of [`Int64`](arrow::datatypes::DataType::Int64) is a sensible
39/// thing to do. However, if the same column is annotated with an extension type like `custom.id`,
40/// the correct interpretation of a column changes. Adding together two `custom.id` values, even
41/// though they are stored as integers, may no longer make sense.
42///
43/// Note that DataFusion's extension type support is still young and therefore might not cover all
44/// relevant use cases. Currently, the following operations can be customized:
45/// - Pretty-printing values in record batches
46///
47/// # Relation to Arrow's [`ExtensionType`](arrow_schema::extension::ExtensionType)
48///
49/// The purpose of Arrow's [`ExtensionType`](arrow_schema::extension::ExtensionType) trait, for the
50/// time being, is to allow reading and writing extension type metadata in a type-safe manner. The
51/// trait does not provide any customization options. Therefore, downstream users (such as
52/// DataFusion) have the flexibility to implement the extension type mechanism according to their
53/// needs. [`DFExtensionType`] is DataFusion's implementation of this extension type mechanism.
54///
55/// Furthermore, the current trait in arrow-rs is not dyn-compatible, which we need for implementing
56/// extension type registries. In the future, the two implementations may increasingly converge.
57///
58/// Another difference is that [`DFExtensionType`] represents a fully resolved extension type that
59/// has a fixed storage type (i.e., [`DataType`]). This is different from arrow-rs, which only
60/// stores the extension type's metadata. For example, an instance of DataFusion's JSON extension
61/// type fixes one of the three possible storage types: [`DataType::Utf8`],
62/// [`DataType::LargeUtf8`], or [`DataType::Utf8View`]. This fixed storaga type is returned in
63/// [`DFExtensionType::storage_type`]. This is not possible in arrow-rs' extension type instances.
64/// This is the reason why we have different types in DataFusion that usually delegate the metadata
65/// processing to the underlying arrow-rs extension type instance
66/// (e.g., [`DFJson`](crate::types::DFJson) instead of [`Json`](arrow_schema::extension::Json)).
67///
68/// # Examples
69///
70/// Examples for using the extension type machinery can be found in the DataFusion examples
71/// directory.
72pub trait DFExtensionType: Debug + Send + Sync {
73    /// Returns the underlying storage type.
74    fn storage_type(&self) -> DataType;
75
76    /// Returns the serialized metadata.
77    fn serialize_metadata(&self) -> Option<String>;
78
79    /// Returns an [`ArrayFormatter`] that can format values of this type.
80    ///
81    /// If `Ok(None)` is returned, the default implementation will be used.
82    /// If an error is returned, there was an error creating the formatter.
83    fn create_array_formatter<'fmt>(
84        &self,
85        _array: &'fmt dyn Array,
86        _options: &FormatOptions<'fmt>,
87    ) -> Result<Option<ArrayFormatter<'fmt>>> {
88        Ok(None)
89    }
90}
datafusion_common/types/extension.rs

datafusion_common/types/
extension.rs
