datafusion_common/scalar/
mod.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! [`ScalarValue`]: stores single  values
19
20mod cache;
21mod consts;
22mod struct_builder;
23
24use std::borrow::Borrow;
25use std::cmp::Ordering;
26use std::collections::{HashMap, HashSet, VecDeque};
27use std::convert::Infallible;
28use std::fmt;
29use std::fmt::Write;
30use std::hash::Hash;
31use std::hash::Hasher;
32use std::iter::repeat_n;
33use std::mem::{size_of, size_of_val};
34use std::str::FromStr;
35use std::sync::Arc;
36
37use crate::assert_or_internal_err;
38use crate::cast::{
39    as_binary_array, as_binary_view_array, as_boolean_array, as_date32_array,
40    as_date64_array, as_decimal32_array, as_decimal64_array, as_decimal128_array,
41    as_decimal256_array, as_dictionary_array, as_duration_microsecond_array,
42    as_duration_millisecond_array, as_duration_nanosecond_array,
43    as_duration_second_array, as_fixed_size_binary_array, as_fixed_size_list_array,
44    as_float16_array, as_float32_array, as_float64_array, as_int8_array, as_int16_array,
45    as_int32_array, as_int64_array, as_interval_dt_array, as_interval_mdn_array,
46    as_interval_ym_array, as_large_binary_array, as_large_list_array,
47    as_large_list_view_array, as_large_string_array, as_list_view_array, as_run_array,
48    as_string_array, as_string_view_array, as_time32_millisecond_array,
49    as_time32_second_array, as_time64_microsecond_array, as_time64_nanosecond_array,
50    as_timestamp_microsecond_array, as_timestamp_millisecond_array,
51    as_timestamp_nanosecond_array, as_timestamp_second_array, as_uint8_array,
52    as_uint16_array, as_uint32_array, as_uint64_array, as_union_array,
53};
54use crate::error::{_exec_err, _internal_err, _not_impl_err, DataFusionError, Result};
55use crate::format::DEFAULT_CAST_OPTIONS;
56use crate::hash_utils::create_hashes;
57use crate::scalar::consts::{
58    DECIMAL32_ONES, DECIMAL64_ONES, DECIMAL128_ONES, DECIMAL256_ONES,
59};
60use crate::utils::SingleRowListArrayBuilder;
61use crate::{_internal_datafusion_err, arrow_datafusion_err};
62use arrow::array::{
63    Array, ArrayData, ArrayDataBuilder, ArrayRef, ArrowNativeTypeOp, ArrowPrimitiveType,
64    AsArray, BinaryArray, BinaryViewArray, BinaryViewBuilder, BooleanArray, Date32Array,
65    Date64Array, Decimal32Array, Decimal64Array, Decimal128Array, Decimal256Array,
66    DictionaryArray, DurationMicrosecondArray, DurationMillisecondArray,
67    DurationNanosecondArray, DurationSecondArray, FixedSizeBinaryArray,
68    FixedSizeListArray, Float16Array, Float32Array, Float64Array, GenericListArray,
69    GenericListViewArray, Int8Array, Int16Array, Int32Array, Int64Array,
70    IntervalDayTimeArray, IntervalMonthDayNanoArray, IntervalYearMonthArray,
71    LargeBinaryArray, LargeListArray, LargeListViewArray, LargeStringArray, ListArray,
72    ListViewArray, MapArray, MutableArrayData, PrimitiveArray, RunArray, Scalar,
73    StringArray, StringViewArray, StringViewBuilder, StructArray, Time32MillisecondArray,
74    Time32SecondArray, Time64MicrosecondArray, Time64NanosecondArray,
75    TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray,
76    TimestampSecondArray, UInt8Array, UInt16Array, UInt32Array, UInt64Array, UnionArray,
77    downcast_run_array, new_empty_array, new_null_array,
78};
79use arrow::buffer::{BooleanBuffer, ScalarBuffer};
80use arrow::compute::kernels::cast::{CastOptions, cast_with_options};
81use arrow::compute::kernels::numeric::{
82    add, add_wrapping, div, mul, mul_wrapping, rem, sub, sub_wrapping,
83};
84use arrow::datatypes::{
85    ArrowDictionaryKeyType, ArrowNativeType, ArrowTimestampType, DataType, Date32Type,
86    Decimal32Type, Decimal64Type, Decimal128Type, Decimal256Type, DecimalType, Field,
87    FieldRef, Float32Type, Int8Type, Int16Type, Int32Type, Int64Type, IntervalDayTime,
88    IntervalDayTimeType, IntervalMonthDayNano, IntervalMonthDayNanoType, IntervalUnit,
89    IntervalYearMonthType, MAX_DECIMAL32_FOR_EACH_PRECISION,
90    MAX_DECIMAL64_FOR_EACH_PRECISION, MAX_DECIMAL128_FOR_EACH_PRECISION,
91    MAX_DECIMAL256_FOR_EACH_PRECISION, MIN_DECIMAL32_FOR_EACH_PRECISION,
92    MIN_DECIMAL64_FOR_EACH_PRECISION, MIN_DECIMAL128_FOR_EACH_PRECISION,
93    MIN_DECIMAL256_FOR_EACH_PRECISION, RunEndIndexType, TimeUnit,
94    TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType,
95    TimestampSecondType, UInt8Type, UInt16Type, UInt32Type, UInt64Type, UnionFields,
96    UnionMode, i256, validate_decimal_precision_and_scale,
97};
98use arrow::util::display::{ArrayFormatter, FormatOptions, array_value_to_string};
99use cache::{get_or_create_cached_key_array, get_or_create_cached_null_array};
100use chrono::{Duration, NaiveDate};
101use half::f16;
102use num_traits::ToPrimitive;
103pub use struct_builder::ScalarStructBuilder;
104
105const SECONDS_PER_DAY: i64 = 86_400;
106const MILLIS_PER_DAY: i64 = SECONDS_PER_DAY * 1_000;
107const MICROS_PER_DAY: i64 = MILLIS_PER_DAY * 1_000;
108const NANOS_PER_DAY: i64 = MICROS_PER_DAY * 1_000;
109const MICROS_PER_MILLISECOND: i64 = 1_000;
110const NANOS_PER_MILLISECOND: i64 = 1_000_000;
111
112/// Returns the multiplier that converts the input date representation into the
113/// desired timestamp unit, if the conversion requires a multiplication that can
114/// overflow an `i64`.
115pub fn date_to_timestamp_multiplier(
116    source_type: &DataType,
117    target_type: &DataType,
118) -> Option<i64> {
119    let DataType::Timestamp(target_unit, _) = target_type else {
120        return None;
121    };
122
123    // Only `Timestamp` target types have a time unit; otherwise no
124    // multiplier applies (handled above). The function returns `Some(m)`
125    // when converting the `source_type` to `target_type` requires a
126    // multiplication that could overflow `i64`. It returns `None` when
127    // the conversion is a division or otherwise doesn't require a
128    // multiplication (e.g. Date64 -> Second).
129    match source_type {
130        // Date32 stores days since epoch. Converting to any timestamp
131        // unit requires multiplying by the per-day factor (seconds,
132        // milliseconds, microseconds, nanoseconds).
133        DataType::Date32 => Some(match target_unit {
134            TimeUnit::Second => SECONDS_PER_DAY,
135            TimeUnit::Millisecond => MILLIS_PER_DAY,
136            TimeUnit::Microsecond => MICROS_PER_DAY,
137            TimeUnit::Nanosecond => NANOS_PER_DAY,
138        }),
139
140        // Date64 stores milliseconds since epoch. Converting to
141        // seconds is a division (no multiplication), so return `None`.
142        // Converting to milliseconds is 1:1 (multiplier 1). Converting
143        // to micro/nano requires multiplying by 1_000 / 1_000_000.
144        DataType::Date64 => match target_unit {
145            TimeUnit::Second => None,
146            // Converting Date64 (ms since epoch) to millisecond timestamps
147            // is an identity conversion and does not require multiplication.
148            // Returning `None` indicates no multiplication-based overflow
149            // check is necessary.
150            TimeUnit::Millisecond => None,
151            TimeUnit::Microsecond => Some(MICROS_PER_MILLISECOND),
152            TimeUnit::Nanosecond => Some(NANOS_PER_MILLISECOND),
153        },
154
155        _ => None,
156    }
157}
158
159/// Returns the multiplier that converts the input timestamp representation into
160/// the desired timestamp unit, if the conversion requires a multiplication that
161/// can overflow an `i64`.
162pub fn timestamp_to_timestamp_multiplier(
163    source_type: &DataType,
164    target_type: &DataType,
165) -> Option<i64> {
166    let (DataType::Timestamp(source_unit, _), DataType::Timestamp(target_unit, _)) =
167        (source_type, target_type)
168    else {
169        return None;
170    };
171
172    match (source_unit, target_unit) {
173        (TimeUnit::Second, TimeUnit::Millisecond) => Some(1_000),
174        (TimeUnit::Second, TimeUnit::Microsecond) => Some(1_000_000),
175        (TimeUnit::Second, TimeUnit::Nanosecond) => Some(1_000_000_000),
176        (TimeUnit::Millisecond, TimeUnit::Microsecond) => Some(1_000),
177        (TimeUnit::Millisecond, TimeUnit::Nanosecond) => Some(1_000_000),
178        (TimeUnit::Microsecond, TimeUnit::Nanosecond) => Some(1_000),
179        _ => None,
180    }
181}
182
183/// Ensures the provided value can be represented as a timestamp with the given
184/// multiplier. Returns an [`DataFusionError::Execution`] when the converted
185/// value would overflow the timestamp range.
186pub fn ensure_timestamp_in_bounds(
187    value: i64,
188    multiplier: i64,
189    source_type: &DataType,
190    target_type: &DataType,
191) -> Result<()> {
192    if multiplier <= 1 {
193        return Ok(());
194    }
195
196    if value.checked_mul(multiplier).is_none() {
197        let target = format_timestamp_type_for_error(target_type);
198        _exec_err!(
199            "Cannot cast {} value {} to {}: converted value exceeds the representable i64 range",
200            source_type,
201            value,
202            target
203        )
204    } else {
205        Ok(())
206    }
207}
208
209/// Format a `DataType::Timestamp` into a short, stable string used in
210/// user-facing error messages.
211pub(crate) fn format_timestamp_type_for_error(target_type: &DataType) -> String {
212    match target_type {
213        DataType::Timestamp(unit, _) => {
214            let s = match unit {
215                TimeUnit::Second => "s",
216                TimeUnit::Millisecond => "ms",
217                TimeUnit::Microsecond => "us",
218                TimeUnit::Nanosecond => "ns",
219            };
220            format!("Timestamp({s})")
221        }
222        other => format!("{other}"),
223    }
224}
225
226/// A dynamically typed, nullable single value.
227///
228/// While an arrow  [`Array`]) stores one or more values of the same type, in a
229/// single column, a `ScalarValue` stores a single value of a single type, the
230/// equivalent of 1 row and one column.
231///
232/// ```text
233///  ┌────────┐
234///  │ value1 │
235///  │ value2 │                  ┌────────┐
236///  │ value3 │                  │ value2 │
237///  │  ...   │                  └────────┘
238///  │ valueN │
239///  └────────┘
240///
241///    Array                     ScalarValue
242///
243/// stores multiple,             stores a single,
244/// possibly null, values of     possible null, value
245/// the same type
246/// ```
247///
248/// # Performance
249///
250/// In general, performance will be better using arrow [`Array`]s rather than
251/// [`ScalarValue`], as it is far more efficient to process multiple values at
252/// once (vectorized processing).
253///
254/// # Example
255/// ```
256/// # use datafusion_common::ScalarValue;
257/// // Create single scalar value for an Int32 value
258/// let s1 = ScalarValue::Int32(Some(10));
259///
260/// // You can also create values using the From impl:
261/// let s2 = ScalarValue::from(10i32);
262/// assert_eq!(s1, s2);
263/// ```
264///
265/// # Null Handling
266///
267/// `ScalarValue` represents null values in the same way as Arrow. Nulls are
268/// "typed" in the sense that a null value in an [`Int32Array`] is different
269/// from a null value in a [`Float64Array`], and is different from the values in
270/// a [`NullArray`].
271///
272/// ```
273/// # fn main() -> datafusion_common::Result<()> {
274/// # use std::collections::hash_set::Difference;
275/// # use datafusion_common::ScalarValue;
276/// # use arrow::datatypes::DataType;
277/// // You can create a 'null' Int32 value directly:
278/// let s1 = ScalarValue::Int32(None);
279///
280/// // You can also create a null value for a given datatype:
281/// let s2 = ScalarValue::try_from(&DataType::Int32)?;
282/// assert_eq!(s1, s2);
283///
284/// // Note that this is DIFFERENT than a `ScalarValue::Null`
285/// let s3 = ScalarValue::Null;
286/// assert_ne!(s1, s3);
287/// # Ok(())
288/// # }
289/// ```
290///
291/// # Nested Types
292///
293/// `List` / `LargeList` / `FixedSizeList` / `ListView` / `LargeListView` / `Struct` / `Map`
294/// are represented as a single element array of the corresponding type.
295///
296/// ## Example: Creating [`ScalarValue::Struct`] using [`ScalarStructBuilder`]
297/// ```
298/// # use std::sync::Arc;
299/// # use arrow::datatypes::{DataType, Field};
300/// # use datafusion_common::{ScalarValue, scalar::ScalarStructBuilder};
301/// // Build a struct like: {a: 1, b: "foo"}
302/// let field_a = Field::new("a", DataType::Int32, false);
303/// let field_b = Field::new("b", DataType::Utf8, false);
304///
305/// let s1 = ScalarStructBuilder::new()
306///     .with_scalar(field_a, ScalarValue::from(1i32))
307///     .with_scalar(field_b, ScalarValue::from("foo"))
308///     .build();
309/// ```
310///
311/// ## Example: Creating a null [`ScalarValue::Struct`] using [`ScalarStructBuilder`]
312/// ```
313/// # use std::sync::Arc;
314/// # use arrow::datatypes::{DataType, Field};
315/// # use datafusion_common::{ScalarValue, scalar::ScalarStructBuilder};
316/// // Build a struct representing a NULL value
317/// let fields = vec![
318///     Field::new("a", DataType::Int32, false),
319///     Field::new("b", DataType::Utf8, false),
320/// ];
321///
322/// let s1 = ScalarStructBuilder::new_null(fields);
323/// ```
324///
325/// ## Example: Creating [`ScalarValue::Struct`] directly
326/// ```
327/// # use std::sync::Arc;
328/// # use arrow::datatypes::{DataType, Field, Fields};
329/// # use arrow::array::{ArrayRef, Int32Array, StructArray, StringArray};
330/// # use datafusion_common::ScalarValue;
331/// // Build a struct like: {a: 1, b: "foo"}
332/// // Field description
333/// let fields = Fields::from(vec![
334///     Field::new("a", DataType::Int32, false),
335///     Field::new("b", DataType::Utf8, false),
336/// ]);
337/// // one row arrays for each field
338/// let arrays: Vec<ArrayRef> = vec![
339///     Arc::new(Int32Array::from(vec![1])),
340///     Arc::new(StringArray::from(vec!["foo"])),
341/// ];
342/// // no nulls for this array
343/// let nulls = None;
344/// let arr = StructArray::new(fields, arrays, nulls);
345///
346/// // Create a ScalarValue::Struct directly
347/// let s1 = ScalarValue::Struct(Arc::new(arr));
348/// ```
349///
350///
351/// # Further Reading
352/// See [datatypes](https://arrow.apache.org/docs/python/api/datatypes.html) for
353/// details on datatypes and the [format](https://github.com/apache/arrow/blob/master/format/Schema.fbs#L354-L375)
354/// for the definitive reference.
355///
356/// [`NullArray`]: arrow::array::NullArray
357#[derive(Clone)]
358pub enum ScalarValue {
359    /// represents `DataType::Null` (castable to/from any other type)
360    Null,
361    /// true or false value
362    Boolean(Option<bool>),
363    /// 16bit float
364    Float16(Option<f16>),
365    /// 32bit float
366    Float32(Option<f32>),
367    /// 64bit float
368    Float64(Option<f64>),
369    /// 32bit decimal, using the i32 to represent the decimal, precision scale
370    Decimal32(Option<i32>, u8, i8),
371    /// 64bit decimal, using the i64 to represent the decimal, precision scale
372    Decimal64(Option<i64>, u8, i8),
373    /// 128bit decimal, using the i128 to represent the decimal, precision scale
374    Decimal128(Option<i128>, u8, i8),
375    /// 256bit decimal, using the i256 to represent the decimal, precision scale
376    Decimal256(Option<i256>, u8, i8),
377    /// signed 8bit int
378    Int8(Option<i8>),
379    /// signed 16bit int
380    Int16(Option<i16>),
381    /// signed 32bit int
382    Int32(Option<i32>),
383    /// signed 64bit int
384    Int64(Option<i64>),
385    /// unsigned 8bit int
386    UInt8(Option<u8>),
387    /// unsigned 16bit int
388    UInt16(Option<u16>),
389    /// unsigned 32bit int
390    UInt32(Option<u32>),
391    /// unsigned 64bit int
392    UInt64(Option<u64>),
393    /// utf-8 encoded string.
394    Utf8(Option<String>),
395    /// utf-8 encoded string but from view types.
396    Utf8View(Option<String>),
397    /// utf-8 encoded string representing a LargeString's arrow type.
398    LargeUtf8(Option<String>),
399    /// binary
400    Binary(Option<Vec<u8>>),
401    /// binary but from view types.
402    BinaryView(Option<Vec<u8>>),
403    /// fixed size binary
404    FixedSizeBinary(i32, Option<Vec<u8>>),
405    /// large binary
406    LargeBinary(Option<Vec<u8>>),
407    /// Fixed size list scalar.
408    ///
409    /// The array must be a FixedSizeListArray with length 1.
410    FixedSizeList(Arc<FixedSizeListArray>),
411    /// Represents a single element of a [`ListArray`] as an [`ArrayRef`]
412    ///
413    /// The array must be a ListArray with length 1.
414    List(Arc<ListArray>),
415    /// The array must be a LargeListArray with length 1.
416    LargeList(Arc<LargeListArray>),
417    /// Represents a single element of a [`ListViewArray`] as an [`ArrayRef`]
418    ///
419    /// The array must be a ListViewArray with length 1.
420    ListView(Arc<ListViewArray>),
421    /// Represents a single element of a [`LargeListViewArray`] as an [`ArrayRef`]
422    ///
423    /// The array must be a LargeListViewArray with length 1.
424    LargeListView(Arc<LargeListViewArray>),
425    /// Represents a single element [`StructArray`] as an [`ArrayRef`]. See
426    /// [`ScalarValue`] for examples of how to create instances of this type.
427    Struct(Arc<StructArray>),
428    /// Represents a single element [`MapArray`] as an [`ArrayRef`].
429    Map(Arc<MapArray>),
430    /// Date stored as a signed 32bit int days since UNIX epoch 1970-01-01
431    Date32(Option<i32>),
432    /// Date stored as a signed 64bit int milliseconds since UNIX epoch 1970-01-01
433    Date64(Option<i64>),
434    /// Time stored as a signed 32bit int as seconds since midnight
435    Time32Second(Option<i32>),
436    /// Time stored as a signed 32bit int as milliseconds since midnight
437    Time32Millisecond(Option<i32>),
438    /// Time stored as a signed 64bit int as microseconds since midnight
439    Time64Microsecond(Option<i64>),
440    /// Time stored as a signed 64bit int as nanoseconds since midnight
441    Time64Nanosecond(Option<i64>),
442    /// Timestamp Second
443    TimestampSecond(Option<i64>, Option<Arc<str>>),
444    /// Timestamp Milliseconds
445    TimestampMillisecond(Option<i64>, Option<Arc<str>>),
446    /// Timestamp Microseconds
447    TimestampMicrosecond(Option<i64>, Option<Arc<str>>),
448    /// Timestamp Nanoseconds
449    TimestampNanosecond(Option<i64>, Option<Arc<str>>),
450    /// Number of elapsed whole months
451    IntervalYearMonth(Option<i32>),
452    /// Number of elapsed days and milliseconds (no leap seconds)
453    /// stored as 2 contiguous 32-bit signed integers
454    IntervalDayTime(Option<IntervalDayTime>),
455    /// A triple of the number of elapsed months, days, and nanoseconds.
456    /// Months and days are encoded as 32-bit signed integers.
457    /// Nanoseconds is encoded as a 64-bit signed integer (no leap seconds).
458    IntervalMonthDayNano(Option<IntervalMonthDayNano>),
459    /// Duration in seconds
460    DurationSecond(Option<i64>),
461    /// Duration in milliseconds
462    DurationMillisecond(Option<i64>),
463    /// Duration in microseconds
464    DurationMicrosecond(Option<i64>),
465    /// Duration in nanoseconds
466    DurationNanosecond(Option<i64>),
467    /// A nested datatype that can represent slots of differing types. Components:
468    /// `.0`: a tuple of union `type_id` and the single value held by this Scalar
469    /// `.1`: the list of fields, zero-to-one of which will by set in `.0`
470    /// `.2`: the physical storage of the source/destination UnionArray from which this Scalar came
471    Union(Option<(i8, Box<ScalarValue>)>, UnionFields, UnionMode),
472    /// Dictionary type: index type and value
473    Dictionary(Box<DataType>, Box<ScalarValue>),
474    /// (run-ends field, value field, value)
475    RunEndEncoded(FieldRef, FieldRef, Box<ScalarValue>),
476}
477
478impl Hash for Fl<f16> {
479    fn hash<H: Hasher>(&self, state: &mut H) {
480        self.0.to_bits().hash(state);
481    }
482}
483
484// manual implementation of `PartialEq`
485impl PartialEq for ScalarValue {
486    fn eq(&self, other: &Self) -> bool {
487        use ScalarValue::*;
488        // This purposely doesn't have a catch-all "(_, _)" so that
489        // any newly added enum variant will require editing this list
490        // or else face a compile error
491        match (self, other) {
492            (Decimal32(v1, p1, s1), Decimal32(v2, p2, s2)) => {
493                v1.eq(v2) && p1.eq(p2) && s1.eq(s2)
494            }
495            (Decimal32(_, _, _), _) => false,
496            (Decimal64(v1, p1, s1), Decimal64(v2, p2, s2)) => {
497                v1.eq(v2) && p1.eq(p2) && s1.eq(s2)
498            }
499            (Decimal64(_, _, _), _) => false,
500            (Decimal128(v1, p1, s1), Decimal128(v2, p2, s2)) => {
501                v1.eq(v2) && p1.eq(p2) && s1.eq(s2)
502            }
503            (Decimal128(_, _, _), _) => false,
504            (Decimal256(v1, p1, s1), Decimal256(v2, p2, s2)) => {
505                v1.eq(v2) && p1.eq(p2) && s1.eq(s2)
506            }
507            (Decimal256(_, _, _), _) => false,
508            (Boolean(v1), Boolean(v2)) => v1.eq(v2),
509            (Boolean(_), _) => false,
510            (Float32(v1), Float32(v2)) => match (v1, v2) {
511                (Some(f1), Some(f2)) => f1.to_bits() == f2.to_bits(),
512                _ => v1.eq(v2),
513            },
514            (Float16(v1), Float16(v2)) => match (v1, v2) {
515                (Some(f1), Some(f2)) => f1.to_bits() == f2.to_bits(),
516                _ => v1.eq(v2),
517            },
518            (Float32(_), _) => false,
519            (Float16(_), _) => false,
520            (Float64(v1), Float64(v2)) => match (v1, v2) {
521                (Some(f1), Some(f2)) => f1.to_bits() == f2.to_bits(),
522                _ => v1.eq(v2),
523            },
524            (Float64(_), _) => false,
525            (Int8(v1), Int8(v2)) => v1.eq(v2),
526            (Int8(_), _) => false,
527            (Int16(v1), Int16(v2)) => v1.eq(v2),
528            (Int16(_), _) => false,
529            (Int32(v1), Int32(v2)) => v1.eq(v2),
530            (Int32(_), _) => false,
531            (Int64(v1), Int64(v2)) => v1.eq(v2),
532            (Int64(_), _) => false,
533            (UInt8(v1), UInt8(v2)) => v1.eq(v2),
534            (UInt8(_), _) => false,
535            (UInt16(v1), UInt16(v2)) => v1.eq(v2),
536            (UInt16(_), _) => false,
537            (UInt32(v1), UInt32(v2)) => v1.eq(v2),
538            (UInt32(_), _) => false,
539            (UInt64(v1), UInt64(v2)) => v1.eq(v2),
540            (UInt64(_), _) => false,
541            (Utf8(v1), Utf8(v2)) => v1.eq(v2),
542            (Utf8(_), _) => false,
543            (Utf8View(v1), Utf8View(v2)) => v1.eq(v2),
544            (Utf8View(_), _) => false,
545            (LargeUtf8(v1), LargeUtf8(v2)) => v1.eq(v2),
546            (LargeUtf8(_), _) => false,
547            (Binary(v1), Binary(v2)) => v1.eq(v2),
548            (Binary(_), _) => false,
549            (BinaryView(v1), BinaryView(v2)) => v1.eq(v2),
550            (BinaryView(_), _) => false,
551            (FixedSizeBinary(_, v1), FixedSizeBinary(_, v2)) => v1.eq(v2),
552            (FixedSizeBinary(_, _), _) => false,
553            (LargeBinary(v1), LargeBinary(v2)) => v1.eq(v2),
554            (LargeBinary(_), _) => false,
555            (FixedSizeList(v1), FixedSizeList(v2)) => v1.eq(v2),
556            (FixedSizeList(_), _) => false,
557            (List(v1), List(v2)) => v1.eq(v2),
558            (List(_), _) => false,
559            (LargeList(v1), LargeList(v2)) => v1.eq(v2),
560            (LargeList(_), _) => false,
561            (ListView(v1), ListView(v2)) => v1.eq(v2),
562            (ListView(_), _) => false,
563            (LargeListView(v1), LargeListView(v2)) => v1.eq(v2),
564            (LargeListView(_), _) => false,
565            (Struct(v1), Struct(v2)) => v1.eq(v2),
566            (Struct(_), _) => false,
567            (Map(v1), Map(v2)) => v1.eq(v2),
568            (Map(_), _) => false,
569            (Date32(v1), Date32(v2)) => v1.eq(v2),
570            (Date32(_), _) => false,
571            (Date64(v1), Date64(v2)) => v1.eq(v2),
572            (Date64(_), _) => false,
573            (Time32Second(v1), Time32Second(v2)) => v1.eq(v2),
574            (Time32Second(_), _) => false,
575            (Time32Millisecond(v1), Time32Millisecond(v2)) => v1.eq(v2),
576            (Time32Millisecond(_), _) => false,
577            (Time64Microsecond(v1), Time64Microsecond(v2)) => v1.eq(v2),
578            (Time64Microsecond(_), _) => false,
579            (Time64Nanosecond(v1), Time64Nanosecond(v2)) => v1.eq(v2),
580            (Time64Nanosecond(_), _) => false,
581            (TimestampSecond(v1, _), TimestampSecond(v2, _)) => v1.eq(v2),
582            (TimestampSecond(_, _), _) => false,
583            (TimestampMillisecond(v1, _), TimestampMillisecond(v2, _)) => v1.eq(v2),
584            (TimestampMillisecond(_, _), _) => false,
585            (TimestampMicrosecond(v1, _), TimestampMicrosecond(v2, _)) => v1.eq(v2),
586            (TimestampMicrosecond(_, _), _) => false,
587            (TimestampNanosecond(v1, _), TimestampNanosecond(v2, _)) => v1.eq(v2),
588            (TimestampNanosecond(_, _), _) => false,
589            (DurationSecond(v1), DurationSecond(v2)) => v1.eq(v2),
590            (DurationSecond(_), _) => false,
591            (DurationMillisecond(v1), DurationMillisecond(v2)) => v1.eq(v2),
592            (DurationMillisecond(_), _) => false,
593            (DurationMicrosecond(v1), DurationMicrosecond(v2)) => v1.eq(v2),
594            (DurationMicrosecond(_), _) => false,
595            (DurationNanosecond(v1), DurationNanosecond(v2)) => v1.eq(v2),
596            (DurationNanosecond(_), _) => false,
597            (IntervalYearMonth(v1), IntervalYearMonth(v2)) => v1.eq(v2),
598            (IntervalYearMonth(_), _) => false,
599            (IntervalDayTime(v1), IntervalDayTime(v2)) => v1.eq(v2),
600            (IntervalDayTime(_), _) => false,
601            (IntervalMonthDayNano(v1), IntervalMonthDayNano(v2)) => v1.eq(v2),
602            (IntervalMonthDayNano(_), _) => false,
603            (Union(val1, fields1, mode1), Union(val2, fields2, mode2)) => {
604                val1.eq(val2) && fields1.eq(fields2) && mode1.eq(mode2)
605            }
606            (Union(_, _, _), _) => false,
607            (Dictionary(k1, v1), Dictionary(k2, v2)) => k1.eq(k2) && v1.eq(v2),
608            (Dictionary(_, _), _) => false,
609            (RunEndEncoded(rf1, vf1, v1), RunEndEncoded(rf2, vf2, v2)) => {
610                rf1.eq(rf2) && vf1.eq(vf2) && v1.eq(v2)
611            }
612            (RunEndEncoded(_, _, _), _) => false,
613            (Null, Null) => true,
614            (Null, _) => false,
615        }
616    }
617}
618
619// manual implementation of `PartialOrd`
620impl PartialOrd for ScalarValue {
621    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
622        use ScalarValue::*;
623        // This purposely doesn't have a catch-all "(_, _)" so that
624        // any newly added enum variant will require editing this list
625        // or else face a compile error
626        match (self, other) {
627            (Decimal32(v1, _, s1), Decimal32(v2, _, s2)) => {
628                if s1.eq(s2) {
629                    // Same scale means the underlying integer values share
630                    // a common interpretation regardless of declared
631                    // precision (arithmetic such as `add_checked` widens
632                    // precision by 1 but does not change the numeric
633                    // meaning).
634                    v1.partial_cmp(v2)
635                } else {
636                    None
637                }
638            }
639            (Decimal32(_, _, _), _) => None,
640            (Decimal64(v1, _, s1), Decimal64(v2, _, s2)) => {
641                if s1.eq(s2) {
642                    v1.partial_cmp(v2)
643                } else {
644                    None
645                }
646            }
647            (Decimal64(_, _, _), _) => None,
648            (Decimal128(v1, _, s1), Decimal128(v2, _, s2)) => {
649                if s1.eq(s2) {
650                    v1.partial_cmp(v2)
651                } else {
652                    None
653                }
654            }
655            (Decimal128(_, _, _), _) => None,
656            (Decimal256(v1, _, s1), Decimal256(v2, _, s2)) => {
657                if s1.eq(s2) {
658                    v1.partial_cmp(v2)
659                } else {
660                    None
661                }
662            }
663            (Decimal256(_, _, _), _) => None,
664            (Boolean(v1), Boolean(v2)) => v1.partial_cmp(v2),
665            (Boolean(_), _) => None,
666            (Float32(v1), Float32(v2)) => match (v1, v2) {
667                (Some(f1), Some(f2)) => Some(f1.total_cmp(f2)),
668                _ => v1.partial_cmp(v2),
669            },
670            (Float16(v1), Float16(v2)) => match (v1, v2) {
671                (Some(f1), Some(f2)) => Some(f1.total_cmp(f2)),
672                _ => v1.partial_cmp(v2),
673            },
674            (Float32(_), _) => None,
675            (Float16(_), _) => None,
676            (Float64(v1), Float64(v2)) => match (v1, v2) {
677                (Some(f1), Some(f2)) => Some(f1.total_cmp(f2)),
678                _ => v1.partial_cmp(v2),
679            },
680            (Float64(_), _) => None,
681            (Int8(v1), Int8(v2)) => v1.partial_cmp(v2),
682            (Int8(_), _) => None,
683            (Int16(v1), Int16(v2)) => v1.partial_cmp(v2),
684            (Int16(_), _) => None,
685            (Int32(v1), Int32(v2)) => v1.partial_cmp(v2),
686            (Int32(_), _) => None,
687            (Int64(v1), Int64(v2)) => v1.partial_cmp(v2),
688            (Int64(_), _) => None,
689            (UInt8(v1), UInt8(v2)) => v1.partial_cmp(v2),
690            (UInt8(_), _) => None,
691            (UInt16(v1), UInt16(v2)) => v1.partial_cmp(v2),
692            (UInt16(_), _) => None,
693            (UInt32(v1), UInt32(v2)) => v1.partial_cmp(v2),
694            (UInt32(_), _) => None,
695            (UInt64(v1), UInt64(v2)) => v1.partial_cmp(v2),
696            (UInt64(_), _) => None,
697            (Utf8(v1), Utf8(v2)) => v1.partial_cmp(v2),
698            (Utf8(_), _) => None,
699            (LargeUtf8(v1), LargeUtf8(v2)) => v1.partial_cmp(v2),
700            (LargeUtf8(_), _) => None,
701            (Utf8View(v1), Utf8View(v2)) => v1.partial_cmp(v2),
702            (Utf8View(_), _) => None,
703            (Binary(v1), Binary(v2)) => v1.partial_cmp(v2),
704            (Binary(_), _) => None,
705            (BinaryView(v1), BinaryView(v2)) => v1.partial_cmp(v2),
706            (BinaryView(_), _) => None,
707            (FixedSizeBinary(_, v1), FixedSizeBinary(_, v2)) => v1.partial_cmp(v2),
708            (FixedSizeBinary(_, _), _) => None,
709            (LargeBinary(v1), LargeBinary(v2)) => v1.partial_cmp(v2),
710            (LargeBinary(_), _) => None,
711            // ScalarValue::List / ScalarValue::FixedSizeList / ScalarValue::LargeList / ScalarValue::ListView / ScalarValue::LargeListView
712            // are guaranteed to have length 1
713            (List(arr1), List(arr2)) => partial_cmp_list(arr1.as_ref(), arr2.as_ref()),
714            (FixedSizeList(arr1), FixedSizeList(arr2)) => {
715                partial_cmp_list(arr1.as_ref(), arr2.as_ref())
716            }
717            (LargeList(arr1), LargeList(arr2)) => {
718                partial_cmp_list(arr1.as_ref(), arr2.as_ref())
719            }
720            (ListView(arr1), ListView(arr2)) => {
721                partial_cmp_list(arr1.as_ref(), arr2.as_ref())
722            }
723            (LargeListView(arr1), LargeListView(arr2)) => {
724                partial_cmp_list(arr1.as_ref(), arr2.as_ref())
725            }
726            (List(_), _)
727            | (LargeList(_), _)
728            | (FixedSizeList(_), _)
729            | (ListView(_), _)
730            | (LargeListView(_), _) => None,
731            (Struct(struct_arr1), Struct(struct_arr2)) => {
732                partial_cmp_struct(struct_arr1.as_ref(), struct_arr2.as_ref())
733            }
734            (Struct(_), _) => None,
735            (Map(map_arr1), Map(map_arr2)) => partial_cmp_map(map_arr1, map_arr2),
736            (Map(_), _) => None,
737            (Date32(v1), Date32(v2)) => v1.partial_cmp(v2),
738            (Date32(_), _) => None,
739            (Date64(v1), Date64(v2)) => v1.partial_cmp(v2),
740            (Date64(_), _) => None,
741            (Time32Second(v1), Time32Second(v2)) => v1.partial_cmp(v2),
742            (Time32Second(_), _) => None,
743            (Time32Millisecond(v1), Time32Millisecond(v2)) => v1.partial_cmp(v2),
744            (Time32Millisecond(_), _) => None,
745            (Time64Microsecond(v1), Time64Microsecond(v2)) => v1.partial_cmp(v2),
746            (Time64Microsecond(_), _) => None,
747            (Time64Nanosecond(v1), Time64Nanosecond(v2)) => v1.partial_cmp(v2),
748            (Time64Nanosecond(_), _) => None,
749            (TimestampSecond(v1, _), TimestampSecond(v2, _)) => v1.partial_cmp(v2),
750            (TimestampSecond(_, _), _) => None,
751            (TimestampMillisecond(v1, _), TimestampMillisecond(v2, _)) => {
752                v1.partial_cmp(v2)
753            }
754            (TimestampMillisecond(_, _), _) => None,
755            (TimestampMicrosecond(v1, _), TimestampMicrosecond(v2, _)) => {
756                v1.partial_cmp(v2)
757            }
758            (TimestampMicrosecond(_, _), _) => None,
759            (TimestampNanosecond(v1, _), TimestampNanosecond(v2, _)) => {
760                v1.partial_cmp(v2)
761            }
762            (TimestampNanosecond(_, _), _) => None,
763            (IntervalYearMonth(v1), IntervalYearMonth(v2)) => v1.partial_cmp(v2),
764            (IntervalYearMonth(_), _) => None,
765            (IntervalDayTime(v1), IntervalDayTime(v2)) => v1.partial_cmp(v2),
766            (IntervalDayTime(_), _) => None,
767            (IntervalMonthDayNano(v1), IntervalMonthDayNano(v2)) => v1.partial_cmp(v2),
768            (IntervalMonthDayNano(_), _) => None,
769            (DurationSecond(v1), DurationSecond(v2)) => v1.partial_cmp(v2),
770            (DurationSecond(_), _) => None,
771            (DurationMillisecond(v1), DurationMillisecond(v2)) => v1.partial_cmp(v2),
772            (DurationMillisecond(_), _) => None,
773            (DurationMicrosecond(v1), DurationMicrosecond(v2)) => v1.partial_cmp(v2),
774            (DurationMicrosecond(_), _) => None,
775            (DurationNanosecond(v1), DurationNanosecond(v2)) => v1.partial_cmp(v2),
776            (DurationNanosecond(_), _) => None,
777            (Union(v1, t1, m1), Union(v2, t2, m2)) => {
778                if t1.eq(t2) && m1.eq(m2) {
779                    v1.partial_cmp(v2)
780                } else {
781                    None
782                }
783            }
784            (Union(_, _, _), _) => None,
785            (Dictionary(k1, v1), Dictionary(k2, v2)) => {
786                // Don't compare if the key types don't match (it is effectively a different datatype)
787                if k1 == k2 { v1.partial_cmp(v2) } else { None }
788            }
789            (Dictionary(_, _), _) => None,
790            (RunEndEncoded(rf1, vf1, v1), RunEndEncoded(rf2, vf2, v2)) => {
791                // Don't compare if the run ends fields don't match (it is effectively a different datatype)
792                if rf1 == rf2 && vf1 == vf2 {
793                    v1.partial_cmp(v2)
794                } else {
795                    None
796                }
797            }
798            (RunEndEncoded(_, _, _), _) => None,
799            (Null, Null) => Some(Ordering::Equal),
800            (Null, _) => None,
801        }
802    }
803}
804
805/// List/LargeList/FixedSizeList/ListView/LargeListView scalars always have a single element
806/// array. This function returns that array
807fn first_array_for_list(arr: &dyn Array) -> ArrayRef {
808    assert_eq!(arr.len(), 1);
809    if let Some(arr) = arr.as_list_opt::<i32>() {
810        arr.value(0)
811    } else if let Some(arr) = arr.as_list_opt::<i64>() {
812        arr.value(0)
813    } else if let Some(arr) = arr.as_fixed_size_list_opt() {
814        arr.value(0)
815    } else if let Some(arr) = arr.as_list_view_opt::<i32>() {
816        arr.value(0)
817    } else if let Some(arr) = arr.as_list_view_opt::<i64>() {
818        arr.value(0)
819    } else {
820        unreachable!(
821            "Since only List / LargeList / FixedSizeList / ListView / LargeListView are supported, this should never happen"
822        )
823    }
824}
825
826/// Compares two List/LargeList/FixedSizeList/ListView/LargeListView scalars
827fn partial_cmp_list(arr1: &dyn Array, arr2: &dyn Array) -> Option<Ordering> {
828    if arr1.data_type() != arr2.data_type() {
829        return None;
830    }
831    let arr1 = first_array_for_list(arr1);
832    let arr2 = first_array_for_list(arr2);
833
834    let min_length = arr1.len().min(arr2.len());
835    let arr1_trimmed = arr1.slice(0, min_length);
836    let arr2_trimmed = arr2.slice(0, min_length);
837
838    let lt_res = arrow::compute::kernels::cmp::lt(&arr1_trimmed, &arr2_trimmed).ok()?;
839    let eq_res = arrow::compute::kernels::cmp::eq(&arr1_trimmed, &arr2_trimmed).ok()?;
840
841    for j in 0..lt_res.len() {
842        // In Postgres, NULL values in lists are always considered to be greater than non-NULL values:
843        //
844        // $ SELECT ARRAY[NULL]::integer[] > ARRAY[1]
845        // true
846        //
847        // These next two if statements are introduced for replicating Postgres behavior, as
848        // arrow::compute does not account for this.
849        if arr1_trimmed.is_null(j) && !arr2_trimmed.is_null(j) {
850            return Some(Ordering::Greater);
851        }
852        if !arr1_trimmed.is_null(j) && arr2_trimmed.is_null(j) {
853            return Some(Ordering::Less);
854        }
855
856        if lt_res.is_valid(j) && lt_res.value(j) {
857            return Some(Ordering::Less);
858        }
859        if eq_res.is_valid(j) && !eq_res.value(j) {
860            return Some(Ordering::Greater);
861        }
862    }
863
864    Some(arr1.len().cmp(&arr2.len()))
865}
866
867fn flatten<'a>(array: &'a StructArray, columns: &mut Vec<&'a ArrayRef>) {
868    for i in 0..array.num_columns() {
869        let column = array.column(i);
870        if let Some(nested_struct) = column.as_any().downcast_ref::<StructArray>() {
871            // If it's a nested struct, recursively expand
872            flatten(nested_struct, columns);
873        } else {
874            // If it's a primitive type, add directly
875            columns.push(column);
876        }
877    }
878}
879
880pub fn partial_cmp_struct(s1: &StructArray, s2: &StructArray) -> Option<Ordering> {
881    if s1.len() != s2.len() {
882        return None;
883    }
884
885    if s1.data_type() != s2.data_type() {
886        return None;
887    }
888
889    let mut expanded_columns1 = Vec::with_capacity(s1.num_columns());
890    let mut expanded_columns2 = Vec::with_capacity(s2.num_columns());
891
892    flatten(s1, &mut expanded_columns1);
893    flatten(s2, &mut expanded_columns2);
894
895    for col_index in 0..expanded_columns1.len() {
896        let arr1 = expanded_columns1[col_index];
897        let arr2 = expanded_columns2[col_index];
898
899        let lt_res = arrow::compute::kernels::cmp::lt(arr1, arr2).ok()?;
900        let eq_res = arrow::compute::kernels::cmp::eq(arr1, arr2).ok()?;
901
902        for j in 0..lt_res.len() {
903            if lt_res.is_valid(j) && lt_res.value(j) {
904                return Some(Ordering::Less);
905            }
906            if eq_res.is_valid(j) && !eq_res.value(j) {
907                return Some(Ordering::Greater);
908            }
909        }
910    }
911    Some(Ordering::Equal)
912}
913
914fn partial_cmp_map(m1: &Arc<MapArray>, m2: &Arc<MapArray>) -> Option<Ordering> {
915    if m1.len() != m2.len() {
916        return None;
917    }
918
919    if m1.data_type() != m2.data_type() {
920        return None;
921    }
922
923    for col_index in 0..m1.len() {
924        let arr1 = m1.entries().column(col_index);
925        let arr2 = m2.entries().column(col_index);
926
927        let lt_res = arrow::compute::kernels::cmp::lt(arr1, arr2).ok()?;
928        let eq_res = arrow::compute::kernels::cmp::eq(arr1, arr2).ok()?;
929
930        for j in 0..lt_res.len() {
931            if lt_res.is_valid(j) && lt_res.value(j) {
932                return Some(Ordering::Less);
933            }
934            if eq_res.is_valid(j) && !eq_res.value(j) {
935                return Some(Ordering::Greater);
936            }
937        }
938    }
939    Some(Ordering::Equal)
940}
941
942impl Eq for ScalarValue {}
943
944//Float wrapper over f32/f64. Just because we cannot build std::hash::Hash for floats directly we have to do it through type wrapper
945struct Fl<T>(T);
946
947macro_rules! hash_float_value {
948    ($(($t:ty, $i:ty)),+) => {
949        $(impl std::hash::Hash for Fl<$t> {
950            #[inline]
951            fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
952                state.write(&<$i>::from_ne_bytes(self.0.to_ne_bytes()).to_ne_bytes())
953            }
954        })+
955    };
956}
957
958hash_float_value!((f64, u64), (f32, u32));
959
960// manual implementation of `Hash`
961//
962// # Panics
963//
964// Panics if there is an error when creating hash values for rows
965impl Hash for ScalarValue {
966    fn hash<H: Hasher>(&self, state: &mut H) {
967        use ScalarValue::*;
968        match self {
969            Decimal32(v, p, s) => {
970                v.hash(state);
971                p.hash(state);
972                s.hash(state)
973            }
974            Decimal64(v, p, s) => {
975                v.hash(state);
976                p.hash(state);
977                s.hash(state)
978            }
979            Decimal128(v, p, s) => {
980                v.hash(state);
981                p.hash(state);
982                s.hash(state)
983            }
984            Decimal256(v, p, s) => {
985                v.hash(state);
986                p.hash(state);
987                s.hash(state)
988            }
989            Boolean(v) => v.hash(state),
990            Float16(v) => v.map(Fl).hash(state),
991            Float32(v) => v.map(Fl).hash(state),
992            Float64(v) => v.map(Fl).hash(state),
993            Int8(v) => v.hash(state),
994            Int16(v) => v.hash(state),
995            Int32(v) => v.hash(state),
996            Int64(v) => v.hash(state),
997            UInt8(v) => v.hash(state),
998            UInt16(v) => v.hash(state),
999            UInt32(v) => v.hash(state),
1000            UInt64(v) => v.hash(state),
1001            Utf8(v) | LargeUtf8(v) | Utf8View(v) => v.hash(state),
1002            Binary(v) | FixedSizeBinary(_, v) | LargeBinary(v) | BinaryView(v) => {
1003                v.hash(state)
1004            }
1005            List(arr) => {
1006                hash_nested_array(arr.to_owned() as ArrayRef, state);
1007            }
1008            LargeList(arr) => {
1009                hash_nested_array(arr.to_owned() as ArrayRef, state);
1010            }
1011            FixedSizeList(arr) => {
1012                hash_nested_array(arr.to_owned() as ArrayRef, state);
1013            }
1014            ListView(arr) => {
1015                hash_nested_array(arr.to_owned() as ArrayRef, state);
1016            }
1017            LargeListView(arr) => {
1018                hash_nested_array(arr.to_owned() as ArrayRef, state);
1019            }
1020            Struct(arr) => {
1021                hash_nested_array(arr.to_owned() as ArrayRef, state);
1022            }
1023            Map(arr) => {
1024                hash_nested_array(arr.to_owned() as ArrayRef, state);
1025            }
1026            Date32(v) => v.hash(state),
1027            Date64(v) => v.hash(state),
1028            Time32Second(v) => v.hash(state),
1029            Time32Millisecond(v) => v.hash(state),
1030            Time64Microsecond(v) => v.hash(state),
1031            Time64Nanosecond(v) => v.hash(state),
1032            TimestampSecond(v, _) => v.hash(state),
1033            TimestampMillisecond(v, _) => v.hash(state),
1034            TimestampMicrosecond(v, _) => v.hash(state),
1035            TimestampNanosecond(v, _) => v.hash(state),
1036            DurationSecond(v) => v.hash(state),
1037            DurationMillisecond(v) => v.hash(state),
1038            DurationMicrosecond(v) => v.hash(state),
1039            DurationNanosecond(v) => v.hash(state),
1040            IntervalYearMonth(v) => v.hash(state),
1041            IntervalDayTime(v) => v.hash(state),
1042            IntervalMonthDayNano(v) => v.hash(state),
1043            Union(v, t, m) => {
1044                v.hash(state);
1045                t.hash(state);
1046                m.hash(state);
1047            }
1048            Dictionary(k, v) => {
1049                k.hash(state);
1050                v.hash(state);
1051            }
1052            RunEndEncoded(rf, vf, v) => {
1053                rf.hash(state);
1054                vf.hash(state);
1055                v.hash(state);
1056            }
1057            // stable hash for Null value
1058            Null => 1.hash(state),
1059        }
1060    }
1061}
1062
1063fn hash_nested_array<H: Hasher>(arr: ArrayRef, state: &mut H) {
1064    let len = arr.len();
1065    let hashes_buffer = &mut vec![0; len];
1066    let random_state = crate::hash_utils::RandomState::with_seed(0);
1067    let hashes = create_hashes(&[arr], &random_state, hashes_buffer)
1068        .expect("hash_nested_array: failed to create row hashes");
1069    // Hash back to std::hash::Hasher
1070    hashes.hash(state);
1071}
1072
1073/// Return a reference to the values array and the index into it for a
1074/// dictionary array
1075///
1076/// # Errors
1077///
1078/// Errors if the array cannot be downcasted to DictionaryArray
1079#[inline]
1080pub fn get_dict_value<K: ArrowDictionaryKeyType>(
1081    array: &dyn Array,
1082    index: usize,
1083) -> Result<(&ArrayRef, Option<usize>)> {
1084    let dict_array = as_dictionary_array::<K>(array)?;
1085    Ok((dict_array.values(), dict_array.key(index)))
1086}
1087
1088/// Create a dictionary array representing `value` repeated `size`
1089/// times
1090fn dict_from_scalar<K: ArrowDictionaryKeyType>(
1091    value: &ScalarValue,
1092    size: usize,
1093) -> Result<ArrayRef> {
1094    // values array is one element long (the value)
1095    let values_array = value.to_array_of_size(1)?;
1096
1097    // Create a key array with `size` elements, each of 0
1098    // Use cache to avoid repeated allocations for the same size
1099    let key_array: PrimitiveArray<K> =
1100        get_or_create_cached_key_array::<K>(size, value.is_null());
1101
1102    // create a new DictionaryArray
1103    //
1104    // Note: this path could be made faster by using the ArrayData
1105    // APIs and skipping validation, if it every comes up in
1106    // performance traces.
1107    Ok(Arc::new(
1108        DictionaryArray::<K>::try_new(key_array, values_array)?, // should always be valid by construction above
1109    ))
1110}
1111
1112/// Create a `DictionaryArray` from the provided values array.
1113///
1114/// Each element gets a unique key (`0..N-1`), without deduplication.
1115/// Useful for wrapping arrays in dictionary form.
1116///
1117/// # Input
1118/// ["alice", "bob", "alice", null, "carol"]
1119///
1120/// # Output
1121/// `DictionaryArray<Int32>`
1122/// {
1123///   keys:   [0, 1, 2, 3, 4],
1124///   values: ["alice", "bob", "alice", null, "carol"]
1125/// }
1126pub fn dict_from_values<K: ArrowDictionaryKeyType>(
1127    values_array: ArrayRef,
1128) -> Result<ArrayRef> {
1129    // Create a key array with `size` elements of 0..array_len for all
1130    // non-null value elements
1131    let key_array: PrimitiveArray<K> = (0..values_array.len())
1132        .map(|index| {
1133            if values_array.is_valid(index) {
1134                let native_index = K::Native::from_usize(index).ok_or_else(|| {
1135                    _internal_datafusion_err!(
1136                        "Can not create index of type {} from value {index}",
1137                        K::DATA_TYPE
1138                    )
1139                })?;
1140                Ok(Some(native_index))
1141            } else {
1142                Ok(None)
1143            }
1144        })
1145        .collect::<Result<Vec<_>>>()?
1146        .into_iter()
1147        .collect();
1148
1149    // create a new DictionaryArray
1150    //
1151    // Note: this path could be made faster by using the ArrayData
1152    // APIs and skipping validation, if it every comes up in
1153    // performance traces.
1154    let dict_array = DictionaryArray::<K>::try_new(key_array, values_array)?;
1155    Ok(Arc::new(dict_array))
1156}
1157
1158macro_rules! typed_cast_tz {
1159    ($array:expr, $index:expr, $array_cast:ident, $SCALAR:ident, $TZ:expr) => {{
1160        let array = $array_cast($array)?;
1161        Ok::<ScalarValue, DataFusionError>(ScalarValue::$SCALAR(
1162            match array.is_null($index) {
1163                true => None,
1164                false => Some(array.value($index).into()),
1165            },
1166            $TZ.clone(),
1167        ))
1168    }};
1169}
1170
1171macro_rules! typed_cast {
1172    ($array:expr, $index:expr, $array_cast:ident, $SCALAR:ident) => {{
1173        let array = $array_cast($array)?;
1174        Ok::<ScalarValue, DataFusionError>(ScalarValue::$SCALAR(
1175            match array.is_null($index) {
1176                true => None,
1177                false => Some(array.value($index).into()),
1178            },
1179        ))
1180    }};
1181}
1182
1183macro_rules! build_array_from_option {
1184    ($DATA_TYPE:ident, $ARRAY_TYPE:ident, $EXPR:expr, $SIZE:expr) => {{
1185        match $EXPR {
1186            Some(value) => Arc::new($ARRAY_TYPE::from_value(*value, $SIZE)),
1187            None => new_null_array(&DataType::$DATA_TYPE, $SIZE),
1188        }
1189    }};
1190    ($DATA_TYPE:ident, $ENUM:expr, $ARRAY_TYPE:ident, $EXPR:expr, $SIZE:expr) => {{
1191        match $EXPR {
1192            Some(value) => Arc::new($ARRAY_TYPE::from_value(*value, $SIZE)),
1193            None => new_null_array(&DataType::$DATA_TYPE($ENUM), $SIZE),
1194        }
1195    }};
1196}
1197
1198macro_rules! build_timestamp_array_from_option {
1199    ($TIME_UNIT:expr, $TZ:expr, $ARRAY_TYPE:ident, $EXPR:expr, $SIZE:expr) => {
1200        match $EXPR {
1201            Some(value) => {
1202                Arc::new($ARRAY_TYPE::from_value(*value, $SIZE).with_timezone_opt($TZ))
1203            }
1204            None => new_null_array(&DataType::Timestamp($TIME_UNIT, $TZ), $SIZE),
1205        }
1206    };
1207}
1208
1209macro_rules! eq_array_primitive {
1210    ($array:expr, $index:expr, $array_cast:ident, $VALUE:expr) => {{
1211        let array = $array_cast($array)?;
1212        let is_valid = array.is_valid($index);
1213        Ok::<bool, DataFusionError>(match $VALUE {
1214            Some(val) => is_valid && &array.value($index) == val,
1215            None => !is_valid,
1216        })
1217    }};
1218}
1219
1220impl ScalarValue {
1221    /// Create a [`Result<ScalarValue>`] with the provided value and datatype
1222    ///
1223    /// # Panics
1224    ///
1225    /// Panics if d is not compatible with T
1226    pub fn new_primitive<T: ArrowPrimitiveType>(
1227        a: Option<T::Native>,
1228        d: &DataType,
1229    ) -> Result<Self> {
1230        match a {
1231            None => d.try_into(),
1232            Some(v) => {
1233                let array = PrimitiveArray::<T>::new(vec![v].into(), None)
1234                    .with_data_type(d.clone());
1235                Self::try_from_array(&array, 0)
1236            }
1237        }
1238    }
1239
1240    /// Create a decimal Scalar from value/precision and scale.
1241    pub fn try_new_decimal128(value: i128, precision: u8, scale: i8) -> Result<Self> {
1242        Self::validate_decimal_or_internal_err::<Decimal128Type>(precision, scale)?;
1243        Ok(ScalarValue::Decimal128(Some(value), precision, scale))
1244    }
1245
1246    /// Create a Null instance of ScalarValue for this datatype
1247    ///
1248    /// Example
1249    /// ```
1250    /// use arrow::datatypes::DataType;
1251    /// use datafusion_common::ScalarValue;
1252    ///
1253    /// let scalar = ScalarValue::try_new_null(&DataType::Int32).unwrap();
1254    /// assert_eq!(scalar.is_null(), true);
1255    /// assert_eq!(scalar.data_type(), DataType::Int32);
1256    /// ```
1257    pub fn try_new_null(data_type: &DataType) -> Result<Self> {
1258        Ok(match data_type {
1259            DataType::Boolean => ScalarValue::Boolean(None),
1260            DataType::Float16 => ScalarValue::Float16(None),
1261            DataType::Float64 => ScalarValue::Float64(None),
1262            DataType::Float32 => ScalarValue::Float32(None),
1263            DataType::Int8 => ScalarValue::Int8(None),
1264            DataType::Int16 => ScalarValue::Int16(None),
1265            DataType::Int32 => ScalarValue::Int32(None),
1266            DataType::Int64 => ScalarValue::Int64(None),
1267            DataType::UInt8 => ScalarValue::UInt8(None),
1268            DataType::UInt16 => ScalarValue::UInt16(None),
1269            DataType::UInt32 => ScalarValue::UInt32(None),
1270            DataType::UInt64 => ScalarValue::UInt64(None),
1271            DataType::Decimal32(precision, scale) => {
1272                ScalarValue::Decimal32(None, *precision, *scale)
1273            }
1274            DataType::Decimal64(precision, scale) => {
1275                ScalarValue::Decimal64(None, *precision, *scale)
1276            }
1277            DataType::Decimal128(precision, scale) => {
1278                ScalarValue::Decimal128(None, *precision, *scale)
1279            }
1280            DataType::Decimal256(precision, scale) => {
1281                ScalarValue::Decimal256(None, *precision, *scale)
1282            }
1283            DataType::Utf8 => ScalarValue::Utf8(None),
1284            DataType::LargeUtf8 => ScalarValue::LargeUtf8(None),
1285            DataType::Utf8View => ScalarValue::Utf8View(None),
1286            DataType::Binary => ScalarValue::Binary(None),
1287            DataType::BinaryView => ScalarValue::BinaryView(None),
1288            DataType::FixedSizeBinary(len) => ScalarValue::FixedSizeBinary(*len, None),
1289            DataType::LargeBinary => ScalarValue::LargeBinary(None),
1290            DataType::Date32 => ScalarValue::Date32(None),
1291            DataType::Date64 => ScalarValue::Date64(None),
1292            DataType::Time32(TimeUnit::Second) => ScalarValue::Time32Second(None),
1293            DataType::Time32(TimeUnit::Millisecond) => {
1294                ScalarValue::Time32Millisecond(None)
1295            }
1296            DataType::Time64(TimeUnit::Microsecond) => {
1297                ScalarValue::Time64Microsecond(None)
1298            }
1299            DataType::Time64(TimeUnit::Nanosecond) => ScalarValue::Time64Nanosecond(None),
1300            DataType::Timestamp(TimeUnit::Second, tz_opt) => {
1301                ScalarValue::TimestampSecond(None, tz_opt.clone())
1302            }
1303            DataType::Timestamp(TimeUnit::Millisecond, tz_opt) => {
1304                ScalarValue::TimestampMillisecond(None, tz_opt.clone())
1305            }
1306            DataType::Timestamp(TimeUnit::Microsecond, tz_opt) => {
1307                ScalarValue::TimestampMicrosecond(None, tz_opt.clone())
1308            }
1309            DataType::Timestamp(TimeUnit::Nanosecond, tz_opt) => {
1310                ScalarValue::TimestampNanosecond(None, tz_opt.clone())
1311            }
1312            DataType::Interval(IntervalUnit::YearMonth) => {
1313                ScalarValue::IntervalYearMonth(None)
1314            }
1315            DataType::Interval(IntervalUnit::DayTime) => {
1316                ScalarValue::IntervalDayTime(None)
1317            }
1318            DataType::Interval(IntervalUnit::MonthDayNano) => {
1319                ScalarValue::IntervalMonthDayNano(None)
1320            }
1321            DataType::Duration(TimeUnit::Second) => ScalarValue::DurationSecond(None),
1322            DataType::Duration(TimeUnit::Millisecond) => {
1323                ScalarValue::DurationMillisecond(None)
1324            }
1325            DataType::Duration(TimeUnit::Microsecond) => {
1326                ScalarValue::DurationMicrosecond(None)
1327            }
1328            DataType::Duration(TimeUnit::Nanosecond) => {
1329                ScalarValue::DurationNanosecond(None)
1330            }
1331            DataType::Dictionary(index_type, value_type) => ScalarValue::Dictionary(
1332                index_type.clone(),
1333                Box::new(value_type.as_ref().try_into()?),
1334            ),
1335            DataType::RunEndEncoded(run_ends_field, value_field) => {
1336                ScalarValue::RunEndEncoded(
1337                    Arc::clone(run_ends_field),
1338                    Arc::clone(value_field),
1339                    Box::new(value_field.data_type().try_into()?),
1340                )
1341            }
1342            // `ScalarValue::List` contains single element `ListArray`.
1343            DataType::List(field_ref) => ScalarValue::List(Arc::new(
1344                GenericListArray::new_null(Arc::clone(field_ref), 1),
1345            )),
1346            // `ScalarValue::LargeList` contains single element `LargeListArray`.
1347            DataType::LargeList(field_ref) => ScalarValue::LargeList(Arc::new(
1348                GenericListArray::new_null(Arc::clone(field_ref), 1),
1349            )),
1350            // `ScalarValue::FixedSizeList` contains single element `FixedSizeList`.
1351            DataType::FixedSizeList(field_ref, fixed_length) => {
1352                ScalarValue::FixedSizeList(Arc::new(FixedSizeListArray::new_null(
1353                    Arc::clone(field_ref),
1354                    *fixed_length,
1355                    1,
1356                )))
1357            }
1358            DataType::ListView(field_ref) => ScalarValue::ListView(Arc::new(
1359                GenericListViewArray::new_null(Arc::clone(field_ref), 1),
1360            )),
1361            DataType::LargeListView(field_ref) => ScalarValue::LargeListView(Arc::new(
1362                GenericListViewArray::new_null(Arc::clone(field_ref), 1),
1363            )),
1364            DataType::Struct(fields) => ScalarValue::Struct(
1365                new_null_array(&DataType::Struct(fields.to_owned()), 1)
1366                    .as_struct()
1367                    .to_owned()
1368                    .into(),
1369            ),
1370            DataType::Map(fields, sorted) => ScalarValue::Map(
1371                new_null_array(&DataType::Map(fields.to_owned(), sorted.to_owned()), 1)
1372                    .as_map()
1373                    .to_owned()
1374                    .into(),
1375            ),
1376            DataType::Union(fields, mode) => {
1377                ScalarValue::Union(None, fields.clone(), *mode)
1378            }
1379            DataType::Null => ScalarValue::Null,
1380            _ => {
1381                return _not_impl_err!(
1382                    "Can't create a null scalar from data_type \"{data_type}\""
1383                );
1384            }
1385        })
1386    }
1387
1388    /// Returns a [`ScalarValue::Utf8`] representing `val`
1389    pub fn new_utf8(val: impl Into<String>) -> Self {
1390        ScalarValue::from(val.into())
1391    }
1392
1393    /// Returns a [`ScalarValue::Utf8View`] representing `val`
1394    pub fn new_utf8view(val: impl Into<String>) -> Self {
1395        ScalarValue::Utf8View(Some(val.into()))
1396    }
1397
1398    /// Returns a [`ScalarValue::IntervalYearMonth`] representing
1399    /// `years` years and `months` months
1400    pub fn new_interval_ym(years: i32, months: i32) -> Self {
1401        let val = IntervalYearMonthType::make_value(years, months);
1402        ScalarValue::IntervalYearMonth(Some(val))
1403    }
1404
1405    /// Returns a [`ScalarValue::IntervalDayTime`] representing
1406    /// `days` days and `millis` milliseconds
1407    pub fn new_interval_dt(days: i32, millis: i32) -> Self {
1408        let val = IntervalDayTimeType::make_value(days, millis);
1409        Self::IntervalDayTime(Some(val))
1410    }
1411
1412    /// Returns a [`ScalarValue::IntervalMonthDayNano`] representing
1413    /// `months` months and `days` days, and `nanos` nanoseconds
1414    pub fn new_interval_mdn(months: i32, days: i32, nanos: i64) -> Self {
1415        let val = IntervalMonthDayNanoType::make_value(months, days, nanos);
1416        ScalarValue::IntervalMonthDayNano(Some(val))
1417    }
1418
1419    /// Returns a [`ScalarValue`] representing
1420    /// `value` and `tz_opt` timezone
1421    pub fn new_timestamp<T: ArrowTimestampType>(
1422        value: Option<i64>,
1423        tz_opt: Option<Arc<str>>,
1424    ) -> Self {
1425        match T::UNIT {
1426            TimeUnit::Second => ScalarValue::TimestampSecond(value, tz_opt),
1427            TimeUnit::Millisecond => ScalarValue::TimestampMillisecond(value, tz_opt),
1428            TimeUnit::Microsecond => ScalarValue::TimestampMicrosecond(value, tz_opt),
1429            TimeUnit::Nanosecond => ScalarValue::TimestampNanosecond(value, tz_opt),
1430        }
1431    }
1432
1433    /// Returns a [`ScalarValue`] representing PI
1434    pub fn new_pi(datatype: &DataType) -> Result<ScalarValue> {
1435        match datatype {
1436            DataType::Float16 => Ok(ScalarValue::from(f16::PI)),
1437            DataType::Float32 => Ok(ScalarValue::from(std::f32::consts::PI)),
1438            DataType::Float64 => Ok(ScalarValue::from(std::f64::consts::PI)),
1439            _ => _internal_err!("PI is not supported for data type: {}", datatype),
1440        }
1441    }
1442
1443    /// Returns a [`ScalarValue`] representing PI's upper bound
1444    pub fn new_pi_upper(datatype: &DataType) -> Result<ScalarValue> {
1445        match datatype {
1446            DataType::Float16 => Ok(ScalarValue::Float16(Some(consts::PI_UPPER_F16))),
1447            DataType::Float32 => Ok(ScalarValue::from(consts::PI_UPPER_F32)),
1448            DataType::Float64 => Ok(ScalarValue::from(consts::PI_UPPER_F64)),
1449            _ => {
1450                _internal_err!("PI_UPPER is not supported for data type: {}", datatype)
1451            }
1452        }
1453    }
1454
1455    /// Returns a [`ScalarValue`] representing -PI's lower bound
1456    pub fn new_negative_pi_lower(datatype: &DataType) -> Result<ScalarValue> {
1457        match datatype {
1458            DataType::Float16 => {
1459                Ok(ScalarValue::Float16(Some(consts::NEGATIVE_PI_LOWER_F16)))
1460            }
1461            DataType::Float32 => Ok(ScalarValue::from(consts::NEGATIVE_PI_LOWER_F32)),
1462            DataType::Float64 => Ok(ScalarValue::from(consts::NEGATIVE_PI_LOWER_F64)),
1463            _ => {
1464                _internal_err!("-PI_LOWER is not supported for data type: {}", datatype)
1465            }
1466        }
1467    }
1468
1469    /// Returns a [`ScalarValue`] representing FRAC_PI_2's upper bound
1470    pub fn new_frac_pi_2_upper(datatype: &DataType) -> Result<ScalarValue> {
1471        match datatype {
1472            DataType::Float16 => {
1473                Ok(ScalarValue::Float16(Some(consts::FRAC_PI_2_UPPER_F16)))
1474            }
1475            DataType::Float32 => Ok(ScalarValue::from(consts::FRAC_PI_2_UPPER_F32)),
1476            DataType::Float64 => Ok(ScalarValue::from(consts::FRAC_PI_2_UPPER_F64)),
1477            _ => {
1478                _internal_err!("PI_UPPER/2 is not supported for data type: {}", datatype)
1479            }
1480        }
1481    }
1482
1483    // Returns a [`ScalarValue`] representing FRAC_PI_2's lower bound
1484    pub fn new_neg_frac_pi_2_lower(datatype: &DataType) -> Result<ScalarValue> {
1485        match datatype {
1486            DataType::Float16 => Ok(ScalarValue::Float16(Some(
1487                consts::NEGATIVE_FRAC_PI_2_LOWER_F16,
1488            ))),
1489            DataType::Float32 => {
1490                Ok(ScalarValue::from(consts::NEGATIVE_FRAC_PI_2_LOWER_F32))
1491            }
1492            DataType::Float64 => {
1493                Ok(ScalarValue::from(consts::NEGATIVE_FRAC_PI_2_LOWER_F64))
1494            }
1495            _ => {
1496                _internal_err!("-PI/2_LOWER is not supported for data type: {}", datatype)
1497            }
1498        }
1499    }
1500
1501    /// Returns a [`ScalarValue`] representing -PI
1502    pub fn new_negative_pi(datatype: &DataType) -> Result<ScalarValue> {
1503        match datatype {
1504            DataType::Float16 => Ok(ScalarValue::from(-f16::PI)),
1505            DataType::Float32 => Ok(ScalarValue::from(-std::f32::consts::PI)),
1506            DataType::Float64 => Ok(ScalarValue::from(-std::f64::consts::PI)),
1507            _ => _internal_err!("-PI is not supported for data type: {}", datatype),
1508        }
1509    }
1510
1511    /// Returns a [`ScalarValue`] representing PI/2
1512    pub fn new_frac_pi_2(datatype: &DataType) -> Result<ScalarValue> {
1513        match datatype {
1514            DataType::Float16 => Ok(ScalarValue::from(f16::FRAC_PI_2)),
1515            DataType::Float32 => Ok(ScalarValue::from(std::f32::consts::FRAC_PI_2)),
1516            DataType::Float64 => Ok(ScalarValue::from(std::f64::consts::FRAC_PI_2)),
1517            _ => _internal_err!("PI/2 is not supported for data type: {}", datatype),
1518        }
1519    }
1520
1521    /// Returns a [`ScalarValue`] representing -PI/2
1522    pub fn new_neg_frac_pi_2(datatype: &DataType) -> Result<ScalarValue> {
1523        match datatype {
1524            DataType::Float16 => Ok(ScalarValue::from(-f16::FRAC_PI_2)),
1525            DataType::Float32 => Ok(ScalarValue::from(-std::f32::consts::FRAC_PI_2)),
1526            DataType::Float64 => Ok(ScalarValue::from(-std::f64::consts::FRAC_PI_2)),
1527            _ => _internal_err!("-PI/2 is not supported for data type: {}", datatype),
1528        }
1529    }
1530
1531    /// Returns a [`ScalarValue`] representing infinity
1532    pub fn new_infinity(datatype: &DataType) -> Result<ScalarValue> {
1533        match datatype {
1534            DataType::Float16 => Ok(ScalarValue::from(f16::INFINITY)),
1535            DataType::Float32 => Ok(ScalarValue::from(f32::INFINITY)),
1536            DataType::Float64 => Ok(ScalarValue::from(f64::INFINITY)),
1537            _ => {
1538                _internal_err!("Infinity is not supported for data type: {}", datatype)
1539            }
1540        }
1541    }
1542
1543    /// Returns a [`ScalarValue`] representing negative infinity
1544    pub fn new_neg_infinity(datatype: &DataType) -> Result<ScalarValue> {
1545        match datatype {
1546            DataType::Float16 => Ok(ScalarValue::from(f16::NEG_INFINITY)),
1547            DataType::Float32 => Ok(ScalarValue::from(f32::NEG_INFINITY)),
1548            DataType::Float64 => Ok(ScalarValue::from(f64::NEG_INFINITY)),
1549            _ => {
1550                _internal_err!(
1551                    "Negative Infinity is not supported for data type: {}",
1552                    datatype
1553                )
1554            }
1555        }
1556    }
1557
1558    /// Create a zero value in the given type.
1559    pub fn new_zero(datatype: &DataType) -> Result<ScalarValue> {
1560        Ok(match datatype {
1561            DataType::Boolean => ScalarValue::Boolean(Some(false)),
1562            DataType::Int8 => ScalarValue::Int8(Some(0)),
1563            DataType::Int16 => ScalarValue::Int16(Some(0)),
1564            DataType::Int32 => ScalarValue::Int32(Some(0)),
1565            DataType::Int64 => ScalarValue::Int64(Some(0)),
1566            DataType::UInt8 => ScalarValue::UInt8(Some(0)),
1567            DataType::UInt16 => ScalarValue::UInt16(Some(0)),
1568            DataType::UInt32 => ScalarValue::UInt32(Some(0)),
1569            DataType::UInt64 => ScalarValue::UInt64(Some(0)),
1570            DataType::Float16 => ScalarValue::Float16(Some(f16::ZERO)),
1571            DataType::Float32 => ScalarValue::Float32(Some(0.0)),
1572            DataType::Float64 => ScalarValue::Float64(Some(0.0)),
1573            DataType::Decimal32(precision, scale) => {
1574                ScalarValue::Decimal32(Some(0), *precision, *scale)
1575            }
1576            DataType::Decimal64(precision, scale) => {
1577                ScalarValue::Decimal64(Some(0), *precision, *scale)
1578            }
1579            DataType::Decimal128(precision, scale) => {
1580                ScalarValue::Decimal128(Some(0), *precision, *scale)
1581            }
1582            DataType::Decimal256(precision, scale) => {
1583                ScalarValue::Decimal256(Some(i256::ZERO), *precision, *scale)
1584            }
1585            DataType::Timestamp(TimeUnit::Second, tz) => {
1586                ScalarValue::TimestampSecond(Some(0), tz.clone())
1587            }
1588            DataType::Timestamp(TimeUnit::Millisecond, tz) => {
1589                ScalarValue::TimestampMillisecond(Some(0), tz.clone())
1590            }
1591            DataType::Timestamp(TimeUnit::Microsecond, tz) => {
1592                ScalarValue::TimestampMicrosecond(Some(0), tz.clone())
1593            }
1594            DataType::Timestamp(TimeUnit::Nanosecond, tz) => {
1595                ScalarValue::TimestampNanosecond(Some(0), tz.clone())
1596            }
1597            DataType::Time32(TimeUnit::Second) => ScalarValue::Time32Second(Some(0)),
1598            DataType::Time32(TimeUnit::Millisecond) => {
1599                ScalarValue::Time32Millisecond(Some(0))
1600            }
1601            DataType::Time64(TimeUnit::Microsecond) => {
1602                ScalarValue::Time64Microsecond(Some(0))
1603            }
1604            DataType::Time64(TimeUnit::Nanosecond) => {
1605                ScalarValue::Time64Nanosecond(Some(0))
1606            }
1607            DataType::Interval(IntervalUnit::YearMonth) => {
1608                ScalarValue::IntervalYearMonth(Some(0))
1609            }
1610            DataType::Interval(IntervalUnit::DayTime) => {
1611                ScalarValue::IntervalDayTime(Some(IntervalDayTime::ZERO))
1612            }
1613            DataType::Interval(IntervalUnit::MonthDayNano) => {
1614                ScalarValue::IntervalMonthDayNano(Some(IntervalMonthDayNano::ZERO))
1615            }
1616            DataType::Duration(TimeUnit::Second) => ScalarValue::DurationSecond(Some(0)),
1617            DataType::Duration(TimeUnit::Millisecond) => {
1618                ScalarValue::DurationMillisecond(Some(0))
1619            }
1620            DataType::Duration(TimeUnit::Microsecond) => {
1621                ScalarValue::DurationMicrosecond(Some(0))
1622            }
1623            DataType::Duration(TimeUnit::Nanosecond) => {
1624                ScalarValue::DurationNanosecond(Some(0))
1625            }
1626            DataType::Date32 => ScalarValue::Date32(Some(0)),
1627            DataType::Date64 => ScalarValue::Date64(Some(0)),
1628            _ => {
1629                return _not_impl_err!(
1630                    "Can't create a zero scalar from data_type \"{datatype}\""
1631                );
1632            }
1633        })
1634    }
1635
1636    /// Returns a default value for the given `DataType`.
1637    ///
1638    /// This function is useful when you need to initialize a column with
1639    /// non-null values in a DataFrame or when you need a "zero" value
1640    /// for a specific data type.
1641    ///
1642    /// # Default Values
1643    ///
1644    /// - **Numeric types**: Returns zero (via [`new_zero`])
1645    /// - **String types**: Returns empty string (`""`)
1646    /// - **Binary types**: Returns empty byte array
1647    /// - **Temporal types**: Returns zero/epoch value
1648    /// - **List types**: Returns empty list
1649    /// - **Struct types**: Returns struct with all fields set to their defaults
1650    /// - **Dictionary types**: Returns dictionary with default value
1651    /// - **Map types**: Returns empty map
1652    /// - **Union types**: Returns first variant with default value
1653    ///
1654    /// # Errors
1655    ///
1656    /// Returns an error for data types that don't have a clear default value
1657    /// or are not yet supported (e.g., `RunEndEncoded`).
1658    ///
1659    /// [`new_zero`]: Self::new_zero
1660    pub fn new_default(datatype: &DataType) -> Result<ScalarValue> {
1661        match datatype {
1662            // Null type
1663            DataType::Null => Ok(ScalarValue::Null),
1664
1665            // Numeric types
1666            DataType::Boolean
1667            | DataType::Int8
1668            | DataType::Int16
1669            | DataType::Int32
1670            | DataType::Int64
1671            | DataType::UInt8
1672            | DataType::UInt16
1673            | DataType::UInt32
1674            | DataType::UInt64
1675            | DataType::Float16
1676            | DataType::Float32
1677            | DataType::Float64
1678            | DataType::Decimal32(_, _)
1679            | DataType::Decimal64(_, _)
1680            | DataType::Decimal128(_, _)
1681            | DataType::Decimal256(_, _)
1682            | DataType::Timestamp(_, _)
1683            | DataType::Time32(_)
1684            | DataType::Time64(_)
1685            | DataType::Interval(_)
1686            | DataType::Duration(_)
1687            | DataType::Date32
1688            | DataType::Date64 => ScalarValue::new_zero(datatype),
1689
1690            // String types
1691            DataType::Utf8 => Ok(ScalarValue::Utf8(Some("".to_string()))),
1692            DataType::LargeUtf8 => Ok(ScalarValue::LargeUtf8(Some("".to_string()))),
1693            DataType::Utf8View => Ok(ScalarValue::Utf8View(Some("".to_string()))),
1694
1695            // Binary types
1696            DataType::Binary => Ok(ScalarValue::Binary(Some(vec![]))),
1697            DataType::LargeBinary => Ok(ScalarValue::LargeBinary(Some(vec![]))),
1698            DataType::BinaryView => Ok(ScalarValue::BinaryView(Some(vec![]))),
1699
1700            // Fixed-size binary
1701            DataType::FixedSizeBinary(size) => Ok(ScalarValue::FixedSizeBinary(
1702                *size,
1703                Some(vec![0; *size as usize]),
1704            )),
1705
1706            // List types
1707            DataType::List(field) => {
1708                let list =
1709                    ScalarValue::new_list(&[], field.data_type(), field.is_nullable());
1710                Ok(ScalarValue::List(list))
1711            }
1712            DataType::FixedSizeList(field, _size) => {
1713                let empty_arr = new_empty_array(field.data_type());
1714                let values = Arc::new(
1715                    SingleRowListArrayBuilder::new(empty_arr)
1716                        .with_field(field)
1717                        .build_fixed_size_list_array(0),
1718                );
1719                Ok(ScalarValue::FixedSizeList(values))
1720            }
1721            DataType::LargeList(field) => {
1722                let list = ScalarValue::new_large_list(&[], field.data_type());
1723                Ok(ScalarValue::LargeList(list))
1724            }
1725            DataType::ListView(field) => {
1726                let empty_arr = new_empty_array(field.data_type());
1727                let values = Arc::new(
1728                    SingleRowListArrayBuilder::new(empty_arr)
1729                        .with_field(field)
1730                        .build_list_view_array(),
1731                );
1732                Ok(ScalarValue::ListView(values))
1733            }
1734            DataType::LargeListView(field) => {
1735                let empty_arr = new_empty_array(field.data_type());
1736                let values = Arc::new(
1737                    SingleRowListArrayBuilder::new(empty_arr)
1738                        .with_field(field)
1739                        .build_large_list_view_array(),
1740                );
1741                Ok(ScalarValue::LargeListView(values))
1742            }
1743
1744            // Struct types
1745            DataType::Struct(fields) => {
1746                if fields.is_empty() {
1747                    Ok(ScalarValue::Struct(Arc::new(
1748                        StructArray::new_empty_fields(1, None),
1749                    )))
1750                } else {
1751                    let values = fields
1752                        .iter()
1753                        .map(|f| ScalarValue::new_default(f.data_type()))
1754                        .collect::<Result<Vec<_>>>()?;
1755                    Ok(ScalarValue::Struct(Arc::new(StructArray::new(
1756                        fields.clone(),
1757                        values
1758                            .into_iter()
1759                            .map(|v| v.to_array())
1760                            .collect::<Result<_>>()?,
1761                        None,
1762                    ))))
1763                }
1764            }
1765
1766            // Dictionary types
1767            DataType::Dictionary(key_type, value_type) => Ok(ScalarValue::Dictionary(
1768                key_type.clone(),
1769                Box::new(ScalarValue::new_default(value_type)?),
1770            )),
1771
1772            DataType::RunEndEncoded(run_ends_field, value_field) => {
1773                Ok(ScalarValue::RunEndEncoded(
1774                    Arc::clone(run_ends_field),
1775                    Arc::clone(value_field),
1776                    Box::new(ScalarValue::new_default(value_field.data_type())?),
1777                ))
1778            }
1779
1780            // Map types
1781            DataType::Map(field, _) => Ok(ScalarValue::Map(Arc::new(MapArray::from(
1782                ArrayData::new_empty(field.data_type()),
1783            )))),
1784
1785            // Union types - return first variant with default value
1786            DataType::Union(fields, mode) => {
1787                if let Some((type_id, field)) = fields.iter().next() {
1788                    let default_value = ScalarValue::new_default(field.data_type())?;
1789                    Ok(ScalarValue::Union(
1790                        Some((type_id, Box::new(default_value))),
1791                        fields.clone(),
1792                        *mode,
1793                    ))
1794                } else {
1795                    _internal_err!("Union type must have at least one field")
1796                }
1797            }
1798        }
1799    }
1800
1801    /// Create an one value in the given type.
1802    pub fn new_one(datatype: &DataType) -> Result<ScalarValue> {
1803        Ok(match datatype {
1804            DataType::Int8 => ScalarValue::Int8(Some(1)),
1805            DataType::Int16 => ScalarValue::Int16(Some(1)),
1806            DataType::Int32 => ScalarValue::Int32(Some(1)),
1807            DataType::Int64 => ScalarValue::Int64(Some(1)),
1808            DataType::UInt8 => ScalarValue::UInt8(Some(1)),
1809            DataType::UInt16 => ScalarValue::UInt16(Some(1)),
1810            DataType::UInt32 => ScalarValue::UInt32(Some(1)),
1811            DataType::UInt64 => ScalarValue::UInt64(Some(1)),
1812            DataType::Float16 => ScalarValue::Float16(Some(f16::ONE)),
1813            DataType::Float32 => ScalarValue::Float32(Some(1.0)),
1814            DataType::Float64 => ScalarValue::Float64(Some(1.0)),
1815            DataType::Decimal32(precision, scale) => {
1816                Self::validate_decimal_or_internal_err::<Decimal32Type>(
1817                    *precision, *scale,
1818                )?;
1819                assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
1820                assert_or_internal_err!(
1821                    *precision != *scale as u8,
1822                    "Can't represent one at scale {} with precision {}",
1823                    *scale,
1824                    *precision
1825                );
1826                let one = DECIMAL32_ONES[*scale as usize];
1827                ScalarValue::Decimal32(Some(one), *precision, *scale)
1828            }
1829            DataType::Decimal64(precision, scale) => {
1830                Self::validate_decimal_or_internal_err::<Decimal64Type>(
1831                    *precision, *scale,
1832                )?;
1833                assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
1834                assert_or_internal_err!(
1835                    *precision != *scale as u8,
1836                    "Can't represent one at scale {} with precision {}",
1837                    *scale,
1838                    *precision
1839                );
1840                let one = DECIMAL64_ONES[*scale as usize];
1841                ScalarValue::Decimal64(Some(one), *precision, *scale)
1842            }
1843            DataType::Decimal128(precision, scale) => {
1844                Self::validate_decimal_or_internal_err::<Decimal128Type>(
1845                    *precision, *scale,
1846                )?;
1847                assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
1848                assert_or_internal_err!(
1849                    *precision != *scale as u8,
1850                    "Can't represent one at scale {} with precision {}",
1851                    *scale,
1852                    *precision
1853                );
1854                let one = DECIMAL128_ONES[*scale as usize];
1855                ScalarValue::Decimal128(Some(one), *precision, *scale)
1856            }
1857            DataType::Decimal256(precision, scale) => {
1858                Self::validate_decimal_or_internal_err::<Decimal256Type>(
1859                    *precision, *scale,
1860                )?;
1861                assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
1862                assert_or_internal_err!(
1863                    *precision != *scale as u8,
1864                    "Can't represent one at scale {} with precision {}",
1865                    *scale,
1866                    *precision
1867                );
1868                let one = DECIMAL256_ONES[*scale as usize];
1869                ScalarValue::Decimal256(Some(one), *precision, *scale)
1870            }
1871            _ => {
1872                return _not_impl_err!(
1873                    "Can't create an one scalar from data_type \"{datatype}\""
1874                );
1875            }
1876        })
1877    }
1878
1879    /// Create a negative one value in the given type.
1880    pub fn new_negative_one(datatype: &DataType) -> Result<ScalarValue> {
1881        Ok(match datatype {
1882            DataType::Int8 => ScalarValue::Int8(Some(-1)),
1883            DataType::Int16 => ScalarValue::Int16(Some(-1)),
1884            DataType::Int32 => ScalarValue::Int32(Some(-1)),
1885            DataType::Int64 => ScalarValue::Int64(Some(-1)),
1886            DataType::Float16 => ScalarValue::Float16(Some(f16::NEG_ONE)),
1887            DataType::Float32 => ScalarValue::Float32(Some(-1.0)),
1888            DataType::Float64 => ScalarValue::Float64(Some(-1.0)),
1889            DataType::Decimal32(precision, scale) => {
1890                Self::validate_decimal_or_internal_err::<Decimal32Type>(
1891                    *precision, *scale,
1892                )?;
1893                assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
1894                assert_or_internal_err!(
1895                    *precision != *scale as u8,
1896                    "Can't represent negative one at scale {} with precision {}",
1897                    *scale,
1898                    *precision
1899                );
1900                let one = DECIMAL32_ONES[*scale as usize];
1901                ScalarValue::Decimal32(Some(-one), *precision, *scale)
1902            }
1903            DataType::Decimal64(precision, scale) => {
1904                Self::validate_decimal_or_internal_err::<Decimal64Type>(
1905                    *precision, *scale,
1906                )?;
1907                assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
1908                assert_or_internal_err!(
1909                    *precision != *scale as u8,
1910                    "Can't represent negative one at scale {} with precision {}",
1911                    *scale,
1912                    *precision
1913                );
1914                let one = DECIMAL64_ONES[*scale as usize];
1915                ScalarValue::Decimal64(Some(-one), *precision, *scale)
1916            }
1917            DataType::Decimal128(precision, scale) => {
1918                Self::validate_decimal_or_internal_err::<Decimal128Type>(
1919                    *precision, *scale,
1920                )?;
1921                assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
1922                assert_or_internal_err!(
1923                    *precision != *scale as u8,
1924                    "Can't represent negative one at scale {} with precision {}",
1925                    *scale,
1926                    *precision
1927                );
1928                let one = DECIMAL128_ONES[*scale as usize];
1929                ScalarValue::Decimal128(Some(-one), *precision, *scale)
1930            }
1931            DataType::Decimal256(precision, scale) => {
1932                Self::validate_decimal_or_internal_err::<Decimal256Type>(
1933                    *precision, *scale,
1934                )?;
1935                assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
1936                assert_or_internal_err!(
1937                    *precision != *scale as u8,
1938                    "Can't represent one at scale {} with precision {}",
1939                    *scale,
1940                    *precision
1941                );
1942                let one = DECIMAL256_ONES[*scale as usize];
1943                ScalarValue::Decimal256(Some(-one), *precision, *scale)
1944            }
1945            _ => {
1946                return _not_impl_err!(
1947                    "Can't create a negative one scalar from data_type \"{datatype}\""
1948                );
1949            }
1950        })
1951    }
1952
1953    pub fn new_ten(datatype: &DataType) -> Result<ScalarValue> {
1954        Ok(match datatype {
1955            DataType::Int8 => ScalarValue::Int8(Some(10)),
1956            DataType::Int16 => ScalarValue::Int16(Some(10)),
1957            DataType::Int32 => ScalarValue::Int32(Some(10)),
1958            DataType::Int64 => ScalarValue::Int64(Some(10)),
1959            DataType::UInt8 => ScalarValue::UInt8(Some(10)),
1960            DataType::UInt16 => ScalarValue::UInt16(Some(10)),
1961            DataType::UInt32 => ScalarValue::UInt32(Some(10)),
1962            DataType::UInt64 => ScalarValue::UInt64(Some(10)),
1963            DataType::Float16 => ScalarValue::Float16(Some(f16::from_f32(10.0))),
1964            DataType::Float32 => ScalarValue::Float32(Some(10.0)),
1965            DataType::Float64 => ScalarValue::Float64(Some(10.0)),
1966            DataType::Decimal32(precision, scale) => {
1967                Self::validate_decimal_or_internal_err::<Decimal32Type>(
1968                    *precision, *scale,
1969                )?;
1970                assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
1971                assert_or_internal_err!(
1972                    (*precision - *scale as u8) > 1,
1973                    "Can't represent ten at scale {} with precision {}",
1974                    *scale,
1975                    *precision
1976                );
1977                // +1 safe since we validate above that scale must be less than
1978                // the max possible scale
1979                let ten = DECIMAL32_ONES[*scale as usize + 1];
1980                ScalarValue::Decimal32(Some(ten), *precision, *scale)
1981            }
1982            DataType::Decimal64(precision, scale) => {
1983                Self::validate_decimal_or_internal_err::<Decimal64Type>(
1984                    *precision, *scale,
1985                )?;
1986                assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
1987                assert_or_internal_err!(
1988                    (*precision - *scale as u8) > 1,
1989                    "Can't represent ten at scale {} with precision {}",
1990                    *scale,
1991                    *precision
1992                );
1993                // +1 safe since we validate above that scale must be less than
1994                // the max possible scale
1995                let ten = DECIMAL64_ONES[*scale as usize + 1];
1996                ScalarValue::Decimal64(Some(ten), *precision, *scale)
1997            }
1998            DataType::Decimal128(precision, scale) => {
1999                Self::validate_decimal_or_internal_err::<Decimal128Type>(
2000                    *precision, *scale,
2001                )?;
2002                assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
2003                assert_or_internal_err!(
2004                    (*precision - *scale as u8) > 1,
2005                    "Can't represent ten at scale {} with precision {}",
2006                    *scale,
2007                    *precision
2008                );
2009                // +1 safe since we validate above that scale must be less than
2010                // the max possible scale
2011                let ten = DECIMAL128_ONES[*scale as usize + 1];
2012                ScalarValue::Decimal128(Some(ten), *precision, *scale)
2013            }
2014            DataType::Decimal256(precision, scale) => {
2015                Self::validate_decimal_or_internal_err::<Decimal256Type>(
2016                    *precision, *scale,
2017                )?;
2018                assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
2019                assert_or_internal_err!(
2020                    (*precision - *scale as u8) > 1,
2021                    "Can't represent ten at scale {} with precision {}",
2022                    *scale,
2023                    *precision
2024                );
2025                // +1 safe since we validate above that scale must be less than
2026                // the max possible scale
2027                let ten = DECIMAL256_ONES[*scale as usize + 1];
2028                ScalarValue::Decimal256(Some(ten), *precision, *scale)
2029            }
2030            _ => {
2031                return _not_impl_err!(
2032                    "Can't create a ten scalar from data_type \"{datatype}\""
2033                );
2034            }
2035        })
2036    }
2037
2038    /// return the [`DataType`] of this `ScalarValue`
2039    pub fn data_type(&self) -> DataType {
2040        match self {
2041            ScalarValue::Boolean(_) => DataType::Boolean,
2042            ScalarValue::UInt8(_) => DataType::UInt8,
2043            ScalarValue::UInt16(_) => DataType::UInt16,
2044            ScalarValue::UInt32(_) => DataType::UInt32,
2045            ScalarValue::UInt64(_) => DataType::UInt64,
2046            ScalarValue::Int8(_) => DataType::Int8,
2047            ScalarValue::Int16(_) => DataType::Int16,
2048            ScalarValue::Int32(_) => DataType::Int32,
2049            ScalarValue::Int64(_) => DataType::Int64,
2050            ScalarValue::Decimal32(_, precision, scale) => {
2051                DataType::Decimal32(*precision, *scale)
2052            }
2053            ScalarValue::Decimal64(_, precision, scale) => {
2054                DataType::Decimal64(*precision, *scale)
2055            }
2056            ScalarValue::Decimal128(_, precision, scale) => {
2057                DataType::Decimal128(*precision, *scale)
2058            }
2059            ScalarValue::Decimal256(_, precision, scale) => {
2060                DataType::Decimal256(*precision, *scale)
2061            }
2062            ScalarValue::TimestampSecond(_, tz_opt) => {
2063                DataType::Timestamp(TimeUnit::Second, tz_opt.clone())
2064            }
2065            ScalarValue::TimestampMillisecond(_, tz_opt) => {
2066                DataType::Timestamp(TimeUnit::Millisecond, tz_opt.clone())
2067            }
2068            ScalarValue::TimestampMicrosecond(_, tz_opt) => {
2069                DataType::Timestamp(TimeUnit::Microsecond, tz_opt.clone())
2070            }
2071            ScalarValue::TimestampNanosecond(_, tz_opt) => {
2072                DataType::Timestamp(TimeUnit::Nanosecond, tz_opt.clone())
2073            }
2074            ScalarValue::Float16(_) => DataType::Float16,
2075            ScalarValue::Float32(_) => DataType::Float32,
2076            ScalarValue::Float64(_) => DataType::Float64,
2077            ScalarValue::Utf8(_) => DataType::Utf8,
2078            ScalarValue::LargeUtf8(_) => DataType::LargeUtf8,
2079            ScalarValue::Utf8View(_) => DataType::Utf8View,
2080            ScalarValue::Binary(_) => DataType::Binary,
2081            ScalarValue::BinaryView(_) => DataType::BinaryView,
2082            ScalarValue::FixedSizeBinary(sz, _) => DataType::FixedSizeBinary(*sz),
2083            ScalarValue::LargeBinary(_) => DataType::LargeBinary,
2084            ScalarValue::List(arr) => arr.data_type().to_owned(),
2085            ScalarValue::LargeList(arr) => arr.data_type().to_owned(),
2086            ScalarValue::FixedSizeList(arr) => arr.data_type().to_owned(),
2087            ScalarValue::ListView(arr) => arr.data_type().to_owned(),
2088            ScalarValue::LargeListView(arr) => arr.data_type().to_owned(),
2089            ScalarValue::Struct(arr) => arr.data_type().to_owned(),
2090            ScalarValue::Map(arr) => arr.data_type().to_owned(),
2091            ScalarValue::Date32(_) => DataType::Date32,
2092            ScalarValue::Date64(_) => DataType::Date64,
2093            ScalarValue::Time32Second(_) => DataType::Time32(TimeUnit::Second),
2094            ScalarValue::Time32Millisecond(_) => DataType::Time32(TimeUnit::Millisecond),
2095            ScalarValue::Time64Microsecond(_) => DataType::Time64(TimeUnit::Microsecond),
2096            ScalarValue::Time64Nanosecond(_) => DataType::Time64(TimeUnit::Nanosecond),
2097            ScalarValue::IntervalYearMonth(_) => {
2098                DataType::Interval(IntervalUnit::YearMonth)
2099            }
2100            ScalarValue::IntervalDayTime(_) => DataType::Interval(IntervalUnit::DayTime),
2101            ScalarValue::IntervalMonthDayNano(_) => {
2102                DataType::Interval(IntervalUnit::MonthDayNano)
2103            }
2104            ScalarValue::DurationSecond(_) => DataType::Duration(TimeUnit::Second),
2105            ScalarValue::DurationMillisecond(_) => {
2106                DataType::Duration(TimeUnit::Millisecond)
2107            }
2108            ScalarValue::DurationMicrosecond(_) => {
2109                DataType::Duration(TimeUnit::Microsecond)
2110            }
2111            ScalarValue::DurationNanosecond(_) => {
2112                DataType::Duration(TimeUnit::Nanosecond)
2113            }
2114            ScalarValue::Union(_, fields, mode) => DataType::Union(fields.clone(), *mode),
2115            ScalarValue::Dictionary(k, v) => {
2116                DataType::Dictionary(k.clone(), Box::new(v.data_type()))
2117            }
2118            ScalarValue::RunEndEncoded(run_ends_field, value_field, _) => {
2119                DataType::RunEndEncoded(
2120                    Arc::clone(run_ends_field),
2121                    Arc::clone(value_field),
2122                )
2123            }
2124            ScalarValue::Null => DataType::Null,
2125        }
2126    }
2127
2128    #[inline]
2129    fn can_use_direct_add(lhs: &ScalarValue, rhs: &ScalarValue) -> bool {
2130        matches!(
2131            (lhs, rhs),
2132            (ScalarValue::Int8(_), ScalarValue::Int8(_))
2133                | (ScalarValue::Int16(_), ScalarValue::Int16(_))
2134                | (ScalarValue::Int32(_), ScalarValue::Int32(_))
2135                | (ScalarValue::Int64(_), ScalarValue::Int64(_))
2136                | (ScalarValue::UInt8(_), ScalarValue::UInt8(_))
2137                | (ScalarValue::UInt16(_), ScalarValue::UInt16(_))
2138                | (ScalarValue::UInt32(_), ScalarValue::UInt32(_))
2139                | (ScalarValue::UInt64(_), ScalarValue::UInt64(_))
2140                | (ScalarValue::Float16(_), ScalarValue::Float16(_))
2141                | (ScalarValue::Float32(_), ScalarValue::Float32(_))
2142                | (ScalarValue::Float64(_), ScalarValue::Float64(_))
2143                | (
2144                    ScalarValue::Decimal32(_, _, _),
2145                    ScalarValue::Decimal32(_, _, _)
2146                )
2147                | (
2148                    ScalarValue::Decimal64(_, _, _),
2149                    ScalarValue::Decimal64(_, _, _)
2150                )
2151                | (
2152                    ScalarValue::Decimal128(_, _, _),
2153                    ScalarValue::Decimal128(_, _, _),
2154                )
2155                | (
2156                    ScalarValue::Decimal256(_, _, _),
2157                    ScalarValue::Decimal256(_, _, _),
2158                )
2159        )
2160    }
2161
2162    #[inline]
2163    fn add_optional<T: ArrowNativeTypeOp>(
2164        lhs: &mut Option<T>,
2165        rhs: Option<T>,
2166        checked: bool,
2167    ) -> Result<()> {
2168        match rhs {
2169            Some(rhs) => {
2170                if let Some(lhs) = lhs.as_mut() {
2171                    *lhs = if checked {
2172                        lhs.add_checked(rhs).map_err(|e| arrow_datafusion_err!(e))?
2173                    } else {
2174                        lhs.add_wrapping(rhs)
2175                    };
2176                }
2177            }
2178            None => *lhs = None,
2179        }
2180        Ok(())
2181    }
2182
2183    #[inline]
2184    fn add_decimal_values<T: DecimalType>(
2185        lhs_value: &mut Option<T::Native>,
2186        lhs_precision: &mut u8,
2187        lhs_scale: &mut i8,
2188        rhs_value: Option<T::Native>,
2189        rhs_precision: u8,
2190        rhs_scale: i8,
2191    ) -> Result<()>
2192    where
2193        T::Native: ArrowNativeTypeOp,
2194    {
2195        Self::validate_decimal_or_internal_err::<T>(*lhs_precision, *lhs_scale)?;
2196        Self::validate_decimal_or_internal_err::<T>(rhs_precision, rhs_scale)?;
2197
2198        let result_scale = (*lhs_scale).max(rhs_scale);
2199        // Decimal scales can be negative, so use a wider signed type for the
2200        // intermediate precision arithmetic.
2201        let lhs_precision_delta = i16::from(*lhs_precision) - i16::from(*lhs_scale);
2202        let rhs_precision_delta = i16::from(rhs_precision) - i16::from(rhs_scale);
2203        let result_precision =
2204            (i16::from(result_scale) + lhs_precision_delta.max(rhs_precision_delta) + 1)
2205                .min(i16::from(T::MAX_PRECISION)) as u8;
2206
2207        Self::validate_decimal_or_internal_err::<T>(result_precision, result_scale)?;
2208
2209        let lhs_mul = T::Native::usize_as(10)
2210            .pow_checked((result_scale - *lhs_scale) as u32)
2211            .map_err(|e| arrow_datafusion_err!(e))?;
2212        let rhs_mul = T::Native::usize_as(10)
2213            .pow_checked((result_scale - rhs_scale) as u32)
2214            .map_err(|e| arrow_datafusion_err!(e))?;
2215
2216        let result_value = match (*lhs_value, rhs_value) {
2217            (Some(lhs_value), Some(rhs_value)) => Some(
2218                lhs_value
2219                    .mul_checked(lhs_mul)
2220                    .and_then(|lhs| {
2221                        rhs_value
2222                            .mul_checked(rhs_mul)
2223                            .and_then(|rhs| lhs.add_checked(rhs))
2224                    })
2225                    .map_err(|e| arrow_datafusion_err!(e))?,
2226            ),
2227            _ => None,
2228        };
2229
2230        *lhs_value = result_value;
2231        *lhs_precision = result_precision;
2232        *lhs_scale = result_scale;
2233
2234        Ok(())
2235    }
2236
2237    #[inline]
2238    fn try_add_in_place_impl(
2239        &mut self,
2240        other: &ScalarValue,
2241        checked: bool,
2242    ) -> Result<bool> {
2243        match (self, other) {
2244            (ScalarValue::Int8(lhs), ScalarValue::Int8(rhs)) => {
2245                Self::add_optional(lhs, *rhs, checked)?;
2246            }
2247            (ScalarValue::Int16(lhs), ScalarValue::Int16(rhs)) => {
2248                Self::add_optional(lhs, *rhs, checked)?;
2249            }
2250            (ScalarValue::Int32(lhs), ScalarValue::Int32(rhs)) => {
2251                Self::add_optional(lhs, *rhs, checked)?;
2252            }
2253            (ScalarValue::Int64(lhs), ScalarValue::Int64(rhs)) => {
2254                Self::add_optional(lhs, *rhs, checked)?;
2255            }
2256            (ScalarValue::UInt8(lhs), ScalarValue::UInt8(rhs)) => {
2257                Self::add_optional(lhs, *rhs, checked)?;
2258            }
2259            (ScalarValue::UInt16(lhs), ScalarValue::UInt16(rhs)) => {
2260                Self::add_optional(lhs, *rhs, checked)?;
2261            }
2262            (ScalarValue::UInt32(lhs), ScalarValue::UInt32(rhs)) => {
2263                Self::add_optional(lhs, *rhs, checked)?;
2264            }
2265            (ScalarValue::UInt64(lhs), ScalarValue::UInt64(rhs)) => {
2266                Self::add_optional(lhs, *rhs, checked)?;
2267            }
2268            (ScalarValue::Float16(lhs), ScalarValue::Float16(rhs)) => {
2269                Self::add_optional(lhs, *rhs, checked)?;
2270            }
2271            (ScalarValue::Float32(lhs), ScalarValue::Float32(rhs)) => {
2272                Self::add_optional(lhs, *rhs, checked)?;
2273            }
2274            (ScalarValue::Float64(lhs), ScalarValue::Float64(rhs)) => {
2275                Self::add_optional(lhs, *rhs, checked)?;
2276            }
2277            (
2278                ScalarValue::Decimal32(lhs, p, s),
2279                ScalarValue::Decimal32(rhs, rhs_p, rhs_s),
2280            ) => {
2281                Self::add_decimal_values::<Decimal32Type>(
2282                    lhs, p, s, *rhs, *rhs_p, *rhs_s,
2283                )?;
2284            }
2285            (
2286                ScalarValue::Decimal64(lhs, p, s),
2287                ScalarValue::Decimal64(rhs, rhs_p, rhs_s),
2288            ) => {
2289                Self::add_decimal_values::<Decimal64Type>(
2290                    lhs, p, s, *rhs, *rhs_p, *rhs_s,
2291                )?;
2292            }
2293            (
2294                ScalarValue::Decimal128(lhs, p, s),
2295                ScalarValue::Decimal128(rhs, rhs_p, rhs_s),
2296            ) => {
2297                Self::add_decimal_values::<Decimal128Type>(
2298                    lhs, p, s, *rhs, *rhs_p, *rhs_s,
2299                )?;
2300            }
2301            (
2302                ScalarValue::Decimal256(lhs, p, s),
2303                ScalarValue::Decimal256(rhs, rhs_p, rhs_s),
2304            ) => {
2305                Self::add_decimal_values::<Decimal256Type>(
2306                    lhs, p, s, *rhs, *rhs_p, *rhs_s,
2307                )?;
2308            }
2309            _ => return Ok(false),
2310        }
2311
2312        Ok(true)
2313    }
2314
2315    #[inline]
2316    pub(crate) fn try_add_wrapping_in_place(
2317        &mut self,
2318        other: &ScalarValue,
2319    ) -> Result<bool> {
2320        self.try_add_in_place_impl(other, false)
2321    }
2322
2323    #[inline]
2324    pub(crate) fn try_add_checked_in_place(
2325        &mut self,
2326        other: &ScalarValue,
2327    ) -> Result<bool> {
2328        self.try_add_in_place_impl(other, true)
2329    }
2330
2331    /// Calculate arithmetic negation for a scalar value
2332    pub fn arithmetic_negate(&self) -> Result<Self> {
2333        fn neg_checked_with_ctx<T: ArrowNativeTypeOp>(
2334            v: T,
2335            ctx: impl Fn() -> String,
2336        ) -> Result<T> {
2337            v.neg_checked()
2338                .map_err(|e| arrow_datafusion_err!(e).context(ctx()))
2339        }
2340        match self {
2341            ScalarValue::Int8(None)
2342            | ScalarValue::Int16(None)
2343            | ScalarValue::Int32(None)
2344            | ScalarValue::Int64(None)
2345            | ScalarValue::Float16(None)
2346            | ScalarValue::Float32(None)
2347            | ScalarValue::Float64(None)
2348            | ScalarValue::IntervalYearMonth(None)
2349            | ScalarValue::IntervalDayTime(None)
2350            | ScalarValue::IntervalMonthDayNano(None)
2351            | ScalarValue::Decimal32(None, _, _)
2352            | ScalarValue::Decimal64(None, _, _)
2353            | ScalarValue::Decimal128(None, _, _)
2354            | ScalarValue::Decimal256(None, _, _)
2355            | ScalarValue::TimestampSecond(None, _)
2356            | ScalarValue::TimestampMillisecond(None, _)
2357            | ScalarValue::TimestampMicrosecond(None, _)
2358            | ScalarValue::TimestampNanosecond(None, _) => Ok(self.clone()),
2359            ScalarValue::Float16(Some(v)) => Ok(ScalarValue::Float16(Some(-v))),
2360            ScalarValue::Float64(Some(v)) => Ok(ScalarValue::Float64(Some(-v))),
2361            ScalarValue::Float32(Some(v)) => Ok(ScalarValue::Float32(Some(-v))),
2362            ScalarValue::Int8(Some(v)) => Ok(ScalarValue::Int8(Some(v.neg_checked()?))),
2363            ScalarValue::Int16(Some(v)) => Ok(ScalarValue::Int16(Some(v.neg_checked()?))),
2364            ScalarValue::Int32(Some(v)) => Ok(ScalarValue::Int32(Some(v.neg_checked()?))),
2365            ScalarValue::Int64(Some(v)) => Ok(ScalarValue::Int64(Some(v.neg_checked()?))),
2366            ScalarValue::IntervalYearMonth(Some(v)) => Ok(
2367                ScalarValue::IntervalYearMonth(Some(neg_checked_with_ctx(*v, || {
2368                    format!("In negation of IntervalYearMonth({v})")
2369                })?)),
2370            ),
2371            ScalarValue::IntervalDayTime(Some(v)) => {
2372                let (days, ms) = IntervalDayTimeType::to_parts(*v);
2373                let val = IntervalDayTimeType::make_value(
2374                    neg_checked_with_ctx(days, || {
2375                        format!("In negation of days {days} in IntervalDayTime")
2376                    })?,
2377                    neg_checked_with_ctx(ms, || {
2378                        format!("In negation of milliseconds {ms} in IntervalDayTime")
2379                    })?,
2380                );
2381                Ok(ScalarValue::IntervalDayTime(Some(val)))
2382            }
2383            ScalarValue::IntervalMonthDayNano(Some(v)) => {
2384                let (months, days, nanos) = IntervalMonthDayNanoType::to_parts(*v);
2385                let val = IntervalMonthDayNanoType::make_value(
2386                    neg_checked_with_ctx(months, || {
2387                        format!("In negation of months {months} of IntervalMonthDayNano")
2388                    })?,
2389                    neg_checked_with_ctx(days, || {
2390                        format!("In negation of days {days} of IntervalMonthDayNano")
2391                    })?,
2392                    neg_checked_with_ctx(nanos, || {
2393                        format!("In negation of nanos {nanos} of IntervalMonthDayNano")
2394                    })?,
2395                );
2396                Ok(ScalarValue::IntervalMonthDayNano(Some(val)))
2397            }
2398            ScalarValue::Decimal32(Some(v), precision, scale) => {
2399                Ok(ScalarValue::Decimal32(
2400                    Some(neg_checked_with_ctx(*v, || {
2401                        format!("In negation of Decimal32({v}, {precision}, {scale})")
2402                    })?),
2403                    *precision,
2404                    *scale,
2405                ))
2406            }
2407            ScalarValue::Decimal64(Some(v), precision, scale) => {
2408                Ok(ScalarValue::Decimal64(
2409                    Some(neg_checked_with_ctx(*v, || {
2410                        format!("In negation of Decimal64({v}, {precision}, {scale})")
2411                    })?),
2412                    *precision,
2413                    *scale,
2414                ))
2415            }
2416            ScalarValue::Decimal128(Some(v), precision, scale) => {
2417                Ok(ScalarValue::Decimal128(
2418                    Some(neg_checked_with_ctx(*v, || {
2419                        format!("In negation of Decimal128({v}, {precision}, {scale})")
2420                    })?),
2421                    *precision,
2422                    *scale,
2423                ))
2424            }
2425            ScalarValue::Decimal256(Some(v), precision, scale) => {
2426                Ok(ScalarValue::Decimal256(
2427                    Some(neg_checked_with_ctx(*v, || {
2428                        format!("In negation of Decimal256({v}, {precision}, {scale})")
2429                    })?),
2430                    *precision,
2431                    *scale,
2432                ))
2433            }
2434            ScalarValue::TimestampSecond(Some(v), tz) => {
2435                Ok(ScalarValue::TimestampSecond(
2436                    Some(neg_checked_with_ctx(*v, || {
2437                        format!("In negation of TimestampSecond({v})")
2438                    })?),
2439                    tz.clone(),
2440                ))
2441            }
2442            ScalarValue::TimestampNanosecond(Some(v), tz) => {
2443                Ok(ScalarValue::TimestampNanosecond(
2444                    Some(neg_checked_with_ctx(*v, || {
2445                        format!("In negation of TimestampNanoSecond({v})")
2446                    })?),
2447                    tz.clone(),
2448                ))
2449            }
2450            ScalarValue::TimestampMicrosecond(Some(v), tz) => {
2451                Ok(ScalarValue::TimestampMicrosecond(
2452                    Some(neg_checked_with_ctx(*v, || {
2453                        format!("In negation of TimestampMicroSecond({v})")
2454                    })?),
2455                    tz.clone(),
2456                ))
2457            }
2458            ScalarValue::TimestampMillisecond(Some(v), tz) => {
2459                Ok(ScalarValue::TimestampMillisecond(
2460                    Some(neg_checked_with_ctx(*v, || {
2461                        format!("In negation of TimestampMilliSecond({v})")
2462                    })?),
2463                    tz.clone(),
2464                ))
2465            }
2466            value => _internal_err!(
2467                "Can not run arithmetic negative on scalar value {value:?}"
2468            ),
2469        }
2470    }
2471
2472    /// Wrapping addition of `ScalarValue`
2473    ///
2474    /// NB: operating on `ScalarValue` directly is not efficient, performance sensitive code
2475    /// should operate on Arrays directly, using vectorized array kernels
2476    pub fn add<T: Borrow<ScalarValue>>(&self, other: T) -> Result<ScalarValue> {
2477        let other = other.borrow();
2478        if Self::can_use_direct_add(self, other) {
2479            let mut result = self.clone();
2480            if result.try_add_wrapping_in_place(other)? {
2481                return Ok(result);
2482            }
2483            debug_assert!(false, "fast-path eligibility drifted from implementation");
2484        }
2485
2486        let r = add_wrapping(&self.to_scalar()?, &other.to_scalar()?)?;
2487        Self::try_from_array(r.as_ref(), 0)
2488    }
2489
2490    /// Checked addition of `ScalarValue`
2491    ///
2492    /// NB: operating on `ScalarValue` directly is not efficient, performance sensitive code
2493    /// should operate on Arrays directly, using vectorized array kernels
2494    pub fn add_checked<T: Borrow<ScalarValue>>(&self, other: T) -> Result<ScalarValue> {
2495        let other = other.borrow();
2496        if Self::can_use_direct_add(self, other) {
2497            let mut result = self.clone();
2498            if result.try_add_checked_in_place(other)? {
2499                return Ok(result);
2500            }
2501            debug_assert!(false, "fast-path eligibility drifted from implementation");
2502        }
2503
2504        let r = add(&self.to_scalar()?, &other.to_scalar()?)?;
2505        Self::try_from_array(r.as_ref(), 0)
2506    }
2507
2508    /// Wrapping subtraction of `ScalarValue`
2509    ///
2510    /// NB: operating on `ScalarValue` directly is not efficient, performance sensitive code
2511    /// should operate on Arrays directly, using vectorized array kernels
2512    pub fn sub<T: Borrow<ScalarValue>>(&self, other: T) -> Result<ScalarValue> {
2513        let r = sub_wrapping(&self.to_scalar()?, &other.borrow().to_scalar()?)?;
2514        Self::try_from_array(r.as_ref(), 0)
2515    }
2516
2517    /// Checked subtraction of `ScalarValue`
2518    ///
2519    /// NB: operating on `ScalarValue` directly is not efficient, performance sensitive code
2520    /// should operate on Arrays directly, using vectorized array kernels
2521    pub fn sub_checked<T: Borrow<ScalarValue>>(&self, other: T) -> Result<ScalarValue> {
2522        let r = sub(&self.to_scalar()?, &other.borrow().to_scalar()?)?;
2523        Self::try_from_array(r.as_ref(), 0)
2524    }
2525
2526    /// Wrapping multiplication of `ScalarValue`
2527    ///
2528    /// NB: operating on `ScalarValue` directly is not efficient, performance sensitive code
2529    /// should operate on Arrays directly, using vectorized array kernels.
2530    pub fn mul<T: Borrow<ScalarValue>>(&self, other: T) -> Result<ScalarValue> {
2531        let r = mul_wrapping(&self.to_scalar()?, &other.borrow().to_scalar()?)?;
2532        Self::try_from_array(r.as_ref(), 0)
2533    }
2534
2535    /// Checked multiplication of `ScalarValue`
2536    ///
2537    /// NB: operating on `ScalarValue` directly is not efficient, performance sensitive code
2538    /// should operate on Arrays directly, using vectorized array kernels.
2539    pub fn mul_checked<T: Borrow<ScalarValue>>(&self, other: T) -> Result<ScalarValue> {
2540        let r = mul(&self.to_scalar()?, &other.borrow().to_scalar()?)?;
2541        Self::try_from_array(r.as_ref(), 0)
2542    }
2543
2544    /// Performs `lhs / rhs`
2545    ///
2546    /// Overflow or division by zero will result in an error, with exception to
2547    /// floating point numbers, which instead follow the IEEE 754 rules.
2548    ///
2549    /// NB: operating on `ScalarValue` directly is not efficient, performance sensitive code
2550    /// should operate on Arrays directly, using vectorized array kernels.
2551    pub fn div<T: Borrow<ScalarValue>>(&self, other: T) -> Result<ScalarValue> {
2552        let r = div(&self.to_scalar()?, &other.borrow().to_scalar()?)?;
2553        Self::try_from_array(r.as_ref(), 0)
2554    }
2555
2556    /// Performs `lhs % rhs`
2557    ///
2558    /// Overflow or division by zero will result in an error, with exception to
2559    /// floating point numbers, which instead follow the IEEE 754 rules.
2560    ///
2561    /// NB: operating on `ScalarValue` directly is not efficient, performance sensitive code
2562    /// should operate on Arrays directly, using vectorized array kernels.
2563    pub fn rem<T: Borrow<ScalarValue>>(&self, other: T) -> Result<ScalarValue> {
2564        let r = rem(&self.to_scalar()?, &other.borrow().to_scalar()?)?;
2565        Self::try_from_array(r.as_ref(), 0)
2566    }
2567
2568    pub fn is_unsigned(&self) -> bool {
2569        matches!(
2570            self,
2571            ScalarValue::UInt8(_)
2572                | ScalarValue::UInt16(_)
2573                | ScalarValue::UInt32(_)
2574                | ScalarValue::UInt64(_)
2575        )
2576    }
2577
2578    /// whether this value is null or not.
2579    pub fn is_null(&self) -> bool {
2580        match self {
2581            ScalarValue::Boolean(v) => v.is_none(),
2582            ScalarValue::Null => true,
2583            ScalarValue::Float16(v) => v.is_none(),
2584            ScalarValue::Float32(v) => v.is_none(),
2585            ScalarValue::Float64(v) => v.is_none(),
2586            ScalarValue::Decimal32(v, _, _) => v.is_none(),
2587            ScalarValue::Decimal64(v, _, _) => v.is_none(),
2588            ScalarValue::Decimal128(v, _, _) => v.is_none(),
2589            ScalarValue::Decimal256(v, _, _) => v.is_none(),
2590            ScalarValue::Int8(v) => v.is_none(),
2591            ScalarValue::Int16(v) => v.is_none(),
2592            ScalarValue::Int32(v) => v.is_none(),
2593            ScalarValue::Int64(v) => v.is_none(),
2594            ScalarValue::UInt8(v) => v.is_none(),
2595            ScalarValue::UInt16(v) => v.is_none(),
2596            ScalarValue::UInt32(v) => v.is_none(),
2597            ScalarValue::UInt64(v) => v.is_none(),
2598            ScalarValue::Utf8(v)
2599            | ScalarValue::Utf8View(v)
2600            | ScalarValue::LargeUtf8(v) => v.is_none(),
2601            ScalarValue::Binary(v)
2602            | ScalarValue::BinaryView(v)
2603            | ScalarValue::FixedSizeBinary(_, v)
2604            | ScalarValue::LargeBinary(v) => v.is_none(),
2605            // arr.len() should be 1 for a list scalar, but we don't seem to
2606            // enforce that anywhere, so we still check against array length.
2607            ScalarValue::List(arr) => arr.len() == arr.null_count(),
2608            ScalarValue::LargeList(arr) => arr.len() == arr.null_count(),
2609            ScalarValue::FixedSizeList(arr) => arr.len() == arr.null_count(),
2610            ScalarValue::ListView(arr) => arr.len() == arr.null_count(),
2611            ScalarValue::LargeListView(arr) => arr.len() == arr.null_count(),
2612            ScalarValue::Struct(arr) => arr.len() == arr.null_count(),
2613            ScalarValue::Map(arr) => arr.len() == arr.null_count(),
2614            ScalarValue::Date32(v) => v.is_none(),
2615            ScalarValue::Date64(v) => v.is_none(),
2616            ScalarValue::Time32Second(v) => v.is_none(),
2617            ScalarValue::Time32Millisecond(v) => v.is_none(),
2618            ScalarValue::Time64Microsecond(v) => v.is_none(),
2619            ScalarValue::Time64Nanosecond(v) => v.is_none(),
2620            ScalarValue::TimestampSecond(v, _) => v.is_none(),
2621            ScalarValue::TimestampMillisecond(v, _) => v.is_none(),
2622            ScalarValue::TimestampMicrosecond(v, _) => v.is_none(),
2623            ScalarValue::TimestampNanosecond(v, _) => v.is_none(),
2624            ScalarValue::IntervalYearMonth(v) => v.is_none(),
2625            ScalarValue::IntervalDayTime(v) => v.is_none(),
2626            ScalarValue::IntervalMonthDayNano(v) => v.is_none(),
2627            ScalarValue::DurationSecond(v) => v.is_none(),
2628            ScalarValue::DurationMillisecond(v) => v.is_none(),
2629            ScalarValue::DurationMicrosecond(v) => v.is_none(),
2630            ScalarValue::DurationNanosecond(v) => v.is_none(),
2631            ScalarValue::Union(v, _, _) => match v {
2632                Some((_, s)) => s.is_null(),
2633                None => true,
2634            },
2635            ScalarValue::Dictionary(_, v) => v.is_null(),
2636            ScalarValue::RunEndEncoded(_, _, v) => v.is_null(),
2637        }
2638    }
2639
2640    /// Absolute distance between two numeric values (of the same type). This method will return
2641    /// None if either one of the arguments are null. It might also return None if the resulting
2642    /// distance is greater than [`usize::MAX`]. If the type is a float, then the distance will be
2643    /// rounded to the nearest integer.
2644    ///
2645    /// Note: the datatype itself must support subtraction.
2646    pub fn distance(&self, other: &ScalarValue) -> Option<usize> {
2647        self.distance_u64(other)
2648            .and_then(|d| usize::try_from(d).ok())
2649    }
2650
2651    /// Helper to convert a rounded float distance to u64, returning None if it exceeds u64::MAX, is negative, or is not finite.
2652    fn rounded_float_distance_u64(diff: f64) -> Option<u64> {
2653        if diff.is_finite() && diff >= 0.0 && diff < u64::MAX as f64 {
2654            Some(diff as u64)
2655        } else {
2656            None
2657        }
2658    }
2659
2660    /// Absolute distance between two numeric values (of the same type). This method will return
2661    /// None if either one of the arguments are null. It might also return None if the resulting
2662    /// distance is greater than [`u64::MAX`]. If the type is a float, then the distance will be
2663    /// rounded to the nearest integer.
2664    ///
2665    /// Note: the datatype itself must support subtraction.
2666    pub fn distance_u64(&self, other: &ScalarValue) -> Option<u64> {
2667        match (self, other) {
2668            (Self::Int8(Some(l)), Self::Int8(Some(r))) => Some(l.abs_diff(*r) as u64),
2669            (Self::Int16(Some(l)), Self::Int16(Some(r))) => Some(l.abs_diff(*r) as u64),
2670            (Self::Int32(Some(l)), Self::Int32(Some(r))) => Some(l.abs_diff(*r) as u64),
2671            (Self::Int64(Some(l)), Self::Int64(Some(r))) => Some(l.abs_diff(*r)),
2672            (Self::UInt8(Some(l)), Self::UInt8(Some(r))) => Some(l.abs_diff(*r) as u64),
2673            (Self::UInt16(Some(l)), Self::UInt16(Some(r))) => Some(l.abs_diff(*r) as u64),
2674            (Self::UInt32(Some(l)), Self::UInt32(Some(r))) => Some(l.abs_diff(*r) as u64),
2675            (Self::UInt64(Some(l)), Self::UInt64(Some(r))) => Some(l.abs_diff(*r)),
2676            // TODO: we might want to look into supporting ceil/floor here for floats.
2677            (Self::Float16(Some(l)), Self::Float16(Some(r))) => {
2678                let diff = (f16::to_f32(*l) - f16::to_f32(*r)).abs().round();
2679                Self::rounded_float_distance_u64(diff as f64)
2680            }
2681            (Self::Float32(Some(l)), Self::Float32(Some(r))) => {
2682                let diff = (l - r).abs().round();
2683                Self::rounded_float_distance_u64(diff as f64)
2684            }
2685            (Self::Float64(Some(l)), Self::Float64(Some(r))) => {
2686                let diff = (l - r).abs().round();
2687                Self::rounded_float_distance_u64(diff)
2688            }
2689            (Self::Date32(Some(l)), Self::Date32(Some(r))) => Some(l.abs_diff(*r) as u64),
2690            (Self::Date64(Some(l)), Self::Date64(Some(r))) => Some(l.abs_diff(*r)),
2691            // Timestamp values are stored as epoch ticks regardless of timezone
2692            // annotation, so the distance is tz-independent (tz is display metadata).
2693            (Self::TimestampSecond(Some(l), _), Self::TimestampSecond(Some(r), _)) => {
2694                Some(l.abs_diff(*r))
2695            }
2696            (
2697                Self::TimestampMillisecond(Some(l), _),
2698                Self::TimestampMillisecond(Some(r), _),
2699            ) => Some(l.abs_diff(*r)),
2700            (
2701                Self::TimestampMicrosecond(Some(l), _),
2702                Self::TimestampMicrosecond(Some(r), _),
2703            ) => Some(l.abs_diff(*r)),
2704            (
2705                Self::TimestampNanosecond(Some(l), _),
2706                Self::TimestampNanosecond(Some(r), _),
2707            ) => Some(l.abs_diff(*r)),
2708            (
2709                Self::Decimal32(Some(l), _, lscale),
2710                Self::Decimal32(Some(r), _, rscale),
2711            ) => {
2712                // In order to be aligned with PartialOrd we only
2713                // check for equal scale, ignoring precision
2714                if lscale == rscale {
2715                    Some(l.abs_diff(*r) as u64)
2716                } else {
2717                    None
2718                }
2719            }
2720            (
2721                Self::Decimal64(Some(l), _, lscale),
2722                Self::Decimal64(Some(r), _, rscale),
2723            ) => {
2724                if lscale == rscale {
2725                    Some(l.abs_diff(*r))
2726                } else {
2727                    None
2728                }
2729            }
2730            (
2731                Self::Decimal128(Some(l), _, lscale),
2732                Self::Decimal128(Some(r), _, rscale),
2733            ) => {
2734                if lscale == rscale {
2735                    l.checked_sub(*r)?.checked_abs()?.to_u64()
2736                } else {
2737                    None
2738                }
2739            }
2740            (
2741                Self::Decimal256(Some(l), _, lscale),
2742                Self::Decimal256(Some(r), _, rscale),
2743            ) => {
2744                if lscale == rscale {
2745                    l.checked_sub(*r)?.checked_abs()?.to_u64()
2746                } else {
2747                    None
2748                }
2749            }
2750            _ => None,
2751        }
2752    }
2753
2754    /// Converts a scalar value into an 1-row array.
2755    ///
2756    /// # Errors
2757    ///
2758    /// Errors if the ScalarValue cannot be converted into a 1-row array
2759    pub fn to_array(&self) -> Result<ArrayRef> {
2760        self.to_array_of_size(1)
2761    }
2762
2763    /// Converts a scalar into an arrow [`Scalar`] (which implements
2764    /// the [`Datum`] interface).
2765    ///
2766    /// This can be used to call arrow compute kernels such as `lt`
2767    ///
2768    /// # Errors
2769    ///
2770    /// Errors if the ScalarValue cannot be converted into a 1-row array
2771    ///
2772    /// # Example
2773    /// ```
2774    /// use arrow::array::{BooleanArray, Int32Array};
2775    /// use datafusion_common::ScalarValue;
2776    ///
2777    /// let arr = Int32Array::from(vec![Some(1), None, Some(10)]);
2778    /// let five = ScalarValue::Int32(Some(5));
2779    ///
2780    /// let result =
2781    ///     arrow::compute::kernels::cmp::lt(&arr, &five.to_scalar().unwrap()).unwrap();
2782    ///
2783    /// let expected = BooleanArray::from(vec![Some(true), None, Some(false)]);
2784    ///
2785    /// assert_eq!(&result, &expected);
2786    /// ```
2787    /// [`Datum`]: arrow::array::Datum
2788    pub fn to_scalar(&self) -> Result<Scalar<ArrayRef>> {
2789        Ok(Scalar::new(self.to_array_of_size(1)?))
2790    }
2791
2792    /// Converts an iterator of references [`ScalarValue`] into an [`ArrayRef`]
2793    /// corresponding to those values. For example, an iterator of
2794    /// [`ScalarValue::Int32`] would be converted to an [`Int32Array`].
2795    ///
2796    /// Returns an error if the iterator is empty or if the
2797    /// [`ScalarValue`]s are not all the same type
2798    ///
2799    /// # Example
2800    /// ```
2801    /// use arrow::array::{ArrayRef, BooleanArray};
2802    /// use datafusion_common::ScalarValue;
2803    ///
2804    /// let scalars = vec![
2805    ///     ScalarValue::Boolean(Some(true)),
2806    ///     ScalarValue::Boolean(None),
2807    ///     ScalarValue::Boolean(Some(false)),
2808    /// ];
2809    ///
2810    /// // Build an Array from the list of ScalarValues
2811    /// let array = ScalarValue::iter_to_array(scalars.into_iter()).unwrap();
2812    ///
2813    /// let expected: ArrayRef =
2814    ///     std::sync::Arc::new(BooleanArray::from(vec![Some(true), None, Some(false)]));
2815    ///
2816    /// assert_eq!(&array, &expected);
2817    /// ```
2818    pub fn iter_to_array(
2819        scalars: impl IntoIterator<Item = ScalarValue>,
2820    ) -> Result<ArrayRef> {
2821        let mut scalars = scalars.into_iter().peekable();
2822
2823        // figure out the type based on the first element
2824        let data_type = match scalars.peek() {
2825            None => {
2826                return _exec_err!("Empty iterator passed to ScalarValue::iter_to_array");
2827            }
2828            Some(sv) => sv.data_type(),
2829        };
2830
2831        /// Creates an array of $ARRAY_TY by unpacking values of
2832        /// SCALAR_TY for primitive types
2833        macro_rules! build_array_primitive {
2834            ($ARRAY_TY:ident, $SCALAR_TY:ident) => {{
2835                {
2836                    let array = scalars
2837                        .map(|sv| {
2838                            if let ScalarValue::$SCALAR_TY(v) = sv {
2839                                Ok(v)
2840                            } else {
2841                                _exec_err!(
2842                                    "Inconsistent types in ScalarValue::iter_to_array. \
2843                                    Expected {:?}, got {:?}",
2844                                    data_type,
2845                                    sv
2846                                )
2847                            }
2848                        })
2849                        .collect::<Result<$ARRAY_TY>>()?;
2850                    Arc::new(array)
2851                }
2852            }};
2853        }
2854
2855        macro_rules! build_array_primitive_tz {
2856            ($ARRAY_TY:ident, $SCALAR_TY:ident, $TZ:expr) => {{
2857                {
2858                    let array = scalars
2859                        .map(|sv| {
2860                            if let ScalarValue::$SCALAR_TY(v, _) = sv {
2861                                Ok(v)
2862                            } else {
2863                                _exec_err!(
2864                                    "Inconsistent types in ScalarValue::iter_to_array. \
2865                                    Expected {:?}, got {:?}",
2866                                    data_type,
2867                                    sv
2868                                )
2869                            }
2870                        })
2871                        .collect::<Result<$ARRAY_TY>>()?;
2872                    Arc::new(array.with_timezone_opt($TZ.clone()))
2873                }
2874            }};
2875        }
2876
2877        /// Creates an array of $ARRAY_TY by unpacking values of
2878        /// SCALAR_TY for "string-like" types.
2879        macro_rules! build_array_string {
2880            ($ARRAY_TY:ident, $SCALAR_TY:ident) => {{
2881                {
2882                    let array = scalars
2883                        .map(|sv| {
2884                            if let ScalarValue::$SCALAR_TY(v) = sv {
2885                                Ok(v)
2886                            } else {
2887                                _exec_err!(
2888                                    "Inconsistent types in ScalarValue::iter_to_array. \
2889                                    Expected {:?}, got {:?}",
2890                                    data_type,
2891                                    sv
2892                                )
2893                            }
2894                        })
2895                        .collect::<Result<$ARRAY_TY>>()?;
2896                    Arc::new(array)
2897                }
2898            }};
2899        }
2900
2901        let array: ArrayRef = match &data_type {
2902            DataType::Decimal32(precision, scale) => {
2903                let decimal_array =
2904                    ScalarValue::iter_to_decimal32_array(scalars, *precision, *scale)?;
2905                Arc::new(decimal_array)
2906            }
2907            DataType::Decimal64(precision, scale) => {
2908                let decimal_array =
2909                    ScalarValue::iter_to_decimal64_array(scalars, *precision, *scale)?;
2910                Arc::new(decimal_array)
2911            }
2912            DataType::Decimal128(precision, scale) => {
2913                let decimal_array =
2914                    ScalarValue::iter_to_decimal128_array(scalars, *precision, *scale)?;
2915                Arc::new(decimal_array)
2916            }
2917            DataType::Decimal256(precision, scale) => {
2918                let decimal_array =
2919                    ScalarValue::iter_to_decimal256_array(scalars, *precision, *scale)?;
2920                Arc::new(decimal_array)
2921            }
2922            DataType::Null => ScalarValue::iter_to_null_array(scalars)?,
2923            DataType::Boolean => build_array_primitive!(BooleanArray, Boolean),
2924            DataType::Float16 => build_array_primitive!(Float16Array, Float16),
2925            DataType::Float32 => build_array_primitive!(Float32Array, Float32),
2926            DataType::Float64 => build_array_primitive!(Float64Array, Float64),
2927            DataType::Int8 => build_array_primitive!(Int8Array, Int8),
2928            DataType::Int16 => build_array_primitive!(Int16Array, Int16),
2929            DataType::Int32 => build_array_primitive!(Int32Array, Int32),
2930            DataType::Int64 => build_array_primitive!(Int64Array, Int64),
2931            DataType::UInt8 => build_array_primitive!(UInt8Array, UInt8),
2932            DataType::UInt16 => build_array_primitive!(UInt16Array, UInt16),
2933            DataType::UInt32 => build_array_primitive!(UInt32Array, UInt32),
2934            DataType::UInt64 => build_array_primitive!(UInt64Array, UInt64),
2935            DataType::Utf8View => build_array_string!(StringViewArray, Utf8View),
2936            DataType::Utf8 => build_array_string!(StringArray, Utf8),
2937            DataType::LargeUtf8 => build_array_string!(LargeStringArray, LargeUtf8),
2938            DataType::BinaryView => build_array_string!(BinaryViewArray, BinaryView),
2939            DataType::Binary => build_array_string!(BinaryArray, Binary),
2940            DataType::LargeBinary => build_array_string!(LargeBinaryArray, LargeBinary),
2941            DataType::Date32 => build_array_primitive!(Date32Array, Date32),
2942            DataType::Date64 => build_array_primitive!(Date64Array, Date64),
2943            DataType::Time32(TimeUnit::Second) => {
2944                build_array_primitive!(Time32SecondArray, Time32Second)
2945            }
2946            DataType::Time32(TimeUnit::Millisecond) => {
2947                build_array_primitive!(Time32MillisecondArray, Time32Millisecond)
2948            }
2949            DataType::Time64(TimeUnit::Microsecond) => {
2950                build_array_primitive!(Time64MicrosecondArray, Time64Microsecond)
2951            }
2952            DataType::Time64(TimeUnit::Nanosecond) => {
2953                build_array_primitive!(Time64NanosecondArray, Time64Nanosecond)
2954            }
2955            DataType::Timestamp(TimeUnit::Second, tz) => {
2956                build_array_primitive_tz!(TimestampSecondArray, TimestampSecond, tz)
2957            }
2958            DataType::Timestamp(TimeUnit::Millisecond, tz) => {
2959                build_array_primitive_tz!(
2960                    TimestampMillisecondArray,
2961                    TimestampMillisecond,
2962                    tz
2963                )
2964            }
2965            DataType::Timestamp(TimeUnit::Microsecond, tz) => {
2966                build_array_primitive_tz!(
2967                    TimestampMicrosecondArray,
2968                    TimestampMicrosecond,
2969                    tz
2970                )
2971            }
2972            DataType::Timestamp(TimeUnit::Nanosecond, tz) => {
2973                build_array_primitive_tz!(
2974                    TimestampNanosecondArray,
2975                    TimestampNanosecond,
2976                    tz
2977                )
2978            }
2979            DataType::Duration(TimeUnit::Second) => {
2980                build_array_primitive!(DurationSecondArray, DurationSecond)
2981            }
2982            DataType::Duration(TimeUnit::Millisecond) => {
2983                build_array_primitive!(DurationMillisecondArray, DurationMillisecond)
2984            }
2985            DataType::Duration(TimeUnit::Microsecond) => {
2986                build_array_primitive!(DurationMicrosecondArray, DurationMicrosecond)
2987            }
2988            DataType::Duration(TimeUnit::Nanosecond) => {
2989                build_array_primitive!(DurationNanosecondArray, DurationNanosecond)
2990            }
2991            DataType::Interval(IntervalUnit::DayTime) => {
2992                build_array_primitive!(IntervalDayTimeArray, IntervalDayTime)
2993            }
2994            DataType::Interval(IntervalUnit::YearMonth) => {
2995                build_array_primitive!(IntervalYearMonthArray, IntervalYearMonth)
2996            }
2997            DataType::Interval(IntervalUnit::MonthDayNano) => {
2998                build_array_primitive!(IntervalMonthDayNanoArray, IntervalMonthDayNano)
2999            }
3000            DataType::FixedSizeList(_, _) => {
3001                // arrow::compute::concat does not allow inconsistent types including the size of FixedSizeList.
3002                // The length of nulls here we got is 1, so we need to resize the length of nulls to
3003                // the length of non-nulls.
3004                let mut arrays =
3005                    scalars.map(|s| s.to_array()).collect::<Result<Vec<_>>>()?;
3006                let first_non_null_data_type = arrays
3007                    .iter()
3008                    .find(|sv| !sv.is_null(0))
3009                    .map(|sv| sv.data_type().to_owned());
3010                if let Some(DataType::FixedSizeList(f, l)) = first_non_null_data_type {
3011                    for array in arrays.iter_mut() {
3012                        if array.is_null(0) {
3013                            *array = Arc::new(FixedSizeListArray::new_null(
3014                                Arc::clone(&f),
3015                                l,
3016                                1,
3017                            ));
3018                        }
3019                    }
3020                }
3021                let arrays = arrays.iter().map(|a| a.as_ref()).collect::<Vec<_>>();
3022                arrow::compute::concat(arrays.as_slice())?
3023            }
3024            DataType::List(_)
3025            | DataType::LargeList(_)
3026            | DataType::ListView(_)
3027            | DataType::LargeListView(_)
3028            | DataType::Map(_, _)
3029            | DataType::Struct(_)
3030            | DataType::Union(_, _) => {
3031                let arrays = scalars.map(|s| s.to_array()).collect::<Result<Vec<_>>>()?;
3032                let arrays = arrays.iter().map(|a| a.as_ref()).collect::<Vec<_>>();
3033                arrow::compute::concat(arrays.as_slice())?
3034            }
3035            DataType::Dictionary(key_type, value_type) => {
3036                // create the values array
3037                let value_scalars = scalars
3038                    .map(|scalar| match scalar {
3039                        ScalarValue::Dictionary(inner_key_type, scalar) => {
3040                            if &inner_key_type == key_type {
3041                                Ok(*scalar)
3042                            } else {
3043                                _exec_err!("Expected inner key type of {key_type} but found: {inner_key_type}, value was ({scalar:?})")
3044                            }
3045                        }
3046                        _ => {
3047                            _exec_err!(
3048                                "Expected scalar of type {value_type} but found: {scalar} {scalar:?}"
3049                            )
3050                        }
3051                    })
3052                    .collect::<Result<Vec<_>>>()?;
3053
3054                let values = Self::iter_to_array(value_scalars)?;
3055                assert_eq!(values.data_type(), value_type.as_ref());
3056
3057                match key_type.as_ref() {
3058                    DataType::Int8 => dict_from_values::<Int8Type>(values)?,
3059                    DataType::Int16 => dict_from_values::<Int16Type>(values)?,
3060                    DataType::Int32 => dict_from_values::<Int32Type>(values)?,
3061                    DataType::Int64 => dict_from_values::<Int64Type>(values)?,
3062                    DataType::UInt8 => dict_from_values::<UInt8Type>(values)?,
3063                    DataType::UInt16 => dict_from_values::<UInt16Type>(values)?,
3064                    DataType::UInt32 => dict_from_values::<UInt32Type>(values)?,
3065                    DataType::UInt64 => dict_from_values::<UInt64Type>(values)?,
3066                    _ => unreachable!("Invalid dictionary keys type: {}", key_type),
3067                }
3068            }
3069            DataType::RunEndEncoded(run_ends_field, value_field) => {
3070                fn make_run_array<R: RunEndIndexType>(
3071                    scalars: impl IntoIterator<Item = ScalarValue>,
3072                    run_ends_field: &FieldRef,
3073                    values_field: &FieldRef,
3074                ) -> Result<ArrayRef> {
3075                    let mut scalars = scalars.into_iter();
3076
3077                    let mut run_ends = vec![];
3078                    let mut value_scalars = vec![];
3079
3080                    let mut len = R::Native::ONE;
3081                    let mut current =
3082                        if let Some(ScalarValue::RunEndEncoded(_, _, scalar)) =
3083                            scalars.next()
3084                        {
3085                            *scalar
3086                        } else {
3087                            // We are guaranteed to have one element of correct
3088                            // type because we peeked above
3089                            unreachable!()
3090                        };
3091                    for scalar in scalars {
3092                        let scalar = match scalar {
3093                            ScalarValue::RunEndEncoded(
3094                                inner_run_ends_field,
3095                                inner_value_field,
3096                                scalar,
3097                            ) if &inner_run_ends_field == run_ends_field
3098                                && &inner_value_field == values_field =>
3099                            {
3100                                *scalar
3101                            }
3102                            _ => {
3103                                return _exec_err!(
3104                                    "Expected RunEndEncoded scalar with run-ends field {run_ends_field} but got: {scalar:?}"
3105                                );
3106                            }
3107                        };
3108
3109                        // new run
3110                        if scalar != current {
3111                            run_ends.push(len);
3112                            value_scalars.push(current);
3113                            current = scalar;
3114                        }
3115
3116                        len = len.add_checked(R::Native::ONE).map_err(|_| {
3117                            DataFusionError::Execution(format!(
3118                                "Cannot construct RunArray: Overflows run-ends type {}",
3119                                run_ends_field.data_type()
3120                            ))
3121                        })?;
3122                    }
3123
3124                    run_ends.push(len);
3125                    value_scalars.push(current);
3126
3127                    let run_ends = PrimitiveArray::<R>::from_iter_values(run_ends);
3128                    let values = ScalarValue::iter_to_array(value_scalars)?;
3129
3130                    // Using ArrayDataBuilder so we can maintain the fields
3131                    let dt = DataType::RunEndEncoded(
3132                        Arc::clone(run_ends_field),
3133                        Arc::clone(values_field),
3134                    );
3135                    let builder = ArrayDataBuilder::new(dt)
3136                        .len(RunArray::logical_len(&run_ends))
3137                        .add_child_data(run_ends.to_data())
3138                        .add_child_data(values.to_data());
3139                    let run_array = RunArray::<R>::from(builder.build()?);
3140
3141                    Ok(Arc::new(run_array))
3142                }
3143
3144                match run_ends_field.data_type() {
3145                    DataType::Int16 => {
3146                        make_run_array::<Int16Type>(scalars, run_ends_field, value_field)?
3147                    }
3148                    DataType::Int32 => {
3149                        make_run_array::<Int32Type>(scalars, run_ends_field, value_field)?
3150                    }
3151                    DataType::Int64 => {
3152                        make_run_array::<Int64Type>(scalars, run_ends_field, value_field)?
3153                    }
3154                    dt => unreachable!("Invalid run-ends type: {dt}"),
3155                }
3156            }
3157            DataType::FixedSizeBinary(size) => {
3158                let array = scalars
3159                    .map(|sv| {
3160                        if let ScalarValue::FixedSizeBinary(_, v) = sv {
3161                            Ok(v)
3162                        } else {
3163                            _exec_err!(
3164                                "Inconsistent types in ScalarValue::iter_to_array. \
3165                                Expected {data_type}, got {sv:?}"
3166                            )
3167                        }
3168                    })
3169                    .collect::<Result<Vec<_>>>()?;
3170                let array = FixedSizeBinaryArray::try_from_sparse_iter_with_size(
3171                    array.into_iter(),
3172                    *size,
3173                )?;
3174                Arc::new(array)
3175            }
3176            // explicitly enumerate unsupported types so newly added
3177            // types must be acknowledged, Time32 and Time64 types are
3178            // not supported if the TimeUnit is not valid (Time32 can
3179            // only be used with Second and Millisecond, Time64 only
3180            // with Microsecond and Nanosecond)
3181            DataType::Time32(TimeUnit::Microsecond)
3182            | DataType::Time32(TimeUnit::Nanosecond)
3183            | DataType::Time64(TimeUnit::Second)
3184            | DataType::Time64(TimeUnit::Millisecond) => {
3185                return _not_impl_err!(
3186                    "Unsupported creation of {:?} array from ScalarValue {:?}",
3187                    data_type,
3188                    scalars.peek()
3189                );
3190            }
3191        };
3192        Ok(array)
3193    }
3194
3195    fn iter_to_null_array(
3196        scalars: impl IntoIterator<Item = ScalarValue>,
3197    ) -> Result<ArrayRef> {
3198        let length = scalars.into_iter().try_fold(
3199            0usize,
3200            |r, element: ScalarValue| match element {
3201                ScalarValue::Null => Ok::<usize, DataFusionError>(r + 1),
3202                s => {
3203                    _internal_err!("Expected ScalarValue::Null element. Received {s:?}")
3204                }
3205            },
3206        )?;
3207        Ok(new_null_array(&DataType::Null, length))
3208    }
3209
3210    fn iter_to_decimal32_array(
3211        scalars: impl IntoIterator<Item = ScalarValue>,
3212        precision: u8,
3213        scale: i8,
3214    ) -> Result<Decimal32Array> {
3215        let array = scalars
3216            .into_iter()
3217            .map(|element: ScalarValue| match element {
3218                ScalarValue::Decimal32(v1, _, _) => Ok(v1),
3219                s => {
3220                    _internal_err!("Expected ScalarValue::Null element. Received {s:?}")
3221                }
3222            })
3223            .collect::<Result<Decimal32Array>>()?
3224            .with_precision_and_scale(precision, scale)?;
3225        Ok(array)
3226    }
3227
3228    fn iter_to_decimal64_array(
3229        scalars: impl IntoIterator<Item = ScalarValue>,
3230        precision: u8,
3231        scale: i8,
3232    ) -> Result<Decimal64Array> {
3233        let array = scalars
3234            .into_iter()
3235            .map(|element: ScalarValue| match element {
3236                ScalarValue::Decimal64(v1, _, _) => Ok(v1),
3237                s => {
3238                    _internal_err!("Expected ScalarValue::Null element. Received {s:?}")
3239                }
3240            })
3241            .collect::<Result<Decimal64Array>>()?
3242            .with_precision_and_scale(precision, scale)?;
3243        Ok(array)
3244    }
3245
3246    fn iter_to_decimal128_array(
3247        scalars: impl IntoIterator<Item = ScalarValue>,
3248        precision: u8,
3249        scale: i8,
3250    ) -> Result<Decimal128Array> {
3251        let array = scalars
3252            .into_iter()
3253            .map(|element: ScalarValue| match element {
3254                ScalarValue::Decimal128(v1, _, _) => Ok(v1),
3255                s => {
3256                    _internal_err!("Expected ScalarValue::Null element. Received {s:?}")
3257                }
3258            })
3259            .collect::<Result<Decimal128Array>>()?
3260            .with_precision_and_scale(precision, scale)?;
3261        Ok(array)
3262    }
3263
3264    fn iter_to_decimal256_array(
3265        scalars: impl IntoIterator<Item = ScalarValue>,
3266        precision: u8,
3267        scale: i8,
3268    ) -> Result<Decimal256Array> {
3269        let array = scalars
3270            .into_iter()
3271            .map(|element: ScalarValue| match element {
3272                ScalarValue::Decimal256(v1, _, _) => Ok(v1),
3273                s => {
3274                    _internal_err!(
3275                        "Expected ScalarValue::Decimal256 element. Received {s:?}"
3276                    )
3277                }
3278            })
3279            .collect::<Result<Decimal256Array>>()?
3280            .with_precision_and_scale(precision, scale)?;
3281        Ok(array)
3282    }
3283
3284    /// Converts `Vec<ScalarValue>` where each element has type corresponding to
3285    /// `data_type`, to a single element [`ListArray`].
3286    ///
3287    /// Example
3288    /// ```
3289    /// use arrow::array::{Int32Array, ListArray};
3290    /// use arrow::datatypes::{DataType, Int32Type};
3291    /// use datafusion_common::cast::as_list_array;
3292    /// use datafusion_common::ScalarValue;
3293    ///
3294    /// let scalars = vec![
3295    ///     ScalarValue::Int32(Some(1)),
3296    ///     ScalarValue::Int32(None),
3297    ///     ScalarValue::Int32(Some(2)),
3298    /// ];
3299    ///
3300    /// let result = ScalarValue::new_list(&scalars, &DataType::Int32, true);
3301    ///
3302    /// let expected = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![Some(vec![
3303    ///     Some(1),
3304    ///     None,
3305    ///     Some(2),
3306    /// ])]);
3307    ///
3308    /// assert_eq!(*result, expected);
3309    /// ```
3310    pub fn new_list(
3311        values: &[ScalarValue],
3312        data_type: &DataType,
3313        nullable: bool,
3314    ) -> Arc<ListArray> {
3315        let values = if values.is_empty() {
3316            new_empty_array(data_type)
3317        } else {
3318            let arr = Self::iter_to_array(values.iter().cloned()).unwrap();
3319            cast_with_options(&arr, data_type, &DEFAULT_CAST_OPTIONS).unwrap()
3320        };
3321        Arc::new(
3322            SingleRowListArrayBuilder::new(values)
3323                .with_nullable(nullable)
3324                .build_list_array(),
3325        )
3326    }
3327
3328    /// Same as [`ScalarValue::new_list`] but with nullable set to true.
3329    pub fn new_list_nullable(
3330        values: &[ScalarValue],
3331        data_type: &DataType,
3332    ) -> Arc<ListArray> {
3333        Self::new_list(values, data_type, true)
3334    }
3335
3336    /// Create ListArray with Null with specific data type
3337    ///
3338    /// - new_null_list(i32, nullable, 1): `ListArray[NULL]`
3339    pub fn new_null_list(data_type: DataType, nullable: bool, null_len: usize) -> Self {
3340        let data_type = DataType::List(Field::new_list_field(data_type, nullable).into());
3341        Self::List(Arc::new(ListArray::from(ArrayData::new_null(
3342            &data_type, null_len,
3343        ))))
3344    }
3345
3346    /// Converts `IntoIterator<Item = ScalarValue>` where each element has type corresponding to
3347    /// `data_type`, to a [`ListArray`].
3348    ///
3349    /// Example
3350    /// ```
3351    /// use arrow::array::{Int32Array, ListArray};
3352    /// use arrow::datatypes::{DataType, Int32Type};
3353    /// use datafusion_common::cast::as_list_array;
3354    /// use datafusion_common::ScalarValue;
3355    ///
3356    /// let scalars = vec![
3357    ///     ScalarValue::Int32(Some(1)),
3358    ///     ScalarValue::Int32(None),
3359    ///     ScalarValue::Int32(Some(2)),
3360    /// ];
3361    ///
3362    /// let result =
3363    ///     ScalarValue::new_list_from_iter(scalars.into_iter(), &DataType::Int32, true);
3364    ///
3365    /// let expected = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![Some(vec![
3366    ///     Some(1),
3367    ///     None,
3368    ///     Some(2),
3369    /// ])]);
3370    ///
3371    /// assert_eq!(*result, expected);
3372    /// ```
3373    pub fn new_list_from_iter(
3374        values: impl IntoIterator<Item = ScalarValue> + ExactSizeIterator,
3375        data_type: &DataType,
3376        nullable: bool,
3377    ) -> Arc<ListArray> {
3378        let values = if values.len() == 0 {
3379            new_empty_array(data_type)
3380        } else {
3381            let arr = Self::iter_to_array(values).unwrap();
3382            cast_with_options(&arr, data_type, &DEFAULT_CAST_OPTIONS).unwrap()
3383        };
3384        Arc::new(
3385            SingleRowListArrayBuilder::new(values)
3386                .with_nullable(nullable)
3387                .build_list_array(),
3388        )
3389    }
3390
3391    /// Converts `Vec<ScalarValue>` where each element has type corresponding to
3392    /// `data_type`, to a [`LargeListArray`].
3393    ///
3394    /// Example
3395    /// ```
3396    /// use arrow::array::{Int32Array, LargeListArray};
3397    /// use arrow::datatypes::{DataType, Int32Type};
3398    /// use datafusion_common::cast::as_large_list_array;
3399    /// use datafusion_common::ScalarValue;
3400    ///
3401    /// let scalars = vec![
3402    ///     ScalarValue::Int32(Some(1)),
3403    ///     ScalarValue::Int32(None),
3404    ///     ScalarValue::Int32(Some(2)),
3405    /// ];
3406    ///
3407    /// let result = ScalarValue::new_large_list(&scalars, &DataType::Int32);
3408    ///
3409    /// let expected =
3410    ///     LargeListArray::from_iter_primitive::<Int32Type, _, _>(vec![Some(vec![
3411    ///         Some(1),
3412    ///         None,
3413    ///         Some(2),
3414    ///     ])]);
3415    ///
3416    /// assert_eq!(*result, expected);
3417    /// ```
3418    pub fn new_large_list(
3419        values: &[ScalarValue],
3420        data_type: &DataType,
3421    ) -> Arc<LargeListArray> {
3422        let values = if values.is_empty() {
3423            new_empty_array(data_type)
3424        } else {
3425            let arr = Self::iter_to_array(values.iter().cloned()).unwrap();
3426            cast_with_options(&arr, data_type, &DEFAULT_CAST_OPTIONS).unwrap()
3427        };
3428        Arc::new(SingleRowListArrayBuilder::new(values).build_large_list_array())
3429    }
3430
3431    /// Converts a scalar value into an array of `size` rows.
3432    ///
3433    /// # Errors
3434    ///
3435    /// Errors if `self` is
3436    /// - a decimal that fails be converted to a decimal array of size
3437    /// - a `FixedSizeList` that fails to be concatenated into an array of size
3438    /// - a `List` that fails to be concatenated into an array of size
3439    /// - a `Dictionary` that fails be converted to a dictionary array of size
3440    pub fn to_array_of_size(&self, size: usize) -> Result<ArrayRef> {
3441        Ok(match self {
3442            ScalarValue::Decimal32(Some(e), precision, scale) => Arc::new(
3443                Decimal32Array::from_value(*e, size)
3444                    .with_precision_and_scale(*precision, *scale)?,
3445            ),
3446            ScalarValue::Decimal32(None, precision, scale) => {
3447                new_null_array(&DataType::Decimal32(*precision, *scale), size)
3448            }
3449            ScalarValue::Decimal64(Some(e), precision, scale) => Arc::new(
3450                Decimal64Array::from_value(*e, size)
3451                    .with_precision_and_scale(*precision, *scale)?,
3452            ),
3453            ScalarValue::Decimal64(None, precision, scale) => {
3454                new_null_array(&DataType::Decimal64(*precision, *scale), size)
3455            }
3456            ScalarValue::Decimal128(Some(e), precision, scale) => Arc::new(
3457                Decimal128Array::from_value(*e, size)
3458                    .with_precision_and_scale(*precision, *scale)?,
3459            ),
3460            ScalarValue::Decimal128(None, precision, scale) => {
3461                new_null_array(&DataType::Decimal128(*precision, *scale), size)
3462            }
3463            ScalarValue::Decimal256(Some(e), precision, scale) => Arc::new(
3464                Decimal256Array::from_value(*e, size)
3465                    .with_precision_and_scale(*precision, *scale)?,
3466            ),
3467            ScalarValue::Decimal256(None, precision, scale) => {
3468                new_null_array(&DataType::Decimal256(*precision, *scale), size)
3469            }
3470
3471            ScalarValue::Boolean(e) => match e {
3472                None => new_null_array(&DataType::Boolean, size),
3473                Some(true) => {
3474                    Arc::new(BooleanArray::new(BooleanBuffer::new_set(size), None))
3475                        as ArrayRef
3476                }
3477                Some(false) => {
3478                    Arc::new(BooleanArray::new(BooleanBuffer::new_unset(size), None))
3479                        as ArrayRef
3480                }
3481            },
3482            ScalarValue::Float64(e) => {
3483                build_array_from_option!(Float64, Float64Array, e, size)
3484            }
3485            ScalarValue::Float32(e) => {
3486                build_array_from_option!(Float32, Float32Array, e, size)
3487            }
3488            ScalarValue::Float16(e) => {
3489                build_array_from_option!(Float16, Float16Array, e, size)
3490            }
3491            ScalarValue::Int8(e) => build_array_from_option!(Int8, Int8Array, e, size),
3492            ScalarValue::Int16(e) => build_array_from_option!(Int16, Int16Array, e, size),
3493            ScalarValue::Int32(e) => build_array_from_option!(Int32, Int32Array, e, size),
3494            ScalarValue::Int64(e) => build_array_from_option!(Int64, Int64Array, e, size),
3495            ScalarValue::UInt8(e) => build_array_from_option!(UInt8, UInt8Array, e, size),
3496            ScalarValue::UInt16(e) => {
3497                build_array_from_option!(UInt16, UInt16Array, e, size)
3498            }
3499            ScalarValue::UInt32(e) => {
3500                build_array_from_option!(UInt32, UInt32Array, e, size)
3501            }
3502            ScalarValue::UInt64(e) => {
3503                build_array_from_option!(UInt64, UInt64Array, e, size)
3504            }
3505            ScalarValue::TimestampSecond(e, tz_opt) => {
3506                build_timestamp_array_from_option!(
3507                    TimeUnit::Second,
3508                    tz_opt.clone(),
3509                    TimestampSecondArray,
3510                    e,
3511                    size
3512                )
3513            }
3514            ScalarValue::TimestampMillisecond(e, tz_opt) => {
3515                build_timestamp_array_from_option!(
3516                    TimeUnit::Millisecond,
3517                    tz_opt.clone(),
3518                    TimestampMillisecondArray,
3519                    e,
3520                    size
3521                )
3522            }
3523
3524            ScalarValue::TimestampMicrosecond(e, tz_opt) => {
3525                build_timestamp_array_from_option!(
3526                    TimeUnit::Microsecond,
3527                    tz_opt.clone(),
3528                    TimestampMicrosecondArray,
3529                    e,
3530                    size
3531                )
3532            }
3533            ScalarValue::TimestampNanosecond(e, tz_opt) => {
3534                build_timestamp_array_from_option!(
3535                    TimeUnit::Nanosecond,
3536                    tz_opt.clone(),
3537                    TimestampNanosecondArray,
3538                    e,
3539                    size
3540                )
3541            }
3542            ScalarValue::Utf8(e) => match e {
3543                Some(value) => Arc::new(StringArray::new_repeated(value, size)),
3544                None => new_null_array(&DataType::Utf8, size),
3545            },
3546            ScalarValue::Utf8View(e) => match e {
3547                Some(value) => {
3548                    let mut builder = StringViewBuilder::with_capacity(size);
3549                    builder.try_append_value_n(value, size)?;
3550                    let array = builder.finish();
3551                    Arc::new(array)
3552                }
3553                None => new_null_array(&DataType::Utf8View, size),
3554            },
3555            ScalarValue::LargeUtf8(e) => match e {
3556                Some(value) => Arc::new(LargeStringArray::new_repeated(value, size)),
3557                None => new_null_array(&DataType::LargeUtf8, size),
3558            },
3559            ScalarValue::Binary(e) => match e {
3560                Some(value) => {
3561                    Arc::new(BinaryArray::new_repeated(value.as_slice(), size))
3562                }
3563                None => new_null_array(&DataType::Binary, size),
3564            },
3565            ScalarValue::BinaryView(e) => match e {
3566                Some(value) => {
3567                    let mut builder = BinaryViewBuilder::with_capacity(size);
3568                    builder.try_append_value_n(value, size)?;
3569                    let array = builder.finish();
3570                    Arc::new(array)
3571                }
3572                None => new_null_array(&DataType::BinaryView, size),
3573            },
3574            ScalarValue::FixedSizeBinary(s, e) => match e {
3575                Some(value) => Arc::new(
3576                    FixedSizeBinaryArray::try_from_sparse_iter_with_size(
3577                        repeat_n(Some(value.as_slice()), size),
3578                        *s,
3579                    )
3580                    .unwrap(),
3581                ),
3582                None => Arc::new(FixedSizeBinaryArray::new_null(*s, size)),
3583            },
3584            ScalarValue::LargeBinary(e) => match e {
3585                Some(value) => {
3586                    Arc::new(LargeBinaryArray::new_repeated(value.as_slice(), size))
3587                }
3588                None => new_null_array(&DataType::LargeBinary, size),
3589            },
3590            ScalarValue::List(arr) => {
3591                if size == 1 {
3592                    return Ok(Arc::clone(arr) as Arc<dyn Array>);
3593                }
3594                Self::list_to_array_of_size(arr.as_ref() as &dyn Array, size)?
3595            }
3596            ScalarValue::LargeList(arr) => {
3597                if size == 1 {
3598                    return Ok(Arc::clone(arr) as Arc<dyn Array>);
3599                }
3600                Self::list_to_array_of_size(arr.as_ref() as &dyn Array, size)?
3601            }
3602            ScalarValue::FixedSizeList(arr) => {
3603                if size == 1 {
3604                    return Ok(Arc::clone(arr) as Arc<dyn Array>);
3605                }
3606                Self::list_to_array_of_size(arr.as_ref() as &dyn Array, size)?
3607            }
3608            ScalarValue::ListView(arr) => {
3609                if size == 1 {
3610                    return Ok(Arc::clone(arr) as Arc<dyn Array>);
3611                }
3612                Self::list_to_array_of_size(arr.as_ref() as &dyn Array, size)?
3613            }
3614            ScalarValue::LargeListView(arr) => {
3615                if size == 1 {
3616                    return Ok(Arc::clone(arr) as Arc<dyn Array>);
3617                }
3618                Self::list_to_array_of_size(arr.as_ref() as &dyn Array, size)?
3619            }
3620            ScalarValue::Struct(arr) => {
3621                if size == 1 {
3622                    return Ok(Arc::clone(arr) as Arc<dyn Array>);
3623                }
3624                Self::list_to_array_of_size(arr.as_ref() as &dyn Array, size)?
3625            }
3626            ScalarValue::Map(arr) => {
3627                if size == 1 {
3628                    return Ok(Arc::clone(arr) as Arc<dyn Array>);
3629                }
3630                Self::list_to_array_of_size(arr.as_ref() as &dyn Array, size)?
3631            }
3632            ScalarValue::Date32(e) => {
3633                build_array_from_option!(Date32, Date32Array, e, size)
3634            }
3635            ScalarValue::Date64(e) => {
3636                build_array_from_option!(Date64, Date64Array, e, size)
3637            }
3638            ScalarValue::Time32Second(e) => {
3639                build_array_from_option!(
3640                    Time32,
3641                    TimeUnit::Second,
3642                    Time32SecondArray,
3643                    e,
3644                    size
3645                )
3646            }
3647            ScalarValue::Time32Millisecond(e) => {
3648                build_array_from_option!(
3649                    Time32,
3650                    TimeUnit::Millisecond,
3651                    Time32MillisecondArray,
3652                    e,
3653                    size
3654                )
3655            }
3656            ScalarValue::Time64Microsecond(e) => {
3657                build_array_from_option!(
3658                    Time64,
3659                    TimeUnit::Microsecond,
3660                    Time64MicrosecondArray,
3661                    e,
3662                    size
3663                )
3664            }
3665            ScalarValue::Time64Nanosecond(e) => {
3666                build_array_from_option!(
3667                    Time64,
3668                    TimeUnit::Nanosecond,
3669                    Time64NanosecondArray,
3670                    e,
3671                    size
3672                )
3673            }
3674            ScalarValue::IntervalDayTime(e) => build_array_from_option!(
3675                Interval,
3676                IntervalUnit::DayTime,
3677                IntervalDayTimeArray,
3678                e,
3679                size
3680            ),
3681            ScalarValue::IntervalYearMonth(e) => build_array_from_option!(
3682                Interval,
3683                IntervalUnit::YearMonth,
3684                IntervalYearMonthArray,
3685                e,
3686                size
3687            ),
3688            ScalarValue::IntervalMonthDayNano(e) => build_array_from_option!(
3689                Interval,
3690                IntervalUnit::MonthDayNano,
3691                IntervalMonthDayNanoArray,
3692                e,
3693                size
3694            ),
3695            ScalarValue::DurationSecond(e) => build_array_from_option!(
3696                Duration,
3697                TimeUnit::Second,
3698                DurationSecondArray,
3699                e,
3700                size
3701            ),
3702            ScalarValue::DurationMillisecond(e) => build_array_from_option!(
3703                Duration,
3704                TimeUnit::Millisecond,
3705                DurationMillisecondArray,
3706                e,
3707                size
3708            ),
3709            ScalarValue::DurationMicrosecond(e) => build_array_from_option!(
3710                Duration,
3711                TimeUnit::Microsecond,
3712                DurationMicrosecondArray,
3713                e,
3714                size
3715            ),
3716            ScalarValue::DurationNanosecond(e) => build_array_from_option!(
3717                Duration,
3718                TimeUnit::Nanosecond,
3719                DurationNanosecondArray,
3720                e,
3721                size
3722            ),
3723            ScalarValue::Union(value, fields, mode) => match value {
3724                Some((v_id, value)) => {
3725                    let mut new_fields = Vec::with_capacity(fields.len());
3726                    let mut child_arrays = Vec::<ArrayRef>::with_capacity(fields.len());
3727                    for (f_id, field) in fields.iter() {
3728                        let ar = if f_id == *v_id {
3729                            value.to_array_of_size(size)?
3730                        } else {
3731                            let dt = field.data_type();
3732                            match mode {
3733                                UnionMode::Sparse => new_null_array(dt, size),
3734                                // In a dense union, only the child with values needs to be
3735                                // allocated
3736                                UnionMode::Dense => new_null_array(dt, 0),
3737                            }
3738                        };
3739                        let field = (**field).clone();
3740                        child_arrays.push(ar);
3741                        new_fields.push(field.clone());
3742                    }
3743                    let type_ids = repeat_n(*v_id, size);
3744                    let type_ids = ScalarBuffer::<i8>::from_iter(type_ids);
3745                    let value_offsets = match mode {
3746                        UnionMode::Sparse => None,
3747                        UnionMode::Dense => Some(ScalarBuffer::from_iter(0..size as i32)),
3748                    };
3749                    let ar = UnionArray::try_new(
3750                        fields.clone(),
3751                        type_ids,
3752                        value_offsets,
3753                        child_arrays,
3754                    )
3755                    .map_err(|e| DataFusionError::ArrowError(Box::new(e), None))?;
3756                    Arc::new(ar)
3757                }
3758                None => new_null_array(&DataType::Union(fields.clone(), *mode), size),
3759            },
3760            ScalarValue::Dictionary(key_type, v) => {
3761                // values array is one element long (the value)
3762                match key_type.as_ref() {
3763                    DataType::Int8 => dict_from_scalar::<Int8Type>(v, size)?,
3764                    DataType::Int16 => dict_from_scalar::<Int16Type>(v, size)?,
3765                    DataType::Int32 => dict_from_scalar::<Int32Type>(v, size)?,
3766                    DataType::Int64 => dict_from_scalar::<Int64Type>(v, size)?,
3767                    DataType::UInt8 => dict_from_scalar::<UInt8Type>(v, size)?,
3768                    DataType::UInt16 => dict_from_scalar::<UInt16Type>(v, size)?,
3769                    DataType::UInt32 => dict_from_scalar::<UInt32Type>(v, size)?,
3770                    DataType::UInt64 => dict_from_scalar::<UInt64Type>(v, size)?,
3771                    _ => unreachable!("Invalid dictionary keys type: {}", key_type),
3772                }
3773            }
3774            ScalarValue::RunEndEncoded(run_ends_field, values_field, value) => {
3775                fn make_run_array<R: RunEndIndexType>(
3776                    run_ends_field: &Arc<Field>,
3777                    values_field: &Arc<Field>,
3778                    value: &ScalarValue,
3779                    size: usize,
3780                ) -> Result<ArrayRef> {
3781                    let size_native = R::Native::from_usize(size)
3782                        .ok_or_else(|| DataFusionError::Execution(format!("Cannot construct RunArray of size {size}: Overflows run-ends type {}", R::DATA_TYPE)))?;
3783                    let values = value.to_array_of_size(1)?;
3784                    let run_ends =
3785                        PrimitiveArray::<R>::new(vec![size_native].into(), None);
3786
3787                    // Using ArrayDataBuilder so we can maintain the fields
3788                    let dt = DataType::RunEndEncoded(
3789                        Arc::clone(run_ends_field),
3790                        Arc::clone(values_field),
3791                    );
3792                    let builder = ArrayDataBuilder::new(dt)
3793                        .len(size)
3794                        .add_child_data(run_ends.to_data())
3795                        .add_child_data(values.to_data());
3796                    let run_array = RunArray::<R>::from(builder.build()?);
3797
3798                    Ok(Arc::new(run_array))
3799                }
3800                match run_ends_field.data_type() {
3801                    DataType::Int16 => make_run_array::<Int16Type>(
3802                        run_ends_field,
3803                        values_field,
3804                        value,
3805                        size,
3806                    )?,
3807                    DataType::Int32 => make_run_array::<Int32Type>(
3808                        run_ends_field,
3809                        values_field,
3810                        value,
3811                        size,
3812                    )?,
3813                    DataType::Int64 => make_run_array::<Int64Type>(
3814                        run_ends_field,
3815                        values_field,
3816                        value,
3817                        size,
3818                    )?,
3819                    dt => unreachable!("Invalid run-ends type: {dt}"),
3820                }
3821            }
3822            ScalarValue::Null => get_or_create_cached_null_array(size),
3823        })
3824    }
3825
3826    fn get_decimal_value_from_array(
3827        array: &dyn Array,
3828        index: usize,
3829        precision: u8,
3830        scale: i8,
3831    ) -> Result<ScalarValue> {
3832        match array.data_type() {
3833            DataType::Decimal32(_, _) => {
3834                let array = as_decimal32_array(array)?;
3835                if array.is_null(index) {
3836                    Ok(ScalarValue::Decimal32(None, precision, scale))
3837                } else {
3838                    let value = array.value(index);
3839                    Ok(ScalarValue::Decimal32(Some(value), precision, scale))
3840                }
3841            }
3842            DataType::Decimal64(_, _) => {
3843                let array = as_decimal64_array(array)?;
3844                if array.is_null(index) {
3845                    Ok(ScalarValue::Decimal64(None, precision, scale))
3846                } else {
3847                    let value = array.value(index);
3848                    Ok(ScalarValue::Decimal64(Some(value), precision, scale))
3849                }
3850            }
3851            DataType::Decimal128(_, _) => {
3852                let array = as_decimal128_array(array)?;
3853                if array.is_null(index) {
3854                    Ok(ScalarValue::Decimal128(None, precision, scale))
3855                } else {
3856                    let value = array.value(index);
3857                    Ok(ScalarValue::Decimal128(Some(value), precision, scale))
3858                }
3859            }
3860            DataType::Decimal256(_, _) => {
3861                let array = as_decimal256_array(array)?;
3862                if array.is_null(index) {
3863                    Ok(ScalarValue::Decimal256(None, precision, scale))
3864                } else {
3865                    let value = array.value(index);
3866                    Ok(ScalarValue::Decimal256(Some(value), precision, scale))
3867                }
3868            }
3869            other => {
3870                unreachable!("Invalid type isn't decimal: {other:?}")
3871            }
3872        }
3873    }
3874
3875    /// Repeats the rows of `arr` `size` times, producing an array with
3876    /// `arr.len() * size` total rows.
3877    fn list_to_array_of_size(arr: &dyn Array, size: usize) -> Result<ArrayRef> {
3878        if size == 0 {
3879            return Ok(arr.slice(0, 0));
3880        }
3881
3882        // Examples: given `arr = [[A, B, C]]` and `size = 3`, `indices = [0, 0, 0]` and
3883        // the result is `[[A, B, C], [A, B, C], [A, B, C]]`.
3884        //
3885        // Given `arr = [[A, B], [C]]` and `size = 2`, `indices = [0, 1, 0, 1]` and the
3886        // result is `[[A, B], [C], [A, B], [C]]`. (But in practice, we are always called
3887        // with `arr.len() == 1`.)
3888        let n = arr.len() as u32;
3889        let indices = UInt32Array::from_iter_values((0..size).flat_map(|_| 0..n));
3890        Ok(arrow::compute::take(arr, &indices, None)?)
3891    }
3892
3893    /// Retrieve ScalarValue for each row in `array`
3894    ///
3895    /// Elements in `array` may be NULL, in which case the corresponding element in the returned vector is None.
3896    ///
3897    /// Example 1: Array (ScalarValue::Int32)
3898    /// ```
3899    /// use arrow::array::ListArray;
3900    /// use arrow::datatypes::{DataType, Int32Type};
3901    /// use datafusion_common::ScalarValue;
3902    ///
3903    /// // Equivalent to [[1,2,3], [4,5]]
3904    /// let list_arr = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
3905    ///     Some(vec![Some(1), Some(2), Some(3)]),
3906    ///     Some(vec![Some(4), Some(5)]),
3907    /// ]);
3908    ///
3909    /// // Convert the array into Scalar Values for each row
3910    /// let scalar_vec = ScalarValue::convert_array_to_scalar_vec(&list_arr).unwrap();
3911    ///
3912    /// let expected = vec![
3913    ///     Some(vec![
3914    ///         ScalarValue::Int32(Some(1)),
3915    ///         ScalarValue::Int32(Some(2)),
3916    ///         ScalarValue::Int32(Some(3)),
3917    ///     ]),
3918    ///     Some(vec![
3919    ///         ScalarValue::Int32(Some(4)),
3920    ///         ScalarValue::Int32(Some(5)),
3921    ///     ]),
3922    /// ];
3923    ///
3924    /// assert_eq!(scalar_vec, expected);
3925    /// ```
3926    ///
3927    /// Example 2: Nested array (ScalarValue::List)
3928    /// ```
3929    /// use arrow::array::ListArray;
3930    /// use arrow::datatypes::{DataType, Int32Type};
3931    /// use datafusion_common::utils::SingleRowListArrayBuilder;
3932    /// use datafusion_common::ScalarValue;
3933    /// use std::sync::Arc;
3934    ///
3935    /// let list_arr = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
3936    ///     Some(vec![Some(1), Some(2), Some(3)]),
3937    ///     Some(vec![Some(4), Some(5)]),
3938    /// ]);
3939    ///
3940    /// // Wrap into another layer of list, we got nested array as [ [[1,2,3], [4,5]] ]
3941    /// let list_arr = SingleRowListArrayBuilder::new(Arc::new(list_arr)).build_list_array();
3942    ///
3943    /// // Convert the array into Scalar Values for each row, we got 1D arrays in this example
3944    /// let scalar_vec = ScalarValue::convert_array_to_scalar_vec(&list_arr).unwrap();
3945    ///
3946    /// let l1 = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![Some(vec![
3947    ///     Some(1),
3948    ///     Some(2),
3949    ///     Some(3),
3950    /// ])]);
3951    /// let l2 = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![Some(vec![
3952    ///     Some(4),
3953    ///     Some(5),
3954    /// ])]);
3955    ///
3956    /// let expected = vec![Some(vec![
3957    ///     ScalarValue::List(Arc::new(l1)),
3958    ///     ScalarValue::List(Arc::new(l2)),
3959    /// ])];
3960    ///
3961    /// assert_eq!(scalar_vec, expected);
3962    /// ```
3963    ///
3964    /// Example 3: Nullable array
3965    /// ```
3966    /// use arrow::array::ListArray;
3967    /// use arrow::datatypes::{DataType, Int32Type};
3968    /// use datafusion_common::ScalarValue;
3969    ///
3970    /// let list_arr = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
3971    ///     Some(vec![Some(1), Some(2), Some(3)]),
3972    ///     None,
3973    ///     Some(vec![Some(4), Some(5)]),
3974    /// ]);
3975    ///
3976    /// // Convert the array into Scalar Values for each row
3977    /// let scalar_vec = ScalarValue::convert_array_to_scalar_vec(&list_arr).unwrap();
3978    ///
3979    /// let expected = vec![
3980    ///     Some(vec![
3981    ///         ScalarValue::Int32(Some(1)),
3982    ///         ScalarValue::Int32(Some(2)),
3983    ///         ScalarValue::Int32(Some(3)),
3984    ///     ]),
3985    ///     None,
3986    ///     Some(vec![
3987    ///         ScalarValue::Int32(Some(4)),
3988    ///         ScalarValue::Int32(Some(5)),
3989    ///     ]),
3990    /// ];
3991    ///
3992    /// assert_eq!(scalar_vec, expected);
3993    /// ```
3994    pub fn convert_array_to_scalar_vec(
3995        array: &dyn Array,
3996    ) -> Result<Vec<Option<Vec<Self>>>> {
3997        fn map_element(
3998            nested_array: Option<ArrayRef>,
3999        ) -> Result<Option<Vec<ScalarValue>>> {
4000            nested_array
4001                .map(|array| {
4002                    (0..array.len())
4003                        .map(|i| ScalarValue::try_from_array(&array, i))
4004                        .collect::<Result<Vec<_>>>()
4005                })
4006                .transpose()
4007        }
4008
4009        match array.data_type() {
4010            DataType::List(_) => array.as_list::<i32>().iter().map(map_element).collect(),
4011            DataType::LargeList(_) => {
4012                array.as_list::<i64>().iter().map(map_element).collect()
4013            }
4014            DataType::ListView(_) => array
4015                .as_list_view::<i32>()
4016                .iter()
4017                .map(map_element)
4018                .collect(),
4019            DataType::LargeListView(_) => array
4020                .as_list_view::<i64>()
4021                .iter()
4022                .map(map_element)
4023                .collect(),
4024            _ => _internal_err!(
4025                "ScalarValue::convert_array_to_scalar_vec input must be a List/LargeList/ListView/LargeListView type"
4026            ),
4027        }
4028    }
4029
4030    #[deprecated(
4031        since = "46.0.0",
4032        note = "This function is obsolete. Use `to_array` instead"
4033    )]
4034    pub fn raw_data(&self) -> Result<ArrayRef> {
4035        match self {
4036            ScalarValue::List(arr) => Ok(arr.to_owned()),
4037            _ => _internal_err!("ScalarValue is not a list"),
4038        }
4039    }
4040
4041    /// Converts a value in `array` at `index` into a ScalarValue
4042    pub fn try_from_array(array: &dyn Array, index: usize) -> Result<Self> {
4043        // handle NULL value
4044        if array.is_null(index) {
4045            return array.data_type().try_into();
4046        }
4047
4048        Ok(match array.data_type() {
4049            DataType::Null => ScalarValue::Null,
4050            DataType::Decimal32(precision, scale) => {
4051                ScalarValue::get_decimal_value_from_array(
4052                    array, index, *precision, *scale,
4053                )?
4054            }
4055            DataType::Decimal64(precision, scale) => {
4056                ScalarValue::get_decimal_value_from_array(
4057                    array, index, *precision, *scale,
4058                )?
4059            }
4060            DataType::Decimal128(precision, scale) => {
4061                ScalarValue::get_decimal_value_from_array(
4062                    array, index, *precision, *scale,
4063                )?
4064            }
4065            DataType::Decimal256(precision, scale) => {
4066                ScalarValue::get_decimal_value_from_array(
4067                    array, index, *precision, *scale,
4068                )?
4069            }
4070            DataType::Boolean => typed_cast!(array, index, as_boolean_array, Boolean)?,
4071            DataType::Float64 => typed_cast!(array, index, as_float64_array, Float64)?,
4072            DataType::Float32 => typed_cast!(array, index, as_float32_array, Float32)?,
4073            DataType::Float16 => typed_cast!(array, index, as_float16_array, Float16)?,
4074            DataType::UInt64 => typed_cast!(array, index, as_uint64_array, UInt64)?,
4075            DataType::UInt32 => typed_cast!(array, index, as_uint32_array, UInt32)?,
4076            DataType::UInt16 => typed_cast!(array, index, as_uint16_array, UInt16)?,
4077            DataType::UInt8 => typed_cast!(array, index, as_uint8_array, UInt8)?,
4078            DataType::Int64 => typed_cast!(array, index, as_int64_array, Int64)?,
4079            DataType::Int32 => typed_cast!(array, index, as_int32_array, Int32)?,
4080            DataType::Int16 => typed_cast!(array, index, as_int16_array, Int16)?,
4081            DataType::Int8 => typed_cast!(array, index, as_int8_array, Int8)?,
4082            DataType::Binary => typed_cast!(array, index, as_binary_array, Binary)?,
4083            DataType::LargeBinary => {
4084                typed_cast!(array, index, as_large_binary_array, LargeBinary)?
4085            }
4086            DataType::BinaryView => {
4087                typed_cast!(array, index, as_binary_view_array, BinaryView)?
4088            }
4089            DataType::Utf8 => typed_cast!(array, index, as_string_array, Utf8)?,
4090            DataType::LargeUtf8 => {
4091                typed_cast!(array, index, as_large_string_array, LargeUtf8)?
4092            }
4093            DataType::Utf8View => {
4094                typed_cast!(array, index, as_string_view_array, Utf8View)?
4095            }
4096            DataType::List(field) => {
4097                let list_array = array.as_list::<i32>();
4098                let nested_array = list_array.value(index);
4099                // Produces a single element `ListArray` with the value at `index`.
4100                SingleRowListArrayBuilder::new(nested_array)
4101                    .with_field(field)
4102                    .build_list_scalar()
4103            }
4104            DataType::LargeList(field) => {
4105                let list_array = as_large_list_array(array)?;
4106                let nested_array = list_array.value(index);
4107                // Produces a single element `LargeListArray` with the value at `index`.
4108                SingleRowListArrayBuilder::new(nested_array)
4109                    .with_field(field)
4110                    .build_large_list_scalar()
4111            }
4112            // TODO: There is no test for FixedSizeList now, add it later
4113            DataType::FixedSizeList(field, _) => {
4114                let list_array = as_fixed_size_list_array(array)?;
4115                let nested_array = list_array.value(index);
4116                // Produces a single element `FixedSizeListArray` with the value at `index`.
4117                let list_size = nested_array.len();
4118                SingleRowListArrayBuilder::new(nested_array)
4119                    .with_field(field)
4120                    .build_fixed_size_list_scalar(list_size)
4121            }
4122            DataType::ListView(field) => {
4123                let list_array = as_list_view_array(array)?;
4124                let nested_array = list_array.value(index);
4125                // Produces a single element `ListViewArray` with the value at `index`.
4126                SingleRowListArrayBuilder::new(nested_array)
4127                    .with_field(field)
4128                    .build_list_view_scalar()
4129            }
4130            DataType::LargeListView(field) => {
4131                let list_array = as_large_list_view_array(array)?;
4132                let nested_array = list_array.value(index);
4133                // Produces a single element `LargeListViewArray` with the value at `index`.
4134                SingleRowListArrayBuilder::new(nested_array)
4135                    .with_field(field)
4136                    .build_large_list_view_scalar()
4137            }
4138            DataType::Date32 => typed_cast!(array, index, as_date32_array, Date32)?,
4139            DataType::Date64 => typed_cast!(array, index, as_date64_array, Date64)?,
4140            DataType::Time32(TimeUnit::Second) => {
4141                typed_cast!(array, index, as_time32_second_array, Time32Second)?
4142            }
4143            DataType::Time32(TimeUnit::Millisecond) => {
4144                typed_cast!(array, index, as_time32_millisecond_array, Time32Millisecond)?
4145            }
4146            DataType::Time64(TimeUnit::Microsecond) => {
4147                typed_cast!(array, index, as_time64_microsecond_array, Time64Microsecond)?
4148            }
4149            DataType::Time64(TimeUnit::Nanosecond) => {
4150                typed_cast!(array, index, as_time64_nanosecond_array, Time64Nanosecond)?
4151            }
4152            DataType::Timestamp(TimeUnit::Second, tz_opt) => typed_cast_tz!(
4153                array,
4154                index,
4155                as_timestamp_second_array,
4156                TimestampSecond,
4157                tz_opt
4158            )?,
4159            DataType::Timestamp(TimeUnit::Millisecond, tz_opt) => typed_cast_tz!(
4160                array,
4161                index,
4162                as_timestamp_millisecond_array,
4163                TimestampMillisecond,
4164                tz_opt
4165            )?,
4166            DataType::Timestamp(TimeUnit::Microsecond, tz_opt) => typed_cast_tz!(
4167                array,
4168                index,
4169                as_timestamp_microsecond_array,
4170                TimestampMicrosecond,
4171                tz_opt
4172            )?,
4173            DataType::Timestamp(TimeUnit::Nanosecond, tz_opt) => typed_cast_tz!(
4174                array,
4175                index,
4176                as_timestamp_nanosecond_array,
4177                TimestampNanosecond,
4178                tz_opt
4179            )?,
4180            DataType::Dictionary(key_type, _) => {
4181                let (values_array, values_index) = match key_type.as_ref() {
4182                    DataType::Int8 => get_dict_value::<Int8Type>(array, index)?,
4183                    DataType::Int16 => get_dict_value::<Int16Type>(array, index)?,
4184                    DataType::Int32 => get_dict_value::<Int32Type>(array, index)?,
4185                    DataType::Int64 => get_dict_value::<Int64Type>(array, index)?,
4186                    DataType::UInt8 => get_dict_value::<UInt8Type>(array, index)?,
4187                    DataType::UInt16 => get_dict_value::<UInt16Type>(array, index)?,
4188                    DataType::UInt32 => get_dict_value::<UInt32Type>(array, index)?,
4189                    DataType::UInt64 => get_dict_value::<UInt64Type>(array, index)?,
4190                    _ => unreachable!("Invalid dictionary keys type: {}", key_type),
4191                };
4192                // look up the index in the values dictionary
4193                let value = match values_index {
4194                    Some(values_index) => {
4195                        ScalarValue::try_from_array(values_array, values_index)
4196                    }
4197                    // else entry was null, so return null
4198                    None => values_array.data_type().try_into(),
4199                }?;
4200
4201                Self::Dictionary(key_type.clone(), Box::new(value))
4202            }
4203            DataType::RunEndEncoded(run_ends_field, value_field) => {
4204                // Explicitly check length here since get_physical_index() doesn't
4205                // bound check for us
4206                if index > array.len() {
4207                    return _exec_err!(
4208                        "Index {index} out of bounds for array of length {}",
4209                        array.len()
4210                    );
4211                }
4212                let scalar = downcast_run_array!(
4213                    array => {
4214                        let index = array.get_physical_index(index);
4215                        ScalarValue::try_from_array(array.values(), index)?
4216                    },
4217                    dt => unreachable!("Invalid run-ends type: {dt}")
4218                );
4219                Self::RunEndEncoded(
4220                    Arc::clone(run_ends_field),
4221                    Arc::clone(value_field),
4222                    Box::new(scalar),
4223                )
4224            }
4225            DataType::Struct(_) => {
4226                let a = array.slice(index, 1);
4227                Self::Struct(Arc::new(a.as_struct().to_owned()))
4228            }
4229            DataType::FixedSizeBinary(_) => {
4230                let array = as_fixed_size_binary_array(array)?;
4231                let size = match array.data_type() {
4232                    DataType::FixedSizeBinary(size) => *size,
4233                    _ => unreachable!(),
4234                };
4235                ScalarValue::FixedSizeBinary(
4236                    size,
4237                    match array.is_null(index) {
4238                        true => None,
4239                        false => Some(array.value(index).into()),
4240                    },
4241                )
4242            }
4243            DataType::Interval(IntervalUnit::DayTime) => {
4244                typed_cast!(array, index, as_interval_dt_array, IntervalDayTime)?
4245            }
4246            DataType::Interval(IntervalUnit::YearMonth) => {
4247                typed_cast!(array, index, as_interval_ym_array, IntervalYearMonth)?
4248            }
4249            DataType::Interval(IntervalUnit::MonthDayNano) => {
4250                typed_cast!(array, index, as_interval_mdn_array, IntervalMonthDayNano)?
4251            }
4252
4253            DataType::Duration(TimeUnit::Second) => {
4254                typed_cast!(array, index, as_duration_second_array, DurationSecond)?
4255            }
4256            DataType::Duration(TimeUnit::Millisecond) => typed_cast!(
4257                array,
4258                index,
4259                as_duration_millisecond_array,
4260                DurationMillisecond
4261            )?,
4262            DataType::Duration(TimeUnit::Microsecond) => typed_cast!(
4263                array,
4264                index,
4265                as_duration_microsecond_array,
4266                DurationMicrosecond
4267            )?,
4268            DataType::Duration(TimeUnit::Nanosecond) => typed_cast!(
4269                array,
4270                index,
4271                as_duration_nanosecond_array,
4272                DurationNanosecond
4273            )?,
4274            DataType::Map(_, _) => {
4275                let a = array.slice(index, 1);
4276                Self::Map(Arc::new(a.as_map().to_owned()))
4277            }
4278            DataType::Union(fields, mode) => {
4279                let array = as_union_array(array)?;
4280                let ti = array.type_id(index);
4281                let index = array.value_offset(index);
4282                let value = ScalarValue::try_from_array(array.child(ti), index)?;
4283                ScalarValue::Union(Some((ti, Box::new(value))), fields.clone(), *mode)
4284            }
4285            other => {
4286                return _not_impl_err!(
4287                    "Can't create a scalar from array of type \"{other:?}\""
4288                );
4289            }
4290        })
4291    }
4292
4293    /// Try to parse `value` into a ScalarValue of type `target_type`
4294    pub fn try_from_string(value: String, target_type: &DataType) -> Result<Self> {
4295        ScalarValue::from(value).cast_to(target_type)
4296    }
4297
4298    /// Returns the Some(`&str`) representation of `ScalarValue` of logical string type
4299    ///
4300    /// Returns `None` if this `ScalarValue` is not a logical string type or the
4301    /// `ScalarValue` represents the `NULL` value.
4302    ///
4303    /// Note you can use [`Option::flatten`] to check for non null logical
4304    /// strings.
4305    ///
4306    /// For example, [`ScalarValue::Utf8`], [`ScalarValue::LargeUtf8`], and
4307    /// [`ScalarValue::Dictionary`] with a logical string value and store
4308    /// strings and can be accessed as `&str` using this method.
4309    ///
4310    /// # Example: logical strings
4311    /// ```
4312    /// # use datafusion_common::ScalarValue;
4313    /// /// non strings return None
4314    /// let scalar = ScalarValue::from(42);
4315    /// assert_eq!(scalar.try_as_str(), None);
4316    /// // Non null logical string returns Some(Some(&str))
4317    /// let scalar = ScalarValue::from("hello");
4318    /// assert_eq!(scalar.try_as_str(), Some(Some("hello")));
4319    /// // Null logical string returns Some(None)
4320    /// let scalar = ScalarValue::Utf8(None);
4321    /// assert_eq!(scalar.try_as_str(), Some(None));
4322    /// ```
4323    ///
4324    /// # Example: use [`Option::flatten`] to check for non-null logical strings
4325    /// ```
4326    /// # use datafusion_common::ScalarValue;
4327    /// // Non null logical string returns Some(Some(&str))
4328    /// let scalar = ScalarValue::from("hello");
4329    /// assert_eq!(scalar.try_as_str().flatten(), Some("hello"));
4330    /// ```
4331    pub fn try_as_str(&self) -> Option<Option<&str>> {
4332        let v = match self {
4333            ScalarValue::Utf8(v) => v,
4334            ScalarValue::LargeUtf8(v) => v,
4335            ScalarValue::Utf8View(v) => v,
4336            ScalarValue::Dictionary(_, v) => return v.try_as_str(),
4337            ScalarValue::RunEndEncoded(_, _, v) => return v.try_as_str(),
4338            _ => return None,
4339        };
4340        Some(v.as_ref().map(|v| v.as_str()))
4341    }
4342
4343    /// Cast this value to a `ScalarValue` of type `target_type` using the
4344    /// default [`CastOptions`].
4345    ///
4346    /// This is a general-purpose cast with the same semantics as the Arrow
4347    /// [`cast_with_options`] kernel and can therefore **lose information** --
4348    /// for example casting the floating point value `123.45` to the integer
4349    /// `123`.
4350    ///
4351    /// Returns an error for casts the Arrow kernel cannot perform.
4352    ///
4353    /// # See Also
4354    /// - [`try_cast_literal_to_type`]: for a *value-preserving* cast
4355    ///
4356    /// [`try_cast_literal_to_type`]: https://docs.rs/datafusion/latest/datafusion/logical_expr_common/casts/fn.try_cast_literal_to_type.html
4357    pub fn cast_to(&self, target_type: &DataType) -> Result<Self> {
4358        self.cast_to_with_options(target_type, &DEFAULT_CAST_OPTIONS)
4359    }
4360
4361    /// Cast this value to type `target_type` with the given [`CastOptions`].
4362    ///
4363    /// # See Also
4364    /// - [`ScalarValue::cast_to`] for more details.
4365    /// - [`try_cast_literal_to_type`]: for a *value-preserving* cast
4366    ///
4367    /// [`try_cast_literal_to_type`]: https://docs.rs/datafusion/latest/datafusion/logical_expr_common/casts/fn.try_cast_literal_to_type.html
4368    pub fn cast_to_with_options(
4369        &self,
4370        target_type: &DataType,
4371        cast_options: &CastOptions<'static>,
4372    ) -> Result<Self> {
4373        let source_type = self.data_type();
4374
4375        // Fast path: an identical target type needs no conversion at all.
4376        if &source_type == target_type {
4377            return Ok(self.clone());
4378        }
4379
4380        // Fast path: conversions among the string types (`Utf8`, `LargeUtf8`,
4381        // `Utf8View`) are value-preserving, so we can rewrap the string
4382        // directly instead of building a single-row array and invoking the
4383        // arrow cast kernel.
4384        if source_type.is_string() && target_type.is_string() {
4385            // `self` is one of the string types, so `try_as_str` returns `Some`
4386            let value = self.try_as_str().flatten().map(|s| s.to_string());
4387            return Ok(match target_type {
4388                DataType::Utf8 => ScalarValue::Utf8(value),
4389                DataType::LargeUtf8 => ScalarValue::LargeUtf8(value),
4390                DataType::Utf8View => ScalarValue::Utf8View(value),
4391                _ => unreachable!("matched a string target type above"),
4392            });
4393        }
4394
4395        if let Some(multiplier) = date_to_timestamp_multiplier(&source_type, target_type)
4396            .or_else(|| timestamp_to_timestamp_multiplier(&source_type, target_type))
4397            && let Some(value) = self.temporal_scalar_value_as_i64()
4398        {
4399            match ensure_timestamp_in_bounds(value, multiplier, &source_type, target_type)
4400            {
4401                Ok(()) => {}
4402                Err(_) if cast_options.safe => {
4403                    return ScalarValue::try_new_null(target_type);
4404                }
4405                Err(e) => return Err(e),
4406            }
4407        }
4408
4409        let scalar_array = self.to_array()?;
4410
4411        // For types that contain structs (including nested inside Lists, Dictionaries,
4412        // etc.), use name-based casting logic that matches struct fields by name and
4413        // recursively casts nested structs.
4414        let cast_arr = if crate::nested_struct::requires_nested_struct_cast(
4415            scalar_array.data_type(),
4416            target_type,
4417        ) {
4418            crate::nested_struct::cast_column(&scalar_array, target_type, cast_options)?
4419        } else {
4420            cast_with_options(&scalar_array, target_type, cast_options)?
4421        };
4422
4423        ScalarValue::try_from_array(&cast_arr, 0)
4424    }
4425
4426    fn temporal_scalar_value_as_i64(&self) -> Option<i64> {
4427        match self {
4428            ScalarValue::Date32(Some(value)) => Some(i64::from(*value)),
4429            ScalarValue::Date64(Some(value)) => Some(*value),
4430            ScalarValue::TimestampSecond(Some(value), _)
4431            | ScalarValue::TimestampMillisecond(Some(value), _)
4432            | ScalarValue::TimestampMicrosecond(Some(value), _)
4433            | ScalarValue::TimestampNanosecond(Some(value), _) => Some(*value),
4434            _ => None,
4435        }
4436    }
4437
4438    fn eq_array_decimal32(
4439        array: &ArrayRef,
4440        index: usize,
4441        value: Option<&i32>,
4442        precision: u8,
4443        scale: i8,
4444    ) -> Result<bool> {
4445        let array = as_decimal32_array(array)?;
4446        if array.precision() != precision || array.scale() != scale {
4447            return Ok(false);
4448        }
4449        let is_null = array.is_null(index);
4450        if let Some(v) = value {
4451            Ok(!array.is_null(index) && array.value(index) == *v)
4452        } else {
4453            Ok(is_null)
4454        }
4455    }
4456
4457    fn eq_array_decimal64(
4458        array: &ArrayRef,
4459        index: usize,
4460        value: Option<&i64>,
4461        precision: u8,
4462        scale: i8,
4463    ) -> Result<bool> {
4464        let array = as_decimal64_array(array)?;
4465        if array.precision() != precision || array.scale() != scale {
4466            return Ok(false);
4467        }
4468        let is_null = array.is_null(index);
4469        if let Some(v) = value {
4470            Ok(!array.is_null(index) && array.value(index) == *v)
4471        } else {
4472            Ok(is_null)
4473        }
4474    }
4475
4476    fn eq_array_decimal(
4477        array: &ArrayRef,
4478        index: usize,
4479        value: Option<&i128>,
4480        precision: u8,
4481        scale: i8,
4482    ) -> Result<bool> {
4483        let array = as_decimal128_array(array)?;
4484        if array.precision() != precision || array.scale() != scale {
4485            return Ok(false);
4486        }
4487        let is_null = array.is_null(index);
4488        if let Some(v) = value {
4489            Ok(!array.is_null(index) && array.value(index) == *v)
4490        } else {
4491            Ok(is_null)
4492        }
4493    }
4494
4495    fn eq_array_decimal256(
4496        array: &ArrayRef,
4497        index: usize,
4498        value: Option<&i256>,
4499        precision: u8,
4500        scale: i8,
4501    ) -> Result<bool> {
4502        let array = as_decimal256_array(array)?;
4503        if array.precision() != precision || array.scale() != scale {
4504            return Ok(false);
4505        }
4506        let is_null = array.is_null(index);
4507        if let Some(v) = value {
4508            Ok(!array.is_null(index) && array.value(index) == *v)
4509        } else {
4510            Ok(is_null)
4511        }
4512    }
4513
4514    /// Compares a single row of array @ index for equality with self,
4515    /// in an optimized fashion.
4516    ///
4517    /// This method implements an optimized version of:
4518    ///
4519    /// ```text
4520    ///     let arr_scalar = Self::try_from_array(array, index).unwrap();
4521    ///     arr_scalar.eq(self)
4522    /// ```
4523    ///
4524    /// *Performance note*: the arrow compute kernels should be
4525    /// preferred over this function if at all possible as they can be
4526    /// vectorized and are generally much faster.
4527    ///
4528    /// This function has a few narrow use cases such as hash table key
4529    /// comparisons where comparing a single row at a time is necessary.
4530    ///
4531    /// # Errors
4532    ///
4533    /// Errors if
4534    /// - it fails to downcast `array` to the data type of `self`
4535    /// - `self` is a `Struct`
4536    ///
4537    /// # Panics
4538    ///
4539    /// Panics if `self` is a dictionary with invalid key type
4540    #[inline]
4541    pub fn eq_array(&self, array: &ArrayRef, index: usize) -> Result<bool> {
4542        Ok(match self {
4543            ScalarValue::Decimal32(v, precision, scale) => {
4544                ScalarValue::eq_array_decimal32(
4545                    array,
4546                    index,
4547                    v.as_ref(),
4548                    *precision,
4549                    *scale,
4550                )?
4551            }
4552            ScalarValue::Decimal64(v, precision, scale) => {
4553                ScalarValue::eq_array_decimal64(
4554                    array,
4555                    index,
4556                    v.as_ref(),
4557                    *precision,
4558                    *scale,
4559                )?
4560            }
4561            ScalarValue::Decimal128(v, precision, scale) => {
4562                ScalarValue::eq_array_decimal(
4563                    array,
4564                    index,
4565                    v.as_ref(),
4566                    *precision,
4567                    *scale,
4568                )?
4569            }
4570            ScalarValue::Decimal256(v, precision, scale) => {
4571                ScalarValue::eq_array_decimal256(
4572                    array,
4573                    index,
4574                    v.as_ref(),
4575                    *precision,
4576                    *scale,
4577                )?
4578            }
4579            ScalarValue::Boolean(val) => {
4580                eq_array_primitive!(array, index, as_boolean_array, val)?
4581            }
4582            ScalarValue::Float16(val) => {
4583                eq_array_primitive!(array, index, as_float16_array, val)?
4584            }
4585            ScalarValue::Float32(val) => {
4586                eq_array_primitive!(array, index, as_float32_array, val)?
4587            }
4588            ScalarValue::Float64(val) => {
4589                eq_array_primitive!(array, index, as_float64_array, val)?
4590            }
4591            ScalarValue::Int8(val) => {
4592                eq_array_primitive!(array, index, as_int8_array, val)?
4593            }
4594            ScalarValue::Int16(val) => {
4595                eq_array_primitive!(array, index, as_int16_array, val)?
4596            }
4597            ScalarValue::Int32(val) => {
4598                eq_array_primitive!(array, index, as_int32_array, val)?
4599            }
4600            ScalarValue::Int64(val) => {
4601                eq_array_primitive!(array, index, as_int64_array, val)?
4602            }
4603            ScalarValue::UInt8(val) => {
4604                eq_array_primitive!(array, index, as_uint8_array, val)?
4605            }
4606            ScalarValue::UInt16(val) => {
4607                eq_array_primitive!(array, index, as_uint16_array, val)?
4608            }
4609            ScalarValue::UInt32(val) => {
4610                eq_array_primitive!(array, index, as_uint32_array, val)?
4611            }
4612            ScalarValue::UInt64(val) => {
4613                eq_array_primitive!(array, index, as_uint64_array, val)?
4614            }
4615            ScalarValue::Utf8(val) => {
4616                eq_array_primitive!(array, index, as_string_array, val)?
4617            }
4618            ScalarValue::Utf8View(val) => {
4619                eq_array_primitive!(array, index, as_string_view_array, val)?
4620            }
4621            ScalarValue::LargeUtf8(val) => {
4622                eq_array_primitive!(array, index, as_large_string_array, val)?
4623            }
4624            ScalarValue::Binary(val) => {
4625                eq_array_primitive!(array, index, as_binary_array, val)?
4626            }
4627            ScalarValue::BinaryView(val) => {
4628                eq_array_primitive!(array, index, as_binary_view_array, val)?
4629            }
4630            ScalarValue::FixedSizeBinary(_, val) => {
4631                eq_array_primitive!(array, index, as_fixed_size_binary_array, val)?
4632            }
4633            ScalarValue::LargeBinary(val) => {
4634                eq_array_primitive!(array, index, as_large_binary_array, val)?
4635            }
4636            ScalarValue::List(arr) => {
4637                Self::eq_array_list(&(arr.to_owned() as ArrayRef), array, index)
4638            }
4639            ScalarValue::LargeList(arr) => {
4640                Self::eq_array_list(&(arr.to_owned() as ArrayRef), array, index)
4641            }
4642            ScalarValue::FixedSizeList(arr) => {
4643                Self::eq_array_list(&(arr.to_owned() as ArrayRef), array, index)
4644            }
4645            ScalarValue::ListView(arr) => {
4646                Self::eq_array_list(&(arr.to_owned() as ArrayRef), array, index)
4647            }
4648            ScalarValue::LargeListView(arr) => {
4649                Self::eq_array_list(&(arr.to_owned() as ArrayRef), array, index)
4650            }
4651            ScalarValue::Struct(arr) => {
4652                Self::eq_array_list(&(arr.to_owned() as ArrayRef), array, index)
4653            }
4654            ScalarValue::Map(arr) => {
4655                Self::eq_array_list(&(arr.to_owned() as ArrayRef), array, index)
4656            }
4657            ScalarValue::Date32(val) => {
4658                eq_array_primitive!(array, index, as_date32_array, val)?
4659            }
4660            ScalarValue::Date64(val) => {
4661                eq_array_primitive!(array, index, as_date64_array, val)?
4662            }
4663            ScalarValue::Time32Second(val) => {
4664                eq_array_primitive!(array, index, as_time32_second_array, val)?
4665            }
4666            ScalarValue::Time32Millisecond(val) => {
4667                eq_array_primitive!(array, index, as_time32_millisecond_array, val)?
4668            }
4669            ScalarValue::Time64Microsecond(val) => {
4670                eq_array_primitive!(array, index, as_time64_microsecond_array, val)?
4671            }
4672            ScalarValue::Time64Nanosecond(val) => {
4673                eq_array_primitive!(array, index, as_time64_nanosecond_array, val)?
4674            }
4675            ScalarValue::TimestampSecond(val, _) => {
4676                eq_array_primitive!(array, index, as_timestamp_second_array, val)?
4677            }
4678            ScalarValue::TimestampMillisecond(val, _) => {
4679                eq_array_primitive!(array, index, as_timestamp_millisecond_array, val)?
4680            }
4681            ScalarValue::TimestampMicrosecond(val, _) => {
4682                eq_array_primitive!(array, index, as_timestamp_microsecond_array, val)?
4683            }
4684            ScalarValue::TimestampNanosecond(val, _) => {
4685                eq_array_primitive!(array, index, as_timestamp_nanosecond_array, val)?
4686            }
4687            ScalarValue::IntervalYearMonth(val) => {
4688                eq_array_primitive!(array, index, as_interval_ym_array, val)?
4689            }
4690            ScalarValue::IntervalDayTime(val) => {
4691                eq_array_primitive!(array, index, as_interval_dt_array, val)?
4692            }
4693            ScalarValue::IntervalMonthDayNano(val) => {
4694                eq_array_primitive!(array, index, as_interval_mdn_array, val)?
4695            }
4696            ScalarValue::DurationSecond(val) => {
4697                eq_array_primitive!(array, index, as_duration_second_array, val)?
4698            }
4699            ScalarValue::DurationMillisecond(val) => {
4700                eq_array_primitive!(array, index, as_duration_millisecond_array, val)?
4701            }
4702            ScalarValue::DurationMicrosecond(val) => {
4703                eq_array_primitive!(array, index, as_duration_microsecond_array, val)?
4704            }
4705            ScalarValue::DurationNanosecond(val) => {
4706                eq_array_primitive!(array, index, as_duration_nanosecond_array, val)?
4707            }
4708            ScalarValue::Union(value, _, _) => {
4709                let array = as_union_array(array)?;
4710                let ti = array.type_id(index);
4711                let index = array.value_offset(index);
4712                if let Some((ti_v, value)) = value {
4713                    ti_v == &ti && value.eq_array(array.child(ti), index)?
4714                } else {
4715                    array.child(ti).is_null(index)
4716                }
4717            }
4718            ScalarValue::Dictionary(key_type, v) => {
4719                let (values_array, values_index) = match key_type.as_ref() {
4720                    DataType::Int8 => get_dict_value::<Int8Type>(array, index)?,
4721                    DataType::Int16 => get_dict_value::<Int16Type>(array, index)?,
4722                    DataType::Int32 => get_dict_value::<Int32Type>(array, index)?,
4723                    DataType::Int64 => get_dict_value::<Int64Type>(array, index)?,
4724                    DataType::UInt8 => get_dict_value::<UInt8Type>(array, index)?,
4725                    DataType::UInt16 => get_dict_value::<UInt16Type>(array, index)?,
4726                    DataType::UInt32 => get_dict_value::<UInt32Type>(array, index)?,
4727                    DataType::UInt64 => get_dict_value::<UInt64Type>(array, index)?,
4728                    _ => unreachable!("Invalid dictionary keys type: {}", key_type),
4729                };
4730                // was the value in the array non null?
4731                match values_index {
4732                    Some(values_index) => v.eq_array(values_array, values_index)?,
4733                    None => v.is_null(),
4734                }
4735            }
4736            ScalarValue::RunEndEncoded(run_ends_field, _, value) => {
4737                // Explicitly check length here since get_physical_index() doesn't
4738                // bound check for us
4739                if index > array.len() {
4740                    return _exec_err!(
4741                        "Index {index} out of bounds for array of length {}",
4742                        array.len()
4743                    );
4744                }
4745                match run_ends_field.data_type() {
4746                    DataType::Int16 => {
4747                        let array = as_run_array::<Int16Type>(array)?;
4748                        let index = array.get_physical_index(index);
4749                        value.eq_array(array.values(), index)?
4750                    }
4751                    DataType::Int32 => {
4752                        let array = as_run_array::<Int32Type>(array)?;
4753                        let index = array.get_physical_index(index);
4754                        value.eq_array(array.values(), index)?
4755                    }
4756                    DataType::Int64 => {
4757                        let array = as_run_array::<Int64Type>(array)?;
4758                        let index = array.get_physical_index(index);
4759                        value.eq_array(array.values(), index)?
4760                    }
4761                    dt => unreachable!("Invalid run-ends type: {dt}"),
4762                }
4763            }
4764            ScalarValue::Null => array.is_null(index),
4765        })
4766    }
4767
4768    fn eq_array_list(arr1: &ArrayRef, arr2: &ArrayRef, index: usize) -> bool {
4769        let right = arr2.slice(index, 1);
4770        arr1 == &right
4771    }
4772
4773    /// Compare `self` with `other` and return an `Ordering`.
4774    ///
4775    /// This is the same as [`PartialOrd`] except that it returns
4776    /// `Err` if the values cannot be compared, e.g., they have incompatible data types.
4777    pub fn try_cmp(&self, other: &Self) -> Result<Ordering> {
4778        self.partial_cmp(other).ok_or_else(|| {
4779            _internal_datafusion_err!("Uncomparable values: {self:?}, {other:?}")
4780        })
4781    }
4782
4783    /// Estimate size if bytes including `Self`. For values with internal containers such as `String`
4784    /// includes the allocated size (`capacity`) rather than the current length (`len`)
4785    pub fn size(&self) -> usize {
4786        size_of_val(self)
4787            + match self {
4788                ScalarValue::Null
4789                | ScalarValue::Boolean(_)
4790                | ScalarValue::Float16(_)
4791                | ScalarValue::Float32(_)
4792                | ScalarValue::Float64(_)
4793                | ScalarValue::Decimal32(_, _, _)
4794                | ScalarValue::Decimal64(_, _, _)
4795                | ScalarValue::Decimal128(_, _, _)
4796                | ScalarValue::Decimal256(_, _, _)
4797                | ScalarValue::Int8(_)
4798                | ScalarValue::Int16(_)
4799                | ScalarValue::Int32(_)
4800                | ScalarValue::Int64(_)
4801                | ScalarValue::UInt8(_)
4802                | ScalarValue::UInt16(_)
4803                | ScalarValue::UInt32(_)
4804                | ScalarValue::UInt64(_)
4805                | ScalarValue::Date32(_)
4806                | ScalarValue::Date64(_)
4807                | ScalarValue::Time32Second(_)
4808                | ScalarValue::Time32Millisecond(_)
4809                | ScalarValue::Time64Microsecond(_)
4810                | ScalarValue::Time64Nanosecond(_)
4811                | ScalarValue::IntervalYearMonth(_)
4812                | ScalarValue::IntervalDayTime(_)
4813                | ScalarValue::IntervalMonthDayNano(_)
4814                | ScalarValue::DurationSecond(_)
4815                | ScalarValue::DurationMillisecond(_)
4816                | ScalarValue::DurationMicrosecond(_)
4817                | ScalarValue::DurationNanosecond(_) => 0,
4818                ScalarValue::Utf8(s)
4819                | ScalarValue::LargeUtf8(s)
4820                | ScalarValue::Utf8View(s) => {
4821                    s.as_ref().map(|s| s.capacity()).unwrap_or_default()
4822                }
4823                ScalarValue::TimestampSecond(_, s)
4824                | ScalarValue::TimestampMillisecond(_, s)
4825                | ScalarValue::TimestampMicrosecond(_, s)
4826                | ScalarValue::TimestampNanosecond(_, s) => {
4827                    s.as_ref().map(|s| s.len()).unwrap_or_default()
4828                }
4829                ScalarValue::Binary(b)
4830                | ScalarValue::FixedSizeBinary(_, b)
4831                | ScalarValue::LargeBinary(b)
4832                | ScalarValue::BinaryView(b) => {
4833                    b.as_ref().map(|b| b.capacity()).unwrap_or_default()
4834                }
4835                ScalarValue::List(arr) => arr.get_array_memory_size(),
4836                ScalarValue::LargeList(arr) => arr.get_array_memory_size(),
4837                ScalarValue::FixedSizeList(arr) => arr.get_array_memory_size(),
4838                ScalarValue::ListView(arr) => arr.get_array_memory_size(),
4839                ScalarValue::LargeListView(arr) => arr.get_array_memory_size(),
4840                ScalarValue::Struct(arr) => arr.get_array_memory_size(),
4841                ScalarValue::Map(arr) => arr.get_array_memory_size(),
4842                ScalarValue::Union(vals, fields, _mode) => {
4843                    vals.as_ref()
4844                        .map(|(_id, sv)| sv.size() - size_of_val(sv))
4845                        .unwrap_or_default()
4846                        // `fields` is boxed, so it is NOT already included in `self`
4847                        + size_of_val(fields)
4848                        + (size_of::<Field>() * fields.len())
4849                        + fields.iter().map(|(_idx, field)| field.size() - size_of_val(field)).sum::<usize>()
4850                }
4851                ScalarValue::Dictionary(dt, sv) => {
4852                    // `dt` and `sv` are boxed, so they are NOT already included in `self`
4853                    dt.size() + sv.size()
4854                }
4855                ScalarValue::RunEndEncoded(rf, vf, v) => rf.size() + vf.size() + v.size(),
4856            }
4857    }
4858
4859    /// Estimates [size](Self::size) of [`Vec`] in bytes.
4860    ///
4861    /// Includes the size of the [`Vec`] container itself.
4862    pub fn size_of_vec(vec: &Vec<Self>) -> usize {
4863        size_of_val(vec)
4864            + (size_of::<ScalarValue>() * vec.capacity())
4865            + vec
4866                .iter()
4867                .map(|sv| sv.size() - size_of_val(sv))
4868                .sum::<usize>()
4869    }
4870
4871    /// Estimates [size](Self::size) of [`VecDeque`] in bytes.
4872    ///
4873    /// Includes the size of the [`VecDeque`] container itself.
4874    pub fn size_of_vec_deque(vec_deque: &VecDeque<Self>) -> usize {
4875        size_of_val(vec_deque)
4876            + (size_of::<ScalarValue>() * vec_deque.capacity())
4877            + vec_deque
4878                .iter()
4879                .map(|sv| sv.size() - size_of_val(sv))
4880                .sum::<usize>()
4881    }
4882
4883    /// Estimates [size](Self::size) of [`HashSet`] in bytes.
4884    ///
4885    /// Includes the size of the [`HashSet`] container itself.
4886    #[allow(clippy::allow_attributes, clippy::mutable_key_type)] // ScalarValue has interior mutability but is intentionally used as hash key
4887    pub fn size_of_hashset<S>(set: &HashSet<Self, S>) -> usize {
4888        size_of_val(set)
4889            + (size_of::<ScalarValue>() * set.capacity())
4890            + set
4891                .iter()
4892                .map(|sv| sv.size() - size_of_val(sv))
4893                .sum::<usize>()
4894    }
4895
4896    /// Estimates [size](Self::size) of [`HashMap`] keyed by [`ScalarValue`] in bytes.
4897    ///
4898    /// Includes the size of the [`HashMap`] container itself. Heap payload of
4899    /// `V` is not accounted for; callers storing heap-backed values should
4900    /// supplement this estimate.
4901    #[allow(clippy::allow_attributes, clippy::mutable_key_type)] // ScalarValue has interior mutability but is intentionally used as hash key
4902    pub fn size_of_hashmap<V, S>(map: &HashMap<Self, V, S>) -> usize {
4903        size_of_val(map)
4904            + ((size_of::<ScalarValue>() + size_of::<V>()) * map.capacity())
4905            + map.keys().map(|k| k.size() - size_of_val(k)).sum::<usize>()
4906    }
4907
4908    /// Compacts the allocation referenced by `self` to the minimum, copying the data if
4909    /// necessary.
4910    ///
4911    /// This can be relevant when `self` is a list or contains a list as a nested value, as
4912    /// a single list holds an Arc to its entire original array buffer.
4913    pub fn compact(&mut self) {
4914        // copy_array_data + compact_view_buffers + downcast back, all in one step.
4915        macro_rules! compact_array {
4916            ($arr:expr, $from_type:ty, $($as_method:tt)+) => {
4917                *Arc::make_mut($arr) = ScalarValue::compact_view_buffers(
4918                    Arc::new(<$from_type>::from(copy_array_data(&$arr.to_data()))) as ArrayRef,
4919                ).$($as_method)+.clone()
4920            };
4921        }
4922        match self {
4923            ScalarValue::Null
4924            | ScalarValue::Boolean(_)
4925            | ScalarValue::Float16(_)
4926            | ScalarValue::Float32(_)
4927            | ScalarValue::Float64(_)
4928            | ScalarValue::Decimal32(_, _, _)
4929            | ScalarValue::Decimal64(_, _, _)
4930            | ScalarValue::Decimal128(_, _, _)
4931            | ScalarValue::Decimal256(_, _, _)
4932            | ScalarValue::Int8(_)
4933            | ScalarValue::Int16(_)
4934            | ScalarValue::Int32(_)
4935            | ScalarValue::Int64(_)
4936            | ScalarValue::UInt8(_)
4937            | ScalarValue::UInt16(_)
4938            | ScalarValue::UInt32(_)
4939            | ScalarValue::UInt64(_)
4940            | ScalarValue::Date32(_)
4941            | ScalarValue::Date64(_)
4942            | ScalarValue::Time32Second(_)
4943            | ScalarValue::Time32Millisecond(_)
4944            | ScalarValue::Time64Microsecond(_)
4945            | ScalarValue::Time64Nanosecond(_)
4946            | ScalarValue::IntervalYearMonth(_)
4947            | ScalarValue::IntervalDayTime(_)
4948            | ScalarValue::IntervalMonthDayNano(_)
4949            | ScalarValue::DurationSecond(_)
4950            | ScalarValue::DurationMillisecond(_)
4951            | ScalarValue::DurationMicrosecond(_)
4952            | ScalarValue::DurationNanosecond(_)
4953            | ScalarValue::Utf8(_)
4954            | ScalarValue::LargeUtf8(_)
4955            | ScalarValue::Utf8View(_)
4956            | ScalarValue::TimestampSecond(_, _)
4957            | ScalarValue::TimestampMillisecond(_, _)
4958            | ScalarValue::TimestampMicrosecond(_, _)
4959            | ScalarValue::TimestampNanosecond(_, _)
4960            | ScalarValue::Binary(_)
4961            | ScalarValue::FixedSizeBinary(_, _)
4962            | ScalarValue::LargeBinary(_)
4963            | ScalarValue::BinaryView(_) => (),
4964            ScalarValue::FixedSizeList(arr) => {
4965                compact_array!(arr, FixedSizeListArray, as_fixed_size_list())
4966            }
4967            ScalarValue::List(arr) => compact_array!(arr, ListArray, as_list::<i32>()),
4968            ScalarValue::LargeList(arr) => {
4969                compact_array!(arr, LargeListArray, as_list::<i64>())
4970            }
4971            ScalarValue::ListView(arr) => {
4972                compact_array!(arr, ListViewArray, as_list_view::<i32>())
4973            }
4974            ScalarValue::LargeListView(arr) => {
4975                compact_array!(arr, LargeListViewArray, as_list_view::<i64>())
4976            }
4977            ScalarValue::Struct(arr) => compact_array!(arr, StructArray, as_struct()),
4978            ScalarValue::Map(arr) => compact_array!(arr, MapArray, as_map()),
4979            ScalarValue::Union(val, _, _) => {
4980                if let Some((_, value)) = val.as_mut() {
4981                    value.compact();
4982                }
4983            }
4984            ScalarValue::Dictionary(_, value) => {
4985                value.compact();
4986            }
4987            ScalarValue::RunEndEncoded(_, _, value) => {
4988                value.compact();
4989            }
4990        }
4991    }
4992
4993    /// Compacts ([ScalarValue::compact]) the current [ScalarValue] and returns it.
4994    pub fn compacted(mut self) -> Self {
4995        self.compact();
4996        self
4997    }
4998
4999    /// Recursively compacts the backing buffers of any [`StringViewArray`] or
5000    /// [`BinaryViewArray`] nested within `array`.
5001    ///
5002    /// View-typed arrays keep an `Arc` reference to their original backing
5003    /// buffers, so a single scalar extracted from a large batch still retains
5004    /// the entire buffer.  Calling [`.gc()`][StringViewArray::gc] copies only
5005    /// the bytes that are actually referenced by the surviving views, releasing
5006    /// the rest.
5007    ///
5008    /// Container types (`List`, `LargeList`, `FixedSizeList`, `ListView`,
5009    /// `LargeListView`, `Struct`, `Map`) are handled by recursing into their
5010    /// child / values arrays and reconstructing the parent with the compacted
5011    /// children.  All other types are returned unchanged.
5012    fn compact_view_buffers(array: ArrayRef) -> ArrayRef {
5013        // Macro for the i32/i64-offset list pair (List / LargeList).
5014        macro_rules! gc_list {
5015            ($field:expr, $offset_type:ty, $array_type:ty) => {{
5016                let list = array.as_list::<$offset_type>();
5017                Arc::new(<$array_type>::new(
5018                    Arc::clone($field),
5019                    list.offsets().clone(),
5020                    ScalarValue::compact_view_buffers(Arc::clone(list.values())),
5021                    list.nulls().cloned(),
5022                )) as ArrayRef
5023            }};
5024        }
5025        // Macro for the i32/i64-offset list-view pair (ListView / LargeListView).
5026        macro_rules! gc_list_view {
5027            ($field:expr, $offset_type:ty, $array_type:ty) => {{
5028                let list = array.as_list_view::<$offset_type>();
5029                Arc::new(<$array_type>::new(
5030                    Arc::clone($field),
5031                    list.offsets().clone(),
5032                    list.sizes().clone(),
5033                    ScalarValue::compact_view_buffers(Arc::clone(list.values())),
5034                    list.nulls().cloned(),
5035                )) as ArrayRef
5036            }};
5037        }
5038
5039        match array.data_type() {
5040            DataType::Utf8View => Arc::new(array.as_string_view().gc()),
5041            DataType::BinaryView => Arc::new(array.as_binary_view().gc()),
5042            DataType::Struct(_) => {
5043                let s = array.as_struct();
5044                if s.fields().is_empty() {
5045                    // Zero-field structs carry no child buffers to compact, so
5046                    // return the input array unchanged.
5047                    array
5048                } else {
5049                    let columns = s
5050                        .columns()
5051                        .iter()
5052                        .map(|c| ScalarValue::compact_view_buffers(Arc::clone(c)))
5053                        .collect();
5054                    Arc::new(StructArray::new(
5055                        s.fields().clone(),
5056                        columns,
5057                        s.nulls().cloned(),
5058                    ))
5059                }
5060            }
5061            DataType::List(field) => gc_list!(field, i32, ListArray),
5062            DataType::LargeList(field) => gc_list!(field, i64, LargeListArray),
5063            DataType::FixedSizeList(field, size) => {
5064                let list = array.as_fixed_size_list();
5065                Arc::new(FixedSizeListArray::new(
5066                    Arc::clone(field),
5067                    *size,
5068                    ScalarValue::compact_view_buffers(Arc::clone(list.values())),
5069                    list.nulls().cloned(),
5070                ))
5071            }
5072            DataType::ListView(field) => gc_list_view!(field, i32, ListViewArray),
5073            DataType::LargeListView(field) => {
5074                gc_list_view!(field, i64, LargeListViewArray)
5075            }
5076            DataType::Map(field, ordered) => {
5077                let map = array.as_map();
5078                let entries = ScalarValue::compact_view_buffers(Arc::new(
5079                    map.entries().clone(),
5080                )
5081                    as ArrayRef);
5082                Arc::new(MapArray::new(
5083                    Arc::clone(field),
5084                    map.offsets().clone(),
5085                    entries.as_struct().clone(),
5086                    map.nulls().cloned(),
5087                    *ordered,
5088                ))
5089            }
5090            _ => array,
5091        }
5092    }
5093
5094    /// Returns the minimum value for the given numeric `DataType`.
5095    ///
5096    /// This function returns the smallest representable value for numeric
5097    /// and temporal data types. For non-numeric types, it returns `None`.
5098    ///
5099    /// # Supported Types
5100    ///
5101    /// - **Integer types**: `i8::MIN`, `i16::MIN`, etc.
5102    /// - **Unsigned types**: Always 0 (`u8::MIN`, `u16::MIN`, etc.)
5103    /// - **Float types**: Negative infinity (IEEE 754)
5104    /// - **Decimal types**: Smallest value based on precision
5105    /// - **Temporal types**: Minimum timestamp/date values
5106    /// - **Time types**: 0 (midnight)
5107    /// - **Duration types**: `i64::MIN`
5108    pub fn min(datatype: &DataType) -> Option<ScalarValue> {
5109        match datatype {
5110            DataType::Int8 => Some(ScalarValue::Int8(Some(i8::MIN))),
5111            DataType::Int16 => Some(ScalarValue::Int16(Some(i16::MIN))),
5112            DataType::Int32 => Some(ScalarValue::Int32(Some(i32::MIN))),
5113            DataType::Int64 => Some(ScalarValue::Int64(Some(i64::MIN))),
5114            DataType::UInt8 => Some(ScalarValue::UInt8(Some(u8::MIN))),
5115            DataType::UInt16 => Some(ScalarValue::UInt16(Some(u16::MIN))),
5116            DataType::UInt32 => Some(ScalarValue::UInt32(Some(u32::MIN))),
5117            DataType::UInt64 => Some(ScalarValue::UInt64(Some(u64::MIN))),
5118            DataType::Float16 => Some(ScalarValue::Float16(Some(f16::NEG_INFINITY))),
5119            DataType::Float32 => Some(ScalarValue::Float32(Some(f32::NEG_INFINITY))),
5120            DataType::Float64 => Some(ScalarValue::Float64(Some(f64::NEG_INFINITY))),
5121            DataType::Decimal32(precision, scale) => {
5122                let min = MIN_DECIMAL32_FOR_EACH_PRECISION[*precision as usize];
5123                Some(ScalarValue::Decimal32(Some(min), *precision, *scale))
5124            }
5125            DataType::Decimal64(precision, scale) => {
5126                let min = MIN_DECIMAL64_FOR_EACH_PRECISION[*precision as usize];
5127                Some(ScalarValue::Decimal64(Some(min), *precision, *scale))
5128            }
5129            DataType::Decimal128(precision, scale) => {
5130                let min = MIN_DECIMAL128_FOR_EACH_PRECISION[*precision as usize];
5131                Some(ScalarValue::Decimal128(Some(min), *precision, *scale))
5132            }
5133            DataType::Decimal256(precision, scale) => {
5134                let min = MIN_DECIMAL256_FOR_EACH_PRECISION[*precision as usize];
5135                Some(ScalarValue::Decimal256(Some(min), *precision, *scale))
5136            }
5137            DataType::Date32 => Some(ScalarValue::Date32(Some(i32::MIN))),
5138            DataType::Date64 => Some(ScalarValue::Date64(Some(i64::MIN))),
5139            DataType::Time32(TimeUnit::Second) => {
5140                Some(ScalarValue::Time32Second(Some(0)))
5141            }
5142            DataType::Time32(TimeUnit::Millisecond) => {
5143                Some(ScalarValue::Time32Millisecond(Some(0)))
5144            }
5145            DataType::Time64(TimeUnit::Microsecond) => {
5146                Some(ScalarValue::Time64Microsecond(Some(0)))
5147            }
5148            DataType::Time64(TimeUnit::Nanosecond) => {
5149                Some(ScalarValue::Time64Nanosecond(Some(0)))
5150            }
5151            DataType::Timestamp(unit, tz) => match unit {
5152                TimeUnit::Second => {
5153                    Some(ScalarValue::TimestampSecond(Some(i64::MIN), tz.clone()))
5154                }
5155                TimeUnit::Millisecond => Some(ScalarValue::TimestampMillisecond(
5156                    Some(i64::MIN),
5157                    tz.clone(),
5158                )),
5159                TimeUnit::Microsecond => Some(ScalarValue::TimestampMicrosecond(
5160                    Some(i64::MIN),
5161                    tz.clone(),
5162                )),
5163                TimeUnit::Nanosecond => {
5164                    Some(ScalarValue::TimestampNanosecond(Some(i64::MIN), tz.clone()))
5165                }
5166            },
5167            DataType::Duration(unit) => match unit {
5168                TimeUnit::Second => Some(ScalarValue::DurationSecond(Some(i64::MIN))),
5169                TimeUnit::Millisecond => {
5170                    Some(ScalarValue::DurationMillisecond(Some(i64::MIN)))
5171                }
5172                TimeUnit::Microsecond => {
5173                    Some(ScalarValue::DurationMicrosecond(Some(i64::MIN)))
5174                }
5175                TimeUnit::Nanosecond => {
5176                    Some(ScalarValue::DurationNanosecond(Some(i64::MIN)))
5177                }
5178            },
5179            _ => None,
5180        }
5181    }
5182
5183    /// Returns the maximum value for the given numeric `DataType`.
5184    ///
5185    /// This function returns the largest representable value for numeric
5186    /// and temporal data types. For non-numeric types, it returns `None`.
5187    ///
5188    /// # Supported Types
5189    ///
5190    /// - **Integer types**: `i8::MAX`, `i16::MAX`, etc.
5191    /// - **Unsigned types**: `u8::MAX`, `u16::MAX`, etc.
5192    /// - **Float types**: Positive infinity (IEEE 754)
5193    /// - **Decimal types**: Largest value based on precision
5194    /// - **Temporal types**: Maximum timestamp/date values
5195    /// - **Time types**: Maximum time in the day (1 day - 1 unit)
5196    /// - **Duration types**: `i64::MAX`
5197    pub fn max(datatype: &DataType) -> Option<ScalarValue> {
5198        match datatype {
5199            DataType::Int8 => Some(ScalarValue::Int8(Some(i8::MAX))),
5200            DataType::Int16 => Some(ScalarValue::Int16(Some(i16::MAX))),
5201            DataType::Int32 => Some(ScalarValue::Int32(Some(i32::MAX))),
5202            DataType::Int64 => Some(ScalarValue::Int64(Some(i64::MAX))),
5203            DataType::UInt8 => Some(ScalarValue::UInt8(Some(u8::MAX))),
5204            DataType::UInt16 => Some(ScalarValue::UInt16(Some(u16::MAX))),
5205            DataType::UInt32 => Some(ScalarValue::UInt32(Some(u32::MAX))),
5206            DataType::UInt64 => Some(ScalarValue::UInt64(Some(u64::MAX))),
5207            DataType::Float16 => Some(ScalarValue::Float16(Some(f16::INFINITY))),
5208            DataType::Float32 => Some(ScalarValue::Float32(Some(f32::INFINITY))),
5209            DataType::Float64 => Some(ScalarValue::Float64(Some(f64::INFINITY))),
5210            DataType::Decimal32(precision, scale) => {
5211                let max = MAX_DECIMAL32_FOR_EACH_PRECISION[*precision as usize];
5212                Some(ScalarValue::Decimal32(Some(max), *precision, *scale))
5213            }
5214            DataType::Decimal64(precision, scale) => {
5215                let max = MAX_DECIMAL64_FOR_EACH_PRECISION[*precision as usize];
5216                Some(ScalarValue::Decimal64(Some(max), *precision, *scale))
5217            }
5218            DataType::Decimal128(precision, scale) => {
5219                let max = MAX_DECIMAL128_FOR_EACH_PRECISION[*precision as usize];
5220                Some(ScalarValue::Decimal128(Some(max), *precision, *scale))
5221            }
5222            DataType::Decimal256(precision, scale) => {
5223                let max = MAX_DECIMAL256_FOR_EACH_PRECISION[*precision as usize];
5224                Some(ScalarValue::Decimal256(Some(max), *precision, *scale))
5225            }
5226            DataType::Date32 => Some(ScalarValue::Date32(Some(i32::MAX))),
5227            DataType::Date64 => Some(ScalarValue::Date64(Some(i64::MAX))),
5228            DataType::Time32(TimeUnit::Second) => {
5229                // 86399 seconds = 23:59:59
5230                Some(ScalarValue::Time32Second(Some(86_399)))
5231            }
5232            DataType::Time32(TimeUnit::Millisecond) => {
5233                // 86_399_999 milliseconds = 23:59:59.999
5234                Some(ScalarValue::Time32Millisecond(Some(86_399_999)))
5235            }
5236            DataType::Time64(TimeUnit::Microsecond) => {
5237                // 86_399_999_999 microseconds = 23:59:59.999999
5238                Some(ScalarValue::Time64Microsecond(Some(86_399_999_999)))
5239            }
5240            DataType::Time64(TimeUnit::Nanosecond) => {
5241                // 86_399_999_999_999 nanoseconds = 23:59:59.999999999
5242                Some(ScalarValue::Time64Nanosecond(Some(86_399_999_999_999)))
5243            }
5244            DataType::Timestamp(unit, tz) => match unit {
5245                TimeUnit::Second => {
5246                    Some(ScalarValue::TimestampSecond(Some(i64::MAX), tz.clone()))
5247                }
5248                TimeUnit::Millisecond => Some(ScalarValue::TimestampMillisecond(
5249                    Some(i64::MAX),
5250                    tz.clone(),
5251                )),
5252                TimeUnit::Microsecond => Some(ScalarValue::TimestampMicrosecond(
5253                    Some(i64::MAX),
5254                    tz.clone(),
5255                )),
5256                TimeUnit::Nanosecond => {
5257                    Some(ScalarValue::TimestampNanosecond(Some(i64::MAX), tz.clone()))
5258                }
5259            },
5260            DataType::Duration(unit) => match unit {
5261                TimeUnit::Second => Some(ScalarValue::DurationSecond(Some(i64::MAX))),
5262                TimeUnit::Millisecond => {
5263                    Some(ScalarValue::DurationMillisecond(Some(i64::MAX)))
5264                }
5265                TimeUnit::Microsecond => {
5266                    Some(ScalarValue::DurationMicrosecond(Some(i64::MAX)))
5267                }
5268                TimeUnit::Nanosecond => {
5269                    Some(ScalarValue::DurationNanosecond(Some(i64::MAX)))
5270                }
5271            },
5272            _ => None,
5273        }
5274    }
5275
5276    /// A thin wrapper on Arrow's validation that throws internal error if validation
5277    /// fails.
5278    fn validate_decimal_or_internal_err<T: DecimalType>(
5279        precision: u8,
5280        scale: i8,
5281    ) -> Result<()> {
5282        validate_decimal_precision_and_scale::<T>(precision, scale).map_err(|err| {
5283            _internal_datafusion_err!(
5284                "Decimal precision/scale invariant violated \
5285                 (precision={precision}, scale={scale}): {err}"
5286            )
5287        })
5288    }
5289}
5290
5291/// Compacts the data of an `ArrayData` into a new `ArrayData`.
5292///
5293/// This is useful when you want to minimize the memory footprint of an
5294/// `ArrayData`. For example, the value returned by [`Array::slice`] still
5295/// points at the same underlying data buffers as the original array, which may
5296/// hold many more values. Calling `copy_array_data` on the sliced array will
5297/// create a new, smaller, `ArrayData` that only contains the data for the
5298/// sliced array.
5299///
5300/// # Example
5301/// ```
5302/// # use arrow::array::{make_array, Array, Int32Array};
5303/// use datafusion_common::scalar::copy_array_data;
5304/// let array = Int32Array::from_iter_values(0..8192);
5305/// // Take only the first 2 elements
5306/// let sliced_array = array.slice(0, 2);
5307/// // The memory footprint of `sliced_array` is close to 8192 * 4 bytes
5308/// assert_eq!(32864, sliced_array.get_array_memory_size());
5309/// // however, we can copy the data to a new `ArrayData`
5310/// let new_array = make_array(copy_array_data(&sliced_array.into_data()));
5311/// // The memory footprint of `new_array` is now only 2 * 4 bytes
5312/// // and overhead:
5313/// assert_eq!(160, new_array.get_array_memory_size());
5314/// ```
5315///
5316/// See also [`ScalarValue::compact`] which applies to `ScalarValue` instances
5317/// as necessary.
5318pub fn copy_array_data(src_data: &ArrayData) -> ArrayData {
5319    let mut copy = MutableArrayData::new(vec![&src_data], true, src_data.len());
5320    copy.try_extend(0, 0, src_data.len())
5321        .expect("copy_array_data failed due to offset overflow");
5322    copy.freeze()
5323}
5324
5325macro_rules! impl_scalar {
5326    ($ty:ty, $scalar:tt) => {
5327        impl From<$ty> for ScalarValue {
5328            fn from(value: $ty) -> Self {
5329                ScalarValue::$scalar(Some(value))
5330            }
5331        }
5332
5333        impl From<Option<$ty>> for ScalarValue {
5334            fn from(value: Option<$ty>) -> Self {
5335                ScalarValue::$scalar(value)
5336            }
5337        }
5338    };
5339}
5340
5341impl_scalar!(f64, Float64);
5342impl_scalar!(f32, Float32);
5343impl_scalar!(f16, Float16);
5344impl_scalar!(i8, Int8);
5345impl_scalar!(i16, Int16);
5346impl_scalar!(i32, Int32);
5347impl_scalar!(i64, Int64);
5348impl_scalar!(bool, Boolean);
5349impl_scalar!(u8, UInt8);
5350impl_scalar!(u16, UInt16);
5351impl_scalar!(u32, UInt32);
5352impl_scalar!(u64, UInt64);
5353
5354impl From<&str> for ScalarValue {
5355    fn from(value: &str) -> Self {
5356        Some(value).into()
5357    }
5358}
5359
5360impl From<Option<&str>> for ScalarValue {
5361    fn from(value: Option<&str>) -> Self {
5362        let value = value.map(|s| s.to_string());
5363        value.into()
5364    }
5365}
5366
5367/// Wrapper to create ScalarValue::Struct for convenience
5368impl From<Vec<(&str, ScalarValue)>> for ScalarValue {
5369    fn from(value: Vec<(&str, ScalarValue)>) -> Self {
5370        value
5371            .into_iter()
5372            .fold(ScalarStructBuilder::new(), |builder, (name, value)| {
5373                builder.with_name_and_scalar(name, value)
5374            })
5375            .build()
5376            .unwrap()
5377    }
5378}
5379
5380impl FromStr for ScalarValue {
5381    type Err = Infallible;
5382
5383    fn from_str(s: &str) -> Result<Self, Self::Err> {
5384        Ok(s.into())
5385    }
5386}
5387
5388impl From<String> for ScalarValue {
5389    fn from(value: String) -> Self {
5390        Some(value).into()
5391    }
5392}
5393
5394impl From<Option<String>> for ScalarValue {
5395    fn from(value: Option<String>) -> Self {
5396        ScalarValue::Utf8(value)
5397    }
5398}
5399
5400macro_rules! impl_try_from {
5401    ($SCALAR:ident, $NATIVE:ident) => {
5402        impl TryFrom<ScalarValue> for $NATIVE {
5403            type Error = DataFusionError;
5404
5405            fn try_from(value: ScalarValue) -> Result<Self> {
5406                match value {
5407                    ScalarValue::$SCALAR(Some(inner_value)) => Ok(inner_value),
5408                    _ => _internal_err!(
5409                        "Cannot convert {:?} to {}",
5410                        value,
5411                        std::any::type_name::<Self>()
5412                    ),
5413                }
5414            }
5415        }
5416    };
5417}
5418
5419impl_try_from!(Int8, i8);
5420impl_try_from!(Int16, i16);
5421
5422// special implementation for i32 because of Date32 and Time32
5423impl TryFrom<ScalarValue> for i32 {
5424    type Error = DataFusionError;
5425
5426    fn try_from(value: ScalarValue) -> Result<Self> {
5427        match value {
5428            ScalarValue::Int32(Some(inner_value))
5429            | ScalarValue::Date32(Some(inner_value))
5430            | ScalarValue::Time32Second(Some(inner_value))
5431            | ScalarValue::Time32Millisecond(Some(inner_value)) => Ok(inner_value),
5432            _ => _internal_err!(
5433                "Cannot convert {:?} to {}",
5434                value,
5435                std::any::type_name::<Self>()
5436            ),
5437        }
5438    }
5439}
5440
5441// special implementation for i64 because of Date64, Time64 and Timestamp
5442impl TryFrom<ScalarValue> for i64 {
5443    type Error = DataFusionError;
5444
5445    fn try_from(value: ScalarValue) -> Result<Self> {
5446        match value {
5447            ScalarValue::Int64(Some(inner_value))
5448            | ScalarValue::Date64(Some(inner_value))
5449            | ScalarValue::Time64Microsecond(Some(inner_value))
5450            | ScalarValue::Time64Nanosecond(Some(inner_value))
5451            | ScalarValue::TimestampNanosecond(Some(inner_value), _)
5452            | ScalarValue::TimestampMicrosecond(Some(inner_value), _)
5453            | ScalarValue::TimestampMillisecond(Some(inner_value), _)
5454            | ScalarValue::TimestampSecond(Some(inner_value), _) => Ok(inner_value),
5455            _ => _internal_err!(
5456                "Cannot convert {:?} to {}",
5457                value,
5458                std::any::type_name::<Self>()
5459            ),
5460        }
5461    }
5462}
5463
5464// special implementation for i128 because of Decimal128
5465impl TryFrom<ScalarValue> for i128 {
5466    type Error = DataFusionError;
5467
5468    fn try_from(value: ScalarValue) -> Result<Self> {
5469        match value {
5470            ScalarValue::Decimal128(Some(inner_value), _, _) => Ok(inner_value),
5471            _ => _internal_err!(
5472                "Cannot convert {:?} to {}",
5473                value,
5474                std::any::type_name::<Self>()
5475            ),
5476        }
5477    }
5478}
5479
5480// special implementation for i256 because of Decimal128
5481impl TryFrom<ScalarValue> for i256 {
5482    type Error = DataFusionError;
5483
5484    fn try_from(value: ScalarValue) -> Result<Self> {
5485        match value {
5486            ScalarValue::Decimal256(Some(inner_value), _, _) => Ok(inner_value),
5487            _ => _internal_err!(
5488                "Cannot convert {:?} to {}",
5489                value,
5490                std::any::type_name::<Self>()
5491            ),
5492        }
5493    }
5494}
5495
5496impl_try_from!(UInt8, u8);
5497impl_try_from!(UInt16, u16);
5498impl_try_from!(UInt32, u32);
5499impl_try_from!(UInt64, u64);
5500impl_try_from!(Float16, f16);
5501impl_try_from!(Float32, f32);
5502impl_try_from!(Float64, f64);
5503impl_try_from!(Boolean, bool);
5504
5505impl TryFrom<DataType> for ScalarValue {
5506    type Error = DataFusionError;
5507
5508    /// Create a Null instance of ScalarValue for this datatype
5509    fn try_from(datatype: DataType) -> Result<Self> {
5510        (&datatype).try_into()
5511    }
5512}
5513
5514impl TryFrom<&DataType> for ScalarValue {
5515    type Error = DataFusionError;
5516
5517    /// Create a Null instance of ScalarValue for this datatype
5518    fn try_from(data_type: &DataType) -> Result<Self> {
5519        Self::try_new_null(data_type)
5520    }
5521}
5522
5523macro_rules! format_option {
5524    ($F:expr, $EXPR:expr) => {{
5525        match $EXPR {
5526            Some(e) => write!($F, "{e}"),
5527            None => write!($F, "NULL"),
5528        }
5529    }};
5530}
5531
5532macro_rules! format_decimal {
5533    ($F:expr, $TYPE:ty, $VALUE:expr, $PRECISION:expr, $SCALE:expr) => {{
5534        match $VALUE {
5535            Some(value) => write!(
5536                $F,
5537                "{}",
5538                <$TYPE>::format_decimal(*value, *$PRECISION, *$SCALE)
5539            ),
5540            None => write!($F, "NULL"),
5541        }
5542    }};
5543}
5544
5545macro_rules! format_decimal_debug {
5546    ($F:expr, $TYPE_NAME:literal, $TYPE:ty, $VALUE:expr, $PRECISION:expr, $SCALE:expr) => {{
5547        match $VALUE {
5548            Some(value) => write!(
5549                $F,
5550                "{}({},{},{})",
5551                $TYPE_NAME,
5552                <$TYPE>::format_decimal(*value, *$PRECISION, *$SCALE),
5553                $PRECISION,
5554                $SCALE
5555            ),
5556            None => write!($F, "{}(NULL,{},{})", $TYPE_NAME, $PRECISION, $SCALE),
5557        }
5558    }};
5559}
5560
5561// Implement Display trait for ScalarValue
5562//
5563// # Panics
5564//
5565// Panics if there is an error when creating a visual representation of columns via `arrow::util::pretty`
5566impl fmt::Display for ScalarValue {
5567    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
5568        match self {
5569            ScalarValue::Decimal32(v, p, s) => {
5570                format_decimal!(f, Decimal32Type, v, p, s)?
5571            }
5572            ScalarValue::Decimal64(v, p, s) => {
5573                format_decimal!(f, Decimal64Type, v, p, s)?
5574            }
5575            ScalarValue::Decimal128(v, p, s) => {
5576                format_decimal!(f, Decimal128Type, v, p, s)?
5577            }
5578            ScalarValue::Decimal256(v, p, s) => {
5579                format_decimal!(f, Decimal256Type, v, p, s)?
5580            }
5581            ScalarValue::Boolean(e) => format_option!(f, e)?,
5582            ScalarValue::Float16(e) => format_option!(f, e)?,
5583            ScalarValue::Float32(e) => format_option!(f, e)?,
5584            ScalarValue::Float64(e) => format_option!(f, e)?,
5585            ScalarValue::Int8(e) => format_option!(f, e)?,
5586            ScalarValue::Int16(e) => format_option!(f, e)?,
5587            ScalarValue::Int32(e) => format_option!(f, e)?,
5588            ScalarValue::Int64(e) => format_option!(f, e)?,
5589            ScalarValue::UInt8(e) => format_option!(f, e)?,
5590            ScalarValue::UInt16(e) => format_option!(f, e)?,
5591            ScalarValue::UInt32(e) => format_option!(f, e)?,
5592            ScalarValue::UInt64(e) => format_option!(f, e)?,
5593            ScalarValue::TimestampSecond(e, _) => format_option!(f, e)?,
5594            ScalarValue::TimestampMillisecond(e, _) => format_option!(f, e)?,
5595            ScalarValue::TimestampMicrosecond(e, _) => format_option!(f, e)?,
5596            ScalarValue::TimestampNanosecond(e, _) => format_option!(f, e)?,
5597            ScalarValue::Utf8(e)
5598            | ScalarValue::LargeUtf8(e)
5599            | ScalarValue::Utf8View(e) => format_option!(f, e)?,
5600            ScalarValue::Binary(e)
5601            | ScalarValue::FixedSizeBinary(_, e)
5602            | ScalarValue::LargeBinary(e)
5603            | ScalarValue::BinaryView(e) => match e {
5604                Some(bytes) => {
5605                    // print up to first 10 bytes, with trailing ... if needed
5606                    const HEX_CHARS_UPPER: &[u8; 16] = b"0123456789ABCDEF";
5607                    for b in bytes.iter().take(10) {
5608                        f.write_char(HEX_CHARS_UPPER[(b >> 4) as usize] as char)?;
5609                        f.write_char(HEX_CHARS_UPPER[(b & 0x0f) as usize] as char)?;
5610                    }
5611                    if bytes.len() > 10 {
5612                        write!(f, "...")?;
5613                    }
5614                }
5615                None => write!(f, "NULL")?,
5616            },
5617            ScalarValue::List(arr) => fmt_list(arr.as_ref(), f)?,
5618            ScalarValue::LargeList(arr) => fmt_list(arr.as_ref(), f)?,
5619            ScalarValue::FixedSizeList(arr) => fmt_list(arr.as_ref(), f)?,
5620            ScalarValue::ListView(arr) => fmt_list(arr.as_ref(), f)?,
5621            ScalarValue::LargeListView(arr) => fmt_list(arr.as_ref(), f)?,
5622            ScalarValue::Date32(e) => format_option!(
5623                f,
5624                e.map(|v| {
5625                    let epoch = NaiveDate::from_ymd_opt(1970, 1, 1).unwrap();
5626                    match epoch.checked_add_signed(Duration::try_days(v as i64).unwrap())
5627                    {
5628                        Some(date) => date.to_string(),
5629                        None => "".to_string(),
5630                    }
5631                })
5632            )?,
5633            ScalarValue::Date64(e) => format_option!(
5634                f,
5635                e.map(|v| {
5636                    let epoch = NaiveDate::from_ymd_opt(1970, 1, 1).unwrap();
5637                    match epoch.checked_add_signed(Duration::try_milliseconds(v).unwrap())
5638                    {
5639                        Some(date) => date.to_string(),
5640                        None => "".to_string(),
5641                    }
5642                })
5643            )?,
5644            ScalarValue::Time32Second(e) => format_option!(f, e)?,
5645            ScalarValue::Time32Millisecond(e) => format_option!(f, e)?,
5646            ScalarValue::Time64Microsecond(e) => format_option!(f, e)?,
5647            ScalarValue::Time64Nanosecond(e) => format_option!(f, e)?,
5648            ScalarValue::IntervalYearMonth(e) => format_option!(f, e)?,
5649            ScalarValue::IntervalMonthDayNano(e) => {
5650                format_option!(f, e.map(|v| format!("{v:?}")))?
5651            }
5652            ScalarValue::IntervalDayTime(e) => {
5653                format_option!(f, e.map(|v| format!("{v:?}")))?;
5654            }
5655            ScalarValue::DurationSecond(e) => format_option!(f, e)?,
5656            ScalarValue::DurationMillisecond(e) => format_option!(f, e)?,
5657            ScalarValue::DurationMicrosecond(e) => format_option!(f, e)?,
5658            ScalarValue::DurationNanosecond(e) => format_option!(f, e)?,
5659            ScalarValue::Struct(struct_arr) => {
5660                // ScalarValue Struct should always have a single element
5661                assert_eq!(struct_arr.len(), 1);
5662
5663                if struct_arr.null_count() == struct_arr.len() {
5664                    write!(f, "NULL")?;
5665                    return Ok(());
5666                }
5667
5668                let columns = struct_arr.columns();
5669                let fields = struct_arr.fields();
5670                let nulls = struct_arr.nulls();
5671
5672                write!(
5673                    f,
5674                    "{{{}}}",
5675                    columns
5676                        .iter()
5677                        .zip(fields.iter())
5678                        .map(|(column, field)| {
5679                            if nulls.is_some_and(|b| b.is_null(0)) {
5680                                format!("{}:NULL", field.name())
5681                            } else if let DataType::Struct(_) = field.data_type() {
5682                                let sv = ScalarValue::Struct(Arc::new(
5683                                    column.as_struct().to_owned(),
5684                                ));
5685                                format!("{}:{sv}", field.name())
5686                            } else {
5687                                let sv = array_value_to_string(column, 0).unwrap();
5688                                format!("{}:{sv}", field.name())
5689                            }
5690                        })
5691                        .collect::<Vec<_>>()
5692                        .join(",")
5693                )?
5694            }
5695            ScalarValue::Map(map_arr) => {
5696                if map_arr.null_count() == map_arr.len() {
5697                    write!(f, "NULL")?;
5698                    return Ok(());
5699                }
5700
5701                write!(
5702                    f,
5703                    "[{}]",
5704                    map_arr
5705                        .iter()
5706                        .map(|struct_array| {
5707                            if let Some(arr) = struct_array {
5708                                let mut buffer = VecDeque::new();
5709                                for i in 0..arr.len() {
5710                                    let key =
5711                                        array_value_to_string(arr.column(0), i).unwrap();
5712                                    let value =
5713                                        array_value_to_string(arr.column(1), i).unwrap();
5714                                    buffer.push_back(format!("{key}:{value}"));
5715                                }
5716                                format!(
5717                                    "{{{}}}",
5718                                    buffer
5719                                        .into_iter()
5720                                        .collect::<Vec<_>>()
5721                                        .join(",")
5722                                        .as_str()
5723                                )
5724                            } else {
5725                                "NULL".to_string()
5726                            }
5727                        })
5728                        .collect::<Vec<_>>()
5729                        .join(",")
5730                )?
5731            }
5732            ScalarValue::Union(val, _fields, _mode) => match val {
5733                Some((id, val)) => write!(f, "{id}:{val}")?,
5734                None => write!(f, "NULL")?,
5735            },
5736            ScalarValue::Dictionary(_k, v) => write!(f, "{v}")?,
5737            ScalarValue::RunEndEncoded(_, _, v) => write!(f, "{v}")?,
5738            ScalarValue::Null => write!(f, "NULL")?,
5739        };
5740        Ok(())
5741    }
5742}
5743
5744fn fmt_list(arr: &dyn Array, f: &mut fmt::Formatter) -> fmt::Result {
5745    // ScalarValue List, LargeList, FixedSizeList, ListView, LargeListView should always have a single element
5746    assert_eq!(arr.len(), 1);
5747    let options = FormatOptions::default().with_display_error(true);
5748    let formatter = ArrayFormatter::try_new(arr, &options).unwrap();
5749    let value_formatter = formatter.value(0);
5750    write!(f, "{value_formatter}")
5751}
5752
5753/// Writes a byte array for ScalarValue Debug formatting.
5754/// `[1, 2, 3]` -> `"1,2,3"`
5755fn fmt_binary_debug(data: &[u8], f: &mut fmt::Formatter) -> fmt::Result {
5756    let mut iter = data.iter();
5757    if let Some(b) = iter.next() {
5758        write!(f, "{b}")?;
5759    }
5760    for b in iter {
5761        write!(f, ",{b}")?;
5762    }
5763    Ok(())
5764}
5765
5766impl fmt::Debug for ScalarValue {
5767    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
5768        match self {
5769            ScalarValue::Decimal32(value, precision, scale) => {
5770                format_decimal_debug!(
5771                    f,
5772                    "Decimal32",
5773                    Decimal32Type,
5774                    value,
5775                    precision,
5776                    scale
5777                )
5778            }
5779            ScalarValue::Decimal64(value, precision, scale) => {
5780                format_decimal_debug!(
5781                    f,
5782                    "Decimal64",
5783                    Decimal64Type,
5784                    value,
5785                    precision,
5786                    scale
5787                )
5788            }
5789            ScalarValue::Decimal128(value, precision, scale) => {
5790                format_decimal_debug!(
5791                    f,
5792                    "Decimal128",
5793                    Decimal128Type,
5794                    value,
5795                    precision,
5796                    scale
5797                )
5798            }
5799            ScalarValue::Decimal256(value, precision, scale) => {
5800                format_decimal_debug!(
5801                    f,
5802                    "Decimal256",
5803                    Decimal256Type,
5804                    value,
5805                    precision,
5806                    scale
5807                )
5808            }
5809            ScalarValue::Boolean(_) => write!(f, "Boolean({self})"),
5810            ScalarValue::Float16(_) => write!(f, "Float16({self})"),
5811            ScalarValue::Float32(_) => write!(f, "Float32({self})"),
5812            ScalarValue::Float64(_) => write!(f, "Float64({self})"),
5813            ScalarValue::Int8(_) => write!(f, "Int8({self})"),
5814            ScalarValue::Int16(_) => write!(f, "Int16({self})"),
5815            ScalarValue::Int32(_) => write!(f, "Int32({self})"),
5816            ScalarValue::Int64(_) => write!(f, "Int64({self})"),
5817            ScalarValue::UInt8(_) => write!(f, "UInt8({self})"),
5818            ScalarValue::UInt16(_) => write!(f, "UInt16({self})"),
5819            ScalarValue::UInt32(_) => write!(f, "UInt32({self})"),
5820            ScalarValue::UInt64(_) => write!(f, "UInt64({self})"),
5821            ScalarValue::TimestampSecond(_, tz_opt) => {
5822                write!(f, "TimestampSecond({self}, {tz_opt:?})")
5823            }
5824            ScalarValue::TimestampMillisecond(_, tz_opt) => {
5825                write!(f, "TimestampMillisecond({self}, {tz_opt:?})")
5826            }
5827            ScalarValue::TimestampMicrosecond(_, tz_opt) => {
5828                write!(f, "TimestampMicrosecond({self}, {tz_opt:?})")
5829            }
5830            ScalarValue::TimestampNanosecond(_, tz_opt) => {
5831                write!(f, "TimestampNanosecond({self}, {tz_opt:?})")
5832            }
5833            ScalarValue::Utf8(None) => write!(f, "Utf8({self})"),
5834            ScalarValue::Utf8(Some(_)) => write!(f, "Utf8(\"{self}\")"),
5835            ScalarValue::Utf8View(None) => write!(f, "Utf8View({self})"),
5836            ScalarValue::Utf8View(Some(_)) => write!(f, "Utf8View(\"{self}\")"),
5837            ScalarValue::LargeUtf8(None) => write!(f, "LargeUtf8({self})"),
5838            ScalarValue::LargeUtf8(Some(_)) => write!(f, "LargeUtf8(\"{self}\")"),
5839            ScalarValue::Binary(None) => write!(f, "Binary({self})"),
5840            ScalarValue::Binary(Some(b)) => {
5841                write!(f, "Binary(\"")?;
5842                fmt_binary_debug(b.as_slice(), f)?;
5843                write!(f, "\")")
5844            }
5845            ScalarValue::BinaryView(None) => write!(f, "BinaryView({self})"),
5846            ScalarValue::BinaryView(Some(b)) => {
5847                write!(f, "BinaryView(\"")?;
5848                fmt_binary_debug(b.as_slice(), f)?;
5849                write!(f, "\")")
5850            }
5851            ScalarValue::FixedSizeBinary(size, None) => {
5852                write!(f, "FixedSizeBinary({size}, {self})")
5853            }
5854            ScalarValue::FixedSizeBinary(size, Some(b)) => {
5855                write!(f, "FixedSizeBinary({size}, \"")?;
5856                fmt_binary_debug(b.as_slice(), f)?;
5857                write!(f, "\")")
5858            }
5859            ScalarValue::LargeBinary(None) => write!(f, "LargeBinary({self})"),
5860            ScalarValue::LargeBinary(Some(b)) => {
5861                write!(f, "LargeBinary(\"")?;
5862                fmt_binary_debug(b.as_slice(), f)?;
5863                write!(f, "\")")
5864            }
5865            ScalarValue::FixedSizeList(_) => write!(f, "FixedSizeList({self})"),
5866            ScalarValue::List(_) => write!(f, "List({self})"),
5867            ScalarValue::LargeList(_) => write!(f, "LargeList({self})"),
5868            ScalarValue::ListView(_) => write!(f, "ListView({self})"),
5869            ScalarValue::LargeListView(_) => write!(f, "LargeListView({self})"),
5870            ScalarValue::Struct(struct_arr) => {
5871                // ScalarValue Struct should always have a single element
5872                assert_eq!(struct_arr.len(), 1);
5873
5874                let columns = struct_arr.columns();
5875                let fields = struct_arr.fields();
5876
5877                write!(
5878                    f,
5879                    "Struct({{{}}})",
5880                    columns
5881                        .iter()
5882                        .zip(fields.iter())
5883                        .map(|(column, field)| {
5884                            let sv = array_value_to_string(column, 0).unwrap();
5885                            let name = field.name();
5886                            format!("{name}:{sv}")
5887                        })
5888                        .collect::<Vec<_>>()
5889                        .join(",")
5890                )
5891            }
5892            ScalarValue::Map(map_arr) => {
5893                write!(
5894                    f,
5895                    "Map([{}])",
5896                    map_arr
5897                        .iter()
5898                        .map(|struct_array| {
5899                            if let Some(arr) = struct_array {
5900                                let buffer: Vec<String> = (0..arr.len())
5901                                    .map(|i| {
5902                                        let key = array_value_to_string(arr.column(0), i)
5903                                            .unwrap();
5904                                        let value =
5905                                            array_value_to_string(arr.column(1), i)
5906                                                .unwrap();
5907                                        format!("{key:?}:{value:?}")
5908                                    })
5909                                    .collect();
5910                                format!("{{{}}}", buffer.join(","))
5911                            } else {
5912                                "NULL".to_string()
5913                            }
5914                        })
5915                        .collect::<Vec<_>>()
5916                        .join(",")
5917                )
5918            }
5919            ScalarValue::Date32(_) => write!(f, "Date32(\"{self}\")"),
5920            ScalarValue::Date64(_) => write!(f, "Date64(\"{self}\")"),
5921            ScalarValue::Time32Second(_) => write!(f, "Time32Second(\"{self}\")"),
5922            ScalarValue::Time32Millisecond(_) => {
5923                write!(f, "Time32Millisecond(\"{self}\")")
5924            }
5925            ScalarValue::Time64Microsecond(_) => {
5926                write!(f, "Time64Microsecond(\"{self}\")")
5927            }
5928            ScalarValue::Time64Nanosecond(_) => {
5929                write!(f, "Time64Nanosecond(\"{self}\")")
5930            }
5931            ScalarValue::IntervalDayTime(_) => {
5932                write!(f, "IntervalDayTime(\"{self}\")")
5933            }
5934            ScalarValue::IntervalYearMonth(_) => {
5935                write!(f, "IntervalYearMonth(\"{self}\")")
5936            }
5937            ScalarValue::IntervalMonthDayNano(_) => {
5938                write!(f, "IntervalMonthDayNano(\"{self}\")")
5939            }
5940            ScalarValue::DurationSecond(_) => write!(f, "DurationSecond(\"{self}\")"),
5941            ScalarValue::DurationMillisecond(_) => {
5942                write!(f, "DurationMillisecond(\"{self}\")")
5943            }
5944            ScalarValue::DurationMicrosecond(_) => {
5945                write!(f, "DurationMicrosecond(\"{self}\")")
5946            }
5947            ScalarValue::DurationNanosecond(_) => {
5948                write!(f, "DurationNanosecond(\"{self}\")")
5949            }
5950            ScalarValue::Union(val, _fields, _mode) => match val {
5951                Some((id, val)) => write!(f, "Union {id}:{val}"),
5952                None => write!(f, "Union(NULL)"),
5953            },
5954            ScalarValue::Dictionary(k, v) => write!(f, "Dictionary({k:?}, {v:?})"),
5955            ScalarValue::RunEndEncoded(rf, vf, v) => {
5956                write!(f, "RunEndEncoded({rf:?}, {vf:?}, {v:?})")
5957            }
5958            ScalarValue::Null => write!(f, "NULL"),
5959        }
5960    }
5961}
5962
5963/// Trait used to map a NativeType to a ScalarValue
5964pub trait ScalarType<T: ArrowNativeType> {
5965    /// returns a scalar from an optional T
5966    fn scalar(r: Option<T>) -> ScalarValue;
5967}
5968
5969impl ScalarType<f32> for Float32Type {
5970    fn scalar(r: Option<f32>) -> ScalarValue {
5971        ScalarValue::Float32(r)
5972    }
5973}
5974
5975impl ScalarType<i64> for TimestampSecondType {
5976    fn scalar(r: Option<i64>) -> ScalarValue {
5977        ScalarValue::TimestampSecond(r, None)
5978    }
5979}
5980
5981impl ScalarType<i64> for TimestampMillisecondType {
5982    fn scalar(r: Option<i64>) -> ScalarValue {
5983        ScalarValue::TimestampMillisecond(r, None)
5984    }
5985}
5986
5987impl ScalarType<i64> for TimestampMicrosecondType {
5988    fn scalar(r: Option<i64>) -> ScalarValue {
5989        ScalarValue::TimestampMicrosecond(r, None)
5990    }
5991}
5992
5993impl ScalarType<i64> for TimestampNanosecondType {
5994    fn scalar(r: Option<i64>) -> ScalarValue {
5995        ScalarValue::TimestampNanosecond(r, None)
5996    }
5997}
5998
5999impl ScalarType<i32> for Date32Type {
6000    fn scalar(r: Option<i32>) -> ScalarValue {
6001        ScalarValue::Date32(r)
6002    }
6003}
6004
6005#[cfg(test)]
6006mod tests {
6007
6008    use super::*;
6009    use crate::cast::{
6010        as_large_list_view_array, as_list_array, as_map_array, as_struct_array,
6011    };
6012    use crate::test_util::batches_to_string;
6013    use arrow::array::{
6014        FixedSizeListBuilder, Int32Builder, LargeListBuilder, LargeListViewBuilder,
6015        ListBuilder, ListViewBuilder, MapBuilder, NullArray, NullBufferBuilder,
6016        OffsetSizeTrait, PrimitiveBuilder, RecordBatch, StringBuilder,
6017        StringDictionaryBuilder, StructBuilder, UnionBuilder,
6018    };
6019    use arrow::buffer::{Buffer, NullBuffer, OffsetBuffer};
6020    use arrow::compute::{is_null, kernels};
6021    use arrow::datatypes::{
6022        ArrowNumericType, DECIMAL128_MAX_PRECISION, DECIMAL256_MAX_PRECISION, Fields,
6023        Float64Type, TimeUnit,
6024    };
6025    use arrow::error::ArrowError;
6026    use arrow::util::pretty::pretty_format_columns;
6027    use insta::assert_snapshot;
6028    use rand::Rng;
6029
6030    #[test]
6031    fn test_scalar_value_from_for_map() {
6032        let string_builder = StringBuilder::new();
6033        let int_builder = Int32Builder::with_capacity(4);
6034        let mut builder = MapBuilder::new(None, string_builder, int_builder);
6035        builder.keys().append_value("joe");
6036        builder.values().append_value(1);
6037        builder.append(true).unwrap();
6038
6039        builder.keys().append_value("blogs");
6040        builder.values().append_value(2);
6041        builder.keys().append_value("foo");
6042        builder.values().append_value(4);
6043        builder.append(true).unwrap();
6044        builder.append(true).unwrap();
6045        builder.append(false).unwrap();
6046
6047        let expected = builder.finish();
6048
6049        let sv = ScalarValue::Map(Arc::new(expected.clone()));
6050        let map_arr = sv.to_array().unwrap();
6051        let actual = as_map_array(&map_arr).unwrap();
6052        assert_eq!(actual, &expected);
6053    }
6054
6055    #[test]
6056    fn test_format_timestamp_type_for_error_and_bounds() {
6057        // format helper
6058        let ts_ns = format_timestamp_type_for_error(&DataType::Timestamp(
6059            TimeUnit::Nanosecond,
6060            None,
6061        ));
6062        assert_eq!(ts_ns, "Timestamp(ns)");
6063
6064        let ts_us = format_timestamp_type_for_error(&DataType::Timestamp(
6065            TimeUnit::Microsecond,
6066            None,
6067        ));
6068        assert_eq!(ts_us, "Timestamp(us)");
6069
6070        // ensure_timestamp_in_bounds: Date32 non-overflow
6071        let ok = ensure_timestamp_in_bounds(
6072            1000,
6073            NANOS_PER_DAY,
6074            &DataType::Date32,
6075            &DataType::Timestamp(TimeUnit::Nanosecond, None),
6076        );
6077        assert!(ok.is_ok());
6078
6079        // Date32 overflow -- known large day value (9999-12-31 -> 2932896)
6080        let err = ensure_timestamp_in_bounds(
6081            2932896,
6082            NANOS_PER_DAY,
6083            &DataType::Date32,
6084            &DataType::Timestamp(TimeUnit::Nanosecond, None),
6085        );
6086        assert!(err.is_err());
6087        let msg = err.unwrap_err().to_string();
6088        assert!(msg.contains("Cannot cast Date32 value 2932896 to Timestamp(ns): converted value exceeds the representable i64 range"));
6089
6090        // Date64 overflow for ns (millis * 1_000_000)
6091        let overflow_millis: i64 = (i64::MAX / NANOS_PER_MILLISECOND) + 1;
6092        let err2 = ensure_timestamp_in_bounds(
6093            overflow_millis,
6094            NANOS_PER_MILLISECOND,
6095            &DataType::Date64,
6096            &DataType::Timestamp(TimeUnit::Nanosecond, None),
6097        );
6098        assert!(err2.is_err());
6099    }
6100
6101    #[test]
6102    fn test_scalar_value_from_for_struct() {
6103        let boolean = Arc::new(BooleanArray::from(vec![false]));
6104        let int = Arc::new(Int32Array::from(vec![42]));
6105
6106        let expected = StructArray::from(vec![
6107            (
6108                Arc::new(Field::new("b", DataType::Boolean, false)),
6109                Arc::clone(&boolean) as ArrayRef,
6110            ),
6111            (
6112                Arc::new(Field::new("c", DataType::Int32, false)),
6113                Arc::clone(&int) as ArrayRef,
6114            ),
6115        ]);
6116
6117        let sv = ScalarStructBuilder::new()
6118            .with_array(Field::new("b", DataType::Boolean, false), boolean)
6119            .with_array(Field::new("c", DataType::Int32, false), int)
6120            .build()
6121            .unwrap();
6122
6123        let struct_arr = sv.to_array().unwrap();
6124        let actual = as_struct_array(&struct_arr).unwrap();
6125        assert_eq!(actual, &expected);
6126    }
6127
6128    #[test]
6129    #[should_panic(
6130        expected = "InvalidArgumentError(\"Incorrect array length for StructArray field \\\"bool\\\", expected 1 got 4\")"
6131    )]
6132    fn test_scalar_value_from_for_struct_should_panic() {
6133        let _ = ScalarStructBuilder::new()
6134            .with_array(
6135                Field::new("bool", DataType::Boolean, false),
6136                Arc::new(BooleanArray::from(vec![false, true, false, false])),
6137            )
6138            .with_array(
6139                Field::new("i32", DataType::Int32, false),
6140                Arc::new(Int32Array::from(vec![42, 28, 19, 31])),
6141            )
6142            .build()
6143            .unwrap();
6144    }
6145
6146    #[test]
6147    fn test_to_array_of_size_for_nested() {
6148        // Struct
6149        let boolean = Arc::new(BooleanArray::from(vec![false, false, true, true]));
6150        let int = Arc::new(Int32Array::from(vec![42, 28, 19, 31]));
6151
6152        let struct_array = StructArray::from(vec![
6153            (
6154                Arc::new(Field::new("b", DataType::Boolean, false)),
6155                Arc::clone(&boolean) as ArrayRef,
6156            ),
6157            (
6158                Arc::new(Field::new("c", DataType::Int32, false)),
6159                Arc::clone(&int) as ArrayRef,
6160            ),
6161        ]);
6162        let sv = ScalarValue::Struct(Arc::new(struct_array));
6163        let actual_arr = sv.to_array_of_size(2).unwrap();
6164
6165        let boolean = Arc::new(BooleanArray::from(vec![
6166            false, false, true, true, false, false, true, true,
6167        ]));
6168        let int = Arc::new(Int32Array::from(vec![42, 28, 19, 31, 42, 28, 19, 31]));
6169
6170        let struct_array = StructArray::from(vec![
6171            (
6172                Arc::new(Field::new("b", DataType::Boolean, false)),
6173                Arc::clone(&boolean) as ArrayRef,
6174            ),
6175            (
6176                Arc::new(Field::new("c", DataType::Int32, false)),
6177                Arc::clone(&int) as ArrayRef,
6178            ),
6179        ]);
6180
6181        let actual = as_struct_array(&actual_arr).unwrap();
6182        assert_eq!(actual, &struct_array);
6183
6184        // List
6185        let arr = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![Some(vec![
6186            Some(1),
6187            None,
6188            Some(2),
6189        ])]);
6190
6191        let sv = ScalarValue::List(Arc::new(arr));
6192        let actual_arr = sv
6193            .to_array_of_size(2)
6194            .expect("Failed to convert to array of size");
6195        let actual_list_arr = actual_arr.as_list::<i32>();
6196
6197        let arr = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
6198            Some(vec![Some(1), None, Some(2)]),
6199            Some(vec![Some(1), None, Some(2)]),
6200        ]);
6201
6202        assert_eq!(&arr, actual_list_arr);
6203
6204        // ListView
6205        let arr =
6206            ListViewArray::from_iter_primitive::<Int32Type, _, _>(vec![Some(vec![
6207                Some(1),
6208                None,
6209                Some(2),
6210            ])]);
6211
6212        let sv = ScalarValue::ListView(Arc::new(arr));
6213        let actual_arr = sv
6214            .to_array_of_size(2)
6215            .expect("Failed to convert to array of size");
6216        let actual_list_arr = actual_arr.as_list_view::<i32>();
6217
6218        let arr = ListViewArray::from_iter_primitive::<Int32Type, _, _>(vec![
6219            Some(vec![Some(1), None, Some(2)]),
6220            Some(vec![Some(1), None, Some(2)]),
6221        ]);
6222
6223        assert_eq!(&arr, actual_list_arr);
6224    }
6225
6226    #[test]
6227    fn test_to_array_of_size_for_fsl() {
6228        let values = Int32Array::from_iter([Some(1), None, Some(2)]);
6229        let field = Arc::new(Field::new_list_field(DataType::Int32, true));
6230        let arr = FixedSizeListArray::new(Arc::clone(&field), 3, Arc::new(values), None);
6231        let sv = ScalarValue::FixedSizeList(Arc::new(arr));
6232        let actual_arr = sv
6233            .to_array_of_size(2)
6234            .expect("Failed to convert to array of size");
6235
6236        let expected_values =
6237            Int32Array::from_iter([Some(1), None, Some(2), Some(1), None, Some(2)]);
6238        let expected_arr =
6239            FixedSizeListArray::new(field, 3, Arc::new(expected_values), None);
6240
6241        assert_eq!(
6242            &expected_arr,
6243            as_fixed_size_list_array(actual_arr.as_ref()).unwrap()
6244        );
6245
6246        let empty_array = sv
6247            .to_array_of_size(0)
6248            .expect("Failed to convert to empty array");
6249
6250        assert_eq!(empty_array.len(), 0);
6251    }
6252
6253    #[test]
6254    fn test_to_array_of_size_list_size_one() {
6255        // size=1 takes the fast path (Arc::clone)
6256        let arr = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![Some(vec![
6257            Some(10),
6258            Some(20),
6259        ])]);
6260        let sv = ScalarValue::List(Arc::new(arr.clone()));
6261        let result = sv.to_array_of_size(1).unwrap();
6262        assert_eq!(result.as_list::<i32>(), &arr);
6263    }
6264
6265    #[test]
6266    fn test_to_array_of_size_list_empty_inner() {
6267        // A list scalar containing an empty list: [[]]
6268        let arr = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![Some(vec![])]);
6269        let sv = ScalarValue::List(Arc::new(arr));
6270        let result = sv.to_array_of_size(3).unwrap();
6271        let result_list = result.as_list::<i32>();
6272        assert_eq!(result_list.len(), 3);
6273        for i in 0..3 {
6274            assert_eq!(result_list.value(i).len(), 0);
6275        }
6276    }
6277
6278    #[test]
6279    fn test_to_array_of_size_large_list() {
6280        let arr =
6281            LargeListArray::from_iter_primitive::<Int32Type, _, _>(vec![Some(vec![
6282                Some(100),
6283                Some(200),
6284            ])]);
6285        let sv = ScalarValue::LargeList(Arc::new(arr));
6286        let result = sv.to_array_of_size(3).unwrap();
6287        let expected = LargeListArray::from_iter_primitive::<Int32Type, _, _>(vec![
6288            Some(vec![Some(100), Some(200)]),
6289            Some(vec![Some(100), Some(200)]),
6290            Some(vec![Some(100), Some(200)]),
6291        ]);
6292        assert_eq!(result.as_list::<i64>(), &expected);
6293    }
6294
6295    #[test]
6296    fn test_list_to_array_of_size_multi_row() {
6297        // Call list_to_array_of_size directly with arr.len() > 1
6298        let arr = Int32Array::from(vec![Some(10), None, Some(30)]);
6299        let result = ScalarValue::list_to_array_of_size(&arr, 3).unwrap();
6300        let result = result.as_primitive::<Int32Type>();
6301        assert_eq!(
6302            result.iter().collect::<Vec<_>>(),
6303            vec![
6304                Some(10),
6305                None,
6306                Some(30),
6307                Some(10),
6308                None,
6309                Some(30),
6310                Some(10),
6311                None,
6312                Some(30),
6313            ]
6314        );
6315    }
6316
6317    #[test]
6318    fn test_to_array_of_size_null_list() {
6319        let dt = DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true)));
6320        let sv = ScalarValue::try_from(&dt).unwrap();
6321        let result = sv.to_array_of_size(3).unwrap();
6322        assert_eq!(result.len(), 3);
6323        assert_eq!(result.null_count(), 3);
6324    }
6325
6326    /// See https://github.com/apache/datafusion/issues/18870
6327    #[test]
6328    fn test_to_array_of_size_for_none_fsb() {
6329        let sv = ScalarValue::FixedSizeBinary(5, None);
6330        let result = sv
6331            .to_array_of_size(2)
6332            .expect("Failed to convert to array of size");
6333        assert_eq!(result.len(), 2);
6334        assert_eq!(result.null_count(), 2);
6335        assert_eq!(result.as_fixed_size_binary().values().len(), 10);
6336    }
6337
6338    #[test]
6339    fn test_list_to_array_string() {
6340        let scalars = vec![
6341            ScalarValue::from("rust"),
6342            ScalarValue::from("arrow"),
6343            ScalarValue::from("data-fusion"),
6344        ];
6345
6346        let result = ScalarValue::new_list_nullable(scalars.as_slice(), &DataType::Utf8);
6347
6348        let expected = single_row_list_array(vec!["rust", "arrow", "data-fusion"]);
6349        assert_eq!(*result, expected);
6350    }
6351
6352    fn single_row_list_array(items: Vec<&str>) -> ListArray {
6353        SingleRowListArrayBuilder::new(Arc::new(StringArray::from(items)))
6354            .build_list_array()
6355    }
6356
6357    fn build_list<O: OffsetSizeTrait>(
6358        values: Vec<Option<Vec<Option<i64>>>>,
6359    ) -> Vec<ScalarValue> {
6360        values
6361            .into_iter()
6362            .map(|v| {
6363                let arr = Arc::new(GenericListArray::<O>::from_iter_primitive::<
6364                    Int64Type,
6365                    _,
6366                    _,
6367                >(vec![v])) as ArrayRef;
6368
6369                if O::IS_LARGE {
6370                    ScalarValue::LargeList(arr.as_list::<i64>().to_owned().into())
6371                } else {
6372                    ScalarValue::List(arr.as_list::<i32>().to_owned().into())
6373                }
6374            })
6375            .collect()
6376    }
6377
6378    fn build_list_view<O: OffsetSizeTrait>(
6379        values: Vec<Option<Vec<Option<i64>>>>,
6380    ) -> Vec<ScalarValue> {
6381        values
6382            .into_iter()
6383            .map(|v| {
6384                let arr = Arc::new(GenericListViewArray::<O>::from_iter_primitive::<
6385                    Int64Type,
6386                    _,
6387                    _,
6388                >(vec![v])) as ArrayRef;
6389
6390                if O::IS_LARGE {
6391                    ScalarValue::LargeListView(
6392                        arr.as_list_view::<i64>().to_owned().into(),
6393                    )
6394                } else {
6395                    ScalarValue::ListView(arr.as_list_view::<i32>().to_owned().into())
6396                }
6397            })
6398            .collect()
6399    }
6400
6401    #[test]
6402    fn test_iter_to_array_fixed_size_list() {
6403        let field = Arc::new(Field::new_list_field(DataType::Int32, true));
6404        let f1 = Arc::new(FixedSizeListArray::new(
6405            Arc::clone(&field),
6406            3,
6407            Arc::new(Int32Array::from(vec![1, 2, 3])),
6408            None,
6409        ));
6410        let f2 = Arc::new(FixedSizeListArray::new(
6411            Arc::clone(&field),
6412            3,
6413            Arc::new(Int32Array::from(vec![4, 5, 6])),
6414            None,
6415        ));
6416        let f_nulls = Arc::new(FixedSizeListArray::new_null(field, 1, 1));
6417
6418        let scalars = vec![
6419            ScalarValue::FixedSizeList(Arc::clone(&f_nulls)),
6420            ScalarValue::FixedSizeList(f1),
6421            ScalarValue::FixedSizeList(f2),
6422            ScalarValue::FixedSizeList(f_nulls),
6423        ];
6424
6425        let array = ScalarValue::iter_to_array(scalars).unwrap();
6426
6427        let expected = FixedSizeListArray::from_iter_primitive::<Int32Type, _, _>(
6428            vec![
6429                None,
6430                Some(vec![Some(1), Some(2), Some(3)]),
6431                Some(vec![Some(4), Some(5), Some(6)]),
6432                None,
6433            ],
6434            3,
6435        );
6436        assert_eq!(array.as_ref(), &expected);
6437    }
6438
6439    #[test]
6440    fn test_iter_to_array_struct() {
6441        let s1 = StructArray::from(vec![
6442            (
6443                Arc::new(Field::new("A", DataType::Boolean, false)),
6444                Arc::new(BooleanArray::from(vec![false])) as ArrayRef,
6445            ),
6446            (
6447                Arc::new(Field::new("B", DataType::Int32, false)),
6448                Arc::new(Int32Array::from(vec![42])) as ArrayRef,
6449            ),
6450        ]);
6451
6452        let s2 = StructArray::from(vec![
6453            (
6454                Arc::new(Field::new("A", DataType::Boolean, false)),
6455                Arc::new(BooleanArray::from(vec![false])) as ArrayRef,
6456            ),
6457            (
6458                Arc::new(Field::new("B", DataType::Int32, false)),
6459                Arc::new(Int32Array::from(vec![42])) as ArrayRef,
6460            ),
6461        ]);
6462
6463        let scalars = vec![
6464            ScalarValue::Struct(Arc::new(s1)),
6465            ScalarValue::Struct(Arc::new(s2)),
6466        ];
6467
6468        let array = ScalarValue::iter_to_array(scalars).unwrap();
6469
6470        let expected = StructArray::from(vec![
6471            (
6472                Arc::new(Field::new("A", DataType::Boolean, false)),
6473                Arc::new(BooleanArray::from(vec![false, false])) as ArrayRef,
6474            ),
6475            (
6476                Arc::new(Field::new("B", DataType::Int32, false)),
6477                Arc::new(Int32Array::from(vec![42, 42])) as ArrayRef,
6478            ),
6479        ]);
6480        assert_eq!(array.as_ref(), &expected);
6481    }
6482
6483    #[test]
6484    fn test_iter_to_array_struct_with_nulls() {
6485        // non-null
6486        let s1 = StructArray::from((
6487            vec![
6488                (
6489                    Arc::new(Field::new("A", DataType::Int32, false)),
6490                    Arc::new(Int32Array::from(vec![1])) as ArrayRef,
6491                ),
6492                (
6493                    Arc::new(Field::new("B", DataType::Int64, false)),
6494                    Arc::new(Int64Array::from(vec![2])) as ArrayRef,
6495                ),
6496            ],
6497            // Present the null mask, 1 is non-null, 0 is null
6498            Buffer::from(&[1]),
6499        ));
6500
6501        // null
6502        let s2 = StructArray::from((
6503            vec![
6504                (
6505                    Arc::new(Field::new("A", DataType::Int32, false)),
6506                    Arc::new(Int32Array::from(vec![3])) as ArrayRef,
6507                ),
6508                (
6509                    Arc::new(Field::new("B", DataType::Int64, false)),
6510                    Arc::new(Int64Array::from(vec![4])) as ArrayRef,
6511                ),
6512            ],
6513            Buffer::from(&[0]),
6514        ));
6515
6516        let scalars = vec![
6517            ScalarValue::Struct(Arc::new(s1)),
6518            ScalarValue::Struct(Arc::new(s2)),
6519        ];
6520
6521        let array = ScalarValue::iter_to_array(scalars).unwrap();
6522        let struct_array = array.as_struct();
6523        assert!(struct_array.is_valid(0));
6524        assert!(struct_array.is_null(1));
6525    }
6526
6527    #[test]
6528    fn iter_to_array_primitive_test() {
6529        // List
6530        // List[[1,2,3]], List[null], List[[4,5]]
6531        let scalars = build_list::<i32>(vec![
6532            Some(vec![Some(1), Some(2), Some(3)]),
6533            None,
6534            Some(vec![Some(4), Some(5)]),
6535        ]);
6536        let array = ScalarValue::iter_to_array(scalars).unwrap();
6537        let list_array = as_list_array(&array).unwrap();
6538        // List[[1,2,3], null, [4,5]]
6539        let expected = ListArray::from_iter_primitive::<Int64Type, _, _>(vec![
6540            Some(vec![Some(1), Some(2), Some(3)]),
6541            None,
6542            Some(vec![Some(4), Some(5)]),
6543        ]);
6544        assert_eq!(list_array, &expected);
6545
6546        // LargeList
6547        // List[[1,2,3]], List[null], List[[4,5]]
6548        let scalars = build_list::<i64>(vec![
6549            Some(vec![Some(1), Some(2), Some(3)]),
6550            None,
6551            Some(vec![Some(4), Some(5)]),
6552        ]);
6553        let array = ScalarValue::iter_to_array(scalars).unwrap();
6554        let large_list_array = as_large_list_array(&array).unwrap();
6555        let expected = LargeListArray::from_iter_primitive::<Int64Type, _, _>(vec![
6556            Some(vec![Some(1), Some(2), Some(3)]),
6557            None,
6558            Some(vec![Some(4), Some(5)]),
6559        ]);
6560        assert_eq!(large_list_array, &expected);
6561
6562        // ListView
6563        // ListView[[1,2,3]], ListView[null], ListView[[4,5]]
6564        let scalars = build_list_view::<i32>(vec![
6565            Some(vec![Some(1), Some(2), Some(3)]),
6566            None,
6567            Some(vec![Some(4), Some(5)]),
6568        ]);
6569
6570        let array = ScalarValue::iter_to_array(scalars).unwrap();
6571        let list_view_array = as_list_view_array(&array).unwrap();
6572        // ListView[[1,2,3], null, [4,5]]
6573        let expected = ListViewArray::from_iter_primitive::<Int64Type, _, _>(vec![
6574            Some(vec![Some(1), Some(2), Some(3)]),
6575            None,
6576            Some(vec![Some(4), Some(5)]),
6577        ]);
6578        assert_eq!(list_view_array, &expected);
6579
6580        // LargeListView
6581        // LargeListView[[1,2,3]], LargeListView[null], LargeListView[[4,5]]
6582        let scalars = build_list_view::<i64>(vec![
6583            Some(vec![Some(1), Some(2), Some(3)]),
6584            None,
6585            Some(vec![Some(4), Some(5)]),
6586        ]);
6587
6588        let array = ScalarValue::iter_to_array(scalars).unwrap();
6589        let large_list_view_array = as_large_list_view_array(&array).unwrap();
6590        // LargeListView[[1,2,3], null, [4,5]]
6591        let expected = LargeListViewArray::from_iter_primitive::<Int64Type, _, _>(vec![
6592            Some(vec![Some(1), Some(2), Some(3)]),
6593            None,
6594            Some(vec![Some(4), Some(5)]),
6595        ]);
6596        assert_eq!(large_list_view_array, &expected);
6597    }
6598
6599    #[test]
6600    fn iter_to_array_string_test() {
6601        let arr1 = single_row_list_array(vec!["foo", "bar", "baz"]);
6602        let arr2 = single_row_list_array(vec!["rust", "world"]);
6603
6604        let scalars = vec![
6605            ScalarValue::List(Arc::new(arr1)),
6606            ScalarValue::List(Arc::new(arr2)),
6607        ];
6608
6609        let array = ScalarValue::iter_to_array(scalars).unwrap();
6610        let result = array.as_list::<i32>();
6611
6612        // build expected array
6613        let string_builder = StringBuilder::with_capacity(5, 25);
6614        let mut list_of_string_builder = ListBuilder::new(string_builder);
6615
6616        list_of_string_builder.values().append_value("foo");
6617        list_of_string_builder.values().append_value("bar");
6618        list_of_string_builder.values().append_value("baz");
6619        list_of_string_builder.append(true);
6620
6621        list_of_string_builder.values().append_value("rust");
6622        list_of_string_builder.values().append_value("world");
6623        list_of_string_builder.append(true);
6624        let expected = list_of_string_builder.finish();
6625
6626        assert_eq!(result, &expected);
6627    }
6628
6629    #[test]
6630    fn test_list_scalar_eq_to_array() {
6631        let list_array: ArrayRef =
6632            Arc::new(ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
6633                Some(vec![Some(0), Some(1), Some(2)]),
6634                None,
6635                Some(vec![None, Some(5)]),
6636            ]));
6637
6638        let fsl_array: ArrayRef =
6639            Arc::new(FixedSizeListArray::from_iter_primitive::<Int32Type, _, _>(
6640                vec![
6641                    Some(vec![Some(0), Some(1), Some(2)]),
6642                    None,
6643                    Some(vec![Some(3), None, Some(5)]),
6644                ],
6645                3,
6646            ));
6647        let list_view_array: ArrayRef =
6648            Arc::new(ListViewArray::from_iter_primitive::<Int32Type, _, _>(vec![
6649                Some(vec![Some(0), Some(1), Some(2)]),
6650                None,
6651                Some(vec![None, Some(5)]),
6652            ]));
6653
6654        for arr in [list_array, fsl_array, list_view_array] {
6655            for i in 0..arr.len() {
6656                let slice = arr.slice(i, 1);
6657                let scalar = match arr.data_type() {
6658                    DataType::List(_) => {
6659                        ScalarValue::List(slice.as_list::<i32>().to_owned().into())
6660                    }
6661                    DataType::FixedSizeList(_, _) => ScalarValue::FixedSizeList(
6662                        slice.as_fixed_size_list().to_owned().into(),
6663                    ),
6664                    DataType::ListView(_) => ScalarValue::ListView(
6665                        slice.as_list_view::<i32>().to_owned().into(),
6666                    ),
6667                    _ => unreachable!(),
6668                };
6669                assert!(scalar.eq_array(&arr, i).unwrap());
6670            }
6671        }
6672    }
6673
6674    #[test]
6675    fn test_eq_array_err_message() {
6676        assert_starts_with(
6677            ScalarValue::Utf8(Some("123".to_string()))
6678                .eq_array(&(Arc::new(Int32Array::from(vec![123])) as ArrayRef), 0)
6679                .unwrap_err()
6680                .message(),
6681            "could not cast array of type Int32 to arrow_array::array::byte_array::GenericByteArray<arrow_array::types::GenericStringType<i32>>",
6682        );
6683    }
6684
6685    #[test]
6686    fn scalar_add_trait_test() -> Result<()> {
6687        let float_value = ScalarValue::Float64(Some(123.));
6688        let float_value_2 = ScalarValue::Float64(Some(123.));
6689        assert_eq!(
6690            (float_value.add(&float_value_2))?,
6691            ScalarValue::Float64(Some(246.))
6692        );
6693        assert_eq!(
6694            (float_value.add(float_value_2))?,
6695            ScalarValue::Float64(Some(246.))
6696        );
6697        Ok(())
6698    }
6699
6700    #[test]
6701    fn scalar_add_trait_null_test() -> Result<()> {
6702        let int_value = ScalarValue::Int32(Some(42));
6703
6704        assert_eq!(
6705            int_value.add(ScalarValue::Int32(None))?,
6706            ScalarValue::Int32(None)
6707        );
6708
6709        Ok(())
6710    }
6711
6712    #[test]
6713    fn scalar_add_trait_wrapping_overflow_test() -> Result<()> {
6714        let int_value = ScalarValue::Int32(Some(i32::MAX));
6715        let one = ScalarValue::Int32(Some(1));
6716
6717        assert_eq!(int_value.add(one)?, ScalarValue::Int32(Some(i32::MIN)));
6718
6719        Ok(())
6720    }
6721
6722    #[test]
6723    fn scalar_add_trait_decimal_scale_test() -> Result<()> {
6724        let decimal = ScalarValue::Decimal128(Some(123), 10, 2);
6725        let decimal_2 = ScalarValue::Decimal128(Some(4), 9, 1);
6726
6727        assert_eq!(
6728            decimal.add(decimal_2)?,
6729            ScalarValue::Decimal128(Some(163), 11, 2)
6730        );
6731
6732        Ok(())
6733    }
6734
6735    #[test]
6736    fn scalar_add_trait_decimal256_scale_test() -> Result<()> {
6737        let decimal = ScalarValue::Decimal256(Some(i256::from(123)), 10, 2);
6738        let decimal_2 = ScalarValue::Decimal256(Some(i256::from(4)), 9, 1);
6739
6740        assert_eq!(
6741            decimal.add(decimal_2)?,
6742            ScalarValue::Decimal256(Some(i256::from(163)), 11, 2)
6743        );
6744
6745        Ok(())
6746    }
6747
6748    #[test]
6749    fn scalar_add_trait_decimal_negative_scale_test() -> Result<()> {
6750        let decimal = ScalarValue::Decimal128(Some(1), DECIMAL128_MAX_PRECISION, i8::MIN);
6751        let decimal_2 =
6752            ScalarValue::Decimal128(Some(1), DECIMAL128_MAX_PRECISION, i8::MIN);
6753
6754        assert_eq!(
6755            decimal.add(decimal_2)?,
6756            ScalarValue::Decimal128(Some(2), DECIMAL128_MAX_PRECISION, i8::MIN)
6757        );
6758
6759        Ok(())
6760    }
6761
6762    #[test]
6763    fn scalar_sub_trait_test() -> Result<()> {
6764        let float_value = ScalarValue::Float64(Some(123.));
6765        let float_value_2 = ScalarValue::Float64(Some(123.));
6766        assert_eq!(
6767            float_value.sub(&float_value_2)?,
6768            ScalarValue::Float64(Some(0.))
6769        );
6770        assert_eq!(
6771            float_value.sub(float_value_2)?,
6772            ScalarValue::Float64(Some(0.))
6773        );
6774        Ok(())
6775    }
6776
6777    #[test]
6778    fn scalar_sub_trait_int32_test() -> Result<()> {
6779        let int_value = ScalarValue::Int32(Some(42));
6780        let int_value_2 = ScalarValue::Int32(Some(100));
6781        assert_eq!(int_value.sub(&int_value_2)?, ScalarValue::Int32(Some(-58)));
6782        assert_eq!(int_value_2.sub(int_value)?, ScalarValue::Int32(Some(58)));
6783        Ok(())
6784    }
6785
6786    #[test]
6787    fn scalar_sub_trait_int32_overflow_test() {
6788        let int_value = ScalarValue::Int32(Some(i32::MAX));
6789        let int_value_2 = ScalarValue::Int32(Some(i32::MIN));
6790        let err = int_value
6791            .sub_checked(&int_value_2)
6792            .unwrap_err()
6793            .strip_backtrace();
6794        assert_eq!(
6795            err,
6796            "Arrow error: Arithmetic overflow: Overflow happened on: 2147483647 - -2147483648"
6797        )
6798    }
6799
6800    #[test]
6801    fn scalar_sub_trait_int64_test() -> Result<()> {
6802        let int_value = ScalarValue::Int64(Some(42));
6803        let int_value_2 = ScalarValue::Int64(Some(100));
6804        assert_eq!(int_value.sub(&int_value_2)?, ScalarValue::Int64(Some(-58)));
6805        assert_eq!(int_value_2.sub(int_value)?, ScalarValue::Int64(Some(58)));
6806        Ok(())
6807    }
6808
6809    #[test]
6810    fn scalar_sub_trait_int64_overflow_test() {
6811        let int_value = ScalarValue::Int64(Some(i64::MAX));
6812        let int_value_2 = ScalarValue::Int64(Some(i64::MIN));
6813        let err = int_value
6814            .sub_checked(&int_value_2)
6815            .unwrap_err()
6816            .strip_backtrace();
6817        assert_eq!(
6818            err,
6819            "Arrow error: Arithmetic overflow: Overflow happened on: 9223372036854775807 - -9223372036854775808"
6820        )
6821    }
6822
6823    #[test]
6824    fn scalar_add_overflow_test() -> Result<()> {
6825        check_scalar_add_overflow::<Int8Type>(
6826            ScalarValue::Int8(Some(i8::MAX)),
6827            ScalarValue::Int8(Some(i8::MAX)),
6828        );
6829        check_scalar_add_overflow::<UInt8Type>(
6830            ScalarValue::UInt8(Some(u8::MAX)),
6831            ScalarValue::UInt8(Some(u8::MAX)),
6832        );
6833        check_scalar_add_overflow::<Int16Type>(
6834            ScalarValue::Int16(Some(i16::MAX)),
6835            ScalarValue::Int16(Some(i16::MAX)),
6836        );
6837        check_scalar_add_overflow::<UInt16Type>(
6838            ScalarValue::UInt16(Some(u16::MAX)),
6839            ScalarValue::UInt16(Some(u16::MAX)),
6840        );
6841        check_scalar_add_overflow::<Int32Type>(
6842            ScalarValue::Int32(Some(i32::MAX)),
6843            ScalarValue::Int32(Some(i32::MAX)),
6844        );
6845        check_scalar_add_overflow::<UInt32Type>(
6846            ScalarValue::UInt32(Some(u32::MAX)),
6847            ScalarValue::UInt32(Some(u32::MAX)),
6848        );
6849        check_scalar_add_overflow::<Int64Type>(
6850            ScalarValue::Int64(Some(i64::MAX)),
6851            ScalarValue::Int64(Some(i64::MAX)),
6852        );
6853        check_scalar_add_overflow::<UInt64Type>(
6854            ScalarValue::UInt64(Some(u64::MAX)),
6855            ScalarValue::UInt64(Some(u64::MAX)),
6856        );
6857
6858        Ok(())
6859    }
6860
6861    #[test]
6862    fn scalar_decimal_add_overflow_test() {
6863        check_scalar_decimal_add_overflow::<Decimal128Type>(
6864            ScalarValue::Decimal128(Some(i128::MAX), DECIMAL128_MAX_PRECISION, 0),
6865            ScalarValue::Decimal128(Some(1), DECIMAL128_MAX_PRECISION, 0),
6866        );
6867        check_scalar_decimal_add_overflow::<Decimal256Type>(
6868            ScalarValue::Decimal256(Some(i256::MAX), DECIMAL256_MAX_PRECISION, 0),
6869            ScalarValue::Decimal256(Some(i256::ONE), DECIMAL256_MAX_PRECISION, 0),
6870        );
6871    }
6872
6873    #[test]
6874    fn scalar_decimal_in_place_add_error_preserves_lhs() {
6875        let mut lhs =
6876            ScalarValue::Decimal128(Some(i128::MAX), DECIMAL128_MAX_PRECISION, 0);
6877        let original = lhs.clone();
6878
6879        let err = lhs
6880            .try_add_checked_in_place(&ScalarValue::Decimal128(
6881                Some(1),
6882                DECIMAL128_MAX_PRECISION,
6883                0,
6884            ))
6885            .unwrap_err()
6886            .strip_backtrace();
6887
6888        assert_eq!(
6889            err,
6890            format!(
6891                "Arrow error: Arithmetic overflow: Overflow happened on: {} + 1",
6892                i128::MAX
6893            )
6894        );
6895        assert_eq!(lhs, original);
6896    }
6897
6898    // Verifies that ScalarValue has the same behavior with compute kernel when it overflows.
6899    fn check_scalar_add_overflow<T>(left: ScalarValue, right: ScalarValue)
6900    where
6901        T: ArrowNumericType,
6902    {
6903        let scalar_result = left.add_checked(&right);
6904
6905        let left_array = left.to_array().expect("Failed to convert to array");
6906        let right_array = right.to_array().expect("Failed to convert to array");
6907        let arrow_left_array = left_array.as_primitive::<T>();
6908        let arrow_right_array = right_array.as_primitive::<T>();
6909        let arrow_result = add(arrow_left_array, arrow_right_array);
6910
6911        assert_eq!(scalar_result.is_ok(), arrow_result.is_ok());
6912    }
6913
6914    // Verifies the decimal fast path preserves the same overflow behavior as Arrow kernels.
6915    fn check_scalar_decimal_add_overflow<T>(left: ScalarValue, right: ScalarValue)
6916    where
6917        T: ArrowPrimitiveType,
6918    {
6919        let scalar_result = left.add(&right);
6920
6921        let left_array = left.to_array().expect("Failed to convert to array");
6922        let right_array = right.to_array().expect("Failed to convert to array");
6923        let arrow_left_array = left_array.as_primitive::<T>();
6924        let arrow_right_array = right_array.as_primitive::<T>();
6925        let arrow_result = add_wrapping(arrow_left_array, arrow_right_array);
6926
6927        assert_eq!(scalar_result.is_ok(), arrow_result.is_ok());
6928    }
6929
6930    #[test]
6931    fn test_interval_add_timestamp() -> Result<()> {
6932        let interval = ScalarValue::IntervalMonthDayNano(Some(IntervalMonthDayNano {
6933            months: 1,
6934            days: 2,
6935            nanoseconds: 3,
6936        }));
6937        let timestamp = ScalarValue::TimestampNanosecond(Some(123), None);
6938        let result = interval.add(&timestamp)?;
6939        let expect = timestamp.add(&interval)?;
6940        assert_eq!(result, expect);
6941
6942        let interval = ScalarValue::IntervalYearMonth(Some(123));
6943        let timestamp = ScalarValue::TimestampNanosecond(Some(123), None);
6944        let result = interval.add(&timestamp)?;
6945        let expect = timestamp.add(&interval)?;
6946        assert_eq!(result, expect);
6947
6948        let interval = ScalarValue::IntervalDayTime(Some(IntervalDayTime {
6949            days: 1,
6950            milliseconds: 23,
6951        }));
6952        let timestamp = ScalarValue::TimestampNanosecond(Some(123), None);
6953        let result = interval.add(&timestamp)?;
6954        let expect = timestamp.add(&interval)?;
6955        assert_eq!(result, expect);
6956        Ok(())
6957    }
6958
6959    #[test]
6960    fn test_try_cmp() {
6961        assert_eq!(
6962            ScalarValue::try_cmp(
6963                &ScalarValue::Int32(Some(1)),
6964                &ScalarValue::Int32(Some(2))
6965            )
6966            .unwrap(),
6967            Ordering::Less
6968        );
6969        assert_eq!(
6970            ScalarValue::try_cmp(&ScalarValue::Int32(None), &ScalarValue::Int32(Some(2)))
6971                .unwrap(),
6972            Ordering::Less
6973        );
6974        assert_starts_with(
6975            ScalarValue::try_cmp(
6976                &ScalarValue::Int32(Some(1)),
6977                &ScalarValue::Int64(Some(2)),
6978            )
6979            .unwrap_err()
6980            .message(),
6981            "Uncomparable values: Int32(1), Int64(2)",
6982        );
6983    }
6984
6985    #[test]
6986    fn scalar_decimal_test() -> Result<()> {
6987        let decimal_value = ScalarValue::Decimal128(Some(123), 10, 1);
6988        assert_eq!(DataType::Decimal128(10, 1), decimal_value.data_type());
6989        let try_into_value: i128 = decimal_value.clone().try_into().unwrap();
6990        assert_eq!(123_i128, try_into_value);
6991        assert!(!decimal_value.is_null());
6992        let neg_decimal_value = decimal_value.arithmetic_negate()?;
6993        match neg_decimal_value {
6994            ScalarValue::Decimal128(v, _, _) => {
6995                assert_eq!(-123, v.unwrap());
6996            }
6997            _ => {
6998                unreachable!();
6999            }
7000        }
7001
7002        // decimal scalar to array
7003        let array = decimal_value
7004            .to_array()
7005            .expect("Failed to convert to array");
7006        let array = as_decimal128_array(&array)?;
7007        assert_eq!(1, array.len());
7008        assert_eq!(DataType::Decimal128(10, 1), array.data_type().clone());
7009        assert_eq!(123i128, array.value(0));
7010
7011        // decimal scalar to array with size
7012        let array = decimal_value
7013            .to_array_of_size(10)
7014            .expect("Failed to convert to array of size");
7015        let array_decimal = as_decimal128_array(&array)?;
7016        assert_eq!(10, array.len());
7017        assert_eq!(DataType::Decimal128(10, 1), array.data_type().clone());
7018        assert_eq!(123i128, array_decimal.value(0));
7019        assert_eq!(123i128, array_decimal.value(9));
7020        // test eq array
7021        assert!(
7022            decimal_value
7023                .eq_array(&array, 1)
7024                .expect("Failed to compare arrays")
7025        );
7026        assert!(
7027            decimal_value
7028                .eq_array(&array, 5)
7029                .expect("Failed to compare arrays")
7030        );
7031        // test try from array
7032        assert_eq!(
7033            decimal_value,
7034            ScalarValue::try_from_array(&array, 5).unwrap()
7035        );
7036
7037        assert_eq!(
7038            decimal_value,
7039            ScalarValue::try_new_decimal128(123, 10, 1).unwrap()
7040        );
7041
7042        // test compare
7043        let left = ScalarValue::Decimal128(Some(123), 10, 2);
7044        let right = ScalarValue::Decimal128(Some(124), 10, 2);
7045        assert!(!left.eq(&right));
7046        let result = left < right;
7047        assert!(result);
7048        let result = left <= right;
7049        assert!(result);
7050        let right = ScalarValue::Decimal128(Some(124), 10, 3);
7051        // make sure that two decimals with diff datatype can't be compared.
7052        let result = left.partial_cmp(&right);
7053        assert_eq!(None, result);
7054
7055        let decimal_vec = vec![
7056            ScalarValue::Decimal128(Some(1), 10, 2),
7057            ScalarValue::Decimal128(Some(2), 10, 2),
7058            ScalarValue::Decimal128(Some(3), 10, 2),
7059        ];
7060        // convert the vec to decimal array and check the result
7061        let array = ScalarValue::iter_to_array(decimal_vec).unwrap();
7062        assert_eq!(3, array.len());
7063        assert_eq!(DataType::Decimal128(10, 2), array.data_type().clone());
7064
7065        let decimal_vec = vec![
7066            ScalarValue::Decimal128(Some(1), 10, 2),
7067            ScalarValue::Decimal128(Some(2), 10, 2),
7068            ScalarValue::Decimal128(Some(3), 10, 2),
7069            ScalarValue::Decimal128(None, 10, 2),
7070        ];
7071        let array = ScalarValue::iter_to_array(decimal_vec).unwrap();
7072        assert_eq!(4, array.len());
7073        assert_eq!(DataType::Decimal128(10, 2), array.data_type().clone());
7074
7075        assert!(
7076            ScalarValue::try_new_decimal128(1, 10, 2)
7077                .unwrap()
7078                .eq_array(&array, 0)
7079                .expect("Failed to compare arrays")
7080        );
7081        assert!(
7082            ScalarValue::try_new_decimal128(2, 10, 2)
7083                .unwrap()
7084                .eq_array(&array, 1)
7085                .expect("Failed to compare arrays")
7086        );
7087        assert!(
7088            ScalarValue::try_new_decimal128(3, 10, 2)
7089                .unwrap()
7090                .eq_array(&array, 2)
7091                .expect("Failed to compare arrays")
7092        );
7093        assert_eq!(
7094            ScalarValue::Decimal128(None, 10, 2),
7095            ScalarValue::try_from_array(&array, 3).unwrap()
7096        );
7097
7098        Ok(())
7099    }
7100
7101    #[test]
7102    fn test_new_one_decimal128() {
7103        assert_eq!(
7104            ScalarValue::new_one(&DataType::Decimal128(5, 0)).unwrap(),
7105            ScalarValue::Decimal128(Some(1), 5, 0)
7106        );
7107        assert_eq!(
7108            ScalarValue::new_one(&DataType::Decimal128(5, 1)).unwrap(),
7109            ScalarValue::Decimal128(Some(10), 5, 1)
7110        );
7111        assert_eq!(
7112            ScalarValue::new_one(&DataType::Decimal128(5, 2)).unwrap(),
7113            ScalarValue::Decimal128(Some(100), 5, 2)
7114        );
7115        // More precision
7116        assert_eq!(
7117            ScalarValue::new_one(&DataType::Decimal128(7, 2)).unwrap(),
7118            ScalarValue::Decimal128(Some(100), 7, 2)
7119        );
7120        // No negative scale
7121        assert!(ScalarValue::new_one(&DataType::Decimal128(5, -1)).is_err());
7122        // Invalid combination
7123        assert!(ScalarValue::new_one(&DataType::Decimal128(0, 2)).is_err());
7124        assert!(ScalarValue::new_one(&DataType::Decimal128(5, 7)).is_err());
7125    }
7126
7127    #[test]
7128    fn test_new_one_decimal256() {
7129        assert_eq!(
7130            ScalarValue::new_one(&DataType::Decimal256(5, 0)).unwrap(),
7131            ScalarValue::Decimal256(Some(1.into()), 5, 0)
7132        );
7133        assert_eq!(
7134            ScalarValue::new_one(&DataType::Decimal256(5, 1)).unwrap(),
7135            ScalarValue::Decimal256(Some(10.into()), 5, 1)
7136        );
7137        assert_eq!(
7138            ScalarValue::new_one(&DataType::Decimal256(5, 2)).unwrap(),
7139            ScalarValue::Decimal256(Some(100.into()), 5, 2)
7140        );
7141        // More precision
7142        assert_eq!(
7143            ScalarValue::new_one(&DataType::Decimal256(7, 2)).unwrap(),
7144            ScalarValue::Decimal256(Some(100.into()), 7, 2)
7145        );
7146        // No negative scale
7147        assert!(ScalarValue::new_one(&DataType::Decimal256(5, -1)).is_err());
7148        // Invalid combination
7149        assert!(ScalarValue::new_one(&DataType::Decimal256(0, 2)).is_err());
7150        assert!(ScalarValue::new_one(&DataType::Decimal256(5, 7)).is_err());
7151    }
7152
7153    #[test]
7154    fn test_new_ten_decimal128() {
7155        assert_eq!(
7156            ScalarValue::new_ten(&DataType::Decimal128(5, 1)).unwrap(),
7157            ScalarValue::Decimal128(Some(100), 5, 1)
7158        );
7159        assert_eq!(
7160            ScalarValue::new_ten(&DataType::Decimal128(5, 2)).unwrap(),
7161            ScalarValue::Decimal128(Some(1000), 5, 2)
7162        );
7163        // More precision
7164        assert_eq!(
7165            ScalarValue::new_ten(&DataType::Decimal128(7, 2)).unwrap(),
7166            ScalarValue::Decimal128(Some(1000), 7, 2)
7167        );
7168        // No negative scale
7169        assert!(ScalarValue::new_ten(&DataType::Decimal128(5, -1)).is_err());
7170        // Invalid combination
7171        assert!(ScalarValue::new_ten(&DataType::Decimal128(0, 2)).is_err());
7172        assert!(ScalarValue::new_ten(&DataType::Decimal128(5, 7)).is_err());
7173    }
7174
7175    #[test]
7176    fn test_new_ten_decimal256() {
7177        assert_eq!(
7178            ScalarValue::new_ten(&DataType::Decimal256(5, 1)).unwrap(),
7179            ScalarValue::Decimal256(Some(100.into()), 5, 1)
7180        );
7181        assert_eq!(
7182            ScalarValue::new_ten(&DataType::Decimal256(5, 2)).unwrap(),
7183            ScalarValue::Decimal256(Some(1000.into()), 5, 2)
7184        );
7185        // More precision
7186        assert_eq!(
7187            ScalarValue::new_ten(&DataType::Decimal256(7, 2)).unwrap(),
7188            ScalarValue::Decimal256(Some(1000.into()), 7, 2)
7189        );
7190        // No negative scale
7191        assert!(ScalarValue::new_ten(&DataType::Decimal256(5, -1)).is_err());
7192        // Invalid combination
7193        assert!(ScalarValue::new_ten(&DataType::Decimal256(0, 2)).is_err());
7194        assert!(ScalarValue::new_ten(&DataType::Decimal256(5, 7)).is_err());
7195    }
7196
7197    #[test]
7198    fn test_new_negative_one_decimal128() {
7199        assert_eq!(
7200            ScalarValue::new_negative_one(&DataType::Decimal128(5, 0)).unwrap(),
7201            ScalarValue::Decimal128(Some(-1), 5, 0)
7202        );
7203        assert_eq!(
7204            ScalarValue::new_negative_one(&DataType::Decimal128(5, 2)).unwrap(),
7205            ScalarValue::Decimal128(Some(-100), 5, 2)
7206        );
7207    }
7208
7209    #[test]
7210    fn test_list_partial_cmp() {
7211        let a =
7212            ScalarValue::List(Arc::new(
7213                ListArray::from_iter_primitive::<Int64Type, _, _>(vec![Some(vec![
7214                    Some(1),
7215                    Some(2),
7216                    Some(3),
7217                ])]),
7218            ));
7219        let b =
7220            ScalarValue::List(Arc::new(
7221                ListArray::from_iter_primitive::<Int64Type, _, _>(vec![Some(vec![
7222                    Some(1),
7223                    Some(2),
7224                    Some(3),
7225                ])]),
7226            ));
7227        assert_eq!(a.partial_cmp(&b), Some(Ordering::Equal));
7228
7229        let a =
7230            ScalarValue::List(Arc::new(
7231                ListArray::from_iter_primitive::<Int64Type, _, _>(vec![Some(vec![
7232                    Some(10),
7233                    Some(2),
7234                    Some(3),
7235                ])]),
7236            ));
7237        let b =
7238            ScalarValue::List(Arc::new(
7239                ListArray::from_iter_primitive::<Int64Type, _, _>(vec![Some(vec![
7240                    Some(1),
7241                    Some(2),
7242                    Some(30),
7243                ])]),
7244            ));
7245        assert_eq!(a.partial_cmp(&b), Some(Ordering::Greater));
7246
7247        let a =
7248            ScalarValue::List(Arc::new(
7249                ListArray::from_iter_primitive::<Int64Type, _, _>(vec![Some(vec![
7250                    Some(10),
7251                    Some(2),
7252                    Some(3),
7253                ])]),
7254            ));
7255        let b =
7256            ScalarValue::List(Arc::new(
7257                ListArray::from_iter_primitive::<Int64Type, _, _>(vec![Some(vec![
7258                    Some(10),
7259                    Some(2),
7260                    Some(30),
7261                ])]),
7262            ));
7263        assert_eq!(a.partial_cmp(&b), Some(Ordering::Less));
7264
7265        let a =
7266            ScalarValue::List(Arc::new(
7267                ListArray::from_iter_primitive::<Int64Type, _, _>(vec![Some(vec![
7268                    Some(1),
7269                    Some(2),
7270                    Some(3),
7271                ])]),
7272            ));
7273        let b =
7274            ScalarValue::List(Arc::new(
7275                ListArray::from_iter_primitive::<Int64Type, _, _>(vec![Some(vec![
7276                    Some(2),
7277                    Some(3),
7278                ])]),
7279            ));
7280        assert_eq!(a.partial_cmp(&b), Some(Ordering::Less));
7281
7282        let a =
7283            ScalarValue::List(Arc::new(
7284                ListArray::from_iter_primitive::<Int64Type, _, _>(vec![Some(vec![
7285                    Some(2),
7286                    Some(3),
7287                    Some(4),
7288                ])]),
7289            ));
7290        let b =
7291            ScalarValue::List(Arc::new(
7292                ListArray::from_iter_primitive::<Int64Type, _, _>(vec![Some(vec![
7293                    Some(1),
7294                    Some(2),
7295                ])]),
7296            ));
7297        assert_eq!(a.partial_cmp(&b), Some(Ordering::Greater));
7298
7299        let a =
7300            ScalarValue::List(Arc::new(
7301                ListArray::from_iter_primitive::<Int64Type, _, _>(vec![Some(vec![
7302                    Some(1),
7303                    Some(2),
7304                    Some(3),
7305                ])]),
7306            ));
7307        let b =
7308            ScalarValue::List(Arc::new(
7309                ListArray::from_iter_primitive::<Int64Type, _, _>(vec![Some(vec![
7310                    Some(1),
7311                    Some(2),
7312                ])]),
7313            ));
7314        assert_eq!(a.partial_cmp(&b), Some(Ordering::Greater));
7315
7316        let a =
7317            ScalarValue::List(Arc::new(
7318                ListArray::from_iter_primitive::<Int64Type, _, _>(vec![Some(vec![
7319                    None,
7320                    Some(2),
7321                    Some(3),
7322                ])]),
7323            ));
7324        let b =
7325            ScalarValue::List(Arc::new(
7326                ListArray::from_iter_primitive::<Int64Type, _, _>(vec![Some(vec![
7327                    Some(1),
7328                    Some(2),
7329                    Some(3),
7330                ])]),
7331            ));
7332        assert_eq!(a.partial_cmp(&b), Some(Ordering::Greater));
7333
7334        let a = ScalarValue::LargeList(Arc::new(LargeListArray::from_iter_primitive::<
7335            Int64Type,
7336            _,
7337            _,
7338        >(vec![Some(vec![
7339            None,
7340            Some(2),
7341            Some(3),
7342        ])])));
7343        let b = ScalarValue::LargeList(Arc::new(LargeListArray::from_iter_primitive::<
7344            Int64Type,
7345            _,
7346            _,
7347        >(vec![Some(vec![
7348            Some(1),
7349            Some(2),
7350            Some(3),
7351        ])])));
7352        assert_eq!(a.partial_cmp(&b), Some(Ordering::Greater));
7353
7354        let a = ScalarValue::FixedSizeList(Arc::new(
7355            FixedSizeListArray::from_iter_primitive::<Int64Type, _, _>(
7356                vec![Some(vec![None, Some(2), Some(3)])],
7357                3,
7358            ),
7359        ));
7360        let b = ScalarValue::FixedSizeList(Arc::new(
7361            FixedSizeListArray::from_iter_primitive::<Int64Type, _, _>(
7362                vec![Some(vec![Some(1), Some(2), Some(3)])],
7363                3,
7364            ),
7365        ));
7366        assert_eq!(a.partial_cmp(&b), Some(Ordering::Greater));
7367
7368        let a = ScalarValue::ListView(Arc::new(ListViewArray::from_iter_primitive::<
7369            Int64Type,
7370            _,
7371            _,
7372        >(vec![Some(vec![
7373            None,
7374            Some(2),
7375            Some(3),
7376        ])])));
7377        let b = ScalarValue::ListView(Arc::new(ListViewArray::from_iter_primitive::<
7378            Int64Type,
7379            _,
7380            _,
7381        >(vec![Some(vec![
7382            Some(1),
7383            Some(2),
7384            Some(3),
7385        ])])));
7386        assert_eq!(a.partial_cmp(&b), Some(Ordering::Greater));
7387
7388        let a =
7389            ScalarValue::LargeListView(Arc::new(
7390                LargeListViewArray::from_iter_primitive::<Int64Type, _, _>(vec![Some(
7391                    vec![None, Some(2), Some(3)],
7392                )]),
7393            ));
7394        let b =
7395            ScalarValue::LargeListView(Arc::new(
7396                LargeListViewArray::from_iter_primitive::<Int64Type, _, _>(vec![Some(
7397                    vec![Some(1), Some(2), Some(3)],
7398                )]),
7399            ));
7400        assert_eq!(a.partial_cmp(&b), Some(Ordering::Greater));
7401    }
7402
7403    #[test]
7404    fn scalar_value_to_array_u64() -> Result<()> {
7405        let value = ScalarValue::UInt64(Some(13u64));
7406        let array = value.to_array().expect("Failed to convert to array");
7407        let array = as_uint64_array(&array)?;
7408        assert_eq!(array.len(), 1);
7409        assert!(!array.is_null(0));
7410        assert_eq!(array.value(0), 13);
7411
7412        let value = ScalarValue::UInt64(None);
7413        let array = value.to_array().expect("Failed to convert to array");
7414        let array = as_uint64_array(&array)?;
7415        assert_eq!(array.len(), 1);
7416        assert!(array.is_null(0));
7417        Ok(())
7418    }
7419
7420    #[test]
7421    fn scalar_value_to_array_u32() -> Result<()> {
7422        let value = ScalarValue::UInt32(Some(13u32));
7423        let array = value.to_array().expect("Failed to convert to array");
7424        let array = as_uint32_array(&array)?;
7425        assert_eq!(array.len(), 1);
7426        assert!(!array.is_null(0));
7427        assert_eq!(array.value(0), 13);
7428
7429        let value = ScalarValue::UInt32(None);
7430        let array = value.to_array().expect("Failed to convert to array");
7431        let array = as_uint32_array(&array)?;
7432        assert_eq!(array.len(), 1);
7433        assert!(array.is_null(0));
7434        Ok(())
7435    }
7436
7437    #[test]
7438    fn scalar_list_null_to_array() {
7439        let list_array = ScalarValue::new_list_nullable(&[], &DataType::UInt64);
7440
7441        assert_eq!(list_array.len(), 1);
7442        assert_eq!(list_array.values().len(), 0);
7443    }
7444
7445    #[test]
7446    fn scalar_large_list_null_to_array() {
7447        let list_array = ScalarValue::new_large_list(&[], &DataType::UInt64);
7448
7449        assert_eq!(list_array.len(), 1);
7450        assert_eq!(list_array.values().len(), 0);
7451    }
7452
7453    #[test]
7454    fn scalar_list_to_array() -> Result<()> {
7455        let values = vec![
7456            ScalarValue::UInt64(Some(100)),
7457            ScalarValue::UInt64(None),
7458            ScalarValue::UInt64(Some(101)),
7459        ];
7460        let list_array = ScalarValue::new_list_nullable(&values, &DataType::UInt64);
7461        assert_eq!(list_array.len(), 1);
7462        assert_eq!(list_array.values().len(), 3);
7463
7464        let prim_array_ref = list_array.value(0);
7465        let prim_array = as_uint64_array(&prim_array_ref)?;
7466        assert_eq!(prim_array.len(), 3);
7467        assert_eq!(prim_array.value(0), 100);
7468        assert!(prim_array.is_null(1));
7469        assert_eq!(prim_array.value(2), 101);
7470        Ok(())
7471    }
7472
7473    #[test]
7474    fn scalar_large_list_to_array() -> Result<()> {
7475        let values = vec![
7476            ScalarValue::UInt64(Some(100)),
7477            ScalarValue::UInt64(None),
7478            ScalarValue::UInt64(Some(101)),
7479        ];
7480        let list_array = ScalarValue::new_large_list(&values, &DataType::UInt64);
7481        assert_eq!(list_array.len(), 1);
7482        assert_eq!(list_array.values().len(), 3);
7483
7484        let prim_array_ref = list_array.value(0);
7485        let prim_array = as_uint64_array(&prim_array_ref)?;
7486        assert_eq!(prim_array.len(), 3);
7487        assert_eq!(prim_array.value(0), 100);
7488        assert!(prim_array.is_null(1));
7489        assert_eq!(prim_array.value(2), 101);
7490        Ok(())
7491    }
7492
7493    /// Creates array directly and via ScalarValue and ensures they are the same
7494    macro_rules! check_scalar_iter {
7495        ($SCALAR_T:ident, $ARRAYTYPE:ident, $INPUT:expr) => {{
7496            let scalars: Vec<_> =
7497                $INPUT.iter().map(|v| ScalarValue::$SCALAR_T(*v)).collect();
7498
7499            let array = ScalarValue::iter_to_array(scalars.into_iter()).unwrap();
7500
7501            let expected: ArrayRef = Arc::new($ARRAYTYPE::from($INPUT));
7502
7503            assert_eq!(&array, &expected);
7504        }};
7505    }
7506
7507    /// Creates array directly and via ScalarValue and ensures they are the same
7508    /// but for variants that carry a timezone field.
7509    macro_rules! check_scalar_iter_tz {
7510        ($SCALAR_T:ident, $ARRAYTYPE:ident, $INPUT:expr) => {{
7511            let scalars: Vec<_> = $INPUT
7512                .iter()
7513                .map(|v| ScalarValue::$SCALAR_T(*v, None))
7514                .collect();
7515
7516            let array = ScalarValue::iter_to_array(scalars.into_iter()).unwrap();
7517
7518            let expected: ArrayRef = Arc::new($ARRAYTYPE::from($INPUT));
7519
7520            assert_eq!(&array, &expected);
7521        }};
7522    }
7523
7524    /// Creates array directly and via ScalarValue and ensures they
7525    /// are the same, for string  arrays
7526    macro_rules! check_scalar_iter_string {
7527        ($SCALAR_T:ident, $ARRAYTYPE:ident, $INPUT:expr) => {{
7528            let scalars: Vec<_> = $INPUT
7529                .iter()
7530                .map(|v| ScalarValue::$SCALAR_T(v.map(|v| v.to_string())))
7531                .collect();
7532
7533            let array = ScalarValue::iter_to_array(scalars.into_iter()).unwrap();
7534
7535            let expected: ArrayRef = Arc::new($ARRAYTYPE::from($INPUT));
7536
7537            assert_eq!(&array, &expected);
7538        }};
7539    }
7540
7541    /// Creates array directly and via ScalarValue and ensures they
7542    /// are the same, for binary arrays
7543    macro_rules! check_scalar_iter_binary {
7544        ($SCALAR_T:ident, $ARRAYTYPE:ident, $INPUT:expr) => {{
7545            let scalars: Vec<_> = $INPUT
7546                .iter()
7547                .map(|v| ScalarValue::$SCALAR_T(v.map(|v| v.to_vec())))
7548                .collect();
7549
7550            let array = ScalarValue::iter_to_array(scalars.into_iter()).unwrap();
7551
7552            let expected: $ARRAYTYPE =
7553                $INPUT.iter().map(|v| v.map(|v| v.to_vec())).collect();
7554
7555            let expected: ArrayRef = Arc::new(expected);
7556
7557            assert_eq!(&array, &expected);
7558        }};
7559    }
7560
7561    #[test]
7562    fn scalar_iter_to_array_boolean() {
7563        check_scalar_iter!(Boolean, BooleanArray, vec![Some(true), None, Some(false)]);
7564        check_scalar_iter!(Float32, Float32Array, vec![Some(1.9), None, Some(-2.1)]);
7565        check_scalar_iter!(Float64, Float64Array, vec![Some(1.9), None, Some(-2.1)]);
7566
7567        check_scalar_iter!(Int8, Int8Array, vec![Some(1), None, Some(3)]);
7568        check_scalar_iter!(Int16, Int16Array, vec![Some(1), None, Some(3)]);
7569        check_scalar_iter!(Int32, Int32Array, vec![Some(1), None, Some(3)]);
7570        check_scalar_iter!(Int64, Int64Array, vec![Some(1), None, Some(3)]);
7571
7572        check_scalar_iter!(UInt8, UInt8Array, vec![Some(1), None, Some(3)]);
7573        check_scalar_iter!(UInt16, UInt16Array, vec![Some(1), None, Some(3)]);
7574        check_scalar_iter!(UInt32, UInt32Array, vec![Some(1), None, Some(3)]);
7575        check_scalar_iter!(UInt64, UInt64Array, vec![Some(1), None, Some(3)]);
7576
7577        check_scalar_iter_tz!(
7578            TimestampSecond,
7579            TimestampSecondArray,
7580            vec![Some(1), None, Some(3)]
7581        );
7582        check_scalar_iter_tz!(
7583            TimestampMillisecond,
7584            TimestampMillisecondArray,
7585            vec![Some(1), None, Some(3)]
7586        );
7587        check_scalar_iter_tz!(
7588            TimestampMicrosecond,
7589            TimestampMicrosecondArray,
7590            vec![Some(1), None, Some(3)]
7591        );
7592        check_scalar_iter_tz!(
7593            TimestampNanosecond,
7594            TimestampNanosecondArray,
7595            vec![Some(1), None, Some(3)]
7596        );
7597
7598        check_scalar_iter_string!(
7599            Utf8,
7600            StringArray,
7601            vec![Some("foo"), None, Some("bar")]
7602        );
7603        check_scalar_iter_string!(
7604            LargeUtf8,
7605            LargeStringArray,
7606            vec![Some("foo"), None, Some("bar")]
7607        );
7608        check_scalar_iter_binary!(
7609            Binary,
7610            BinaryArray,
7611            [Some(b"foo"), None, Some(b"bar")]
7612        );
7613        check_scalar_iter_binary!(
7614            LargeBinary,
7615            LargeBinaryArray,
7616            [Some(b"foo"), None, Some(b"bar")]
7617        );
7618    }
7619
7620    #[test]
7621    fn scalar_iter_to_array_empty() {
7622        let scalars = vec![] as Vec<ScalarValue>;
7623
7624        let result = ScalarValue::iter_to_array(scalars).unwrap_err();
7625        assert!(
7626            result
7627                .to_string()
7628                .contains("Empty iterator passed to ScalarValue::iter_to_array"),
7629            "{}",
7630            result
7631        );
7632    }
7633
7634    #[test]
7635    fn scalar_iter_to_dictionary() {
7636        fn make_val(v: Option<String>) -> ScalarValue {
7637            let key_type = DataType::Int32;
7638            let value = ScalarValue::Utf8(v);
7639            ScalarValue::Dictionary(Box::new(key_type), Box::new(value))
7640        }
7641
7642        let scalars = [
7643            make_val(Some("Foo".into())),
7644            make_val(None),
7645            make_val(Some("Bar".into())),
7646        ];
7647
7648        let array = ScalarValue::iter_to_array(scalars).unwrap();
7649        let array = as_dictionary_array::<Int32Type>(&array).unwrap();
7650        let values_array = as_string_array(array.values()).unwrap();
7651
7652        let values = array
7653            .keys_iter()
7654            .map(|k| {
7655                k.map(|k| {
7656                    assert!(values_array.is_valid(k));
7657                    values_array.value(k)
7658                })
7659            })
7660            .collect::<Vec<_>>();
7661
7662        let expected = vec![Some("Foo"), None, Some("Bar")];
7663        assert_eq!(values, expected);
7664    }
7665
7666    #[test]
7667    fn scalar_iter_to_array_mismatched_types() {
7668        use ScalarValue::*;
7669        // If the scalar values are not all the correct type, error here
7670        let scalars = [Boolean(Some(true)), Int32(Some(5))];
7671
7672        let result = ScalarValue::iter_to_array(scalars).unwrap_err();
7673        assert!(result.to_string().contains("Inconsistent types in ScalarValue::iter_to_array. Expected Boolean, got Int32(5)"),
7674                "{}", result);
7675    }
7676
7677    #[test]
7678    fn scalar_try_from_array_null() {
7679        let array = vec![Some(33), None].into_iter().collect::<Int64Array>();
7680        let array: ArrayRef = Arc::new(array);
7681
7682        assert_eq!(
7683            ScalarValue::Int64(Some(33)),
7684            ScalarValue::try_from_array(&array, 0).unwrap()
7685        );
7686        assert_eq!(
7687            ScalarValue::Int64(None),
7688            ScalarValue::try_from_array(&array, 1).unwrap()
7689        );
7690    }
7691
7692    #[test]
7693    fn scalar_try_from_array_list_array_null() {
7694        let list = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
7695            Some(vec![Some(1), Some(2)]),
7696            None,
7697        ]);
7698
7699        let non_null_list_scalar = ScalarValue::try_from_array(&list, 0).unwrap();
7700        let null_list_scalar = ScalarValue::try_from_array(&list, 1).unwrap();
7701
7702        let data_type =
7703            DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true)));
7704
7705        assert_eq!(non_null_list_scalar.data_type(), data_type);
7706        assert_eq!(null_list_scalar.data_type(), data_type);
7707    }
7708
7709    #[test]
7710    fn scalar_try_from_list_datatypes() {
7711        let inner_field = Arc::new(Field::new_list_field(DataType::Int32, true));
7712
7713        // Test for List
7714        let data_type = &DataType::List(Arc::clone(&inner_field));
7715        let scalar: ScalarValue = data_type.try_into().unwrap();
7716        let expected = ScalarValue::List(
7717            new_null_array(data_type, 1)
7718                .as_list::<i32>()
7719                .to_owned()
7720                .into(),
7721        );
7722        assert_eq!(expected, scalar);
7723        assert!(expected.is_null());
7724
7725        // Test for LargeList
7726        let data_type = &DataType::LargeList(Arc::clone(&inner_field));
7727        let scalar: ScalarValue = data_type.try_into().unwrap();
7728        let expected = ScalarValue::LargeList(
7729            new_null_array(data_type, 1)
7730                .as_list::<i64>()
7731                .to_owned()
7732                .into(),
7733        );
7734        assert_eq!(expected, scalar);
7735        assert!(expected.is_null());
7736
7737        // Test for FixedSizeList(5)
7738        let data_type = &DataType::FixedSizeList(Arc::clone(&inner_field), 5);
7739        let scalar: ScalarValue = data_type.try_into().unwrap();
7740        let expected = ScalarValue::FixedSizeList(
7741            new_null_array(data_type, 1)
7742                .as_fixed_size_list()
7743                .to_owned()
7744                .into(),
7745        );
7746        assert_eq!(expected, scalar);
7747        assert!(expected.is_null());
7748
7749        // Test for ListView
7750        let data_type = &DataType::ListView(Arc::clone(&inner_field));
7751        let scalar: ScalarValue = data_type.try_into().unwrap();
7752        let expected = ScalarValue::ListView(
7753            new_null_array(data_type, 1)
7754                .as_list_view::<i32>()
7755                .to_owned()
7756                .into(),
7757        );
7758        assert_eq!(expected, scalar);
7759        assert!(expected.is_null());
7760
7761        // Test for LargeListView
7762        let data_type = &DataType::LargeListView(Arc::clone(&inner_field));
7763        let scalar: ScalarValue = data_type.try_into().unwrap();
7764        let expected = ScalarValue::LargeListView(
7765            new_null_array(data_type, 1)
7766                .as_list_view::<i64>()
7767                .to_owned()
7768                .into(),
7769        );
7770        assert_eq!(expected, scalar);
7771        assert!(expected.is_null());
7772    }
7773
7774    #[test]
7775    fn scalar_try_from_list_of_list() {
7776        let data_type = DataType::List(Arc::new(Field::new_list_field(
7777            DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true))),
7778            true,
7779        )));
7780        let data_type = &data_type;
7781        let scalar: ScalarValue = data_type.try_into().unwrap();
7782
7783        let expected = ScalarValue::List(
7784            new_null_array(
7785                &DataType::List(Arc::new(Field::new_list_field(
7786                    DataType::List(Arc::new(Field::new_list_field(
7787                        DataType::Int32,
7788                        true,
7789                    ))),
7790                    true,
7791                ))),
7792                1,
7793            )
7794            .as_list::<i32>()
7795            .to_owned()
7796            .into(),
7797        );
7798
7799        assert_eq!(expected, scalar)
7800    }
7801
7802    #[test]
7803    fn scalar_try_from_not_equal_list_nested_list() {
7804        let list_data_type =
7805            DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true)));
7806        let data_type = &list_data_type;
7807        let list_scalar: ScalarValue = data_type.try_into().unwrap();
7808
7809        let nested_list_data_type = DataType::List(Arc::new(Field::new_list_field(
7810            DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true))),
7811            true,
7812        )));
7813        let data_type = &nested_list_data_type;
7814        let nested_list_scalar: ScalarValue = data_type.try_into().unwrap();
7815
7816        assert_ne!(list_scalar, nested_list_scalar);
7817    }
7818
7819    #[test]
7820    fn scalar_try_from_dict_datatype() {
7821        let data_type =
7822            DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8));
7823        let data_type = &data_type;
7824        let expected = ScalarValue::Dictionary(
7825            Box::new(DataType::Int8),
7826            Box::new(ScalarValue::Utf8(None)),
7827        );
7828        assert_eq!(expected, data_type.try_into().unwrap())
7829    }
7830
7831    #[test]
7832    fn size_of_scalar() {
7833        // Since ScalarValues are used in a non trivial number of places,
7834        // making it larger means significant more memory consumption
7835        // per distinct value.
7836        //
7837        // Thus this test ensures that no code change makes ScalarValue larger
7838        //
7839        // The alignment requirements differ across architectures and
7840        // thus the size of the enum appears to as well
7841
7842        // The value may also change depending on rust version
7843        assert_eq!(size_of::<ScalarValue>(), 64);
7844    }
7845
7846    #[test]
7847    fn memory_size() {
7848        let sv = ScalarValue::Binary(Some(Vec::with_capacity(10)));
7849        assert_eq!(sv.size(), size_of::<ScalarValue>() + 10,);
7850        let sv_size = sv.size();
7851
7852        let mut v = Vec::with_capacity(10);
7853        // do NOT clone `sv` here because this may shrink the vector capacity
7854        v.push(sv);
7855        assert_eq!(v.capacity(), 10);
7856        assert_eq!(
7857            ScalarValue::size_of_vec(&v),
7858            size_of::<Vec<ScalarValue>>() + (9 * size_of::<ScalarValue>()) + sv_size,
7859        );
7860
7861        #[allow(clippy::allow_attributes, clippy::mutable_key_type)]
7862        // ScalarValue has interior mutability but is intentionally used as hash key
7863        let mut s = HashSet::with_capacity(0);
7864        // do NOT clone `sv` here because this may shrink the vector capacity
7865        s.insert(v.pop().unwrap());
7866        // hashsets may easily grow during insert, so capacity is dynamic
7867        let s_capacity = s.capacity();
7868        assert_eq!(
7869            ScalarValue::size_of_hashset(&s),
7870            size_of::<HashSet<ScalarValue>>()
7871                + ((s_capacity - 1) * size_of::<ScalarValue>())
7872                + sv_size,
7873        );
7874    }
7875
7876    #[test]
7877    fn scalar_eq_array() {
7878        // Validate that eq_array has the same semantics as ScalarValue::eq
7879        macro_rules! make_typed_vec {
7880            ($INPUT:expr, $TYPE:ident) => {{
7881                $INPUT
7882                    .iter()
7883                    .map(|v| v.map(|v| v as $TYPE))
7884                    .collect::<Vec<_>>()
7885            }};
7886        }
7887
7888        let bool_vals = [Some(true), None, Some(false)];
7889        let f32_vals = [Some(-1.0), None, Some(1.0)];
7890        let f64_vals = make_typed_vec!(f32_vals, f64);
7891
7892        let i8_vals = [Some(-1), None, Some(1)];
7893        let i16_vals = make_typed_vec!(i8_vals, i16);
7894        let i32_vals = make_typed_vec!(i8_vals, i32);
7895        let i64_vals = make_typed_vec!(i8_vals, i64);
7896
7897        let u8_vals = [Some(0), None, Some(1)];
7898        let u16_vals = make_typed_vec!(u8_vals, u16);
7899        let u32_vals = make_typed_vec!(u8_vals, u32);
7900        let u64_vals = make_typed_vec!(u8_vals, u64);
7901
7902        let str_vals = [Some("foo"), None, Some("bar")];
7903
7904        let interval_dt_vals = [
7905            Some(IntervalDayTime::MINUS_ONE),
7906            None,
7907            Some(IntervalDayTime::ONE),
7908        ];
7909        let interval_mdn_vals = [
7910            Some(IntervalMonthDayNano::MINUS_ONE),
7911            None,
7912            Some(IntervalMonthDayNano::ONE),
7913        ];
7914
7915        /// Test each value in `scalar` with the corresponding element
7916        /// at `array`. Assumes each element is unique (aka not equal
7917        /// with all other indexes)
7918        #[derive(Debug)]
7919        struct TestCase {
7920            array: ArrayRef,
7921            scalars: Vec<ScalarValue>,
7922        }
7923
7924        /// Create a test case for casing the input to the specified array type
7925        macro_rules! make_test_case {
7926            ($INPUT:expr, $ARRAY_TY:ident, $SCALAR_TY:ident) => {{
7927                TestCase {
7928                    array: Arc::new($INPUT.iter().collect::<$ARRAY_TY>()),
7929                    scalars: $INPUT.iter().map(|v| ScalarValue::$SCALAR_TY(*v)).collect(),
7930                }
7931            }};
7932
7933            ($INPUT:expr, $ARRAY_TY:ident, $SCALAR_TY:ident, $TZ:expr) => {{
7934                let tz = $TZ;
7935                TestCase {
7936                    array: Arc::new($INPUT.iter().collect::<$ARRAY_TY>()),
7937                    scalars: $INPUT
7938                        .iter()
7939                        .map(|v| ScalarValue::$SCALAR_TY(*v, tz.clone()))
7940                        .collect(),
7941                }
7942            }};
7943        }
7944
7945        macro_rules! make_str_test_case {
7946            ($INPUT:expr, $ARRAY_TY:ident, $SCALAR_TY:ident) => {{
7947                TestCase {
7948                    array: Arc::new($INPUT.iter().cloned().collect::<$ARRAY_TY>()),
7949                    scalars: $INPUT
7950                        .iter()
7951                        .map(|v| ScalarValue::$SCALAR_TY(v.map(|v| v.to_string())))
7952                        .collect(),
7953                }
7954            }};
7955        }
7956
7957        macro_rules! make_binary_test_case {
7958            ($INPUT:expr, $ARRAY_TY:ident, $SCALAR_TY:ident) => {{
7959                TestCase {
7960                    array: Arc::new($INPUT.iter().cloned().collect::<$ARRAY_TY>()),
7961                    scalars: $INPUT
7962                        .iter()
7963                        .map(|v| {
7964                            ScalarValue::$SCALAR_TY(v.map(|v| v.as_bytes().to_vec()))
7965                        })
7966                        .collect(),
7967                }
7968            }};
7969        }
7970
7971        /// create a test case for DictionaryArray<$INDEX_TY>
7972        macro_rules! make_str_dict_test_case {
7973            ($INPUT:expr, $INDEX_TY:ident) => {{
7974                TestCase {
7975                    array: Arc::new(
7976                        $INPUT
7977                            .iter()
7978                            .cloned()
7979                            .collect::<DictionaryArray<$INDEX_TY>>(),
7980                    ),
7981                    scalars: $INPUT
7982                        .iter()
7983                        .map(|v| {
7984                            ScalarValue::Dictionary(
7985                                Box::new($INDEX_TY::DATA_TYPE),
7986                                Box::new(ScalarValue::Utf8(v.map(|v| v.to_string()))),
7987                            )
7988                        })
7989                        .collect(),
7990                }
7991            }};
7992        }
7993
7994        let cases = vec![
7995            make_test_case!(bool_vals, BooleanArray, Boolean),
7996            make_test_case!(f32_vals, Float32Array, Float32),
7997            make_test_case!(f64_vals, Float64Array, Float64),
7998            make_test_case!(i8_vals, Int8Array, Int8),
7999            make_test_case!(i16_vals, Int16Array, Int16),
8000            make_test_case!(i32_vals, Int32Array, Int32),
8001            make_test_case!(i64_vals, Int64Array, Int64),
8002            make_test_case!(u8_vals, UInt8Array, UInt8),
8003            make_test_case!(u16_vals, UInt16Array, UInt16),
8004            make_test_case!(u32_vals, UInt32Array, UInt32),
8005            make_test_case!(u64_vals, UInt64Array, UInt64),
8006            make_str_test_case!(str_vals, StringArray, Utf8),
8007            make_str_test_case!(str_vals, LargeStringArray, LargeUtf8),
8008            make_binary_test_case!(str_vals, BinaryArray, Binary),
8009            make_binary_test_case!(str_vals, LargeBinaryArray, LargeBinary),
8010            make_test_case!(i32_vals, Date32Array, Date32),
8011            make_test_case!(i64_vals, Date64Array, Date64),
8012            make_test_case!(i32_vals, Time32SecondArray, Time32Second),
8013            make_test_case!(i32_vals, Time32MillisecondArray, Time32Millisecond),
8014            make_test_case!(i64_vals, Time64MicrosecondArray, Time64Microsecond),
8015            make_test_case!(i64_vals, Time64NanosecondArray, Time64Nanosecond),
8016            make_test_case!(i64_vals, TimestampSecondArray, TimestampSecond, None),
8017            make_test_case!(
8018                i64_vals,
8019                TimestampSecondArray,
8020                TimestampSecond,
8021                Some("UTC".into())
8022            ),
8023            make_test_case!(
8024                i64_vals,
8025                TimestampMillisecondArray,
8026                TimestampMillisecond,
8027                None
8028            ),
8029            make_test_case!(
8030                i64_vals,
8031                TimestampMillisecondArray,
8032                TimestampMillisecond,
8033                Some("UTC".into())
8034            ),
8035            make_test_case!(
8036                i64_vals,
8037                TimestampMicrosecondArray,
8038                TimestampMicrosecond,
8039                None
8040            ),
8041            make_test_case!(
8042                i64_vals,
8043                TimestampMicrosecondArray,
8044                TimestampMicrosecond,
8045                Some("UTC".into())
8046            ),
8047            make_test_case!(
8048                i64_vals,
8049                TimestampNanosecondArray,
8050                TimestampNanosecond,
8051                None
8052            ),
8053            make_test_case!(
8054                i64_vals,
8055                TimestampNanosecondArray,
8056                TimestampNanosecond,
8057                Some("UTC".into())
8058            ),
8059            make_test_case!(i32_vals, IntervalYearMonthArray, IntervalYearMonth),
8060            make_test_case!(interval_dt_vals, IntervalDayTimeArray, IntervalDayTime),
8061            make_test_case!(
8062                interval_mdn_vals,
8063                IntervalMonthDayNanoArray,
8064                IntervalMonthDayNano
8065            ),
8066            make_str_dict_test_case!(str_vals, Int8Type),
8067            make_str_dict_test_case!(str_vals, Int16Type),
8068            make_str_dict_test_case!(str_vals, Int32Type),
8069            make_str_dict_test_case!(str_vals, Int64Type),
8070            make_str_dict_test_case!(str_vals, UInt8Type),
8071            make_str_dict_test_case!(str_vals, UInt16Type),
8072            make_str_dict_test_case!(str_vals, UInt32Type),
8073            make_str_dict_test_case!(str_vals, UInt64Type),
8074        ];
8075
8076        for case in cases {
8077            println!("**** Test Case *****");
8078            let TestCase { array, scalars } = case;
8079            println!("Input array type: {}", array.data_type());
8080            println!("Input scalars: {scalars:#?}");
8081            assert_eq!(array.len(), scalars.len());
8082
8083            for (index, scalar) in scalars.into_iter().enumerate() {
8084                assert!(
8085                    scalar
8086                        .eq_array(&array, index)
8087                        .expect("Failed to compare arrays"),
8088                    "Expected {scalar:?} to be equal to {array:?} at index {index}"
8089                );
8090
8091                // test that all other elements are *not* equal
8092                for other_index in 0..array.len() {
8093                    if index != other_index {
8094                        assert!(
8095                            !scalar
8096                                .eq_array(&array, other_index)
8097                                .expect("Failed to compare arrays"),
8098                            "Expected {scalar:?} to be NOT equal to {array:?} at index {other_index}"
8099                        );
8100                    }
8101                }
8102            }
8103        }
8104    }
8105
8106    #[test]
8107    fn scalar_partial_ordering() {
8108        use ScalarValue::*;
8109
8110        assert_eq!(
8111            Int64(Some(33)).partial_cmp(&Int64(Some(0))),
8112            Some(Ordering::Greater)
8113        );
8114        assert_eq!(
8115            Int64(Some(0)).partial_cmp(&Int64(Some(33))),
8116            Some(Ordering::Less)
8117        );
8118        assert_eq!(
8119            Int64(Some(33)).partial_cmp(&Int64(Some(33))),
8120            Some(Ordering::Equal)
8121        );
8122        // For different data type, `partial_cmp` returns None.
8123        assert_eq!(Int64(Some(33)).partial_cmp(&Int32(Some(33))), None);
8124        assert_eq!(Int32(Some(33)).partial_cmp(&Int64(Some(33))), None);
8125
8126        assert_eq!(
8127            ScalarValue::from(vec![
8128                ("A", ScalarValue::from(1.0)),
8129                ("B", ScalarValue::from("Z")),
8130            ])
8131            .partial_cmp(&ScalarValue::from(vec![
8132                ("A", ScalarValue::from(2.0)),
8133                ("B", ScalarValue::from("A")),
8134            ])),
8135            Some(Ordering::Less)
8136        );
8137
8138        // For different struct fields, `partial_cmp` returns None.
8139        assert_eq!(
8140            ScalarValue::from(vec![
8141                ("A", ScalarValue::from(1.0)),
8142                ("B", ScalarValue::from("Z")),
8143            ])
8144            .partial_cmp(&ScalarValue::from(vec![
8145                ("a", ScalarValue::from(2.0)),
8146                ("b", ScalarValue::from("A")),
8147            ])),
8148            None
8149        );
8150    }
8151
8152    #[test]
8153    fn test_scalar_value_from_string() {
8154        let scalar = ScalarValue::from("foo");
8155        assert_eq!(scalar, ScalarValue::Utf8(Some("foo".to_string())));
8156        let scalar = ScalarValue::from("foo".to_string());
8157        assert_eq!(scalar, ScalarValue::Utf8(Some("foo".to_string())));
8158        let scalar = ScalarValue::from_str("foo").unwrap();
8159        assert_eq!(scalar, ScalarValue::Utf8(Some("foo".to_string())));
8160    }
8161
8162    #[test]
8163    fn test_scalar_struct() {
8164        let field_a = Arc::new(Field::new("A", DataType::Int32, false));
8165        let field_b = Arc::new(Field::new("B", DataType::Boolean, false));
8166        let field_c = Arc::new(Field::new("C", DataType::Utf8, false));
8167
8168        let field_e = Arc::new(Field::new("e", DataType::Int16, false));
8169        let field_f = Arc::new(Field::new("f", DataType::Int64, false));
8170        let field_d = Arc::new(Field::new(
8171            "D",
8172            DataType::Struct(vec![Arc::clone(&field_e), Arc::clone(&field_f)].into()),
8173            false,
8174        ));
8175
8176        let struct_array = StructArray::from(vec![
8177            (
8178                Arc::clone(&field_e),
8179                Arc::new(Int16Array::from(vec![2])) as ArrayRef,
8180            ),
8181            (
8182                Arc::clone(&field_f),
8183                Arc::new(Int64Array::from(vec![3])) as ArrayRef,
8184            ),
8185        ]);
8186
8187        let struct_array = StructArray::from(vec![
8188            (
8189                Arc::clone(&field_a),
8190                Arc::new(Int32Array::from(vec![23])) as ArrayRef,
8191            ),
8192            (
8193                Arc::clone(&field_b),
8194                Arc::new(BooleanArray::from(vec![false])) as ArrayRef,
8195            ),
8196            (
8197                Arc::clone(&field_c),
8198                Arc::new(StringArray::from(vec!["Hello"])) as ArrayRef,
8199            ),
8200            (Arc::clone(&field_d), Arc::new(struct_array) as ArrayRef),
8201        ]);
8202        let scalar = ScalarValue::Struct(Arc::new(struct_array));
8203
8204        let array = scalar
8205            .to_array_of_size(2)
8206            .expect("Failed to convert to array of size");
8207
8208        let expected = Arc::new(StructArray::from(vec![
8209            (
8210                Arc::clone(&field_a),
8211                Arc::new(Int32Array::from(vec![23, 23])) as ArrayRef,
8212            ),
8213            (
8214                Arc::clone(&field_b),
8215                Arc::new(BooleanArray::from(vec![false, false])) as ArrayRef,
8216            ),
8217            (
8218                Arc::clone(&field_c),
8219                Arc::new(StringArray::from(vec!["Hello", "Hello"])) as ArrayRef,
8220            ),
8221            (
8222                Arc::clone(&field_d),
8223                Arc::new(StructArray::from(vec![
8224                    (
8225                        Arc::clone(&field_e),
8226                        Arc::new(Int16Array::from(vec![2, 2])) as ArrayRef,
8227                    ),
8228                    (
8229                        Arc::clone(&field_f),
8230                        Arc::new(Int64Array::from(vec![3, 3])) as ArrayRef,
8231                    ),
8232                ])) as ArrayRef,
8233            ),
8234        ])) as ArrayRef;
8235
8236        assert_eq!(&array, &expected);
8237
8238        // Construct from second element of ArrayRef
8239        let constructed = ScalarValue::try_from_array(&expected, 1).unwrap();
8240        assert_eq!(constructed, scalar);
8241
8242        // None version
8243        let none_scalar = ScalarValue::try_from(array.data_type()).unwrap();
8244        assert!(none_scalar.is_null());
8245        assert_eq!(
8246            format!("{none_scalar:?}"),
8247            String::from("Struct({A:,B:,C:,D:})")
8248        );
8249
8250        // Construct with convenience From<Vec<(&str, ScalarValue)>>
8251        let constructed = ScalarValue::from(vec![
8252            ("A", ScalarValue::from(23)),
8253            ("B", ScalarValue::from(false)),
8254            ("C", ScalarValue::from("Hello")),
8255            (
8256                "D",
8257                ScalarValue::from(vec![
8258                    ("e", ScalarValue::from(2i16)),
8259                    ("f", ScalarValue::from(3i64)),
8260                ]),
8261            ),
8262        ]);
8263        assert_eq!(constructed, scalar);
8264
8265        // Build Array from Vec of structs
8266        let scalars = vec![
8267            ScalarValue::from(vec![
8268                ("A", ScalarValue::from(23)),
8269                ("B", ScalarValue::from(false)),
8270                ("C", ScalarValue::from("Hello")),
8271                (
8272                    "D",
8273                    ScalarValue::from(vec![
8274                        ("e", ScalarValue::from(2i16)),
8275                        ("f", ScalarValue::from(3i64)),
8276                    ]),
8277                ),
8278            ]),
8279            ScalarValue::from(vec![
8280                ("A", ScalarValue::from(7)),
8281                ("B", ScalarValue::from(true)),
8282                ("C", ScalarValue::from("World")),
8283                (
8284                    "D",
8285                    ScalarValue::from(vec![
8286                        ("e", ScalarValue::from(4i16)),
8287                        ("f", ScalarValue::from(5i64)),
8288                    ]),
8289                ),
8290            ]),
8291            ScalarValue::from(vec![
8292                ("A", ScalarValue::from(-1000)),
8293                ("B", ScalarValue::from(true)),
8294                ("C", ScalarValue::from("!!!!!")),
8295                (
8296                    "D",
8297                    ScalarValue::from(vec![
8298                        ("e", ScalarValue::from(6i16)),
8299                        ("f", ScalarValue::from(7i64)),
8300                    ]),
8301                ),
8302            ]),
8303        ];
8304        let array = ScalarValue::iter_to_array(scalars).unwrap();
8305
8306        let expected = Arc::new(StructArray::from(vec![
8307            (
8308                Arc::clone(&field_a),
8309                Arc::new(Int32Array::from(vec![23, 7, -1000])) as ArrayRef,
8310            ),
8311            (
8312                Arc::clone(&field_b),
8313                Arc::new(BooleanArray::from(vec![false, true, true])) as ArrayRef,
8314            ),
8315            (
8316                Arc::clone(&field_c),
8317                Arc::new(StringArray::from(vec!["Hello", "World", "!!!!!"])) as ArrayRef,
8318            ),
8319            (
8320                Arc::clone(&field_d),
8321                Arc::new(StructArray::from(vec![
8322                    (
8323                        Arc::clone(&field_e),
8324                        Arc::new(Int16Array::from(vec![2, 4, 6])) as ArrayRef,
8325                    ),
8326                    (
8327                        Arc::clone(&field_f),
8328                        Arc::new(Int64Array::from(vec![3, 5, 7])) as ArrayRef,
8329                    ),
8330                ])) as ArrayRef,
8331            ),
8332        ])) as ArrayRef;
8333
8334        assert_eq!(&array, &expected);
8335    }
8336
8337    #[test]
8338    fn round_trip() {
8339        // Each array type should be able to round tripped through a scalar
8340        let cases: Vec<ArrayRef> = vec![
8341            // int
8342            Arc::new(Int8Array::from(vec![Some(1), None, Some(3)])),
8343            Arc::new(Int16Array::from(vec![Some(1), None, Some(3)])),
8344            Arc::new(Int32Array::from(vec![Some(1), None, Some(3)])),
8345            Arc::new(Int64Array::from(vec![Some(1), None, Some(3)])),
8346            Arc::new(UInt8Array::from(vec![Some(1), None, Some(3)])),
8347            Arc::new(UInt16Array::from(vec![Some(1), None, Some(3)])),
8348            Arc::new(UInt32Array::from(vec![Some(1), None, Some(3)])),
8349            Arc::new(UInt64Array::from(vec![Some(1), None, Some(3)])),
8350            // bool
8351            Arc::new(BooleanArray::from(vec![Some(true), None, Some(false)])),
8352            // float
8353            Arc::new(Float32Array::from(vec![Some(1.0), None, Some(3.0)])),
8354            Arc::new(Float64Array::from(vec![Some(1.0), None, Some(3.0)])),
8355            // string array
8356            Arc::new(StringArray::from(vec![Some("foo"), None, Some("bar")])),
8357            Arc::new(LargeStringArray::from(vec![Some("foo"), None, Some("bar")])),
8358            Arc::new(StringViewArray::from(vec![Some("foo"), None, Some("bar")])),
8359            // string dictionary
8360            {
8361                let mut builder = StringDictionaryBuilder::<Int32Type>::new();
8362                builder.append("foo").unwrap();
8363                builder.append_null();
8364                builder.append("bar").unwrap();
8365                Arc::new(builder.finish())
8366            },
8367            // binary array
8368            Arc::new(BinaryArray::from_iter(vec![
8369                Some(b"foo"),
8370                None,
8371                Some(b"bar"),
8372            ])),
8373            Arc::new(LargeBinaryArray::from_iter(vec![
8374                Some(b"foo"),
8375                None,
8376                Some(b"bar"),
8377            ])),
8378            Arc::new(BinaryViewArray::from_iter(vec![
8379                Some(b"foo"),
8380                None,
8381                Some(b"bar"),
8382            ])),
8383            // timestamp
8384            Arc::new(TimestampSecondArray::from(vec![Some(1), None, Some(3)])),
8385            Arc::new(TimestampMillisecondArray::from(vec![
8386                Some(1),
8387                None,
8388                Some(3),
8389            ])),
8390            Arc::new(TimestampMicrosecondArray::from(vec![
8391                Some(1),
8392                None,
8393                Some(3),
8394            ])),
8395            Arc::new(TimestampNanosecondArray::from(vec![Some(1), None, Some(3)])),
8396            // timestamp with timezone
8397            Arc::new(
8398                TimestampSecondArray::from(vec![Some(1), None, Some(3)])
8399                    .with_timezone_opt(Some("UTC")),
8400            ),
8401            Arc::new(
8402                TimestampMillisecondArray::from(vec![Some(1), None, Some(3)])
8403                    .with_timezone_opt(Some("UTC")),
8404            ),
8405            Arc::new(
8406                TimestampMicrosecondArray::from(vec![Some(1), None, Some(3)])
8407                    .with_timezone_opt(Some("UTC")),
8408            ),
8409            Arc::new(
8410                TimestampNanosecondArray::from(vec![Some(1), None, Some(3)])
8411                    .with_timezone_opt(Some("UTC")),
8412            ),
8413            // date
8414            Arc::new(Date32Array::from(vec![Some(1), None, Some(3)])),
8415            Arc::new(Date64Array::from(vec![Some(1), None, Some(3)])),
8416            // time
8417            Arc::new(Time32SecondArray::from(vec![Some(1), None, Some(3)])),
8418            Arc::new(Time32MillisecondArray::from(vec![Some(1), None, Some(3)])),
8419            Arc::new(Time64MicrosecondArray::from(vec![Some(1), None, Some(3)])),
8420            Arc::new(Time64NanosecondArray::from(vec![Some(1), None, Some(3)])),
8421            // null array
8422            Arc::new(NullArray::new(3)),
8423            // dense union
8424            {
8425                let mut builder = UnionBuilder::new_dense();
8426                builder.append::<Int32Type>("a", 1).unwrap();
8427                builder.append::<Float64Type>("b", 3.4).unwrap();
8428                Arc::new(builder.build().unwrap())
8429            },
8430            // sparse union
8431            {
8432                let mut builder = UnionBuilder::new_sparse();
8433                builder.append::<Int32Type>("a", 1).unwrap();
8434                builder.append::<Float64Type>("b", 3.4).unwrap();
8435                Arc::new(builder.build().unwrap())
8436            },
8437            // list array
8438            {
8439                let values_builder = StringBuilder::new();
8440                let mut builder = ListBuilder::new(values_builder);
8441                // [A, B]
8442                builder.values().append_value("A");
8443                builder.values().append_value("B");
8444                builder.append(true);
8445                // [ ] (empty list)
8446                builder.append(true);
8447                // Null
8448                builder.values().append_value("?"); // irrelevant
8449                builder.append(false);
8450                Arc::new(builder.finish())
8451            },
8452            // large list array
8453            {
8454                let values_builder = StringBuilder::new();
8455                let mut builder = LargeListBuilder::new(values_builder);
8456                // [A, B]
8457                builder.values().append_value("A");
8458                builder.values().append_value("B");
8459                builder.append(true);
8460                // [ ] (empty list)
8461                builder.append(true);
8462                // Null
8463                builder.append(false);
8464                Arc::new(builder.finish())
8465            },
8466            // fixed size list array
8467            {
8468                let values_builder = Int32Builder::new();
8469                let mut builder = FixedSizeListBuilder::new(values_builder, 3);
8470
8471                //  [[0, 1, 2], null, [3, null, 5]
8472                builder.values().append_value(0);
8473                builder.values().append_value(1);
8474                builder.values().append_value(2);
8475                builder.append(true);
8476                builder.values().append_null();
8477                builder.values().append_null();
8478                builder.values().append_null();
8479                builder.append(false);
8480                builder.values().append_value(3);
8481                builder.values().append_null();
8482                builder.values().append_value(5);
8483                builder.append(true);
8484                Arc::new(builder.finish())
8485            },
8486            // list view array
8487            {
8488                let values_builder = StringBuilder::new();
8489                let mut builder = ListViewBuilder::new(values_builder);
8490                // [A, B]
8491                builder.values().append_value("A");
8492                builder.values().append_value("B");
8493                builder.append(true);
8494                // [ ] (empty list)
8495                builder.append(true);
8496                // Null
8497                builder.append(false);
8498                Arc::new(builder.finish())
8499            },
8500            // large list view array
8501            {
8502                let values_builder = StringBuilder::new();
8503                let mut builder = LargeListViewBuilder::new(values_builder);
8504                // [A, B]
8505                builder.values().append_value("A");
8506                builder.values().append_value("B");
8507                builder.append(true);
8508                // [ ] (empty list)
8509                builder.append(true);
8510                // Null
8511                builder.append(false);
8512                Arc::new(builder.finish())
8513            },
8514            // map
8515            {
8516                let string_builder = StringBuilder::new();
8517                let int_builder = Int32Builder::with_capacity(4);
8518
8519                let mut builder = MapBuilder::new(None, string_builder, int_builder);
8520                // {"joe": 1}
8521                builder.keys().append_value("joe");
8522                builder.values().append_value(1);
8523                builder.append(true).unwrap();
8524                // {}
8525                builder.append(true).unwrap();
8526                // null
8527                builder.append(false).unwrap();
8528
8529                Arc::new(builder.finish())
8530            },
8531        ];
8532
8533        for arr in cases {
8534            round_trip_through_scalar(arr);
8535        }
8536    }
8537
8538    /// for each row in `arr`:
8539    /// 1. convert to a `ScalarValue`
8540    /// 2. Convert `ScalarValue` back to an `ArrayRef`
8541    /// 3. Compare the original array (sliced) and new array for equality
8542    fn round_trip_through_scalar(arr: ArrayRef) {
8543        for i in 0..arr.len() {
8544            // convert Scalar --> Array
8545            let scalar = ScalarValue::try_from_array(&arr, i).unwrap();
8546            let array = scalar.to_array_of_size(1).unwrap();
8547            assert_eq!(array.len(), 1);
8548            assert_eq!(array.data_type(), arr.data_type());
8549            assert_eq!(array.as_ref(), arr.slice(i, 1).as_ref());
8550        }
8551    }
8552
8553    #[test]
8554    fn roundtrip_run_array() {
8555        // Comparison logic in round_trip_through_scalar doesn't work for RunArrays
8556        // so we have a custom test for them
8557        // TODO: https://github.com/apache/arrow-rs/pull/9213 might fix this ^
8558        let run_ends = Int16Array::from(vec![2, 3]);
8559        let values = Int64Array::from(vec![Some(1), None]);
8560        let run_array = RunArray::try_new(&run_ends, &values).unwrap();
8561        let run_array = run_array.downcast::<Int64Array>().unwrap();
8562
8563        let expected_values = run_array.into_iter().collect::<Vec<_>>();
8564
8565        for i in 0..run_array.len() {
8566            let scalar = ScalarValue::try_from_array(&run_array, i).unwrap();
8567            let array = scalar.to_array_of_size(1).unwrap();
8568            assert_eq!(array.data_type(), run_array.data_type());
8569            let array = array.as_run::<Int16Type>();
8570            let array = array.downcast::<Int64Array>().unwrap();
8571            assert_eq!(
8572                array.into_iter().collect::<Vec<_>>(),
8573                expected_values[i..i + 1]
8574            );
8575        }
8576    }
8577
8578    #[test]
8579    fn test_scalar_union_sparse() {
8580        let field_a = Arc::new(Field::new("A", DataType::Int32, true));
8581        let field_b = Arc::new(Field::new("B", DataType::Boolean, true));
8582        let field_c = Arc::new(Field::new("C", DataType::Utf8, true));
8583        let fields = UnionFields::from_iter([(0, field_a), (1, field_b), (2, field_c)]);
8584
8585        let mut values_a = vec![None; 6];
8586        values_a[0] = Some(42);
8587        let mut values_b = vec![None; 6];
8588        values_b[1] = Some(true);
8589        let mut values_c = vec![None; 6];
8590        values_c[2] = Some("foo");
8591        let children: Vec<ArrayRef> = vec![
8592            Arc::new(Int32Array::from(values_a)),
8593            Arc::new(BooleanArray::from(values_b)),
8594            Arc::new(StringArray::from(values_c)),
8595        ];
8596
8597        let type_ids = ScalarBuffer::from(vec![0, 1, 2, 0, 1, 2]);
8598        let array: ArrayRef = Arc::new(
8599            UnionArray::try_new(fields.clone(), type_ids, None, children)
8600                .expect("UnionArray"),
8601        );
8602
8603        let expected = [
8604            (0, ScalarValue::from(42)),
8605            (1, ScalarValue::from(true)),
8606            (2, ScalarValue::from("foo")),
8607            (0, ScalarValue::Int32(None)),
8608            (1, ScalarValue::Boolean(None)),
8609            (2, ScalarValue::Utf8(None)),
8610        ];
8611
8612        for (i, (ti, value)) in expected.into_iter().enumerate() {
8613            let is_null = value.is_null();
8614            let value = Some((ti, Box::new(value)));
8615            let expected = ScalarValue::Union(value, fields.clone(), UnionMode::Sparse);
8616            let actual = ScalarValue::try_from_array(&array, i).expect("try_from_array");
8617
8618            assert_eq!(
8619                actual, expected,
8620                "[{i}] {actual} was not equal to {expected}"
8621            );
8622
8623            assert!(
8624                expected.eq_array(&array, i).expect("eq_array"),
8625                "[{i}] {expected}.eq_array was false"
8626            );
8627
8628            if is_null {
8629                assert!(actual.is_null(), "[{i}] {actual} was not null")
8630            }
8631        }
8632    }
8633
8634    #[test]
8635    fn test_scalar_union_dense() {
8636        let field_a = Arc::new(Field::new("A", DataType::Int32, true));
8637        let field_b = Arc::new(Field::new("B", DataType::Boolean, true));
8638        let field_c = Arc::new(Field::new("C", DataType::Utf8, true));
8639        let fields = UnionFields::from_iter([(0, field_a), (1, field_b), (2, field_c)]);
8640        let children: Vec<ArrayRef> = vec![
8641            Arc::new(Int32Array::from(vec![Some(42), None])),
8642            Arc::new(BooleanArray::from(vec![Some(true), None])),
8643            Arc::new(StringArray::from(vec![Some("foo"), None])),
8644        ];
8645
8646        let type_ids = ScalarBuffer::from(vec![0, 1, 2, 0, 1, 2]);
8647        let offsets = ScalarBuffer::from(vec![0, 0, 0, 1, 1, 1]);
8648        let array: ArrayRef = Arc::new(
8649            UnionArray::try_new(fields.clone(), type_ids, Some(offsets), children)
8650                .expect("UnionArray"),
8651        );
8652
8653        let expected = [
8654            (0, ScalarValue::from(42)),
8655            (1, ScalarValue::from(true)),
8656            (2, ScalarValue::from("foo")),
8657            (0, ScalarValue::Int32(None)),
8658            (1, ScalarValue::Boolean(None)),
8659            (2, ScalarValue::Utf8(None)),
8660        ];
8661
8662        for (i, (ti, value)) in expected.into_iter().enumerate() {
8663            let is_null = value.is_null();
8664            let value = Some((ti, Box::new(value)));
8665            let expected = ScalarValue::Union(value, fields.clone(), UnionMode::Dense);
8666            let actual = ScalarValue::try_from_array(&array, i).expect("try_from_array");
8667
8668            assert_eq!(
8669                actual, expected,
8670                "[{i}] {actual} was not equal to {expected}"
8671            );
8672
8673            assert!(
8674                expected.eq_array(&array, i).expect("eq_array"),
8675                "[{i}] {expected}.eq_array was false"
8676            );
8677
8678            if is_null {
8679                assert!(actual.is_null(), "[{i}] {actual} was not null")
8680            }
8681        }
8682    }
8683
8684    #[test]
8685    fn test_lists_in_struct() {
8686        let field_a = Arc::new(Field::new("A", DataType::Utf8, false));
8687        let field_primitive_list = Arc::new(Field::new(
8688            "primitive_list",
8689            DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true))),
8690            false,
8691        ));
8692
8693        // Define primitive list scalars
8694        let l0 =
8695            ScalarValue::List(Arc::new(
8696                ListArray::from_iter_primitive::<Int32Type, _, _>(vec![Some(vec![
8697                    Some(1),
8698                    Some(2),
8699                    Some(3),
8700                ])]),
8701            ));
8702        let l1 =
8703            ScalarValue::List(Arc::new(
8704                ListArray::from_iter_primitive::<Int32Type, _, _>(vec![Some(vec![
8705                    Some(4),
8706                    Some(5),
8707                ])]),
8708            ));
8709        let l2 = ScalarValue::List(Arc::new(ListArray::from_iter_primitive::<
8710            Int32Type,
8711            _,
8712            _,
8713        >(vec![Some(vec![Some(6)])])));
8714
8715        // Define struct scalars
8716        let s0 = ScalarValue::from(vec![
8717            ("A", ScalarValue::from("First")),
8718            ("primitive_list", l0),
8719        ]);
8720
8721        let s1 = ScalarValue::from(vec![
8722            ("A", ScalarValue::from("Second")),
8723            ("primitive_list", l1),
8724        ]);
8725
8726        let s2 = ScalarValue::from(vec![
8727            ("A", ScalarValue::from("Third")),
8728            ("primitive_list", l2),
8729        ]);
8730
8731        // iter_to_array for struct scalars
8732        let array =
8733            ScalarValue::iter_to_array(vec![s0.clone(), s1.clone(), s2.clone()]).unwrap();
8734
8735        let array = as_struct_array(&array).unwrap();
8736        let expected = StructArray::from(vec![
8737            (
8738                Arc::clone(&field_a),
8739                Arc::new(StringArray::from(vec!["First", "Second", "Third"])) as ArrayRef,
8740            ),
8741            (
8742                Arc::clone(&field_primitive_list),
8743                Arc::new(ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
8744                    Some(vec![Some(1), Some(2), Some(3)]),
8745                    Some(vec![Some(4), Some(5)]),
8746                    Some(vec![Some(6)]),
8747                ])),
8748            ),
8749        ]);
8750
8751        assert_eq!(array, &expected);
8752
8753        // Define list-of-structs scalars
8754
8755        let nl0_array = ScalarValue::iter_to_array(vec![s0, s1.clone()]).unwrap();
8756        let nl0 = SingleRowListArrayBuilder::new(nl0_array).build_list_scalar();
8757
8758        let nl1_array = ScalarValue::iter_to_array(vec![s2]).unwrap();
8759        let nl1 = SingleRowListArrayBuilder::new(nl1_array).build_list_scalar();
8760
8761        let nl2_array = ScalarValue::iter_to_array(vec![s1]).unwrap();
8762        let nl2 = SingleRowListArrayBuilder::new(nl2_array).build_list_scalar();
8763
8764        // iter_to_array for list-of-struct
8765        let array = ScalarValue::iter_to_array(vec![nl0, nl1, nl2]).unwrap();
8766        let array = array.as_list::<i32>();
8767
8768        // Construct expected array with array builders
8769        let field_a_builder = StringBuilder::with_capacity(4, 1024);
8770        let primitive_value_builder = Int32Array::builder(8);
8771        let field_primitive_list_builder = ListBuilder::new(primitive_value_builder);
8772
8773        let element_builder = StructBuilder::new(
8774            vec![field_a, field_primitive_list],
8775            vec![
8776                Box::new(field_a_builder),
8777                Box::new(field_primitive_list_builder),
8778            ],
8779        );
8780
8781        let mut list_builder = ListBuilder::new(element_builder);
8782
8783        list_builder
8784            .values()
8785            .field_builder::<StringBuilder>(0)
8786            .unwrap()
8787            .append_value("First");
8788        list_builder
8789            .values()
8790            .field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
8791            .unwrap()
8792            .values()
8793            .append_value(1);
8794        list_builder
8795            .values()
8796            .field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
8797            .unwrap()
8798            .values()
8799            .append_value(2);
8800        list_builder
8801            .values()
8802            .field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
8803            .unwrap()
8804            .values()
8805            .append_value(3);
8806        list_builder
8807            .values()
8808            .field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
8809            .unwrap()
8810            .append(true);
8811        list_builder.values().append(true);
8812
8813        list_builder
8814            .values()
8815            .field_builder::<StringBuilder>(0)
8816            .unwrap()
8817            .append_value("Second");
8818        list_builder
8819            .values()
8820            .field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
8821            .unwrap()
8822            .values()
8823            .append_value(4);
8824        list_builder
8825            .values()
8826            .field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
8827            .unwrap()
8828            .values()
8829            .append_value(5);
8830        list_builder
8831            .values()
8832            .field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
8833            .unwrap()
8834            .append(true);
8835        list_builder.values().append(true);
8836        list_builder.append(true);
8837
8838        list_builder
8839            .values()
8840            .field_builder::<StringBuilder>(0)
8841            .unwrap()
8842            .append_value("Third");
8843        list_builder
8844            .values()
8845            .field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
8846            .unwrap()
8847            .values()
8848            .append_value(6);
8849        list_builder
8850            .values()
8851            .field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
8852            .unwrap()
8853            .append(true);
8854        list_builder.values().append(true);
8855        list_builder.append(true);
8856
8857        list_builder
8858            .values()
8859            .field_builder::<StringBuilder>(0)
8860            .unwrap()
8861            .append_value("Second");
8862        list_builder
8863            .values()
8864            .field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
8865            .unwrap()
8866            .values()
8867            .append_value(4);
8868        list_builder
8869            .values()
8870            .field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
8871            .unwrap()
8872            .values()
8873            .append_value(5);
8874        list_builder
8875            .values()
8876            .field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
8877            .unwrap()
8878            .append(true);
8879        list_builder.values().append(true);
8880        list_builder.append(true);
8881
8882        let expected = list_builder.finish();
8883
8884        assert_eq!(array, &expected);
8885    }
8886
8887    fn build_2d_list(data: Vec<Option<i32>>) -> ListArray {
8888        let a1 = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![Some(data)]);
8889        ListArray::new(
8890            Arc::new(Field::new_list_field(
8891                DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true))),
8892                true,
8893            )),
8894            OffsetBuffer::<i32>::from_lengths([1]),
8895            Arc::new(a1),
8896            None,
8897        )
8898    }
8899
8900    #[test]
8901    fn test_nested_lists() {
8902        // Define inner list scalars
8903        let arr1 = build_2d_list(vec![Some(1), Some(2), Some(3)]);
8904        let arr2 = build_2d_list(vec![Some(4), Some(5)]);
8905        let arr3 = build_2d_list(vec![Some(6)]);
8906
8907        let array = ScalarValue::iter_to_array(vec![
8908            ScalarValue::List(Arc::new(arr1)),
8909            ScalarValue::List(Arc::new(arr2)),
8910            ScalarValue::List(Arc::new(arr3)),
8911        ])
8912        .unwrap();
8913        let array = array.as_list::<i32>();
8914
8915        // Construct expected array with array builders
8916        let inner_builder = Int32Array::builder(6);
8917        let middle_builder = ListBuilder::new(inner_builder);
8918        let mut outer_builder = ListBuilder::new(middle_builder);
8919
8920        outer_builder.values().values().append_value(1);
8921        outer_builder.values().values().append_value(2);
8922        outer_builder.values().values().append_value(3);
8923        outer_builder.values().append(true);
8924        outer_builder.append(true);
8925
8926        outer_builder.values().values().append_value(4);
8927        outer_builder.values().values().append_value(5);
8928        outer_builder.values().append(true);
8929        outer_builder.append(true);
8930
8931        outer_builder.values().values().append_value(6);
8932        outer_builder.values().append(true);
8933        outer_builder.append(true);
8934
8935        let expected = outer_builder.finish();
8936
8937        assert_eq!(array, &expected);
8938    }
8939
8940    #[test]
8941    fn scalar_timestamp_ns_utc_timezone() {
8942        let scalar = ScalarValue::TimestampNanosecond(
8943            Some(1599566400000000000),
8944            Some("UTC".into()),
8945        );
8946
8947        assert_eq!(
8948            scalar.data_type(),
8949            DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into()))
8950        );
8951
8952        let array = scalar.to_array().expect("Failed to convert to array");
8953        assert_eq!(array.len(), 1);
8954        assert_eq!(
8955            array.data_type(),
8956            &DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into()))
8957        );
8958
8959        let new_scalar = ScalarValue::try_from_array(&array, 0).unwrap();
8960        assert_eq!(
8961            new_scalar.data_type(),
8962            DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into()))
8963        );
8964    }
8965
8966    #[test]
8967    fn cast_round_trip() {
8968        check_scalar_cast(ScalarValue::Int8(Some(5)), DataType::Int16);
8969        check_scalar_cast(ScalarValue::Int8(None), DataType::Int16);
8970
8971        check_scalar_cast(ScalarValue::Float64(Some(5.5)), DataType::Int16);
8972
8973        check_scalar_cast(ScalarValue::Float64(None), DataType::Int16);
8974
8975        check_scalar_cast(
8976            ScalarValue::from("foo"),
8977            DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
8978        );
8979
8980        check_scalar_cast(
8981            ScalarValue::Utf8(None),
8982            DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
8983        );
8984
8985        check_scalar_cast(ScalarValue::Utf8(None), DataType::Utf8View);
8986        check_scalar_cast(ScalarValue::from("foo"), DataType::Utf8View);
8987        check_scalar_cast(
8988            ScalarValue::from("larger than 12 bytes string"),
8989            DataType::Utf8View,
8990        );
8991
8992        // Cases also covered by `try_cast_literal_to_type` in datafusion-expr-common
8993
8994        // identity casts (exercise the no-conversion fast path in `cast_to`)
8995        check_scalar_cast(ScalarValue::Int32(Some(5)), DataType::Int32);
8996        check_scalar_cast(ScalarValue::from("foo"), DataType::Utf8);
8997        check_scalar_cast(ScalarValue::Utf8(None), DataType::Utf8);
8998        check_scalar_cast(
8999            ScalarValue::Dictionary(
9000                Box::new(DataType::Int32),
9001                Box::new(ScalarValue::from("foo")),
9002            ),
9003            DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
9004        );
9005
9006        // integer widening / narrowing (in range)
9007        check_scalar_cast(ScalarValue::Int32(Some(123)), DataType::Int64);
9008        check_scalar_cast(ScalarValue::Int64(Some(123)), DataType::Int32);
9009        check_scalar_cast(ScalarValue::UInt32(Some(123)), DataType::Int64);
9010        check_scalar_cast(ScalarValue::Int32(Some(123)), DataType::UInt64);
9011
9012        // integer <-> decimal
9013        check_scalar_cast(ScalarValue::Int32(Some(123)), DataType::Decimal128(10, 0));
9014        check_scalar_cast(ScalarValue::Decimal128(Some(123), 3, 0), DataType::Int64);
9015        // decimal rescale
9016        check_scalar_cast(
9017            ScalarValue::Decimal128(Some(12300), 5, 2),
9018            DataType::Decimal128(8, 5),
9019        );
9020
9021        // timestamp unit conversion
9022        check_scalar_cast(
9023            ScalarValue::TimestampNanosecond(Some(123456), None),
9024            DataType::Timestamp(TimeUnit::Microsecond, None),
9025        );
9026        // timestamp timezone conversion
9027        check_scalar_cast(
9028            ScalarValue::TimestampSecond(Some(12345), None),
9029            DataType::Timestamp(TimeUnit::Second, Some("+00:00".into())),
9030        );
9031        // int64 <-> timestamp
9032        check_scalar_cast(
9033            ScalarValue::Int64(Some(12345)),
9034            DataType::Timestamp(TimeUnit::Nanosecond, None),
9035        );
9036        check_scalar_cast(
9037            ScalarValue::TimestampSecond(Some(12345), Some("+00:00".into())),
9038            DataType::Int64,
9039        );
9040
9041        // additional string conversions
9042        check_scalar_cast(ScalarValue::from("foo"), DataType::LargeUtf8);
9043        check_scalar_cast(ScalarValue::LargeUtf8(Some("foo".into())), DataType::Utf8);
9044        check_scalar_cast(
9045            ScalarValue::LargeUtf8(Some("foo".into())),
9046            DataType::Utf8View,
9047        );
9048        check_scalar_cast(ScalarValue::Utf8View(Some("foo".into())), DataType::Utf8);
9049
9050        // dictionary unwrap
9051        check_scalar_cast(
9052            ScalarValue::Dictionary(
9053                Box::new(DataType::Int32),
9054                Box::new(ScalarValue::from("foo")),
9055            ),
9056            DataType::Utf8,
9057        );
9058
9059        // binary -> fixed size binary
9060        check_scalar_cast(
9061            ScalarValue::Binary(Some(vec![1, 2, 3])),
9062            DataType::FixedSizeBinary(3),
9063        );
9064
9065        check_scalar_cast(
9066            {
9067                let element_field =
9068                    Arc::new(Field::new("element", DataType::Int32, true));
9069
9070                let mut builder =
9071                    ListBuilder::new(Int32Builder::new()).with_field(element_field);
9072                builder.append_value([Some(1)]);
9073                builder.append(true);
9074
9075                ScalarValue::List(Arc::new(builder.finish()))
9076            },
9077            DataType::List(Arc::new(Field::new("element", DataType::Int64, true))),
9078        );
9079        check_scalar_cast(
9080            {
9081                let element_field =
9082                    Arc::new(Field::new("element", DataType::Int32, true));
9083
9084                let mut builder = FixedSizeListBuilder::new(Int32Builder::new(), 1)
9085                    .with_field(element_field);
9086                builder.values().append_value(1);
9087                builder.append(true);
9088
9089                ScalarValue::FixedSizeList(Arc::new(builder.finish()))
9090            },
9091            DataType::FixedSizeList(
9092                Arc::new(Field::new("element", DataType::Int64, true)),
9093                1,
9094            ),
9095        );
9096        check_scalar_cast(
9097            {
9098                let element_field =
9099                    Arc::new(Field::new("element", DataType::Int32, true));
9100
9101                let mut builder =
9102                    LargeListBuilder::new(Int32Builder::new()).with_field(element_field);
9103                builder.append_value([Some(1)]);
9104                builder.append(true);
9105
9106                ScalarValue::LargeList(Arc::new(builder.finish()))
9107            },
9108            DataType::LargeList(Arc::new(Field::new("element", DataType::Int64, true))),
9109        );
9110        check_scalar_cast(
9111            {
9112                let element_field =
9113                    Arc::new(Field::new("element", DataType::Int32, true));
9114
9115                let mut builder =
9116                    ListViewBuilder::new(Int32Builder::new()).with_field(element_field);
9117                builder.append_value([Some(1)]);
9118                builder.append(true);
9119
9120                ScalarValue::ListView(Arc::new(builder.finish()))
9121            },
9122            DataType::ListView(Arc::new(Field::new("element", DataType::Int64, true))),
9123        );
9124        check_scalar_cast(
9125            {
9126                let element_field =
9127                    Arc::new(Field::new("element", DataType::Int32, true));
9128
9129                let mut builder = LargeListViewBuilder::new(Int32Builder::new())
9130                    .with_field(element_field);
9131                builder.append_value([Some(1)]);
9132                builder.append(true);
9133
9134                ScalarValue::LargeListView(Arc::new(builder.finish()))
9135            },
9136            DataType::LargeListView(Arc::new(Field::new(
9137                "element",
9138                DataType::Int64,
9139                true,
9140            ))),
9141        );
9142    }
9143
9144    // mimics how casting work on scalar values by `casting` `scalar` to `desired_type`
9145    fn check_scalar_cast(scalar: ScalarValue, desired_type: DataType) {
9146        // convert from scalar --> Array to call cast
9147        let scalar_array = scalar.to_array().expect("Failed to convert to array");
9148        // cast the actual value
9149        let cast_array = kernels::cast::cast(&scalar_array, &desired_type).unwrap();
9150
9151        // turn it back to a scalar
9152        let cast_scalar = ScalarValue::try_from_array(&cast_array, 0).unwrap();
9153        assert_eq!(cast_scalar.data_type(), desired_type);
9154
9155        // `ScalarValue::cast_to` (which has array-free fast paths) must produce
9156        // exactly the same result as casting through the arrow kernel above.
9157        let cast_to_scalar = scalar
9158            .cast_to(&desired_type)
9159            .expect("Failed to cast_to scalar");
9160        assert_eq!(
9161            cast_to_scalar, cast_scalar,
9162            "cast_to({scalar:?} -> {desired_type:?}) disagreed with the arrow cast kernel"
9163        );
9164
9165        // Some time later the "cast" scalar is turned back into an array:
9166        let array = cast_scalar
9167            .to_array_of_size(10)
9168            .expect("Failed to convert to array of size");
9169
9170        // The datatype should be "Dictionary" but is actually Utf8!!!
9171        assert_eq!(array.data_type(), &desired_type)
9172    }
9173
9174    #[test]
9175    fn test_scalar_negative() -> Result<()> {
9176        // positive test
9177        let value = ScalarValue::Int32(Some(12));
9178        assert_eq!(ScalarValue::Int32(Some(-12)), value.arithmetic_negate()?);
9179        let value = ScalarValue::Int32(None);
9180        assert_eq!(ScalarValue::Int32(None), value.arithmetic_negate()?);
9181
9182        // negative test
9183        let value = ScalarValue::UInt8(Some(12));
9184        assert!(value.arithmetic_negate().is_err());
9185        let value = ScalarValue::Boolean(None);
9186        assert!(value.arithmetic_negate().is_err());
9187        Ok(())
9188    }
9189
9190    #[test]
9191    fn test_scalar_negative_overflows() -> Result<()> {
9192        macro_rules! test_overflow_on_value {
9193            ($($val:expr),* $(,)?) => {$(
9194                {
9195                    let value: ScalarValue = $val;
9196                    let err = value.arithmetic_negate().expect_err("Should receive overflow error on negating {value:?}");
9197                    let root_err = err.find_root();
9198                    match  root_err{
9199                        DataFusionError::ArrowError(err, _) if matches!(err.as_ref(), ArrowError::ArithmeticOverflow(_)) => {}
9200                        _ => return Err(err),
9201                    };
9202                }
9203            )*};
9204        }
9205        test_overflow_on_value!(
9206            // the integers
9207            i8::MIN.into(),
9208            i16::MIN.into(),
9209            i32::MIN.into(),
9210            i64::MIN.into(),
9211            // for decimals, only value needs to be tested
9212            ScalarValue::try_new_decimal128(i128::MIN, 10, 5)?,
9213            ScalarValue::Decimal256(Some(i256::MIN), 20, 5),
9214            // interval, check all possible values
9215            ScalarValue::IntervalYearMonth(Some(i32::MIN)),
9216            ScalarValue::new_interval_dt(i32::MIN, 999),
9217            ScalarValue::new_interval_dt(1, i32::MIN),
9218            ScalarValue::new_interval_mdn(i32::MIN, 15, 123_456),
9219            ScalarValue::new_interval_mdn(12, i32::MIN, 123_456),
9220            ScalarValue::new_interval_mdn(12, 15, i64::MIN),
9221            // tz doesn't matter when negating
9222            ScalarValue::TimestampSecond(Some(i64::MIN), None),
9223            ScalarValue::TimestampMillisecond(Some(i64::MIN), None),
9224            ScalarValue::TimestampMicrosecond(Some(i64::MIN), None),
9225            ScalarValue::TimestampNanosecond(Some(i64::MIN), None),
9226        );
9227
9228        let float_cases = [
9229            (
9230                ScalarValue::Float16(Some(f16::MIN)),
9231                ScalarValue::Float16(Some(f16::MAX)),
9232            ),
9233            (
9234                ScalarValue::Float16(Some(f16::MAX)),
9235                ScalarValue::Float16(Some(f16::MIN)),
9236            ),
9237            (f32::MIN.into(), f32::MAX.into()),
9238            (f32::MAX.into(), f32::MIN.into()),
9239            (f64::MIN.into(), f64::MAX.into()),
9240            (f64::MAX.into(), f64::MIN.into()),
9241        ];
9242        // skip float 16 because they aren't supported
9243        for (test, expected) in float_cases.into_iter().skip(2) {
9244            assert_eq!(test.arithmetic_negate()?, expected);
9245        }
9246        Ok(())
9247    }
9248
9249    #[test]
9250    fn f16_test_overflow() {
9251        // TODO: if negate supports f16, add these cases to `test_scalar_negative_overflows` test case
9252        let cases = [
9253            (
9254                ScalarValue::Float16(Some(f16::MIN)),
9255                ScalarValue::Float16(Some(f16::MAX)),
9256            ),
9257            (
9258                ScalarValue::Float16(Some(f16::MAX)),
9259                ScalarValue::Float16(Some(f16::MIN)),
9260            ),
9261        ];
9262
9263        for (test, expected) in cases {
9264            assert_eq!(test.arithmetic_negate().unwrap(), expected);
9265        }
9266    }
9267
9268    macro_rules! expect_operation_error {
9269        ($TEST_NAME:ident, $FUNCTION:ident, $EXPECTED_ERROR:expr) => {
9270            #[test]
9271            fn $TEST_NAME() {
9272                let lhs = ScalarValue::UInt64(Some(12));
9273                let rhs = ScalarValue::Int32(Some(-3));
9274                match lhs.$FUNCTION(&rhs) {
9275                    Ok(_result) => {
9276                        panic!(
9277                            "Expected binary operation error between lhs: '{:?}', rhs: {:?}",
9278                            lhs, rhs
9279                        );
9280                    }
9281                    Err(e) => {
9282                        let error_message = e.to_string();
9283                        assert!(
9284                            error_message.contains($EXPECTED_ERROR),
9285                            "Expected error '{}' not found in actual error '{}'",
9286                            $EXPECTED_ERROR,
9287                            error_message
9288                        );
9289                    }
9290                }
9291            }
9292        };
9293    }
9294
9295    expect_operation_error!(
9296        expect_add_error,
9297        add,
9298        "Invalid arithmetic operation: UInt64 + Int32"
9299    );
9300    expect_operation_error!(
9301        expect_sub_error,
9302        sub,
9303        "Invalid arithmetic operation: UInt64 - Int32"
9304    );
9305
9306    macro_rules! decimal_op_test_cases {
9307    ($OPERATION:ident, [$([$L_VALUE:expr, $L_PRECISION:expr, $L_SCALE:expr, $R_VALUE:expr, $R_PRECISION:expr, $R_SCALE:expr, $O_VALUE:expr, $O_PRECISION:expr, $O_SCALE:expr]),+]) => {
9308            $(
9309
9310                let left = ScalarValue::Decimal128($L_VALUE, $L_PRECISION, $L_SCALE);
9311                let right = ScalarValue::Decimal128($R_VALUE, $R_PRECISION, $R_SCALE);
9312                let result = left.$OPERATION(&right).unwrap();
9313                assert_eq!(ScalarValue::Decimal128($O_VALUE, $O_PRECISION, $O_SCALE), result);
9314
9315            )+
9316        };
9317    }
9318
9319    #[test]
9320    fn decimal_operations() {
9321        decimal_op_test_cases!(
9322            add,
9323            [
9324                [Some(123), 10, 2, Some(124), 10, 2, Some(123 + 124), 11, 2],
9325                // test sum decimal with diff scale
9326                [
9327                    Some(123),
9328                    10,
9329                    3,
9330                    Some(124),
9331                    10,
9332                    2,
9333                    Some(123 + 124 * 10_i128.pow(1)),
9334                    12,
9335                    3
9336                ],
9337                // diff precision and scale for decimal data type
9338                [
9339                    Some(123),
9340                    10,
9341                    2,
9342                    Some(124),
9343                    11,
9344                    3,
9345                    Some(123 * 10_i128.pow(3 - 2) + 124),
9346                    12,
9347                    3
9348                ]
9349            ]
9350        );
9351    }
9352
9353    #[test]
9354    fn decimal_operations_with_nulls() {
9355        decimal_op_test_cases!(
9356            add,
9357            [
9358                // Case: (None, Some, 0)
9359                [None, 10, 2, Some(123), 10, 2, None, 11, 2],
9360                // Case: (Some, None, 0)
9361                [Some(123), 10, 2, None, 10, 2, None, 11, 2],
9362                // Case: (Some, None, _) + Side=False
9363                [Some(123), 8, 2, None, 10, 3, None, 11, 3],
9364                // Case: (None, Some, _) + Side=False
9365                [None, 8, 2, Some(123), 10, 3, None, 11, 3],
9366                // Case: (Some, None, _) + Side=True
9367                [Some(123), 8, 4, None, 10, 3, None, 12, 4],
9368                // Case: (None, Some, _) + Side=True
9369                [None, 10, 3, Some(123), 8, 4, None, 12, 4]
9370            ]
9371        );
9372    }
9373
9374    #[test]
9375    fn test_scalar_distance() {
9376        let cases = [
9377            // scalar (lhs), scalar (rhs), expected distance
9378            // ---------------------------------------------
9379            (ScalarValue::Int8(Some(1)), ScalarValue::Int8(Some(2)), 1),
9380            (ScalarValue::Int8(Some(2)), ScalarValue::Int8(Some(1)), 1),
9381            (
9382                ScalarValue::Int16(Some(-5)),
9383                ScalarValue::Int16(Some(5)),
9384                10,
9385            ),
9386            (
9387                ScalarValue::Int16(Some(5)),
9388                ScalarValue::Int16(Some(-5)),
9389                10,
9390            ),
9391            (ScalarValue::Int32(Some(0)), ScalarValue::Int32(Some(0)), 0),
9392            (
9393                ScalarValue::Int32(Some(-5)),
9394                ScalarValue::Int32(Some(-10)),
9395                5,
9396            ),
9397            (
9398                ScalarValue::Int64(Some(-10)),
9399                ScalarValue::Int64(Some(-5)),
9400                5,
9401            ),
9402            (ScalarValue::UInt8(Some(1)), ScalarValue::UInt8(Some(2)), 1),
9403            (ScalarValue::UInt8(Some(0)), ScalarValue::UInt8(Some(0)), 0),
9404            (
9405                ScalarValue::UInt16(Some(5)),
9406                ScalarValue::UInt16(Some(10)),
9407                5,
9408            ),
9409            (
9410                ScalarValue::UInt32(Some(10)),
9411                ScalarValue::UInt32(Some(5)),
9412                5,
9413            ),
9414            (
9415                ScalarValue::UInt64(Some(5)),
9416                ScalarValue::UInt64(Some(10)),
9417                5,
9418            ),
9419            (
9420                ScalarValue::Float16(Some(f16::from_f32(1.1))),
9421                ScalarValue::Float16(Some(f16::from_f32(1.9))),
9422                1,
9423            ),
9424            (
9425                ScalarValue::Float16(Some(f16::from_f32(-5.3))),
9426                ScalarValue::Float16(Some(f16::from_f32(-9.2))),
9427                4,
9428            ),
9429            (
9430                ScalarValue::Float16(Some(f16::from_f32(-5.3))),
9431                ScalarValue::Float16(Some(f16::from_f32(-9.7))),
9432                4,
9433            ),
9434            (
9435                ScalarValue::Float32(Some(1.0)),
9436                ScalarValue::Float32(Some(2.0)),
9437                1,
9438            ),
9439            (
9440                ScalarValue::Float32(Some(2.0)),
9441                ScalarValue::Float32(Some(1.0)),
9442                1,
9443            ),
9444            (
9445                ScalarValue::Float64(Some(0.0)),
9446                ScalarValue::Float64(Some(0.0)),
9447                0,
9448            ),
9449            (
9450                ScalarValue::Float64(Some(-5.0)),
9451                ScalarValue::Float64(Some(-10.0)),
9452                5,
9453            ),
9454            (
9455                ScalarValue::Float64(Some(-10.0)),
9456                ScalarValue::Float64(Some(-5.0)),
9457                5,
9458            ),
9459            // Floats are currently special cased to f64/f32 and the result is rounded
9460            // rather than ceiled/floored. In the future we might want to take a mode
9461            // which specified the rounding behavior.
9462            (
9463                ScalarValue::Float32(Some(1.2)),
9464                ScalarValue::Float32(Some(1.3)),
9465                0,
9466            ),
9467            (
9468                ScalarValue::Float32(Some(1.1)),
9469                ScalarValue::Float32(Some(1.9)),
9470                1,
9471            ),
9472            (
9473                ScalarValue::Float64(Some(-5.3)),
9474                ScalarValue::Float64(Some(-9.2)),
9475                4,
9476            ),
9477            (
9478                ScalarValue::Float64(Some(-5.3)),
9479                ScalarValue::Float64(Some(-9.7)),
9480                4,
9481            ),
9482            (
9483                ScalarValue::Float64(Some(-5.3)),
9484                ScalarValue::Float64(Some(-9.9)),
9485                5,
9486            ),
9487            (
9488                ScalarValue::Decimal128(Some(10), 1, 0),
9489                ScalarValue::Decimal128(Some(5), 1, 0),
9490                5,
9491            ),
9492            (
9493                ScalarValue::Decimal128(Some(5), 1, 0),
9494                ScalarValue::Decimal128(Some(10), 1, 0),
9495                5,
9496            ),
9497            (
9498                ScalarValue::Decimal256(Some(10.into()), 1, 0),
9499                ScalarValue::Decimal256(Some(5.into()), 1, 0),
9500                5,
9501            ),
9502            (
9503                ScalarValue::Decimal256(Some(5.into()), 1, 0),
9504                ScalarValue::Decimal256(Some(10.into()), 1, 0),
9505                5,
9506            ),
9507            // Temporal types
9508            (
9509                ScalarValue::Date32(Some(0)),
9510                ScalarValue::Date32(Some(10)),
9511                10,
9512            ),
9513            (
9514                ScalarValue::Date32(Some(10)),
9515                ScalarValue::Date32(Some(0)),
9516                10,
9517            ),
9518            (
9519                ScalarValue::Date64(Some(1000)),
9520                ScalarValue::Date64(Some(5000)),
9521                4000,
9522            ),
9523            (
9524                ScalarValue::TimestampSecond(Some(100), None),
9525                ScalarValue::TimestampSecond(Some(200), None),
9526                100,
9527            ),
9528            (
9529                ScalarValue::TimestampMillisecond(Some(1000), None),
9530                ScalarValue::TimestampMillisecond(Some(5000), None),
9531                4000,
9532            ),
9533            (
9534                ScalarValue::TimestampMicrosecond(Some(0), None),
9535                ScalarValue::TimestampMicrosecond(Some(1_000_000), None),
9536                1_000_000,
9537            ),
9538            (
9539                ScalarValue::TimestampNanosecond(Some(1_000_000_000), None),
9540                ScalarValue::TimestampNanosecond(Some(2_000_000_000), None),
9541                1_000_000_000,
9542            ),
9543        ];
9544        for (lhs, rhs, expected) in cases.iter() {
9545            let distance = lhs.distance_u64(rhs).unwrap();
9546            assert_eq!(distance, *expected as u64);
9547        }
9548    }
9549
9550    #[test]
9551    fn test_distance_none() {
9552        let cases = [
9553            (
9554                ScalarValue::Decimal128(Some(i128::MAX), DECIMAL128_MAX_PRECISION, 0),
9555                ScalarValue::Decimal128(Some(-i128::MAX), DECIMAL128_MAX_PRECISION, 0),
9556            ),
9557            (
9558                ScalarValue::Decimal256(Some(i256::MAX), DECIMAL256_MAX_PRECISION, 0),
9559                ScalarValue::Decimal256(Some(-i256::MAX), DECIMAL256_MAX_PRECISION, 0),
9560            ),
9561        ];
9562        for (lhs, rhs) in cases.iter() {
9563            let distance = lhs.distance_u64(rhs);
9564            assert!(distance.is_none(), "{lhs} vs {rhs}");
9565        }
9566    }
9567
9568    #[test]
9569    fn test_scalar_distance_invalid() {
9570        let cases = [
9571            // scalar (lhs), scalar (rhs)
9572            // --------------------------
9573            // Same type but with nulls
9574            (ScalarValue::Int8(None), ScalarValue::Int8(None)),
9575            (ScalarValue::Int8(None), ScalarValue::Int8(Some(1))),
9576            (ScalarValue::Int8(Some(1)), ScalarValue::Int8(None)),
9577            // Different type
9578            (ScalarValue::Int8(Some(1)), ScalarValue::Int16(Some(1))),
9579            (ScalarValue::Int8(Some(1)), ScalarValue::Float32(Some(1.0))),
9580            (
9581                ScalarValue::Float16(Some(f16::from_f32(1.0))),
9582                ScalarValue::Float32(Some(1.0)),
9583            ),
9584            (
9585                ScalarValue::Float16(Some(f16::from_f32(1.0))),
9586                ScalarValue::Int32(Some(1)),
9587            ),
9588            (
9589                ScalarValue::Float64(Some(1.1)),
9590                ScalarValue::Float32(Some(2.2)),
9591            ),
9592            (
9593                ScalarValue::UInt64(Some(777)),
9594                ScalarValue::Int32(Some(111)),
9595            ),
9596            // Different types with nulls
9597            (ScalarValue::Int8(None), ScalarValue::Int16(Some(1))),
9598            (ScalarValue::Int8(Some(1)), ScalarValue::Int16(None)),
9599            // Unsupported types
9600            (ScalarValue::from("foo"), ScalarValue::from("bar")),
9601            (
9602                ScalarValue::Boolean(Some(true)),
9603                ScalarValue::Boolean(Some(false)),
9604            ),
9605            (
9606                ScalarValue::Decimal128(Some(123), 5, 5),
9607                ScalarValue::Decimal128(Some(120), 5, 3),
9608            ),
9609            (
9610                ScalarValue::Decimal256(Some(123.into()), 5, 5),
9611                ScalarValue::Decimal256(Some(120.into()), 5, 3),
9612            ),
9613            // Distance 2 * 2^50 is larger than usize
9614            (
9615                ScalarValue::Decimal256(
9616                    Some(i256::from_parts(0, 2_i64.pow(50).into())),
9617                    1,
9618                    0,
9619                ),
9620                ScalarValue::Decimal256(
9621                    Some(i256::from_parts(0, (-(2_i64).pow(50)).into())),
9622                    1,
9623                    0,
9624                ),
9625            ),
9626            // Distance overflow
9627            (
9628                ScalarValue::Decimal256(Some(i256::from_parts(0, i128::MAX)), 1, 0),
9629                ScalarValue::Decimal256(Some(i256::from_parts(0, -i128::MAX)), 1, 0),
9630            ),
9631        ];
9632        for (lhs, rhs) in cases {
9633            let distance = lhs.distance_u64(&rhs);
9634            assert!(distance.is_none());
9635        }
9636    }
9637
9638    #[test]
9639    fn test_scalar_distance_u64_boundaries() {
9640        // 1. Full-domain integer ranges
9641        // i64::MIN to i64::MAX -> distance is u64::MAX
9642        let lhs = ScalarValue::Int64(Some(i64::MIN));
9643        let rhs = ScalarValue::Int64(Some(i64::MAX));
9644        assert_eq!(lhs.distance_u64(&rhs), Some(u64::MAX));
9645        assert_eq!(rhs.distance_u64(&lhs), Some(u64::MAX));
9646
9647        // u64::MIN to u64::MAX -> distance is u64::MAX
9648        let lhs = ScalarValue::UInt64(Some(u64::MIN));
9649        let rhs = ScalarValue::UInt64(Some(u64::MAX));
9650        assert_eq!(lhs.distance_u64(&rhs), Some(u64::MAX));
9651        assert_eq!(rhs.distance_u64(&lhs), Some(u64::MAX));
9652
9653        // 2. Decimal128 overflow edges (around u64::MAX)
9654        // distance equal to u64::MAX fits
9655        let lhs = ScalarValue::Decimal128(Some(0), 20, 0);
9656        let rhs = ScalarValue::Decimal128(Some(u64::MAX as i128), 20, 0);
9657        assert_eq!(lhs.distance_u64(&rhs), Some(u64::MAX));
9658
9659        // distance greater than u64::MAX overflows
9660        let lhs = ScalarValue::Decimal128(Some(0), 20, 0);
9661        let rhs = ScalarValue::Decimal128(Some(u64::MAX as i128 + 1), 20, 0);
9662        assert_eq!(lhs.distance_u64(&rhs), None);
9663
9664        // 3. Decimal256 overflow edges (around u64::MAX)
9665        // distance equal to u64::MAX fits
9666        let lhs = ScalarValue::Decimal256(Some(i256::from_parts(0, 0)), 20, 0);
9667        let rhs =
9668            ScalarValue::Decimal256(Some(i256::from_parts(u64::MAX as u128, 0)), 20, 0);
9669        assert_eq!(lhs.distance_u64(&rhs), Some(u64::MAX));
9670
9671        // distance greater than u64::MAX overflows
9672        let lhs = ScalarValue::Decimal256(Some(i256::from_parts(0, 0)), 20, 0);
9673        let rhs = ScalarValue::Decimal256(
9674            Some(i256::from_parts(u64::MAX as u128 + 1, 0)),
9675            20,
9676            0,
9677        );
9678        assert_eq!(lhs.distance_u64(&rhs), None);
9679
9680        // 4. Float64 overflow edges (around u64::MAX)
9681        let lhs = ScalarValue::Float64(Some(0.0));
9682        let val: f64 = 18446744073709500000.0;
9683        let rhs = ScalarValue::Float64(Some(val));
9684        assert_eq!(lhs.distance_u64(&rhs), Some(18446744073709500416));
9685
9686        // float value > u64::MAX overflows
9687        let rhs = ScalarValue::Float64(Some(1.9e19));
9688        assert_eq!(lhs.distance_u64(&rhs), None);
9689
9690        // exact 2^64 boundary (18446744073709551616.0) is greater than u64::MAX, so it should return None
9691        let exact_2_64_f64 = ScalarValue::Float64(Some(18446744073709551616.0));
9692        assert_eq!(lhs.distance_u64(&exact_2_64_f64), None);
9693
9694        // exact 2^64 boundary as Float32 should also return None
9695        let lhs_f32 = ScalarValue::Float32(Some(0.0));
9696        let exact_2_64_f32 = ScalarValue::Float32(Some(18446744073709551616.0));
9697        assert_eq!(lhs_f32.distance_u64(&exact_2_64_f32), None);
9698
9699        // largest float32 value below 2^64 (2^64 - 2^41 = 18446741874686296064.0) should fit
9700        let below_2_64_f32 = ScalarValue::Float32(Some(18446741874686296064.0));
9701        assert_eq!(
9702            lhs_f32.distance_u64(&below_2_64_f32),
9703            Some(18446741874686296064)
9704        );
9705
9706        // Inf, NegInf, NaN
9707        let inf = ScalarValue::Float64(Some(f64::INFINITY));
9708        let neg_inf = ScalarValue::Float64(Some(f64::NEG_INFINITY));
9709        let nan = ScalarValue::Float64(Some(f64::NAN));
9710        assert_eq!(lhs.distance_u64(&inf), None);
9711        assert_eq!(lhs.distance_u64(&neg_inf), None);
9712        assert_eq!(lhs.distance_u64(&nan), None);
9713
9714        let inf_f32 = ScalarValue::Float32(Some(f32::INFINITY));
9715        let neg_inf_f32 = ScalarValue::Float32(Some(f32::NEG_INFINITY));
9716        let nan_f32 = ScalarValue::Float32(Some(f32::NAN));
9717        assert_eq!(lhs_f32.distance_u64(&inf_f32), None);
9718        assert_eq!(lhs_f32.distance_u64(&neg_inf_f32), None);
9719        assert_eq!(lhs_f32.distance_u64(&nan_f32), None);
9720
9721        let lhs_f16 = ScalarValue::Float16(Some(f16::ZERO));
9722        let inf_f16 = ScalarValue::Float16(Some(f16::INFINITY));
9723        let neg_inf_f16 = ScalarValue::Float16(Some(f16::NEG_INFINITY));
9724        let nan_f16 = ScalarValue::Float16(Some(f16::NAN));
9725        assert_eq!(lhs_f16.distance_u64(&inf_f16), None);
9726        assert_eq!(lhs_f16.distance_u64(&neg_inf_f16), None);
9727        assert_eq!(lhs_f16.distance_u64(&nan_f16), None);
9728
9729        // 5. Date and Timestamp boundaries
9730        // Date32: i32::MIN to i32::MAX
9731        let lhs = ScalarValue::Date32(Some(i32::MIN));
9732        let rhs = ScalarValue::Date32(Some(i32::MAX));
9733        assert_eq!(lhs.distance_u64(&rhs), Some(u32::MAX as u64));
9734
9735        // TimestampSecond: i64::MIN to i64::MAX
9736        let lhs = ScalarValue::TimestampSecond(Some(i64::MIN), None);
9737        let rhs = ScalarValue::TimestampSecond(Some(i64::MAX), None);
9738        assert_eq!(lhs.distance_u64(&rhs), Some(u64::MAX));
9739
9740        // 6. Decimal scale matching (ignoring precision)
9741        let lhs = ScalarValue::Decimal128(Some(100), 10, 2);
9742        let rhs = ScalarValue::Decimal128(Some(150), 15, 2);
9743        assert_eq!(lhs.distance_u64(&rhs), Some(50));
9744        assert_eq!(rhs.distance_u64(&lhs), Some(50));
9745
9746        let lhs = ScalarValue::Decimal128(Some(100), 10, 2);
9747        let rhs = ScalarValue::Decimal128(Some(150), 10, 3);
9748        assert_eq!(lhs.distance_u64(&rhs), None);
9749    }
9750
9751    #[test]
9752    fn test_scalar_interval_negate() {
9753        let cases = [
9754            (
9755                ScalarValue::new_interval_ym(1, 12),
9756                ScalarValue::new_interval_ym(-1, -12),
9757            ),
9758            (
9759                ScalarValue::new_interval_dt(1, 999),
9760                ScalarValue::new_interval_dt(-1, -999),
9761            ),
9762            (
9763                ScalarValue::new_interval_mdn(12, 15, 123_456),
9764                ScalarValue::new_interval_mdn(-12, -15, -123_456),
9765            ),
9766        ];
9767        for (expr, expected) in cases.iter() {
9768            let result = expr.arithmetic_negate().unwrap();
9769            assert_eq!(*expected, result, "-expr:{expr:?}");
9770        }
9771    }
9772
9773    #[test]
9774    fn test_scalar_interval_add() {
9775        let cases = [
9776            (
9777                ScalarValue::new_interval_ym(1, 12),
9778                ScalarValue::new_interval_ym(1, 12),
9779                ScalarValue::new_interval_ym(2, 24),
9780            ),
9781            (
9782                ScalarValue::new_interval_dt(1, 999),
9783                ScalarValue::new_interval_dt(1, 999),
9784                ScalarValue::new_interval_dt(2, 1998),
9785            ),
9786            (
9787                ScalarValue::new_interval_mdn(12, 15, 123_456),
9788                ScalarValue::new_interval_mdn(12, 15, 123_456),
9789                ScalarValue::new_interval_mdn(24, 30, 246_912),
9790            ),
9791        ];
9792        for (lhs, rhs, expected) in cases.iter() {
9793            let result = lhs.add(rhs).unwrap();
9794            let result_commute = rhs.add(lhs).unwrap();
9795            assert_eq!(*expected, result, "lhs:{lhs:?} + rhs:{rhs:?}");
9796            assert_eq!(*expected, result_commute, "lhs:{rhs:?} + rhs:{lhs:?}");
9797        }
9798    }
9799
9800    #[test]
9801    fn test_scalar_interval_sub() {
9802        let cases = [
9803            (
9804                ScalarValue::new_interval_ym(1, 12),
9805                ScalarValue::new_interval_ym(1, 12),
9806                ScalarValue::new_interval_ym(0, 0),
9807            ),
9808            (
9809                ScalarValue::new_interval_dt(1, 999),
9810                ScalarValue::new_interval_dt(1, 999),
9811                ScalarValue::new_interval_dt(0, 0),
9812            ),
9813            (
9814                ScalarValue::new_interval_mdn(12, 15, 123_456),
9815                ScalarValue::new_interval_mdn(12, 15, 123_456),
9816                ScalarValue::new_interval_mdn(0, 0, 0),
9817            ),
9818        ];
9819        for (lhs, rhs, expected) in cases.iter() {
9820            let result = lhs.sub(rhs).unwrap();
9821            assert_eq!(*expected, result, "lhs:{lhs:?} - rhs:{rhs:?}");
9822        }
9823    }
9824
9825    #[test]
9826    fn timestamp_op_random_tests() {
9827        // timestamp1 + (or -) interval = timestamp2
9828        // timestamp2 - timestamp1 (or timestamp1 - timestamp2) = interval ?
9829        let sample_size = 1000;
9830        let timestamps1 = get_random_timestamps(sample_size);
9831        let intervals = get_random_intervals(sample_size);
9832        // ts(sec) + interval(ns) = ts(sec); however,
9833        // ts(sec) - ts(sec) cannot be = interval(ns). Therefore,
9834        // timestamps are more precise than intervals in tests.
9835        for (idx, ts1) in timestamps1.iter().enumerate() {
9836            if idx % 2 == 0 {
9837                let timestamp2 = ts1.add(intervals[idx].clone()).unwrap();
9838                let back = timestamp2.sub(intervals[idx].clone()).unwrap();
9839                assert_eq!(ts1, &back);
9840            } else {
9841                let timestamp2 = ts1.sub(intervals[idx].clone()).unwrap();
9842                let back = timestamp2.add(intervals[idx].clone()).unwrap();
9843                assert_eq!(ts1, &back);
9844            };
9845        }
9846    }
9847
9848    #[test]
9849    fn test_struct_nulls() {
9850        let fields_b = Fields::from(vec![
9851            Field::new("ba", DataType::UInt64, true),
9852            Field::new("bb", DataType::UInt64, true),
9853        ]);
9854        let fields = Fields::from(vec![
9855            Field::new("a", DataType::UInt64, true),
9856            Field::new("b", DataType::Struct(fields_b.clone()), true),
9857        ]);
9858
9859        let struct_value = vec![
9860            (
9861                Arc::clone(&fields[0]),
9862                Arc::new(UInt64Array::from(vec![Some(1)])) as ArrayRef,
9863            ),
9864            (
9865                Arc::clone(&fields[1]),
9866                Arc::new(StructArray::from(vec![
9867                    (
9868                        Arc::clone(&fields_b[0]),
9869                        Arc::new(UInt64Array::from(vec![Some(2)])) as ArrayRef,
9870                    ),
9871                    (
9872                        Arc::clone(&fields_b[1]),
9873                        Arc::new(UInt64Array::from(vec![Some(3)])) as ArrayRef,
9874                    ),
9875                ])) as ArrayRef,
9876            ),
9877        ];
9878
9879        let struct_value_with_nulls = vec![
9880            (
9881                Arc::clone(&fields[0]),
9882                Arc::new(UInt64Array::from(vec![Some(1)])) as ArrayRef,
9883            ),
9884            (
9885                Arc::clone(&fields[1]),
9886                Arc::new(StructArray::from((
9887                    vec![
9888                        (
9889                            Arc::clone(&fields_b[0]),
9890                            Arc::new(UInt64Array::from(vec![Some(2)])) as ArrayRef,
9891                        ),
9892                        (
9893                            Arc::clone(&fields_b[1]),
9894                            Arc::new(UInt64Array::from(vec![Some(3)])) as ArrayRef,
9895                        ),
9896                    ],
9897                    Buffer::from(&[0]),
9898                ))) as ArrayRef,
9899            ),
9900        ];
9901
9902        let scalars = vec![
9903            // all null
9904            ScalarValue::Struct(Arc::new(StructArray::from((
9905                struct_value.clone(),
9906                Buffer::from(&[0]),
9907            )))),
9908            // field 1 valid, field 2 null
9909            ScalarValue::Struct(Arc::new(StructArray::from((
9910                struct_value_with_nulls.clone(),
9911                Buffer::from(&[1]),
9912            )))),
9913            // all valid
9914            ScalarValue::Struct(Arc::new(StructArray::from((
9915                struct_value.clone(),
9916                Buffer::from(&[1]),
9917            )))),
9918        ];
9919
9920        let check_array = |array: Arc<dyn Array>| {
9921            let is_null = is_null(&array).unwrap();
9922            assert_eq!(is_null, BooleanArray::from(vec![true, false, false]));
9923
9924            let formatted = pretty_format_columns("col", &[array]).unwrap().to_string();
9925            let formatted = formatted.split('\n').collect::<Vec<_>>();
9926            let expected = vec![
9927                "+---------------------------+",
9928                "| col                       |",
9929                "+---------------------------+",
9930                "|                           |",
9931                "| {a: 1, b: }               |",
9932                "| {a: 1, b: {ba: 2, bb: 3}} |",
9933                "+---------------------------+",
9934            ];
9935            assert_eq!(
9936                formatted, expected,
9937                "Actual:\n{formatted:#?}\n\nExpected:\n{expected:#?}"
9938            );
9939        };
9940
9941        // test `ScalarValue::iter_to_array`
9942        let array = ScalarValue::iter_to_array(scalars.clone()).unwrap();
9943        check_array(array);
9944
9945        // test `ScalarValue::to_array` / `ScalarValue::to_array_of_size`
9946        let arrays = scalars
9947            .iter()
9948            .map(ScalarValue::to_array)
9949            .collect::<Result<Vec<_>>>()
9950            .expect("Failed to convert to array");
9951        let arrays = arrays.iter().map(|a| a.as_ref()).collect::<Vec<_>>();
9952        let array = arrow::compute::concat(&arrays).unwrap();
9953        check_array(array);
9954    }
9955
9956    #[test]
9957    fn test_struct_display() {
9958        let field_a = Field::new("a", DataType::Int32, true);
9959        let field_b = Field::new("b", DataType::Utf8, true);
9960
9961        let s = ScalarStructBuilder::new()
9962            .with_scalar(field_a, ScalarValue::from(1i32))
9963            .with_scalar(field_b, ScalarValue::Utf8(None))
9964            .build()
9965            .unwrap();
9966
9967        assert_eq!(s.to_string(), "{a:1,b:}");
9968        assert_eq!(format!("{s:?}"), r#"Struct({a:1,b:})"#);
9969
9970        let ScalarValue::Struct(arr) = s else {
9971            panic!("Expected struct");
9972        };
9973
9974        //verify compared to arrow display
9975        let batch = RecordBatch::try_from_iter(vec![("s", arr as _)]).unwrap();
9976        assert_snapshot!(batches_to_string(&[batch]), @r"
9977        +-------------+
9978        | s           |
9979        +-------------+
9980        | {a: 1, b: } |
9981        +-------------+
9982        ");
9983    }
9984
9985    #[test]
9986    fn test_list_view_display() {
9987        let s = ScalarValue::ListView(
9988            ListViewArray::from_iter_primitive::<Int64Type, _, _>(vec![Some(vec![
9989                Some(1),
9990                None,
9991                Some(3),
9992            ])])
9993            .into(),
9994        );
9995
9996        assert_eq!(s.to_string(), "[1, , 3]");
9997        assert_eq!(format!("{s:?}"), "ListView([1, , 3])");
9998    }
9999
10000    #[test]
10001    fn test_null_bug() {
10002        let field_a = Field::new("a", DataType::Int32, true);
10003        let field_b = Field::new("b", DataType::Int32, true);
10004        let fields = Fields::from(vec![field_a, field_b]);
10005
10006        let array_a = Arc::new(Int32Array::from_iter_values([1]));
10007        let array_b = Arc::new(Int32Array::from_iter_values([2]));
10008        let arrays: Vec<ArrayRef> = vec![array_a, array_b];
10009
10010        let mut not_nulls = NullBufferBuilder::new(1);
10011
10012        not_nulls.append_non_null();
10013
10014        let ar = StructArray::new(fields, arrays, not_nulls.finish());
10015        let s = ScalarValue::Struct(Arc::new(ar));
10016
10017        assert_eq!(s.to_string(), "{a:1,b:2}");
10018        assert_eq!(format!("{s:?}"), r#"Struct({a:1,b:2})"#);
10019
10020        let ScalarValue::Struct(arr) = s else {
10021            panic!("Expected struct");
10022        };
10023
10024        //verify compared to arrow display
10025        let batch = RecordBatch::try_from_iter(vec![("s", arr as _)]).unwrap();
10026        assert_snapshot!(batches_to_string(&[batch]), @r"
10027        +--------------+
10028        | s            |
10029        +--------------+
10030        | {a: 1, b: 2} |
10031        +--------------+
10032        ");
10033    }
10034
10035    #[test]
10036    fn test_display_date64_large_values() {
10037        assert_eq!(
10038            format!("{}", ScalarValue::Date64(Some(790179464505))),
10039            "1995-01-15"
10040        );
10041        // This used to panic, see https://github.com/apache/arrow-rs/issues/7728
10042        assert_eq!(
10043            format!("{}", ScalarValue::Date64(Some(-790179464505600000))),
10044            ""
10045        );
10046    }
10047
10048    #[test]
10049    fn test_decimal_display_and_debug() {
10050        let decimal32 = ScalarValue::Decimal32(Some(123), 3, 2);
10051        assert_eq!(decimal32.to_string(), "1.23");
10052        assert_eq!(format!("{decimal32:?}"), "Decimal32(1.23,3,2)");
10053
10054        let decimal64 = ScalarValue::Decimal64(Some(-12345), 5, 3);
10055        assert_eq!(decimal64.to_string(), "-12.345");
10056        assert_eq!(format!("{decimal64:?}"), "Decimal64(-12.345,5,3)");
10057
10058        let decimal128 = ScalarValue::Decimal128(Some(1), 1, 1);
10059        assert_eq!(decimal128.to_string(), "0.1");
10060        assert_eq!(format!("{decimal128:?}"), "Decimal128(0.1,1,1)");
10061
10062        let decimal128_trailing_zero = ScalarValue::Decimal128(Some(120), 3, 2);
10063        assert_eq!(decimal128_trailing_zero.to_string(), "1.20");
10064        assert_eq!(
10065            format!("{decimal128_trailing_zero:?}"),
10066            "Decimal128(1.20,3,2)"
10067        );
10068
10069        let decimal256 = ScalarValue::Decimal256(Some(i256::from(100123)), 28, 3);
10070        assert_eq!(decimal256.to_string(), "100.123");
10071        assert_eq!(format!("{decimal256:?}"), "Decimal256(100.123,28,3)");
10072
10073        let null_decimal = ScalarValue::Decimal128(None, 10, 2);
10074        assert_eq!(null_decimal.to_string(), "NULL");
10075        assert_eq!(format!("{null_decimal:?}"), "Decimal128(NULL,10,2)");
10076    }
10077
10078    #[test]
10079    fn test_struct_display_null() {
10080        let fields = vec![Field::new("a", DataType::Int32, false)];
10081        let s = ScalarStructBuilder::new_null(fields);
10082        assert_eq!(s.to_string(), "NULL");
10083
10084        let ScalarValue::Struct(arr) = s else {
10085            panic!("Expected struct");
10086        };
10087
10088        //verify compared to arrow display
10089        let batch = RecordBatch::try_from_iter(vec![("s", arr as _)]).unwrap();
10090
10091        assert_snapshot!(batches_to_string(&[batch]), @r"
10092        +---+
10093        | s |
10094        +---+
10095        |   |
10096        +---+
10097        ");
10098    }
10099
10100    #[test]
10101    fn test_map_display_and_debug() {
10102        let string_builder = StringBuilder::new();
10103        let int_builder = Int32Builder::with_capacity(4);
10104        let mut builder = MapBuilder::new(None, string_builder, int_builder);
10105        builder.keys().append_value("joe");
10106        builder.values().append_value(1);
10107        builder.append(true).unwrap();
10108
10109        builder.keys().append_value("blogs");
10110        builder.values().append_value(2);
10111        builder.keys().append_value("foo");
10112        builder.values().append_value(4);
10113        builder.append(true).unwrap();
10114        builder.append(true).unwrap();
10115        builder.append(false).unwrap();
10116
10117        let map_value = ScalarValue::Map(Arc::new(builder.finish()));
10118
10119        assert_eq!(map_value.to_string(), "[{joe:1},{blogs:2,foo:4},{},NULL]");
10120        assert_eq!(
10121            format!("{map_value:?}"),
10122            r#"Map([{"joe":"1"},{"blogs":"2","foo":"4"},{},NULL])"#
10123        );
10124
10125        let ScalarValue::Map(arr) = map_value else {
10126            panic!("Expected map");
10127        };
10128
10129        //verify compared to arrow display
10130        let batch = RecordBatch::try_from_iter(vec![("m", arr as _)]).unwrap();
10131        assert_snapshot!(batches_to_string(&[batch]), @r"
10132        +--------------------+
10133        | m                  |
10134        +--------------------+
10135        | {joe: 1}           |
10136        | {blogs: 2, foo: 4} |
10137        | {}                 |
10138        |                    |
10139        +--------------------+
10140        ");
10141    }
10142
10143    #[test]
10144    fn test_binary_display() {
10145        let no_binary_value = ScalarValue::Binary(None);
10146        assert_eq!(format!("{no_binary_value}"), "NULL");
10147        let single_binary_value = ScalarValue::Binary(Some(vec![42u8]));
10148        assert_eq!(format!("{single_binary_value}"), "2A");
10149        let small_binary_value = ScalarValue::Binary(Some(vec![1u8, 2, 3]));
10150        assert_eq!(format!("{small_binary_value}"), "010203");
10151        let large_binary_value =
10152            ScalarValue::Binary(Some(vec![1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]));
10153        assert_eq!(format!("{large_binary_value}"), "0102030405060708090A...");
10154
10155        let no_binary_value = ScalarValue::BinaryView(None);
10156        assert_eq!(format!("{no_binary_value}"), "NULL");
10157        let small_binary_value = ScalarValue::BinaryView(Some(vec![1u8, 2, 3]));
10158        assert_eq!(format!("{small_binary_value}"), "010203");
10159        let large_binary_value =
10160            ScalarValue::BinaryView(Some(vec![1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]));
10161        assert_eq!(format!("{large_binary_value}"), "0102030405060708090A...");
10162
10163        let no_binary_value = ScalarValue::LargeBinary(None);
10164        assert_eq!(format!("{no_binary_value}"), "NULL");
10165        let small_binary_value = ScalarValue::LargeBinary(Some(vec![1u8, 2, 3]));
10166        assert_eq!(format!("{small_binary_value}"), "010203");
10167        let large_binary_value =
10168            ScalarValue::LargeBinary(Some(vec![1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]));
10169        assert_eq!(format!("{large_binary_value}"), "0102030405060708090A...");
10170
10171        let no_binary_value = ScalarValue::FixedSizeBinary(3, None);
10172        assert_eq!(format!("{no_binary_value}"), "NULL");
10173        let small_binary_value = ScalarValue::FixedSizeBinary(3, Some(vec![1u8, 2, 3]));
10174        assert_eq!(format!("{small_binary_value}"), "010203");
10175        let large_binary_value = ScalarValue::FixedSizeBinary(
10176            11,
10177            Some(vec![1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]),
10178        );
10179        assert_eq!(format!("{large_binary_value}"), "0102030405060708090A...");
10180    }
10181
10182    #[test]
10183    fn test_binary_debug() {
10184        let no_binary_value = ScalarValue::Binary(None);
10185        assert_eq!(format!("{no_binary_value:?}"), "Binary(NULL)");
10186        let single_binary_value = ScalarValue::Binary(Some(vec![42u8]));
10187        assert_eq!(format!("{single_binary_value:?}"), "Binary(\"42\")");
10188        let small_binary_value = ScalarValue::Binary(Some(vec![1u8, 2, 3]));
10189        assert_eq!(format!("{small_binary_value:?}"), "Binary(\"1,2,3\")");
10190        let large_binary_value =
10191            ScalarValue::Binary(Some(vec![1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]));
10192        assert_eq!(
10193            format!("{large_binary_value:?}"),
10194            "Binary(\"1,2,3,4,5,6,7,8,9,10,11\")"
10195        );
10196
10197        let no_binary_value = ScalarValue::BinaryView(None);
10198        assert_eq!(format!("{no_binary_value:?}"), "BinaryView(NULL)");
10199        let small_binary_value = ScalarValue::BinaryView(Some(vec![1u8, 2, 3]));
10200        assert_eq!(format!("{small_binary_value:?}"), "BinaryView(\"1,2,3\")");
10201        let large_binary_value =
10202            ScalarValue::BinaryView(Some(vec![1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]));
10203        assert_eq!(
10204            format!("{large_binary_value:?}"),
10205            "BinaryView(\"1,2,3,4,5,6,7,8,9,10,11\")"
10206        );
10207
10208        let no_binary_value = ScalarValue::LargeBinary(None);
10209        assert_eq!(format!("{no_binary_value:?}"), "LargeBinary(NULL)");
10210        let small_binary_value = ScalarValue::LargeBinary(Some(vec![1u8, 2, 3]));
10211        assert_eq!(format!("{small_binary_value:?}"), "LargeBinary(\"1,2,3\")");
10212        let large_binary_value =
10213            ScalarValue::LargeBinary(Some(vec![1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]));
10214        assert_eq!(
10215            format!("{large_binary_value:?}"),
10216            "LargeBinary(\"1,2,3,4,5,6,7,8,9,10,11\")"
10217        );
10218
10219        let no_binary_value = ScalarValue::FixedSizeBinary(3, None);
10220        assert_eq!(format!("{no_binary_value:?}"), "FixedSizeBinary(3, NULL)");
10221        let small_binary_value = ScalarValue::FixedSizeBinary(3, Some(vec![1u8, 2, 3]));
10222        assert_eq!(
10223            format!("{small_binary_value:?}"),
10224            "FixedSizeBinary(3, \"1,2,3\")"
10225        );
10226        let large_binary_value = ScalarValue::FixedSizeBinary(
10227            11,
10228            Some(vec![1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]),
10229        );
10230        assert_eq!(
10231            format!("{large_binary_value:?}"),
10232            "FixedSizeBinary(11, \"1,2,3,4,5,6,7,8,9,10,11\")"
10233        );
10234    }
10235
10236    #[test]
10237    fn test_build_timestamp_millisecond_list() {
10238        let values = vec![ScalarValue::TimestampMillisecond(Some(1), None)];
10239        let arr = ScalarValue::new_list_nullable(
10240            &values,
10241            &DataType::Timestamp(TimeUnit::Millisecond, None),
10242        );
10243        assert_eq!(1, arr.len());
10244    }
10245
10246    #[test]
10247    fn test_newlist_timestamp_zone() {
10248        let s: &'static str = "UTC";
10249        let values = vec![ScalarValue::TimestampMillisecond(Some(1), Some(s.into()))];
10250        let arr = ScalarValue::new_list_nullable(
10251            &values,
10252            &DataType::Timestamp(TimeUnit::Millisecond, Some(s.into())),
10253        );
10254        assert_eq!(1, arr.len());
10255        assert_eq!(
10256            arr.data_type(),
10257            &DataType::List(Arc::new(Field::new_list_field(
10258                DataType::Timestamp(TimeUnit::Millisecond, Some(s.into())),
10259                true,
10260            )))
10261        );
10262    }
10263
10264    fn get_random_timestamps(sample_size: u64) -> Vec<ScalarValue> {
10265        let vector_size = sample_size;
10266        let mut timestamp = vec![];
10267        let mut rng = rand::rng();
10268        for i in 0..vector_size {
10269            let year = rng.random_range(1995..=2050);
10270            let month = rng.random_range(1..=12);
10271            let day = rng.random_range(1..=28); // to exclude invalid dates
10272            let hour = rng.random_range(0..=23);
10273            let minute = rng.random_range(0..=59);
10274            let second = rng.random_range(0..=59);
10275            if i % 4 == 0 {
10276                timestamp.push(ScalarValue::TimestampSecond(
10277                    Some(
10278                        NaiveDate::from_ymd_opt(year, month, day)
10279                            .unwrap()
10280                            .and_hms_opt(hour, minute, second)
10281                            .unwrap()
10282                            .and_utc()
10283                            .timestamp(),
10284                    ),
10285                    None,
10286                ))
10287            } else if i % 4 == 1 {
10288                let millisec = rng.random_range(0..=999);
10289                timestamp.push(ScalarValue::TimestampMillisecond(
10290                    Some(
10291                        NaiveDate::from_ymd_opt(year, month, day)
10292                            .unwrap()
10293                            .and_hms_milli_opt(hour, minute, second, millisec)
10294                            .unwrap()
10295                            .and_utc()
10296                            .timestamp_millis(),
10297                    ),
10298                    None,
10299                ))
10300            } else if i % 4 == 2 {
10301                let microsec = rng.random_range(0..=999_999);
10302                timestamp.push(ScalarValue::TimestampMicrosecond(
10303                    Some(
10304                        NaiveDate::from_ymd_opt(year, month, day)
10305                            .unwrap()
10306                            .and_hms_micro_opt(hour, minute, second, microsec)
10307                            .unwrap()
10308                            .and_utc()
10309                            .timestamp_micros(),
10310                    ),
10311                    None,
10312                ))
10313            } else if i % 4 == 3 {
10314                let nanosec = rng.random_range(0..=999_999_999);
10315                timestamp.push(ScalarValue::TimestampNanosecond(
10316                    Some(
10317                        NaiveDate::from_ymd_opt(year, month, day)
10318                            .unwrap()
10319                            .and_hms_nano_opt(hour, minute, second, nanosec)
10320                            .unwrap()
10321                            .and_utc()
10322                            .timestamp_nanos_opt()
10323                            .unwrap(),
10324                    ),
10325                    None,
10326                ))
10327            }
10328        }
10329        timestamp
10330    }
10331
10332    fn get_random_intervals(sample_size: u64) -> Vec<ScalarValue> {
10333        const MILLISECS_IN_ONE_DAY: i64 = 86_400_000;
10334        const NANOSECS_IN_ONE_DAY: i64 = 86_400_000_000_000;
10335
10336        let vector_size = sample_size;
10337        let mut intervals = vec![];
10338        let mut rng = rand::rng();
10339        const SECS_IN_ONE_DAY: i32 = 86_400;
10340        const MICROSECS_IN_ONE_DAY: i64 = 86_400_000_000;
10341        for i in 0..vector_size {
10342            if i % 4 == 0 {
10343                let days = rng.random_range(0..5000);
10344                // to not break second precision
10345                let millis = rng.random_range(0..SECS_IN_ONE_DAY) * 1000;
10346                intervals.push(ScalarValue::new_interval_dt(days, millis));
10347            } else if i % 4 == 1 {
10348                let days = rng.random_range(0..5000);
10349                let millisec = rng.random_range(0..(MILLISECS_IN_ONE_DAY as i32));
10350                intervals.push(ScalarValue::new_interval_dt(days, millisec));
10351            } else if i % 4 == 2 {
10352                let days = rng.random_range(0..5000);
10353                // to not break microsec precision
10354                let nanosec = rng.random_range(0..MICROSECS_IN_ONE_DAY) * 1000;
10355                intervals.push(ScalarValue::new_interval_mdn(0, days, nanosec));
10356            } else {
10357                let days = rng.random_range(0..5000);
10358                let nanosec = rng.random_range(0..NANOSECS_IN_ONE_DAY);
10359                intervals.push(ScalarValue::new_interval_mdn(0, days, nanosec));
10360            }
10361        }
10362        intervals
10363    }
10364
10365    fn union_fields() -> UnionFields {
10366        [
10367            (0, Arc::new(Field::new("A", DataType::Int32, true))),
10368            (1, Arc::new(Field::new("B", DataType::Float64, true))),
10369        ]
10370        .into_iter()
10371        .collect()
10372    }
10373
10374    #[test]
10375    fn sparse_scalar_union_is_null() {
10376        let sparse_scalar = ScalarValue::Union(
10377            Some((0_i8, Box::new(ScalarValue::Int32(None)))),
10378            union_fields(),
10379            UnionMode::Sparse,
10380        );
10381        assert!(sparse_scalar.is_null());
10382    }
10383
10384    #[test]
10385    fn dense_scalar_union_is_null() {
10386        let dense_scalar = ScalarValue::Union(
10387            Some((0_i8, Box::new(ScalarValue::Int32(None)))),
10388            union_fields(),
10389            UnionMode::Dense,
10390        );
10391        assert!(dense_scalar.is_null());
10392    }
10393
10394    #[test]
10395    fn cast_date_to_timestamp_overflow_returns_error() {
10396        let scalar = ScalarValue::Date32(Some(i32::MAX));
10397        let err = scalar
10398            .cast_to(&DataType::Timestamp(TimeUnit::Nanosecond, None))
10399            .expect_err("expected cast to fail");
10400        assert!(
10401            err.to_string()
10402                .contains("converted value exceeds the representable i64 range"),
10403            "unexpected error: {err}"
10404        );
10405    }
10406
10407    #[test]
10408    fn safe_cast_date_to_timestamp_overflow_returns_null() {
10409        let scalar = ScalarValue::Date32(Some(i32::MAX));
10410        let safe_options = CastOptions {
10411            safe: true,
10412            ..DEFAULT_CAST_OPTIONS
10413        };
10414
10415        let casted = scalar
10416            .cast_to_with_options(
10417                &DataType::Timestamp(TimeUnit::Nanosecond, None),
10418                &safe_options,
10419            )
10420            .expect("expected safe cast to return null");
10421
10422        assert_eq!(casted, ScalarValue::TimestampNanosecond(None, None));
10423    }
10424
10425    #[test]
10426    fn cast_timestamp_to_timestamp_overflow_returns_error() {
10427        let scalar = ScalarValue::TimestampSecond(Some(i64::MAX), None);
10428        let err = scalar
10429            .cast_to(&DataType::Timestamp(TimeUnit::Nanosecond, None))
10430            .expect_err("expected cast to fail");
10431        assert!(
10432            err.to_string()
10433                .contains("converted value exceeds the representable i64 range"),
10434            "unexpected error: {err}"
10435        );
10436    }
10437
10438    #[test]
10439    fn safe_cast_timestamp_to_timestamp_overflow_returns_null() {
10440        let scalar = ScalarValue::TimestampSecond(Some(i64::MAX), None);
10441        let safe_options = CastOptions {
10442            safe: true,
10443            ..DEFAULT_CAST_OPTIONS
10444        };
10445
10446        let casted = scalar
10447            .cast_to_with_options(
10448                &DataType::Timestamp(TimeUnit::Nanosecond, None),
10449                &safe_options,
10450            )
10451            .expect("expected safe cast to return null");
10452
10453        assert_eq!(casted, ScalarValue::TimestampNanosecond(None, None));
10454    }
10455
10456    #[test]
10457    fn null_dictionary_scalar_produces_null_dictionary_array() {
10458        let dictionary_scalar = ScalarValue::Dictionary(
10459            Box::new(DataType::Int32),
10460            Box::new(ScalarValue::Null),
10461        );
10462        assert!(dictionary_scalar.is_null());
10463        let dictionary_array = dictionary_scalar.to_array().unwrap();
10464        assert!(dictionary_array.is_null(0));
10465    }
10466
10467    #[test]
10468    fn test_scalar_value_try_new_null() {
10469        let scalars = vec![
10470            ScalarValue::try_new_null(&DataType::Boolean).unwrap(),
10471            ScalarValue::try_new_null(&DataType::Int8).unwrap(),
10472            ScalarValue::try_new_null(&DataType::Int16).unwrap(),
10473            ScalarValue::try_new_null(&DataType::Int32).unwrap(),
10474            ScalarValue::try_new_null(&DataType::Int64).unwrap(),
10475            ScalarValue::try_new_null(&DataType::UInt8).unwrap(),
10476            ScalarValue::try_new_null(&DataType::UInt16).unwrap(),
10477            ScalarValue::try_new_null(&DataType::UInt32).unwrap(),
10478            ScalarValue::try_new_null(&DataType::UInt64).unwrap(),
10479            ScalarValue::try_new_null(&DataType::Float16).unwrap(),
10480            ScalarValue::try_new_null(&DataType::Float32).unwrap(),
10481            ScalarValue::try_new_null(&DataType::Float64).unwrap(),
10482            ScalarValue::try_new_null(&DataType::Decimal128(42, 42)).unwrap(),
10483            ScalarValue::try_new_null(&DataType::Decimal256(42, 42)).unwrap(),
10484            ScalarValue::try_new_null(&DataType::Utf8).unwrap(),
10485            ScalarValue::try_new_null(&DataType::LargeUtf8).unwrap(),
10486            ScalarValue::try_new_null(&DataType::Utf8View).unwrap(),
10487            ScalarValue::try_new_null(&DataType::Binary).unwrap(),
10488            ScalarValue::try_new_null(&DataType::BinaryView).unwrap(),
10489            ScalarValue::try_new_null(&DataType::FixedSizeBinary(42)).unwrap(),
10490            ScalarValue::try_new_null(&DataType::LargeBinary).unwrap(),
10491            ScalarValue::try_new_null(&DataType::Date32).unwrap(),
10492            ScalarValue::try_new_null(&DataType::Date64).unwrap(),
10493            ScalarValue::try_new_null(&DataType::Time32(TimeUnit::Second)).unwrap(),
10494            ScalarValue::try_new_null(&DataType::Time32(TimeUnit::Millisecond)).unwrap(),
10495            ScalarValue::try_new_null(&DataType::Time64(TimeUnit::Microsecond)).unwrap(),
10496            ScalarValue::try_new_null(&DataType::Time64(TimeUnit::Nanosecond)).unwrap(),
10497            ScalarValue::try_new_null(&DataType::Timestamp(TimeUnit::Second, None))
10498                .unwrap(),
10499            ScalarValue::try_new_null(&DataType::Timestamp(TimeUnit::Millisecond, None))
10500                .unwrap(),
10501            ScalarValue::try_new_null(&DataType::Timestamp(TimeUnit::Microsecond, None))
10502                .unwrap(),
10503            ScalarValue::try_new_null(&DataType::Timestamp(TimeUnit::Nanosecond, None))
10504                .unwrap(),
10505            ScalarValue::try_new_null(&DataType::Interval(IntervalUnit::YearMonth))
10506                .unwrap(),
10507            ScalarValue::try_new_null(&DataType::Interval(IntervalUnit::DayTime))
10508                .unwrap(),
10509            ScalarValue::try_new_null(&DataType::Interval(IntervalUnit::MonthDayNano))
10510                .unwrap(),
10511            ScalarValue::try_new_null(&DataType::Duration(TimeUnit::Second)).unwrap(),
10512            ScalarValue::try_new_null(&DataType::Duration(TimeUnit::Microsecond))
10513                .unwrap(),
10514            ScalarValue::try_new_null(&DataType::Duration(TimeUnit::Nanosecond)).unwrap(),
10515            ScalarValue::try_new_null(&DataType::Null).unwrap(),
10516        ];
10517        assert!(scalars.iter().all(|s| s.is_null()));
10518
10519        let field_ref = Arc::new(Field::new("foo", DataType::Int32, true));
10520        let map_field_ref = Arc::new(Field::new(
10521            "foo",
10522            DataType::Struct(Fields::from(vec![
10523                Field::new("bar", DataType::Utf8, true),
10524                Field::new("baz", DataType::Int32, true),
10525            ])),
10526            true,
10527        ));
10528        let scalars = [
10529            ScalarValue::try_new_null(&DataType::List(Arc::clone(&field_ref))).unwrap(),
10530            ScalarValue::try_new_null(&DataType::LargeList(Arc::clone(&field_ref)))
10531                .unwrap(),
10532            ScalarValue::try_new_null(&DataType::FixedSizeList(
10533                Arc::clone(&field_ref),
10534                42,
10535            ))
10536            .unwrap(),
10537            ScalarValue::try_new_null(&DataType::ListView(Arc::clone(&field_ref)))
10538                .unwrap(),
10539            ScalarValue::try_new_null(&DataType::LargeListView(Arc::clone(&field_ref)))
10540                .unwrap(),
10541            ScalarValue::try_new_null(&DataType::Struct(
10542                vec![Arc::clone(&field_ref)].into(),
10543            ))
10544            .unwrap(),
10545            ScalarValue::try_new_null(&DataType::Map(map_field_ref, false)).unwrap(),
10546            ScalarValue::try_new_null(&DataType::Union(
10547                UnionFields::try_new(vec![42], vec![field_ref]).unwrap(),
10548                UnionMode::Dense,
10549            ))
10550            .unwrap(),
10551        ];
10552        assert!(scalars.iter().all(|s| s.is_null()));
10553    }
10554
10555    // `err.to_string()` depends on backtrace being present (may have backtrace appended)
10556    // `err.strip_backtrace()` also depends on backtrace being present (may have "This was likely caused by ..." stripped)
10557    fn assert_starts_with(actual: impl AsRef<str>, expected_prefix: impl AsRef<str>) {
10558        let actual = actual.as_ref();
10559        let expected_prefix = expected_prefix.as_ref();
10560        assert!(
10561            actual.starts_with(expected_prefix),
10562            "Expected '{actual}' to start with '{expected_prefix}'"
10563        );
10564    }
10565
10566    #[test]
10567    fn test_new_default() {
10568        // Test numeric types
10569        assert_eq!(
10570            ScalarValue::new_default(&DataType::Int32).unwrap(),
10571            ScalarValue::Int32(Some(0))
10572        );
10573        assert_eq!(
10574            ScalarValue::new_default(&DataType::Float64).unwrap(),
10575            ScalarValue::Float64(Some(0.0))
10576        );
10577        assert_eq!(
10578            ScalarValue::new_default(&DataType::Boolean).unwrap(),
10579            ScalarValue::Boolean(Some(false))
10580        );
10581
10582        // Test string types
10583        assert_eq!(
10584            ScalarValue::new_default(&DataType::Utf8).unwrap(),
10585            ScalarValue::Utf8(Some("".to_string()))
10586        );
10587        assert_eq!(
10588            ScalarValue::new_default(&DataType::LargeUtf8).unwrap(),
10589            ScalarValue::LargeUtf8(Some("".to_string()))
10590        );
10591
10592        // Test binary types
10593        assert_eq!(
10594            ScalarValue::new_default(&DataType::Binary).unwrap(),
10595            ScalarValue::Binary(Some(vec![]))
10596        );
10597
10598        // Test fixed size binary
10599        assert_eq!(
10600            ScalarValue::new_default(&DataType::FixedSizeBinary(5)).unwrap(),
10601            ScalarValue::FixedSizeBinary(5, Some(vec![0, 0, 0, 0, 0]))
10602        );
10603
10604        // Test temporal types
10605        assert_eq!(
10606            ScalarValue::new_default(&DataType::Date32).unwrap(),
10607            ScalarValue::Date32(Some(0))
10608        );
10609        assert_eq!(
10610            ScalarValue::new_default(&DataType::Time32(TimeUnit::Second)).unwrap(),
10611            ScalarValue::Time32Second(Some(0))
10612        );
10613
10614        // Test decimal types
10615        assert_eq!(
10616            ScalarValue::new_default(&DataType::Decimal128(10, 2)).unwrap(),
10617            ScalarValue::Decimal128(Some(0), 10, 2)
10618        );
10619
10620        // Test list type
10621        let list_field = Field::new_list_field(DataType::Int32, true);
10622        let list_result =
10623            ScalarValue::new_default(&DataType::List(Arc::new(list_field.clone())))
10624                .unwrap();
10625        match list_result {
10626            ScalarValue::List(arr) => {
10627                assert_eq!(arr.len(), 1);
10628                assert_eq!(arr.value_length(0), 0); // empty list
10629            }
10630            _ => panic!("Expected List"),
10631        }
10632
10633        let list_field = Field::new_list_field(DataType::Int32, true);
10634        let list_result =
10635            ScalarValue::new_default(&DataType::LargeList(Arc::new(list_field.clone())))
10636                .unwrap();
10637        match list_result {
10638            ScalarValue::LargeList(arr) => {
10639                assert_eq!(arr.len(), 1);
10640                assert_eq!(arr.value_length(0), 0); // empty list
10641            }
10642            _ => panic!("Expected LargeList"),
10643        }
10644
10645        let list_result =
10646            ScalarValue::new_default(&DataType::ListView(Arc::new(list_field.clone())))
10647                .unwrap();
10648        match list_result {
10649            ScalarValue::ListView(arr) => {
10650                assert_eq!(arr.len(), 1);
10651                assert_eq!(arr.value_size(0), 0); // empty list
10652            }
10653            _ => panic!("Expected ListView"),
10654        }
10655
10656        let list_result = ScalarValue::new_default(&DataType::LargeListView(Arc::new(
10657            list_field.clone(),
10658        )))
10659        .unwrap();
10660        match list_result {
10661            ScalarValue::LargeListView(arr) => {
10662                assert_eq!(arr.len(), 1);
10663                assert_eq!(arr.value_size(0), 0); // empty list
10664            }
10665            _ => panic!("Expected LargeListView"),
10666        }
10667
10668        // Test struct type
10669        let struct_fields = Fields::from(vec![
10670            Field::new("a", DataType::Int32, false),
10671            Field::new("b", DataType::Utf8, false),
10672        ]);
10673        let struct_result =
10674            ScalarValue::new_default(&DataType::Struct(struct_fields.clone())).unwrap();
10675        match struct_result {
10676            ScalarValue::Struct(arr) => {
10677                assert_eq!(arr.len(), 1);
10678                assert_eq!(arr.column(0).as_primitive::<Int32Type>().value(0), 0);
10679                assert_eq!(arr.column(1).as_string::<i32>().value(0), "");
10680            }
10681            _ => panic!("Expected Struct"),
10682        }
10683
10684        // Test union type
10685        let union_fields = UnionFields::try_new(
10686            vec![0, 1],
10687            vec![
10688                Field::new("i32", DataType::Int32, false),
10689                Field::new("f64", DataType::Float64, false),
10690            ],
10691        )
10692        .unwrap();
10693        let union_result = ScalarValue::new_default(&DataType::Union(
10694            union_fields.clone(),
10695            UnionMode::Sparse,
10696        ))
10697        .unwrap();
10698        match union_result {
10699            ScalarValue::Union(Some((type_id, value)), _, _) => {
10700                assert_eq!(type_id, 0);
10701                assert_eq!(*value, ScalarValue::Int32(Some(0)));
10702            }
10703            _ => panic!("Expected Union"),
10704        }
10705    }
10706
10707    #[test]
10708    fn test_scalar_min() {
10709        // Test integer types
10710        assert_eq!(
10711            ScalarValue::min(&DataType::Int8),
10712            Some(ScalarValue::Int8(Some(i8::MIN)))
10713        );
10714        assert_eq!(
10715            ScalarValue::min(&DataType::Int32),
10716            Some(ScalarValue::Int32(Some(i32::MIN)))
10717        );
10718        assert_eq!(
10719            ScalarValue::min(&DataType::UInt8),
10720            Some(ScalarValue::UInt8(Some(0)))
10721        );
10722        assert_eq!(
10723            ScalarValue::min(&DataType::UInt64),
10724            Some(ScalarValue::UInt64(Some(0)))
10725        );
10726
10727        // Test float types
10728        assert_eq!(
10729            ScalarValue::min(&DataType::Float32),
10730            Some(ScalarValue::Float32(Some(f32::NEG_INFINITY)))
10731        );
10732        assert_eq!(
10733            ScalarValue::min(&DataType::Float64),
10734            Some(ScalarValue::Float64(Some(f64::NEG_INFINITY)))
10735        );
10736
10737        // Test decimal types
10738        let decimal_min = ScalarValue::min(&DataType::Decimal128(5, 2)).unwrap();
10739        match decimal_min {
10740            ScalarValue::Decimal128(Some(val), 5, 2) => {
10741                assert_eq!(val, -99999); // -999.99 with scale 2
10742            }
10743            _ => panic!("Expected Decimal128"),
10744        }
10745
10746        // Test temporal types
10747        assert_eq!(
10748            ScalarValue::min(&DataType::Date32),
10749            Some(ScalarValue::Date32(Some(i32::MIN)))
10750        );
10751        assert_eq!(
10752            ScalarValue::min(&DataType::Time32(TimeUnit::Second)),
10753            Some(ScalarValue::Time32Second(Some(0)))
10754        );
10755        assert_eq!(
10756            ScalarValue::min(&DataType::Timestamp(TimeUnit::Nanosecond, None)),
10757            Some(ScalarValue::TimestampNanosecond(Some(i64::MIN), None))
10758        );
10759
10760        // Test duration types
10761        assert_eq!(
10762            ScalarValue::min(&DataType::Duration(TimeUnit::Second)),
10763            Some(ScalarValue::DurationSecond(Some(i64::MIN)))
10764        );
10765
10766        // Test unsupported types
10767        assert_eq!(ScalarValue::min(&DataType::Utf8), None);
10768        assert_eq!(ScalarValue::min(&DataType::Binary), None);
10769        assert_eq!(
10770            ScalarValue::min(&DataType::List(Arc::new(Field::new(
10771                "item",
10772                DataType::Int32,
10773                true
10774            )))),
10775            None
10776        );
10777        assert_eq!(
10778            ScalarValue::min(&DataType::LargeList(Arc::new(Field::new(
10779                "item",
10780                DataType::Int32,
10781                true
10782            )))),
10783            None
10784        );
10785        assert_eq!(
10786            ScalarValue::min(&DataType::ListView(Arc::new(Field::new(
10787                "item",
10788                DataType::Int32,
10789                true
10790            )))),
10791            None
10792        );
10793        assert_eq!(
10794            ScalarValue::min(&DataType::LargeListView(Arc::new(Field::new(
10795                "item",
10796                DataType::Int32,
10797                true
10798            )))),
10799            None
10800        );
10801    }
10802
10803    #[test]
10804    fn test_scalar_max() {
10805        // Test integer types
10806        assert_eq!(
10807            ScalarValue::max(&DataType::Int8),
10808            Some(ScalarValue::Int8(Some(i8::MAX)))
10809        );
10810        assert_eq!(
10811            ScalarValue::max(&DataType::Int32),
10812            Some(ScalarValue::Int32(Some(i32::MAX)))
10813        );
10814        assert_eq!(
10815            ScalarValue::max(&DataType::UInt8),
10816            Some(ScalarValue::UInt8(Some(u8::MAX)))
10817        );
10818        assert_eq!(
10819            ScalarValue::max(&DataType::UInt64),
10820            Some(ScalarValue::UInt64(Some(u64::MAX)))
10821        );
10822
10823        // Test float types
10824        assert_eq!(
10825            ScalarValue::max(&DataType::Float32),
10826            Some(ScalarValue::Float32(Some(f32::INFINITY)))
10827        );
10828        assert_eq!(
10829            ScalarValue::max(&DataType::Float64),
10830            Some(ScalarValue::Float64(Some(f64::INFINITY)))
10831        );
10832
10833        // Test decimal types
10834        let decimal_max = ScalarValue::max(&DataType::Decimal128(5, 2)).unwrap();
10835        match decimal_max {
10836            ScalarValue::Decimal128(Some(val), 5, 2) => {
10837                assert_eq!(val, 99999); // 999.99 with scale 2
10838            }
10839            _ => panic!("Expected Decimal128"),
10840        }
10841
10842        // Test temporal types
10843        assert_eq!(
10844            ScalarValue::max(&DataType::Date32),
10845            Some(ScalarValue::Date32(Some(i32::MAX)))
10846        );
10847        assert_eq!(
10848            ScalarValue::max(&DataType::Time32(TimeUnit::Second)),
10849            Some(ScalarValue::Time32Second(Some(86_399))) // 23:59:59
10850        );
10851        assert_eq!(
10852            ScalarValue::max(&DataType::Time64(TimeUnit::Microsecond)),
10853            Some(ScalarValue::Time64Microsecond(Some(86_399_999_999))) // 23:59:59.999999
10854        );
10855        assert_eq!(
10856            ScalarValue::max(&DataType::Timestamp(TimeUnit::Nanosecond, None)),
10857            Some(ScalarValue::TimestampNanosecond(Some(i64::MAX), None))
10858        );
10859
10860        // Test duration types
10861        assert_eq!(
10862            ScalarValue::max(&DataType::Duration(TimeUnit::Millisecond)),
10863            Some(ScalarValue::DurationMillisecond(Some(i64::MAX)))
10864        );
10865
10866        // Test unsupported types
10867        assert_eq!(ScalarValue::max(&DataType::Utf8), None);
10868        assert_eq!(ScalarValue::max(&DataType::Binary), None);
10869        assert_eq!(
10870            ScalarValue::max(&DataType::Struct(Fields::from(vec![Field::new(
10871                "field",
10872                DataType::Int32,
10873                true
10874            )]))),
10875            None
10876        );
10877        assert_eq!(
10878            ScalarValue::max(&DataType::ListView(Arc::new(Field::new(
10879                "item",
10880                DataType::Int32,
10881                true
10882            )))),
10883            None
10884        );
10885        assert_eq!(
10886            ScalarValue::max(&DataType::LargeListView(Arc::new(Field::new(
10887                "item",
10888                DataType::Int32,
10889                true
10890            )))),
10891            None
10892        );
10893    }
10894
10895    #[test]
10896    fn test_min_max_float16() {
10897        // Test Float16 min and max
10898        let min_f16 = ScalarValue::min(&DataType::Float16).unwrap();
10899        match min_f16 {
10900            ScalarValue::Float16(Some(val)) => {
10901                assert_eq!(val, f16::NEG_INFINITY);
10902            }
10903            _ => panic!("Expected Float16"),
10904        }
10905
10906        let max_f16 = ScalarValue::max(&DataType::Float16).unwrap();
10907        match max_f16 {
10908            ScalarValue::Float16(Some(val)) => {
10909                assert_eq!(val, f16::INFINITY);
10910            }
10911            _ => panic!("Expected Float16"),
10912        }
10913    }
10914
10915    #[test]
10916    fn test_new_default_interval() {
10917        // Test all interval types
10918        assert_eq!(
10919            ScalarValue::new_default(&DataType::Interval(IntervalUnit::YearMonth))
10920                .unwrap(),
10921            ScalarValue::IntervalYearMonth(Some(0))
10922        );
10923        assert_eq!(
10924            ScalarValue::new_default(&DataType::Interval(IntervalUnit::DayTime)).unwrap(),
10925            ScalarValue::IntervalDayTime(Some(IntervalDayTime::ZERO))
10926        );
10927        assert_eq!(
10928            ScalarValue::new_default(&DataType::Interval(IntervalUnit::MonthDayNano))
10929                .unwrap(),
10930            ScalarValue::IntervalMonthDayNano(Some(IntervalMonthDayNano::ZERO))
10931        );
10932    }
10933
10934    #[test]
10935    fn test_min_max_with_timezone() {
10936        let tz = Some(Arc::from("UTC"));
10937
10938        // Test timestamp with timezone
10939        let min_ts =
10940            ScalarValue::min(&DataType::Timestamp(TimeUnit::Second, tz.clone())).unwrap();
10941        match min_ts {
10942            ScalarValue::TimestampSecond(Some(val), Some(tz_str)) => {
10943                assert_eq!(val, i64::MIN);
10944                assert_eq!(tz_str.as_ref(), "UTC");
10945            }
10946            _ => panic!("Expected TimestampSecond with timezone"),
10947        }
10948
10949        let max_ts =
10950            ScalarValue::max(&DataType::Timestamp(TimeUnit::Millisecond, tz.clone()))
10951                .unwrap();
10952        match max_ts {
10953            ScalarValue::TimestampMillisecond(Some(val), Some(tz_str)) => {
10954                assert_eq!(val, i64::MAX);
10955                assert_eq!(tz_str.as_ref(), "UTC");
10956            }
10957            _ => panic!("Expected TimestampMillisecond with timezone"),
10958        }
10959    }
10960
10961    #[test]
10962    fn test_views_minimize_memory() {
10963        let value = "this string is longer than 12 bytes".to_string();
10964
10965        let scalar = ScalarValue::Utf8View(Some(value.clone()));
10966        let array = scalar.to_array_of_size(10).unwrap();
10967        let array = array.as_string_view();
10968        let buffers = array.data_buffers();
10969        assert_eq!(1, buffers.len());
10970        // Ensure we only have a single copy of the value string
10971        assert_eq!(value.len(), buffers[0].len());
10972
10973        // Same but for BinaryView
10974        let scalar = ScalarValue::BinaryView(Some(value.bytes().collect()));
10975        let array = scalar.to_array_of_size(10).unwrap();
10976        let array = array.as_binary_view();
10977        let buffers = array.data_buffers();
10978        assert_eq!(1, buffers.len());
10979        assert_eq!(value.len(), buffers[0].len());
10980    }
10981
10982    #[test]
10983    fn test_to_array_of_size_run_end_encoded() {
10984        fn run_test<R: RunEndIndexType>() {
10985            let value = Box::new(ScalarValue::Float32(Some(1.0)));
10986            let size = 5;
10987            let scalar = ScalarValue::RunEndEncoded(
10988                Field::new("run_ends", R::DATA_TYPE, false).into(),
10989                Field::new("values", DataType::Float32, true).into(),
10990                value.clone(),
10991            );
10992            let array = scalar.to_array_of_size(size).unwrap();
10993            let array = array.as_run::<R>();
10994            let array = array.downcast::<Float32Array>().unwrap();
10995            assert_eq!(vec![Some(1.0); size], array.into_iter().collect::<Vec<_>>());
10996            assert_eq!(1, array.values().len());
10997        }
10998
10999        run_test::<Int16Type>();
11000        run_test::<Int32Type>();
11001        run_test::<Int64Type>();
11002
11003        let scalar = ScalarValue::RunEndEncoded(
11004            Field::new("run_ends", DataType::Int16, false).into(),
11005            Field::new("values", DataType::Float32, true).into(),
11006            Box::new(ScalarValue::Float32(Some(1.0))),
11007        );
11008        let err = scalar.to_array_of_size(i16::MAX as usize + 10).unwrap_err();
11009        assert_eq!(
11010            "Execution error: Cannot construct RunArray of size 32777: Overflows run-ends type Int16",
11011            err.to_string()
11012        )
11013    }
11014
11015    #[test]
11016    fn test_eq_array_run_end_encoded() {
11017        let run_ends = Int16Array::from(vec![1, 3]);
11018        let values = Float32Array::from(vec![None, Some(1.0)]);
11019        let run_array =
11020            Arc::new(RunArray::try_new(&run_ends, &values).unwrap()) as ArrayRef;
11021
11022        let scalar = ScalarValue::RunEndEncoded(
11023            Field::new("run_ends", DataType::Int16, false).into(),
11024            Field::new("values", DataType::Float32, true).into(),
11025            Box::new(ScalarValue::Float32(None)),
11026        );
11027        assert!(scalar.eq_array(&run_array, 0).unwrap());
11028
11029        let scalar = ScalarValue::RunEndEncoded(
11030            Field::new("run_ends", DataType::Int16, false).into(),
11031            Field::new("values", DataType::Float32, true).into(),
11032            Box::new(ScalarValue::Float32(Some(1.0))),
11033        );
11034        assert!(scalar.eq_array(&run_array, 1).unwrap());
11035        assert!(scalar.eq_array(&run_array, 2).unwrap());
11036
11037        // value types must match
11038        let scalar = ScalarValue::RunEndEncoded(
11039            Field::new("run_ends", DataType::Int16, false).into(),
11040            Field::new("values", DataType::Float64, true).into(),
11041            Box::new(ScalarValue::Float64(Some(1.0))),
11042        );
11043        let err = scalar.eq_array(&run_array, 1).unwrap_err();
11044        let expected = "Internal error: could not cast array of type Float32 to arrow_array::array::primitive_array::PrimitiveArray<arrow_array::types::Float64Type>";
11045        assert!(err.to_string().starts_with(expected));
11046
11047        // run ends type must match
11048        let scalar = ScalarValue::RunEndEncoded(
11049            Field::new("run_ends", DataType::Int32, false).into(),
11050            Field::new("values", DataType::Float32, true).into(),
11051            Box::new(ScalarValue::Float32(None)),
11052        );
11053        let err = scalar.eq_array(&run_array, 0).unwrap_err();
11054        let expected = "Internal error: could not cast array of type RunEndEncoded(\"run_ends\": non-null Int16, \"values\": Float32) to arrow_array::array::run_array::RunArray<arrow_array::types::Int32Type>";
11055        assert!(err.to_string().starts_with(expected));
11056    }
11057
11058    #[test]
11059    fn test_iter_to_array_run_end_encoded() {
11060        let run_ends_field = Arc::new(Field::new("run_ends", DataType::Int16, false));
11061        let values_field = Arc::new(Field::new("values", DataType::Int64, true));
11062        let scalars = vec![
11063            ScalarValue::RunEndEncoded(
11064                Arc::clone(&run_ends_field),
11065                Arc::clone(&values_field),
11066                Box::new(ScalarValue::Int64(Some(1))),
11067            ),
11068            ScalarValue::RunEndEncoded(
11069                Arc::clone(&run_ends_field),
11070                Arc::clone(&values_field),
11071                Box::new(ScalarValue::Int64(Some(1))),
11072            ),
11073            ScalarValue::RunEndEncoded(
11074                Arc::clone(&run_ends_field),
11075                Arc::clone(&values_field),
11076                Box::new(ScalarValue::Int64(None)),
11077            ),
11078            ScalarValue::RunEndEncoded(
11079                Arc::clone(&run_ends_field),
11080                Arc::clone(&values_field),
11081                Box::new(ScalarValue::Int64(Some(2))),
11082            ),
11083            ScalarValue::RunEndEncoded(
11084                Arc::clone(&run_ends_field),
11085                Arc::clone(&values_field),
11086                Box::new(ScalarValue::Int64(Some(2))),
11087            ),
11088            ScalarValue::RunEndEncoded(
11089                Arc::clone(&run_ends_field),
11090                Arc::clone(&values_field),
11091                Box::new(ScalarValue::Int64(Some(2))),
11092            ),
11093        ];
11094
11095        let run_array = ScalarValue::iter_to_array(scalars).unwrap();
11096        let expected = RunArray::try_new(
11097            &Int16Array::from(vec![2, 3, 6]),
11098            &Int64Array::from(vec![Some(1), None, Some(2)]),
11099        )
11100        .unwrap();
11101        assert_eq!(&expected as &dyn Array, run_array.as_ref());
11102
11103        // inconsistent run-ends type
11104        let scalars = vec![
11105            ScalarValue::RunEndEncoded(
11106                Arc::clone(&run_ends_field),
11107                Arc::clone(&values_field),
11108                Box::new(ScalarValue::Int64(Some(1))),
11109            ),
11110            ScalarValue::RunEndEncoded(
11111                Field::new("run_ends", DataType::Int32, false).into(),
11112                Arc::clone(&values_field),
11113                Box::new(ScalarValue::Int64(Some(1))),
11114            ),
11115        ];
11116        let err = ScalarValue::iter_to_array(scalars).unwrap_err();
11117        let expected = "Execution error: Expected RunEndEncoded scalar with run-ends field Field { \"run_ends\": Int16 } but got: RunEndEncoded(Field { name: \"run_ends\", data_type: Int32 }, Field { name: \"values\", data_type: Int64, nullable: true }, Int64(1))";
11118        assert!(err.to_string().starts_with(expected));
11119
11120        // inconsistent value type
11121        let scalars = vec![
11122            ScalarValue::RunEndEncoded(
11123                Arc::clone(&run_ends_field),
11124                Arc::clone(&values_field),
11125                Box::new(ScalarValue::Int64(Some(1))),
11126            ),
11127            ScalarValue::RunEndEncoded(
11128                Arc::clone(&run_ends_field),
11129                Field::new("values", DataType::Int32, true).into(),
11130                Box::new(ScalarValue::Int32(Some(1))),
11131            ),
11132        ];
11133        let err = ScalarValue::iter_to_array(scalars).unwrap_err();
11134        let expected = "Execution error: Expected RunEndEncoded scalar with run-ends field Field { \"run_ends\": Int16 } but got: RunEndEncoded(Field { name: \"run_ends\", data_type: Int16 }, Field { name: \"values\", data_type: Int32, nullable: true }, Int32(1))";
11135        assert!(err.to_string().starts_with(expected));
11136
11137        // inconsistent scalars type
11138        let scalars = vec![
11139            ScalarValue::RunEndEncoded(
11140                Arc::clone(&run_ends_field),
11141                Arc::clone(&values_field),
11142                Box::new(ScalarValue::Int64(Some(1))),
11143            ),
11144            ScalarValue::Int64(Some(1)),
11145        ];
11146        let err = ScalarValue::iter_to_array(scalars).unwrap_err();
11147        let expected = "Execution error: Expected RunEndEncoded scalar with run-ends field Field { \"run_ends\": Int16 } but got: Int64(1)";
11148        assert!(err.to_string().starts_with(expected));
11149    }
11150
11151    #[test]
11152    fn test_convert_array_to_scalar_vec() {
11153        // 1: Regular ListArray
11154        let list = ListArray::from_iter_primitive::<Int64Type, _, _>(vec![
11155            Some(vec![Some(1), Some(2)]),
11156            None,
11157            Some(vec![Some(3), None, Some(4)]),
11158        ]);
11159        let converted = ScalarValue::convert_array_to_scalar_vec(&list).unwrap();
11160        assert_eq!(
11161            converted,
11162            vec![
11163                Some(vec![
11164                    ScalarValue::Int64(Some(1)),
11165                    ScalarValue::Int64(Some(2))
11166                ]),
11167                None,
11168                Some(vec![
11169                    ScalarValue::Int64(Some(3)),
11170                    ScalarValue::Int64(None),
11171                    ScalarValue::Int64(Some(4))
11172                ]),
11173            ]
11174        );
11175
11176        // 2: Regular LargeListArray
11177        let large_list = LargeListArray::from_iter_primitive::<Int64Type, _, _>(vec![
11178            Some(vec![Some(1), Some(2)]),
11179            None,
11180            Some(vec![Some(3), None, Some(4)]),
11181        ]);
11182        let converted = ScalarValue::convert_array_to_scalar_vec(&large_list).unwrap();
11183        assert_eq!(
11184            converted,
11185            vec![
11186                Some(vec![
11187                    ScalarValue::Int64(Some(1)),
11188                    ScalarValue::Int64(Some(2))
11189                ]),
11190                None,
11191                Some(vec![
11192                    ScalarValue::Int64(Some(3)),
11193                    ScalarValue::Int64(None),
11194                    ScalarValue::Int64(Some(4))
11195                ]),
11196            ]
11197        );
11198
11199        // 3: Funky (null slot has non-zero list offsets)
11200        // Offsets + Values looks like this: [[1, 2], [3, 4], [5]]
11201        // But with NullBuffer it's like this: [[1, 2], NULL, [5]]
11202        let funky = ListArray::new(
11203            Field::new_list_field(DataType::Int64, true).into(),
11204            OffsetBuffer::new(vec![0, 2, 4, 5].into()),
11205            Arc::new(Int64Array::from(vec![1, 2, 3, 4, 5, 6])),
11206            Some(NullBuffer::from(vec![true, false, true])),
11207        );
11208        let converted = ScalarValue::convert_array_to_scalar_vec(&funky).unwrap();
11209        assert_eq!(
11210            converted,
11211            vec![
11212                Some(vec![
11213                    ScalarValue::Int64(Some(1)),
11214                    ScalarValue::Int64(Some(2))
11215                ]),
11216                None,
11217                Some(vec![ScalarValue::Int64(Some(5))]),
11218            ]
11219        );
11220
11221        // 4: Offsets + Values looks like this: [[1, 2], [], [5]]
11222        // But with NullBuffer it's like this: [[1, 2], NULL, [5]]
11223        // The converted result is: [[1, 2], None, [5]]
11224        let array4 = ListArray::new(
11225            Field::new_list_field(DataType::Int64, true).into(),
11226            OffsetBuffer::new(vec![0, 2, 2, 5].into()),
11227            Arc::new(Int64Array::from(vec![1, 2, 3, 4, 5, 6])),
11228            Some(NullBuffer::from(vec![true, false, true])),
11229        );
11230        let converted = ScalarValue::convert_array_to_scalar_vec(&array4).unwrap();
11231        assert_eq!(
11232            converted,
11233            vec![
11234                Some(vec![
11235                    ScalarValue::Int64(Some(1)),
11236                    ScalarValue::Int64(Some(2))
11237                ]),
11238                None,
11239                Some(vec![
11240                    ScalarValue::Int64(Some(3)),
11241                    ScalarValue::Int64(Some(4)),
11242                    ScalarValue::Int64(Some(5)),
11243                ]),
11244            ]
11245        );
11246
11247        // 5: Offsets + Values looks like this: [[1, 2], [], [5]]
11248        // Same as 4, but the middle array is not null, so after conversion it's empty.
11249        let array5 = ListArray::new(
11250            Field::new_list_field(DataType::Int64, true).into(),
11251            OffsetBuffer::new(vec![0, 2, 2, 5].into()),
11252            Arc::new(Int64Array::from(vec![1, 2, 3, 4, 5, 6])),
11253            Some(NullBuffer::from(vec![true, true, true])),
11254        );
11255        let converted = ScalarValue::convert_array_to_scalar_vec(&array5).unwrap();
11256        assert_eq!(
11257            converted,
11258            vec![
11259                Some(vec![
11260                    ScalarValue::Int64(Some(1)),
11261                    ScalarValue::Int64(Some(2))
11262                ]),
11263                Some(vec![]),
11264                Some(vec![
11265                    ScalarValue::Int64(Some(3)),
11266                    ScalarValue::Int64(Some(4)),
11267                    ScalarValue::Int64(Some(5)),
11268                ]),
11269            ]
11270        );
11271
11272        // 6: Regular ListViewArray
11273        let list = ListViewArray::from_iter_primitive::<Int64Type, _, _>(vec![
11274            Some(vec![Some(1), Some(2)]),
11275            None,
11276            Some(vec![Some(3), None, Some(4)]),
11277        ]);
11278        let converted = ScalarValue::convert_array_to_scalar_vec(&list).unwrap();
11279        assert_eq!(
11280            converted,
11281            vec![
11282                Some(vec![
11283                    ScalarValue::Int64(Some(1)),
11284                    ScalarValue::Int64(Some(2))
11285                ]),
11286                None,
11287                Some(vec![
11288                    ScalarValue::Int64(Some(3)),
11289                    ScalarValue::Int64(None),
11290                    ScalarValue::Int64(Some(4))
11291                ]),
11292            ]
11293        );
11294
11295        // 7: Regular LargeListViewArray
11296        let large_list =
11297            LargeListViewArray::from_iter_primitive::<Int64Type, _, _>(vec![
11298                Some(vec![Some(1), Some(2)]),
11299                None,
11300                Some(vec![Some(3), None, Some(4)]),
11301            ]);
11302        let converted = ScalarValue::convert_array_to_scalar_vec(&large_list).unwrap();
11303        assert_eq!(
11304            converted,
11305            vec![
11306                Some(vec![
11307                    ScalarValue::Int64(Some(1)),
11308                    ScalarValue::Int64(Some(2))
11309                ]),
11310                None,
11311                Some(vec![
11312                    ScalarValue::Int64(Some(3)),
11313                    ScalarValue::Int64(None),
11314                    ScalarValue::Int64(Some(4))
11315                ]),
11316            ]
11317        );
11318    }
11319
11320    // ── compact / compact_view_buffers ───────────────────────────────────────
11321
11322    /// Builds a `StringViewArray` with `n` strings that are all longer than
11323    /// 12 bytes so they are stored in backing buffers rather than inline.
11324    fn make_long_strings(n: usize) -> StringViewArray {
11325        let mut b = StringViewBuilder::new();
11326        for i in 0..n {
11327            b.append_value(format!("long_string_value_pad_{i:04}"));
11328        }
11329        b.finish()
11330    }
11331
11332    /// Total bytes across all backing buffers of a `StringViewArray`.
11333    fn utf8view_buffer_bytes(a: &StringViewArray) -> usize {
11334        a.data_buffers().iter().map(|b| b.len()).sum()
11335    }
11336
11337    #[test]
11338    fn test_compact_list_utf8view() {
11339        const N: usize = 50;
11340        let strings = make_long_strings(N);
11341        let one_len = strings.value(0).len();
11342        assert!(utf8view_buffer_bytes(&strings) >= N * one_len);
11343
11344        let single_row_list_array =
11345            SingleRowListArrayBuilder::new(Arc::new(strings.slice(0, 1)) as ArrayRef)
11346                .build_list_array();
11347        let mut scalar = ScalarValue::List(Arc::new(single_row_list_array));
11348        scalar.compact();
11349
11350        let ScalarValue::List(arr) = &scalar else {
11351            panic!("expected List")
11352        };
11353        assert_eq!(
11354            utf8view_buffer_bytes(arr.values().as_string_view()),
11355            one_len
11356        );
11357        assert_eq!(arr.values().as_string_view().value(0), strings.value(0));
11358    }
11359
11360    #[test]
11361    fn test_compact_large_list_utf8view() {
11362        const N: usize = 50;
11363        let strings = make_long_strings(N);
11364        let one_len = strings.value(0).len();
11365        assert!(utf8view_buffer_bytes(&strings) >= N * one_len);
11366
11367        let single_row_list_array =
11368            SingleRowListArrayBuilder::new(Arc::new(strings.slice(0, 1)) as ArrayRef)
11369                .build_large_list_array();
11370        let mut scalar = ScalarValue::LargeList(Arc::new(single_row_list_array));
11371        scalar.compact();
11372
11373        let ScalarValue::LargeList(arr) = &scalar else {
11374            panic!("expected LargeList")
11375        };
11376        assert_eq!(
11377            utf8view_buffer_bytes(arr.values().as_string_view()),
11378            one_len
11379        );
11380        assert_eq!(arr.values().as_string_view().value(0), strings.value(0));
11381    }
11382
11383    #[test]
11384    fn test_compact_fixed_size_list_utf8view() {
11385        const N: usize = 50;
11386        let strings = make_long_strings(N);
11387        let one_len = strings.value(0).len();
11388        assert!(utf8view_buffer_bytes(&strings) >= N * one_len);
11389
11390        let single_row_list_array =
11391            SingleRowListArrayBuilder::new(Arc::new(strings.slice(0, 1)) as ArrayRef)
11392                .build_fixed_size_list_array(1);
11393        let mut scalar = ScalarValue::FixedSizeList(Arc::new(single_row_list_array));
11394        scalar.compact();
11395
11396        let ScalarValue::FixedSizeList(arr) = &scalar else {
11397            panic!("expected FixedSizeList")
11398        };
11399        assert_eq!(
11400            utf8view_buffer_bytes(arr.values().as_string_view()),
11401            one_len
11402        );
11403        assert_eq!(arr.values().as_string_view().value(0), strings.value(0));
11404    }
11405
11406    #[test]
11407    fn test_compact_list_view_utf8view() {
11408        const N: usize = 50;
11409        let strings = make_long_strings(N);
11410        let one_len = strings.value(0).len();
11411        assert!(utf8view_buffer_bytes(&strings) >= N * one_len);
11412
11413        let single_row_list_array =
11414            SingleRowListArrayBuilder::new(Arc::new(strings.slice(0, 1)) as ArrayRef)
11415                .build_list_view_array();
11416        let mut scalar = ScalarValue::ListView(Arc::new(single_row_list_array));
11417        scalar.compact();
11418
11419        let ScalarValue::ListView(arr) = &scalar else {
11420            panic!("expected ListView")
11421        };
11422        assert_eq!(
11423            utf8view_buffer_bytes(arr.values().as_string_view()),
11424            one_len
11425        );
11426        assert_eq!(arr.values().as_string_view().value(0), strings.value(0));
11427    }
11428
11429    #[test]
11430    fn test_compact_large_list_view_utf8view() {
11431        const N: usize = 50;
11432        let strings = make_long_strings(N);
11433        let one_len = strings.value(0).len();
11434        assert!(utf8view_buffer_bytes(&strings) >= N * one_len);
11435
11436        let single_row_list_array =
11437            SingleRowListArrayBuilder::new(Arc::new(strings.slice(0, 1)) as ArrayRef)
11438                .build_large_list_view_array();
11439        let mut scalar = ScalarValue::LargeListView(Arc::new(single_row_list_array));
11440        scalar.compact();
11441
11442        let ScalarValue::LargeListView(arr) = &scalar else {
11443            panic!("expected LargeListView")
11444        };
11445        assert_eq!(
11446            utf8view_buffer_bytes(arr.values().as_string_view()),
11447            one_len
11448        );
11449        assert_eq!(arr.values().as_string_view().value(0), strings.value(0));
11450    }
11451
11452    #[test]
11453    fn test_compact_struct_utf8view() {
11454        const N: usize = 50;
11455        let strings = make_long_strings(N);
11456        let one_len = strings.value(0).len();
11457
11458        let field = Arc::new(Field::new("name", DataType::Utf8View, true));
11459        let struct_arr = StructArray::new(
11460            Fields::from(vec![Arc::clone(&field)]),
11461            vec![Arc::new(strings.slice(0, 1)) as ArrayRef],
11462            None,
11463        );
11464
11465        let mut scalar = ScalarValue::Struct(Arc::new(struct_arr));
11466        scalar.compact();
11467
11468        let ScalarValue::Struct(arr) = &scalar else {
11469            panic!("expected Struct")
11470        };
11471        let col = arr.column(0).as_string_view();
11472        assert_eq!(utf8view_buffer_bytes(col), one_len);
11473        assert_eq!(col.value(0), strings.value(0));
11474    }
11475
11476    #[test]
11477    fn test_compact_map_utf8view() {
11478        const N: usize = 50;
11479        let strings = make_long_strings(N);
11480        let one_len = strings.value(0).len();
11481
11482        let key_field = Arc::new(Field::new("key", DataType::Utf8View, false));
11483        let val_field = Arc::new(Field::new("value", DataType::Int32, true));
11484        let entries = StructArray::new(
11485            Fields::from(vec![Arc::clone(&key_field), Arc::clone(&val_field)]),
11486            vec![
11487                Arc::new(strings.slice(0, 1)) as ArrayRef,
11488                Arc::new(Int32Array::from(vec![1i32])) as ArrayRef,
11489            ],
11490            None,
11491        );
11492        let entries_field = Arc::new(Field::new(
11493            "entries",
11494            DataType::Struct(Fields::from(vec![key_field, val_field])),
11495            false,
11496        ));
11497        let map = MapArray::new(
11498            entries_field,
11499            OffsetBuffer::new(vec![0i32, 1].into()),
11500            entries,
11501            None,
11502            false,
11503        );
11504
11505        let mut scalar = ScalarValue::Map(Arc::new(map));
11506        scalar.compact();
11507
11508        let ScalarValue::Map(arr) = &scalar else {
11509            panic!("expected Map")
11510        };
11511        let keys = arr.entries().column(0).as_string_view();
11512        assert_eq!(utf8view_buffer_bytes(keys), one_len);
11513        assert_eq!(keys.value(0), strings.value(0));
11514    }
11515
11516    #[test]
11517    fn test_zero_size_fsl() {
11518        let s = ScalarValue::new_default(&DataType::FixedSizeList(
11519            Field::new("a", DataType::Int32, true).into(),
11520            0,
11521        ))
11522        .unwrap();
11523        assert_eq!(s.to_string(), "[]");
11524    }
11525
11526    #[test]
11527    fn test_decimal_value_bounds() {
11528        fn run_tests<D: DecimalType>() {
11529            // 0.1111, 0.2222, etc.
11530            let max_scale = D::TYPE_CONSTRUCTOR(D::MAX_PRECISION, D::MAX_SCALE);
11531            // 1.111, 2.222, etc.
11532            let max_scale_less_one =
11533                D::TYPE_CONSTRUCTOR(D::MAX_PRECISION, D::MAX_SCALE - 1);
11534            // 11.11, 22.22, etc.
11535            let max_scale_less_two =
11536                D::TYPE_CONSTRUCTOR(D::MAX_PRECISION, D::MAX_SCALE - 2);
11537
11538            // Invalid (can't represent the value)
11539            assert!(ScalarValue::new_one(&max_scale).is_err());
11540            assert!(ScalarValue::new_negative_one(&max_scale).is_err());
11541            assert!(ScalarValue::new_ten(&max_scale).is_err());
11542            assert!(ScalarValue::new_ten(&max_scale_less_one).is_err());
11543
11544            // Valid
11545            let one = ScalarValue::Int32(Some(1));
11546            let neg_one = ScalarValue::Int32(Some(-1));
11547            let ten = ScalarValue::Int32(Some(10));
11548
11549            let num = ScalarValue::new_one(&max_scale_less_one).unwrap();
11550            assert_eq!(num.cast_to(&DataType::Int32).unwrap(), one);
11551            let num = ScalarValue::new_negative_one(&max_scale_less_one).unwrap();
11552            assert_eq!(num.cast_to(&DataType::Int32).unwrap(), neg_one);
11553            let num = ScalarValue::new_ten(&max_scale_less_two).unwrap();
11554            assert_eq!(num.cast_to(&DataType::Int32).unwrap(), ten);
11555        }
11556
11557        run_tests::<Decimal32Type>();
11558        run_tests::<Decimal64Type>();
11559        run_tests::<Decimal128Type>();
11560        run_tests::<Decimal256Type>();
11561    }
11562
11563    #[test]
11564    fn test_new_list_nested_nullability_mismatch_issue_24022() {
11565        // requested element type: Struct(n: Int32 nullable=true)
11566        let requested_element_type =
11567            DataType::Struct(Fields::from(vec![Field::new("n", DataType::Int32, true)]));
11568
11569        // inferred from concrete values: Struct(n: Int32 nullable=false)
11570        let inferred_field = Field::new("n", DataType::Int32, false);
11571
11572        let value = ScalarValue::Struct(Arc::new(StructArray::from(vec![(
11573            Arc::new(inferred_field),
11574            Arc::new(Int32Array::from(vec![1])) as ArrayRef,
11575        )])));
11576
11577        let expected_struct_array = StructArray::from(vec![(
11578            Arc::new(Field::new("n", DataType::Int32, true)),
11579            Arc::new(Int32Array::from(vec![1])) as ArrayRef,
11580        )]);
11581        let expected_array = Arc::new(expected_struct_array) as ArrayRef;
11582
11583        // Test new_list
11584        let list = ScalarValue::new_list(
11585            std::slice::from_ref(&value),
11586            &requested_element_type,
11587            true,
11588        );
11589        assert_eq!(
11590            list.data_type(),
11591            &DataType::List(Arc::new(Field::new_list_field(
11592                requested_element_type.clone(),
11593                true
11594            )))
11595        );
11596        assert_eq!(&list.value(0), &expected_array);
11597
11598        // Test new_list_from_iter
11599        let list_from_iter = ScalarValue::new_list_from_iter(
11600            std::iter::once(value.clone()),
11601            &requested_element_type,
11602            true,
11603        );
11604        assert_eq!(
11605            list_from_iter.data_type(),
11606            &DataType::List(Arc::new(Field::new_list_field(
11607                requested_element_type.clone(),
11608                true
11609            )))
11610        );
11611        assert_eq!(&list_from_iter.value(0), &expected_array);
11612
11613        // Test new_large_list
11614        let large_list = ScalarValue::new_large_list(&[value], &requested_element_type);
11615        assert_eq!(
11616            large_list.data_type(),
11617            &DataType::LargeList(Arc::new(Field::new(
11618                "item",
11619                requested_element_type.clone(),
11620                true
11621            )))
11622        );
11623        assert_eq!(&large_list.value(0), &expected_array);
11624    }
11625
11626    #[test]
11627    fn test_compact_empty_struct() {
11628        // A struct scalar wraps a single-row StructArray; use a null row to also
11629        // exercise null-buffer preservation.
11630        let nulls = NullBuffer::from(vec![false]);
11631        let empty_struct = Arc::new(StructArray::new_empty_fields(1, Some(nulls)));
11632        let mut scalar = ScalarValue::Struct(empty_struct);
11633
11634        // Before fix: panics inside compact_view_buffers calling StructArray::new on 0 fields
11635        scalar.compact();
11636
11637        let ScalarValue::Struct(arr) = &scalar else {
11638            panic!("expected Struct")
11639        };
11640        assert_eq!(arr.len(), 1);
11641        assert_eq!(arr.num_columns(), 0);
11642        assert_eq!(arr.null_count(), 1);
11643        assert!(arr.is_null(0));
11644    }
11645
11646    #[test]
11647    fn test_compact_nested_empty_struct() {
11648        // 1. List of empty structs
11649        let inner_struct_field =
11650            Arc::new(Field::new("item", DataType::Struct(Fields::empty()), true));
11651        let inner_struct_arr =
11652            Arc::new(StructArray::new_empty_fields(2, None)) as ArrayRef;
11653        let list_arr = ListArray::new(
11654            inner_struct_field,
11655            OffsetBuffer::new(vec![0i32, 2].into()),
11656            inner_struct_arr,
11657            None,
11658        );
11659        let mut list_scalar = ScalarValue::List(Arc::new(list_arr));
11660        list_scalar.compact();
11661
11662        let ScalarValue::List(res_list) = &list_scalar else {
11663            panic!("expected List")
11664        };
11665        assert_eq!(res_list.len(), 1);
11666        assert_eq!(res_list.values().len(), 2);
11667
11668        // 2. Struct containing an empty struct field
11669        let empty_field = Arc::new(Field::new(
11670            "empty_child",
11671            DataType::Struct(Fields::empty()),
11672            true,
11673        ));
11674        let int_field = Arc::new(Field::new("int_child", DataType::Int32, true));
11675        let outer_struct = StructArray::new(
11676            Fields::from(vec![Arc::clone(&empty_field), Arc::clone(&int_field)]),
11677            vec![
11678                Arc::new(StructArray::new_empty_fields(2, None)) as ArrayRef,
11679                Arc::new(Int32Array::from(vec![10, 20])) as ArrayRef,
11680            ],
11681            None,
11682        );
11683        let mut outer_scalar = ScalarValue::Struct(Arc::new(outer_struct));
11684        outer_scalar.compact();
11685
11686        let ScalarValue::Struct(res_outer) = &outer_scalar else {
11687            panic!("expected Struct")
11688        };
11689        assert_eq!(res_outer.len(), 2);
11690        let child_empty = res_outer.column(0).as_struct();
11691        assert_eq!(child_empty.len(), 2);
11692    }
11693
11694    #[test]
11695    fn test_new_default_empty_struct() {
11696        let empty_struct_type = DataType::Struct(Fields::empty());
11697        let scalar = ScalarValue::new_default(&empty_struct_type).unwrap();
11698
11699        let ScalarValue::Struct(arr) = &scalar else {
11700            panic!("expected Struct")
11701        };
11702        assert_eq!(arr.len(), 1);
11703        assert_eq!(arr.null_count(), 0);
11704        assert_eq!(arr.num_columns(), 0);
11705    }
11706}
datafusion_common/scalar/mod.rs

datafusion_common/scalar/
mod.rs
