Skip to main content

vortex_array/scalar/
proto.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4//! Protobuf serialization and deserialization for scalars.
5
6use num_traits::ToBytes;
7use num_traits::ToPrimitive;
8use prost::Message;
9use vortex_buffer::BufferString;
10use vortex_buffer::ByteBuffer;
11use vortex_error::VortexExpect;
12use vortex_error::VortexResult;
13use vortex_error::vortex_bail;
14use vortex_error::vortex_ensure;
15use vortex_error::vortex_ensure_eq;
16use vortex_error::vortex_err;
17use vortex_session::VortexSession;
18
19use crate::dtype::DType;
20use crate::dtype::PType;
21use crate::dtype::half::f16;
22use crate::dtype::i256;
23use crate::proto::scalar as pb;
24use crate::proto::scalar::ListValue;
25use crate::proto::scalar::UnionValue as PbUnionValue;
26use crate::proto::scalar::scalar_value::Kind;
27use crate::scalar::DecimalValue;
28use crate::scalar::PValue;
29use crate::scalar::Scalar;
30use crate::scalar::ScalarValue;
31use crate::scalar::UnionValue;
32
33////////////////////////////////////////////////////////////////////////////////////////////////////
34// Serialize INTO proto.
35////////////////////////////////////////////////////////////////////////////////////////////////////
36
37impl From<&Scalar> for pb::Scalar {
38    fn from(value: &Scalar) -> Self {
39        pb::Scalar {
40            dtype: Some(
41                (value.dtype())
42                    .try_into()
43                    .vortex_expect("Failed to convert DType to proto"),
44            ),
45            value: Some(Box::new(ScalarValue::to_proto(value.value()))),
46        }
47    }
48}
49
50impl ScalarValue {
51    /// Ideally, we would not have this function and instead implement this `From` implementation:
52    ///
53    /// ```ignore
54    /// impl From<Option<&ScalarValue>> for pb::ScalarValue { ... }
55    /// ```
56    ///
57    /// However, we are not allowed to do this because of the Orphan rule (`Option` and
58    /// `pb::ScalarValue` are not types defined in this crate). So we must make this a method on
59    /// `vortex_array::scalar::ScalarValue` directly.
60    pub fn to_proto(this: Option<&Self>) -> pb::ScalarValue {
61        match this {
62            None => pb::ScalarValue {
63                kind: Some(Kind::NullValue(0)),
64            },
65            Some(this) => pb::ScalarValue::from(this),
66        }
67    }
68
69    /// Serialize an optional [`ScalarValue`] to protobuf bytes (handles null values).
70    pub fn to_proto_bytes<B: Default + bytes::BufMut>(value: Option<&ScalarValue>) -> B {
71        let proto = Self::to_proto(value);
72        let mut buf = B::default();
73        proto
74            .encode(&mut buf)
75            .vortex_expect("Failed to encode scalar value");
76        buf
77    }
78}
79
80impl From<&ScalarValue> for pb::ScalarValue {
81    fn from(value: &ScalarValue) -> Self {
82        match value {
83            ScalarValue::Bool(v) => pb::ScalarValue {
84                kind: Some(Kind::BoolValue(*v)),
85            },
86            ScalarValue::Primitive(v) => pb::ScalarValue::from(v),
87            ScalarValue::Decimal(v) => {
88                let inner_value = match v {
89                    DecimalValue::I8(v) => v.to_le_bytes().to_vec(),
90                    DecimalValue::I16(v) => v.to_le_bytes().to_vec(),
91                    DecimalValue::I32(v) => v.to_le_bytes().to_vec(),
92                    DecimalValue::I64(v) => v.to_le_bytes().to_vec(),
93                    DecimalValue::I128(v128) => v128.to_le_bytes().to_vec(),
94                    DecimalValue::I256(v256) => v256.to_le_bytes().to_vec(),
95                };
96
97                pb::ScalarValue {
98                    kind: Some(Kind::BytesValue(inner_value)),
99                }
100            }
101            ScalarValue::Utf8(v) => pb::ScalarValue {
102                kind: Some(Kind::StringValue(v.to_string())),
103            },
104            ScalarValue::Binary(v) => pb::ScalarValue {
105                kind: Some(Kind::BytesValue(v.to_vec())),
106            },
107            ScalarValue::Tuple(v) => {
108                let mut values = Vec::with_capacity(v.len());
109                for elem in v.iter() {
110                    values.push(ScalarValue::to_proto(elem.as_ref()));
111                }
112                pb::ScalarValue {
113                    kind: Some(Kind::ListValue(ListValue { values })),
114                }
115            }
116            ScalarValue::Union(v) => pb::ScalarValue {
117                kind: Some(Kind::UnionValue(Box::new(PbUnionValue {
118                    type_id: u32::from(v.type_id()),
119                    value: Some(Box::new(ScalarValue::to_proto(v.child_value()))),
120                }))),
121            },
122            ScalarValue::Variant(v) => pb::ScalarValue {
123                kind: Some(Kind::VariantValue(Box::new(pb::Scalar::from(v.as_ref())))),
124            },
125        }
126    }
127}
128
129impl From<&PValue> for pb::ScalarValue {
130    fn from(value: &PValue) -> Self {
131        match value {
132            PValue::I8(v) => pb::ScalarValue {
133                kind: Some(Kind::Int64Value(*v as i64)),
134            },
135            PValue::I16(v) => pb::ScalarValue {
136                kind: Some(Kind::Int64Value(*v as i64)),
137            },
138            PValue::I32(v) => pb::ScalarValue {
139                kind: Some(Kind::Int64Value(*v as i64)),
140            },
141            PValue::I64(v) => pb::ScalarValue {
142                kind: Some(Kind::Int64Value(*v)),
143            },
144            PValue::U8(v) => pb::ScalarValue {
145                kind: Some(Kind::Uint64Value(*v as u64)),
146            },
147            PValue::U16(v) => pb::ScalarValue {
148                kind: Some(Kind::Uint64Value(*v as u64)),
149            },
150            PValue::U32(v) => pb::ScalarValue {
151                kind: Some(Kind::Uint64Value(*v as u64)),
152            },
153            PValue::U64(v) => pb::ScalarValue {
154                kind: Some(Kind::Uint64Value(*v)),
155            },
156            PValue::F16(v) => pb::ScalarValue {
157                kind: Some(Kind::F16Value(v.to_bits() as u64)),
158            },
159            PValue::F32(v) => pb::ScalarValue {
160                kind: Some(Kind::F32Value(*v)),
161            },
162            PValue::F64(v) => pb::ScalarValue {
163                kind: Some(Kind::F64Value(*v)),
164            },
165        }
166    }
167}
168
169////////////////////////////////////////////////////////////////////////////////////////////////////
170// Serialize FROM proto.
171////////////////////////////////////////////////////////////////////////////////////////////////////
172
173impl Scalar {
174    /// Creates a [`Scalar`] from a [protobuf `ScalarValue`](pb::ScalarValue) representation.
175    ///
176    /// Note that we need to provide a [`DType`] since protobuf serialization only supports 64-bit
177    /// integers, and serializing _into_ protobuf loses that type information.
178    ///
179    /// # Errors
180    ///
181    /// Returns an error if type validation fails.
182    pub fn from_proto_value(
183        value: &pb::ScalarValue,
184        dtype: &DType,
185        session: &VortexSession,
186    ) -> VortexResult<Self> {
187        let scalar_value = ScalarValue::from_proto(value, dtype, session)?;
188
189        Scalar::try_new(dtype.clone(), scalar_value)
190    }
191
192    /// Creates a [`Scalar`] from its [protobuf](pb::Scalar) representation.
193    ///
194    /// # Errors
195    ///
196    /// Returns an error if the protobuf is missing required fields or if type validation fails.
197    pub fn from_proto(value: &pb::Scalar, session: &VortexSession) -> VortexResult<Self> {
198        let dtype = DType::from_proto(
199            value
200                .dtype
201                .as_ref()
202                .ok_or_else(|| vortex_err!("Scalar missing dtype"))?,
203            session,
204        )?;
205
206        let pb_scalar_value: &pb::ScalarValue = value
207            .value
208            .as_ref()
209            .ok_or_else(|| vortex_err!("Scalar missing value"))?;
210
211        let value: Option<ScalarValue> = ScalarValue::from_proto(pb_scalar_value, &dtype, session)?;
212
213        Scalar::try_new(dtype, value)
214    }
215}
216
217impl ScalarValue {
218    /// Deserialize a [`ScalarValue`] from protobuf bytes.
219    ///
220    /// Note that we need to provide a [`DType`] since protobuf serialization only supports 64-bit
221    /// integers, and serializing _into_ protobuf loses that type information.
222    ///
223    /// # Errors
224    ///
225    /// Returns an error if decoding or type validation fails.
226    pub fn from_proto_bytes(
227        bytes: &[u8],
228        dtype: &DType,
229        session: &VortexSession,
230    ) -> VortexResult<Option<Self>> {
231        let proto = pb::ScalarValue::decode(bytes)?;
232        Self::from_proto(&proto, dtype, session)
233    }
234
235    /// Creates a [`ScalarValue`] from its [protobuf](pb::ScalarValue) representation.
236    ///
237    /// Note that we need to provide a [`DType`] since protobuf serialization only supports 64-bit
238    /// integers, and serializing _into_ protobuf loses that type information.
239    ///
240    /// # Errors
241    ///
242    /// Returns an error if the protobuf value cannot be converted to the given [`DType`].
243    pub fn from_proto(
244        value: &pb::ScalarValue,
245        dtype: &DType,
246        session: &VortexSession,
247    ) -> VortexResult<Option<Self>> {
248        let kind = value
249            .kind
250            .as_ref()
251            .ok_or_else(|| vortex_err!("Scalar value missing kind"))?;
252
253        // `DType::Extension` store their serialized values using the storage `DType`.
254        let dtype = match dtype {
255            DType::Extension(ext) => ext.storage_dtype(),
256            _ => dtype,
257        };
258
259        Ok(match kind {
260            Kind::NullValue(_) => None,
261            Kind::BoolValue(v) => Some(bool_from_proto(*v, dtype)?),
262            Kind::Int64Value(v) => Some(int64_from_proto(*v, dtype)?),
263            Kind::Uint64Value(v) => Some(uint64_from_proto(*v, dtype)?),
264            Kind::F16Value(v) => Some(f16_from_proto(*v, dtype)?),
265            Kind::F32Value(v) => Some(f32_from_proto(*v, dtype)?),
266            Kind::F64Value(v) => Some(f64_from_proto(*v, dtype)?),
267            Kind::StringValue(s) => Some(string_from_proto(s, dtype)?),
268            Kind::BytesValue(b) => Some(bytes_from_proto(b, dtype)?),
269            Kind::ListValue(v) => Some(list_from_proto(v, dtype, session)?),
270            Kind::UnionValue(v) => Some(union_from_proto(v, dtype, session)?),
271            Kind::VariantValue(v) => match dtype {
272                DType::Variant(_) => Some(ScalarValue::Variant(Box::new(Scalar::from_proto(
273                    v, session,
274                )?))),
275                _ => vortex_bail!("expected non-Variant scalar proto for dtype {dtype}"),
276            },
277        })
278    }
279}
280
281/// Deserialize a [`ScalarValue::Bool`] from a protobuf `BoolValue`.
282fn bool_from_proto(v: bool, dtype: &DType) -> VortexResult<ScalarValue> {
283    vortex_ensure!(
284        dtype.is_boolean(),
285        "expected Bool dtype for BoolValue, got {dtype}"
286    );
287
288    Ok(ScalarValue::Bool(v))
289}
290
291/// Deserialize a [`ScalarValue::Primitive`] from a protobuf `Int64Value`.
292///
293/// Protobuf consolidates all signed integers into `i64`, so we narrow back to the original
294/// type using the provided [`DType`].
295fn int64_from_proto(v: i64, dtype: &DType) -> VortexResult<ScalarValue> {
296    vortex_ensure!(
297        dtype.is_primitive(),
298        "expected Primitive dtype for Int64Value, got {dtype}"
299    );
300
301    let pvalue = match dtype.as_ptype() {
302        PType::I8 => v.to_i8().map(PValue::I8),
303        PType::I16 => v.to_i16().map(PValue::I16),
304        PType::I32 => v.to_i32().map(PValue::I32),
305        PType::I64 => Some(PValue::I64(v)),
306        // It was previously possible for unsigned types to get their stats serialised as signed,
307        // so we allow casting back to unsigned for backwards compatibility.
308        PType::U8 => v.to_u8().map(PValue::U8),
309        PType::U16 => v.to_u16().map(PValue::U16),
310        PType::U32 => v.to_u32().map(PValue::U32),
311        PType::U64 => v.to_u64().map(PValue::U64),
312        ftype @ (PType::F16 | PType::F32 | PType::F64) => vortex_bail!(
313            "expected signed integer ptype for serialized Int64Value, got float {ftype}"
314        ),
315    }
316    .ok_or_else(|| vortex_err!("Int64 value {v} out of range for dtype {dtype}"))?;
317
318    Ok(ScalarValue::Primitive(pvalue))
319}
320
321/// Deserialize a [`ScalarValue::Primitive`] from a protobuf `Uint64Value`.
322///
323/// Protobuf consolidates all unsigned integers into `u64`, so we narrow back to the original
324/// type using the provided [`DType`]. Also handles the backwards-compatible case where `f16`
325/// values were serialized as `u64` (via `f16::to_bits() as u64`).
326fn uint64_from_proto(v: u64, dtype: &DType) -> VortexResult<ScalarValue> {
327    vortex_ensure!(
328        dtype.is_primitive(),
329        "expected Primitive dtype for Uint64Value, got {dtype}"
330    );
331
332    let pvalue = match dtype.as_ptype() {
333        PType::U8 => v.to_u8().map(PValue::U8),
334        PType::U16 => v.to_u16().map(PValue::U16),
335        PType::U32 => v.to_u32().map(PValue::U32),
336        PType::U64 => Some(PValue::U64(v)),
337        // It was previously possible for signed types to get their stats serialised as unsigned,
338        // so we allow casting back to signed for backwards compatibility.
339        PType::I8 => v.to_i8().map(PValue::I8),
340        PType::I16 => v.to_i16().map(PValue::I16),
341        PType::I32 => v.to_i32().map(PValue::I32),
342        PType::I64 => v.to_i64().map(PValue::I64),
343        // f16 values used to be serialized as u64, so we need to be able to read an f16 from a u64.
344        PType::F16 => v.to_u16().map(f16::from_bits).map(PValue::F16),
345        ftype @ (PType::F32 | PType::F64) => {
346            vortex_bail!("expected unsigned integer ptype for serialized Uint64Value, got {ftype}")
347        }
348    }
349    .ok_or_else(|| vortex_err!("Uint64 value {v} out of range for dtype {dtype}"))?;
350
351    Ok(ScalarValue::Primitive(pvalue))
352}
353
354/// Deserialize a [`ScalarValue::Primitive`] from a protobuf `F16Value`.
355fn f16_from_proto(v: u64, dtype: &DType) -> VortexResult<ScalarValue> {
356    vortex_ensure!(
357        matches!(dtype, DType::Primitive(PType::F16, _)),
358        "expected F16 dtype for F16Value, got {dtype}"
359    );
360
361    let bits = u16::try_from(v)
362        .map_err(|_| vortex_err!("f16 bitwise representation has more than 16 bits: {v}"))?;
363
364    Ok(ScalarValue::Primitive(PValue::F16(f16::from_bits(bits))))
365}
366
367/// Deserialize a [`ScalarValue::Primitive`] from a protobuf `F32Value`.
368fn f32_from_proto(v: f32, dtype: &DType) -> VortexResult<ScalarValue> {
369    vortex_ensure!(
370        matches!(dtype, DType::Primitive(PType::F32, _)),
371        "expected F32 dtype for F32Value, got {dtype}"
372    );
373
374    Ok(ScalarValue::Primitive(PValue::F32(v)))
375}
376
377/// Deserialize a [`ScalarValue::Primitive`] from a protobuf `F64Value`.
378fn f64_from_proto(v: f64, dtype: &DType) -> VortexResult<ScalarValue> {
379    vortex_ensure!(
380        matches!(dtype, DType::Primitive(PType::F64, _)),
381        "expected F64 dtype for F64Value, got {dtype}"
382    );
383
384    Ok(ScalarValue::Primitive(PValue::F64(v)))
385}
386
387/// Deserialize a [`ScalarValue::Utf8`] or [`ScalarValue::Binary`] from a protobuf
388/// `StringValue`.
389fn string_from_proto(s: &str, dtype: &DType) -> VortexResult<ScalarValue> {
390    match dtype {
391        DType::Utf8(_) => Ok(ScalarValue::Utf8(BufferString::from(s))),
392        DType::Binary(_) => Ok(ScalarValue::Binary(ByteBuffer::copy_from(s.as_bytes()))),
393        _ => vortex_bail!("expected Utf8 or Binary dtype for StringValue, got {dtype}"),
394    }
395}
396
397/// Deserialize a [`ScalarValue`] from a protobuf bytes and a `DType`.
398///
399/// Handles [`Utf8`](ScalarValue::Utf8), [`Binary`](ScalarValue::Binary), and
400/// [`Decimal`](ScalarValue::Decimal) dtypes.
401fn bytes_from_proto(bytes: &[u8], dtype: &DType) -> VortexResult<ScalarValue> {
402    match dtype {
403        DType::Utf8(_) => Ok(ScalarValue::Utf8(BufferString::try_from(bytes)?)),
404        DType::Binary(_) => Ok(ScalarValue::Binary(ByteBuffer::copy_from(bytes))),
405        // TODO(connor): This is incorrect, we need to verify this matches the inner decimal_dtype.
406        DType::Decimal(..) => Ok(ScalarValue::Decimal(match bytes.len() {
407            1 => DecimalValue::I8(bytes[0] as i8),
408            2 => DecimalValue::I16(i16::from_le_bytes(
409                bytes
410                    .try_into()
411                    .ok()
412                    .vortex_expect("Buffer has invalid number of bytes"),
413            )),
414            4 => DecimalValue::I32(i32::from_le_bytes(
415                bytes
416                    .try_into()
417                    .ok()
418                    .vortex_expect("Buffer has invalid number of bytes"),
419            )),
420            8 => DecimalValue::I64(i64::from_le_bytes(
421                bytes
422                    .try_into()
423                    .ok()
424                    .vortex_expect("Buffer has invalid number of bytes"),
425            )),
426            16 => DecimalValue::I128(i128::from_le_bytes(
427                bytes
428                    .try_into()
429                    .ok()
430                    .vortex_expect("Buffer has invalid number of bytes"),
431            )),
432            32 => DecimalValue::I256(i256::from_le_bytes(
433                bytes
434                    .try_into()
435                    .ok()
436                    .vortex_expect("Buffer has invalid number of bytes"),
437            )),
438            l => vortex_bail!("invalid decimal byte length: {l}"),
439        })),
440        _ => vortex_bail!("expected Utf8, Binary, or Decimal dtype for BytesValue, got {dtype}"),
441    }
442}
443
444/// Deserialize a [`ScalarValue::Tuple`] from a protobuf `ListValue`.
445fn list_from_proto(
446    v: &ListValue,
447    dtype: &DType,
448    session: &VortexSession,
449) -> VortexResult<ScalarValue> {
450    let values = match dtype {
451        DType::List(element_dtype, _) | DType::FixedSizeList(element_dtype, ..) => v
452            .values
453            .iter()
454            .map(|elem| ScalarValue::from_proto(elem, element_dtype.as_ref(), session))
455            .collect::<VortexResult<Vec<_>>>()?,
456        DType::Struct(fields, _) => {
457            vortex_ensure_eq!(
458                v.values.len(),
459                fields.nfields(),
460                "ListValue has the wrong number of struct fields"
461            );
462
463            v.values
464                .iter()
465                .zip(fields.fields())
466                .map(|(value, field_dtype)| ScalarValue::from_proto(value, &field_dtype, session))
467                .collect::<VortexResult<Vec<_>>>()?
468        }
469        DType::Map(map, _) => {
470            let entry_dtype = map.entries_dtype();
471            v.values
472                .iter()
473                .map(|entry| ScalarValue::from_proto(entry, &entry_dtype, session))
474                .collect::<VortexResult<Vec<_>>>()?
475        }
476        _ => vortex_bail!("expected a tuple-backed dtype for ListValue, got {dtype}"),
477    };
478    Ok(ScalarValue::Tuple(values))
479}
480
481/// Deserialize a present union scalar value.
482fn union_from_proto(
483    value: &PbUnionValue,
484    dtype: &DType,
485    session: &VortexSession,
486) -> VortexResult<ScalarValue> {
487    let DType::Union(variants, _) = dtype else {
488        vortex_bail!("expected Union dtype for UnionValue, got {dtype}");
489    };
490
491    let type_id = u8::try_from(value.type_id)
492        .map_err(|_| vortex_err!("union type ID {} is outside the u8 range", value.type_id))?;
493
494    let child_index = variants.tag_to_child_index(type_id).ok_or_else(|| {
495        vortex_err!(
496            "union type ID {type_id} is not present in {:?}",
497            variants.type_ids()
498        )
499    })?;
500
501    let child_dtype = variants
502        .variant_by_index(child_index)
503        .ok_or_else(|| vortex_err!("union type ID {type_id} resolved out of bounds"))?;
504
505    let child_proto = value
506        .value
507        .as_deref()
508        .ok_or_else(|| vortex_err!("UnionValue missing child value"))?;
509
510    let child_value = ScalarValue::from_proto(child_proto, &child_dtype, session)?;
511    Scalar::validate(&child_dtype, child_value.as_ref()).map_err(|error| {
512        error.with_context(format!(
513            "union type ID {type_id} has invalid child for dtype {child_dtype}"
514        ))
515    })?;
516
517    Ok(ScalarValue::Union(UnionValue::new(type_id, child_value)))
518}
519
520#[cfg(test)]
521mod tests {
522    use std::f32;
523    use std::f64;
524    use std::sync::Arc;
525
526    use vortex_buffer::BufferString;
527    use vortex_error::vortex_panic;
528    use vortex_session::VortexSession;
529
530    use super::*;
531    use crate::dtype::DType;
532    use crate::dtype::DecimalDType;
533    use crate::dtype::Nullability;
534    use crate::dtype::PType;
535    use crate::dtype::UnionVariants;
536    use crate::dtype::half::f16;
537    use crate::proto::scalar as pb;
538    use crate::scalar::DecimalValue;
539    use crate::scalar::Scalar;
540    use crate::scalar::ScalarValue;
541
542    fn session() -> VortexSession {
543        VortexSession::empty()
544    }
545
546    fn round_trip(scalar: Scalar) {
547        let decoded = Scalar::from_proto(&pb::Scalar::from(&scalar), &session()).unwrap();
548        assert_eq!(decoded.dtype(), scalar.dtype());
549        assert_eq!(decoded, scalar);
550    }
551
552    #[test]
553    fn test_null() {
554        round_trip(Scalar::null(DType::Null));
555    }
556
557    #[test]
558    fn test_bool() {
559        round_trip(Scalar::new(
560            DType::Bool(Nullability::Nullable),
561            Some(ScalarValue::Bool(true)),
562        ));
563    }
564
565    #[test]
566    fn test_primitive() {
567        round_trip(Scalar::new(
568            DType::Primitive(PType::I32, Nullability::Nullable),
569            Some(ScalarValue::Primitive(42i32.into())),
570        ));
571    }
572
573    #[test]
574    fn test_buffer() {
575        round_trip(Scalar::new(
576            DType::Binary(Nullability::Nullable),
577            Some(ScalarValue::Binary(vec![1, 2, 3].into())),
578        ));
579    }
580
581    #[test]
582    fn test_buffer_string() {
583        round_trip(Scalar::new(
584            DType::Utf8(Nullability::Nullable),
585            Some(ScalarValue::Utf8(BufferString::from("hello".to_string()))),
586        ));
587    }
588
589    #[test]
590    fn test_list() {
591        round_trip(Scalar::new(
592            DType::List(
593                Arc::new(DType::Primitive(PType::I32, Nullability::Nullable)),
594                Nullability::Nullable,
595            ),
596            Some(ScalarValue::Tuple(vec![
597                Some(ScalarValue::Primitive(42i32.into())),
598                Some(ScalarValue::Primitive(43i32.into())),
599            ])),
600        ));
601    }
602
603    #[test]
604    fn test_map() {
605        let dtype = DType::map(
606            DType::Primitive(PType::I32, Nullability::NonNullable),
607            DType::Utf8(Nullability::Nullable),
608            true,
609            Nullability::Nullable,
610        )
611        .unwrap();
612        round_trip(
613            Scalar::try_map(
614                dtype.clone(),
615                [
616                    (
617                        Scalar::primitive(1i32, Nullability::NonNullable),
618                        Scalar::utf8("one", Nullability::Nullable),
619                    ),
620                    (
621                        Scalar::primitive(2i32, Nullability::NonNullable),
622                        Scalar::null(DType::Utf8(Nullability::Nullable)),
623                    ),
624                ],
625            )
626            .unwrap(),
627        );
628        round_trip(Scalar::null(dtype));
629    }
630
631    #[test]
632    fn test_f16() {
633        round_trip(Scalar::primitive(
634            f16::from_f32(0.42),
635            Nullability::Nullable,
636        ));
637    }
638
639    #[test]
640    fn test_i8() {
641        round_trip(Scalar::new(
642            DType::Primitive(PType::I8, Nullability::Nullable),
643            Some(ScalarValue::Primitive(i8::MIN.into())),
644        ));
645
646        round_trip(Scalar::new(
647            DType::Primitive(PType::I8, Nullability::Nullable),
648            Some(ScalarValue::Primitive(0i8.into())),
649        ));
650
651        round_trip(Scalar::new(
652            DType::Primitive(PType::I8, Nullability::Nullable),
653            Some(ScalarValue::Primitive(i8::MAX.into())),
654        ));
655    }
656
657    #[test]
658    fn test_decimal_i32_roundtrip() {
659        // A typical decimal with moderate precision and scale.
660        round_trip(Scalar::decimal(
661            DecimalValue::I32(123_456),
662            DecimalDType::new(10, 2),
663            Nullability::NonNullable,
664        ));
665    }
666
667    #[test]
668    fn test_decimal_i128_roundtrip() {
669        // A large decimal value that requires i128 storage.
670        round_trip(Scalar::decimal(
671            DecimalValue::I128(99_999_999_999_999_999_999),
672            DecimalDType::new(38, 6),
673            Nullability::Nullable,
674        ));
675    }
676
677    #[test]
678    fn test_decimal_null_roundtrip() {
679        round_trip(Scalar::null(DType::Decimal(
680            DecimalDType::new(10, 2),
681            Nullability::Nullable,
682        )));
683    }
684
685    #[test]
686    fn test_scalar_value_serde_roundtrip_binary() {
687        round_trip(Scalar::binary(
688            ByteBuffer::copy_from(b"hello"),
689            Nullability::NonNullable,
690        ));
691    }
692
693    #[test]
694    fn test_scalar_value_serde_roundtrip_utf8() {
695        round_trip(Scalar::utf8("hello", Nullability::NonNullable));
696    }
697
698    #[test]
699    fn test_variant_scalar_roundtrip() {
700        let nums = Scalar::list(
701            Arc::new(DType::Variant(Nullability::NonNullable)),
702            vec![
703                Scalar::variant(Scalar::primitive(-7_i16, Nullability::NonNullable)),
704                Scalar::variant(Scalar::primitive(42_u32, Nullability::NonNullable)),
705                Scalar::variant(Scalar::decimal(
706                    DecimalValue::I128(123_456_789),
707                    DecimalDType::new(18, 0),
708                    Nullability::NonNullable,
709                )),
710            ],
711            Nullability::NonNullable,
712        );
713
714        let nested = Scalar::list(
715            Arc::new(DType::Variant(Nullability::NonNullable)),
716            vec![
717                Scalar::variant(Scalar::from(true)),
718                Scalar::variant(nums),
719                Scalar::variant(Scalar::binary(
720                    ByteBuffer::copy_from(b"abc"),
721                    Nullability::NonNullable,
722                )),
723                Scalar::variant(Scalar::null(DType::Null)),
724            ],
725            Nullability::NonNullable,
726        );
727
728        round_trip(Scalar::variant(nested));
729    }
730
731    #[test]
732    fn test_variant_scalar_proto_preserves_scalar_null_vs_variant_null() {
733        let scalar_null = Scalar::null(DType::Variant(Nullability::Nullable));
734        let variant_null = Scalar::variant(Scalar::null(DType::Null));
735
736        let scalar_null_pb = pb::Scalar::from(&scalar_null);
737        let variant_null_pb = pb::Scalar::from(&variant_null);
738
739        assert_ne!(scalar_null_pb, variant_null_pb);
740        assert_eq!(
741            Scalar::from_proto(&scalar_null_pb, &session()).unwrap(),
742            scalar_null,
743        );
744        assert_eq!(
745            Scalar::from_proto(&variant_null_pb, &session()).unwrap(),
746            variant_null,
747        );
748    }
749
750    #[test]
751    fn test_union_scalar_roundtrip() -> VortexResult<()> {
752        let variants = UnionVariants::try_new(
753            ["int", "string"].into(),
754            vec![
755                DType::Primitive(PType::I32, Nullability::Nullable),
756                DType::Utf8(Nullability::NonNullable),
757            ],
758            vec![5, 9],
759        )?;
760
761        round_trip(Scalar::union(
762            variants.clone(),
763            5,
764            Scalar::primitive(42_i32, Nullability::Nullable),
765            Nullability::Nullable,
766        )?);
767
768        let inner_null = Scalar::union(
769            variants.clone(),
770            5,
771            Scalar::null(DType::Primitive(PType::I32, Nullability::Nullable)),
772            Nullability::Nullable,
773        )?;
774        let inner_null_proto = pb::Scalar::from(&inner_null);
775
776        assert!(matches!(
777            inner_null_proto
778                .value
779                .as_deref()
780                .and_then(|value| value.kind.as_ref()),
781            Some(Kind::UnionValue(union_value))
782                if matches!(
783                    union_value
784                        .value
785                        .as_deref()
786                        .and_then(|value| value.kind.as_ref()),
787                    Some(Kind::NullValue(_))
788                )
789        ));
790        let outer_null = Scalar::null(DType::Union(variants, Nullability::Nullable));
791        let outer_null_proto = pb::Scalar::from(&outer_null);
792        assert!(matches!(
793            outer_null_proto
794                .value
795                .as_deref()
796                .and_then(|value| value.kind.as_ref()),
797            Some(Kind::NullValue(_))
798        ));
799
800        assert_ne!(inner_null_proto, outer_null_proto);
801        round_trip(inner_null);
802        round_trip(outer_null);
803
804        let struct_dtype = DType::struct_(
805            [
806                (
807                    "number",
808                    DType::Primitive(PType::I32, Nullability::NonNullable),
809                ),
810                ("label", DType::Utf8(Nullability::Nullable)),
811            ],
812            Nullability::NonNullable,
813        );
814        let struct_scalar = Scalar::struct_(
815            struct_dtype.clone(),
816            [
817                Scalar::primitive(42_i32, Nullability::NonNullable),
818                Scalar::utf8("answer", Nullability::Nullable),
819            ],
820        );
821        let struct_variants =
822            UnionVariants::try_new(["record"].into(), vec![struct_dtype], vec![13])?;
823        round_trip(Scalar::union(
824            struct_variants,
825            13,
826            struct_scalar,
827            Nullability::NonNullable,
828        )?);
829
830        Ok(())
831    }
832
833    #[test]
834    fn test_union_proto_rejects_malformed_values() -> VortexResult<()> {
835        let variants = UnionVariants::try_new(
836            ["int"].into(),
837            vec![DType::Primitive(PType::I32, Nullability::Nullable)],
838            vec![5],
839        )?;
840        let dtype = DType::Union(variants, Nullability::NonNullable);
841
842        let unknown_tag = pb::ScalarValue {
843            kind: Some(Kind::UnionValue(Box::new(PbUnionValue {
844                type_id: 7,
845                value: Some(Box::new(ScalarValue::to_proto(
846                    Scalar::primitive(42_i32, Nullability::Nullable).value(),
847                ))),
848            }))),
849        };
850
851        assert!(ScalarValue::from_proto(&unknown_tag, &dtype, &session()).is_err());
852
853        let missing_child = pb::ScalarValue {
854            kind: Some(Kind::UnionValue(Box::new(PbUnionValue {
855                type_id: 5,
856                value: None,
857            }))),
858        };
859
860        let err = ScalarValue::from_proto(&missing_child, &dtype, &session())
861            .unwrap_err()
862            .to_string();
863        assert!(err.contains("UnionValue missing child value"), "{err}");
864
865        let wrong_child_value = pb::ScalarValue {
866            kind: Some(Kind::UnionValue(Box::new(PbUnionValue {
867                type_id: 5,
868                value: Some(Box::new(ScalarValue::to_proto(
869                    Scalar::utf8("wrong", Nullability::NonNullable).value(),
870                ))),
871            }))),
872        };
873
874        let err = ScalarValue::from_proto(&wrong_child_value, &dtype, &session())
875            .unwrap_err()
876            .to_string();
877        assert!(
878            err.contains("expected Utf8 or Binary dtype for StringValue, got i32?"),
879            "{err}"
880        );
881
882        Ok(())
883    }
884
885    #[test]
886    fn test_backcompat_f16_serialized_as_u64() {
887        // Backwards compatibility test for the legacy f16 serialization format.
888        //
889        // Previously, f16 ScalarValues were serialized as `Uint64Value(v.to_bits() as u64)` because
890        // the proto schema only had 64-bit integer types, and f16's underlying representation is
891        // u16 which got widened to u64.
892        //
893        // The current implementation uses a dedicated `F16Value` proto field, but we must still be
894        // able to deserialize the old format. This test verifies that:
895        //
896        // 1. A `Uint64Value` containing f16 bits can be read as a U64 primitive (the raw bits).
897        // 2. When wrapped in a Scalar with F16 dtype, the value is correctly interpreted as f16.
898        //
899        // This ensures data written with the old serialization format remains readable.
900
901        // Simulate the old serialization: f16(0.42) stored as Uint64Value with its bit pattern.
902        let f16_value = f16::from_f32(0.42);
903        let f16_bits_as_u64 = f16_value.to_bits() as u64; // 14008
904
905        let pb_scalar_value = pb::ScalarValue {
906            kind: Some(Kind::Uint64Value(f16_bits_as_u64)),
907        };
908
909        // Step 1: Verify the normal U64 scalar.
910        let scalar_value = ScalarValue::from_proto(
911            &pb_scalar_value,
912            &DType::Primitive(PType::U64, Nullability::NonNullable),
913            &session(),
914        )
915        .unwrap();
916        assert_eq!(
917            scalar_value.as_ref().map(|v| v.as_primitive()),
918            Some(&PValue::U64(14008u64)),
919        );
920
921        // Step 2: Verify that when we use F16 dtype, the Uint64Value is correctly interpreted.
922        let scalar_value_f16 = ScalarValue::from_proto(
923            &pb_scalar_value,
924            &DType::Primitive(PType::F16, Nullability::Nullable),
925            &session(),
926        )
927        .unwrap();
928
929        let scalar = Scalar::new(
930            DType::Primitive(PType::F16, Nullability::Nullable),
931            scalar_value_f16,
932        );
933
934        assert_eq!(
935            scalar.as_primitive().pvalue().unwrap(),
936            PValue::F16(f16::from_f32(0.42)),
937            "Uint64Value should be correctly interpreted as f16 when dtype is F16"
938        );
939    }
940
941    #[test]
942    fn test_scalar_value_direct_roundtrip_f16() {
943        // Test that ScalarValue with f16 roundtrips correctly without going through Scalar.
944        let f16_values = vec![
945            f16::from_f32(0.0),
946            f16::from_f32(1.0),
947            f16::from_f32(-1.0),
948            f16::from_f32(0.42),
949            f16::from_f32(5.722046e-6),
950            f16::from_f32(f32::consts::PI),
951            f16::INFINITY,
952            f16::NEG_INFINITY,
953            f16::NAN,
954        ];
955
956        for f16_val in f16_values {
957            let scalar_value = ScalarValue::Primitive(PValue::F16(f16_val));
958            let pb_value = ScalarValue::to_proto(Some(&scalar_value));
959            let read_back = ScalarValue::from_proto(
960                &pb_value,
961                &DType::Primitive(PType::F16, Nullability::NonNullable),
962                &session(),
963            )
964            .unwrap();
965
966            match (&scalar_value, read_back.as_ref()) {
967                (
968                    ScalarValue::Primitive(PValue::F16(original)),
969                    Some(ScalarValue::Primitive(PValue::F16(roundtripped))),
970                ) => {
971                    if original.is_nan() && roundtripped.is_nan() {
972                        // NaN values are equal for our purposes.
973                        continue;
974                    }
975                    assert_eq!(
976                        original, roundtripped,
977                        "F16 value {original:?} did not roundtrip correctly"
978                    );
979                }
980                _ => {
981                    vortex_panic!(
982                        "Expected f16 primitive values, got {scalar_value:?} and {read_back:?}"
983                    )
984                }
985            }
986        }
987    }
988
989    #[test]
990    fn test_scalar_value_direct_roundtrip_preserves_values() {
991        // Test that ScalarValue roundtripping preserves values (but not necessarily exact types).
992        // Note: Proto encoding consolidates integer types (u8/u16/u32 → u64, i8/i16/i32 → i64).
993
994        // Test cases that should roundtrip exactly.
995        let exact_roundtrip_cases: Vec<(&str, Option<ScalarValue>, DType)> = vec![
996            ("null", None, DType::Null),
997            (
998                "bool_true",
999                Some(ScalarValue::Bool(true)),
1000                DType::Bool(Nullability::Nullable),
1001            ),
1002            (
1003                "bool_false",
1004                Some(ScalarValue::Bool(false)),
1005                DType::Bool(Nullability::Nullable),
1006            ),
1007            (
1008                "u64",
1009                Some(ScalarValue::Primitive(PValue::U64(18446744073709551615))),
1010                DType::Primitive(PType::U64, Nullability::Nullable),
1011            ),
1012            (
1013                "i64",
1014                Some(ScalarValue::Primitive(PValue::I64(-9223372036854775808))),
1015                DType::Primitive(PType::I64, Nullability::Nullable),
1016            ),
1017            (
1018                "f32",
1019                Some(ScalarValue::Primitive(PValue::F32(f32::consts::E))),
1020                DType::Primitive(PType::F32, Nullability::Nullable),
1021            ),
1022            (
1023                "f64",
1024                Some(ScalarValue::Primitive(PValue::F64(f64::consts::PI))),
1025                DType::Primitive(PType::F64, Nullability::Nullable),
1026            ),
1027            (
1028                "string",
1029                Some(ScalarValue::Utf8(BufferString::from("test"))),
1030                DType::Utf8(Nullability::Nullable),
1031            ),
1032            (
1033                "bytes",
1034                Some(ScalarValue::Binary(vec![1, 2, 3, 4, 5].into())),
1035                DType::Binary(Nullability::Nullable),
1036            ),
1037        ];
1038
1039        for (name, value, dtype) in exact_roundtrip_cases {
1040            let pb_value = ScalarValue::to_proto(value.as_ref());
1041            let read_back = ScalarValue::from_proto(&pb_value, &dtype, &session()).unwrap();
1042
1043            let original_debug = format!("{value:?}");
1044            let roundtrip_debug = format!("{read_back:?}");
1045            assert_eq!(
1046                original_debug, roundtrip_debug,
1047                "ScalarValue {name} did not roundtrip exactly"
1048            );
1049        }
1050
1051        // The wire format widens integers; decoding must restore their original widths.
1052        let unsigned_cases = vec![
1053            (
1054                "u8",
1055                ScalarValue::Primitive(PValue::U8(255)),
1056                DType::Primitive(PType::U8, Nullability::Nullable),
1057                255u64,
1058            ),
1059            (
1060                "u16",
1061                ScalarValue::Primitive(PValue::U16(65535)),
1062                DType::Primitive(PType::U16, Nullability::Nullable),
1063                65535u64,
1064            ),
1065            (
1066                "u32",
1067                ScalarValue::Primitive(PValue::U32(4294967295)),
1068                DType::Primitive(PType::U32, Nullability::Nullable),
1069                4294967295u64,
1070            ),
1071        ];
1072
1073        for (name, value, dtype, expected) in unsigned_cases {
1074            let pb_value = ScalarValue::to_proto(Some(&value));
1075            let read_back = ScalarValue::from_proto(&pb_value, &dtype, &session()).unwrap();
1076
1077            match read_back.as_ref() {
1078                Some(ScalarValue::Primitive(pv)) => {
1079                    assert_eq!(pv.ptype(), dtype.as_ptype(), "{name} primitive width");
1080                    let v = match pv {
1081                        PValue::U8(v) => *v as u64,
1082                        PValue::U16(v) => *v as u64,
1083                        PValue::U32(v) => *v as u64,
1084                        PValue::U64(v) => *v,
1085                        _ => {
1086                            vortex_panic!("Unexpected primitive type for {name}: {pv:?}")
1087                        }
1088                    };
1089                    assert_eq!(
1090                        v, expected,
1091                        "ScalarValue {name} value not preserved: expected {expected}, got {v}"
1092                    );
1093                }
1094                _ => {
1095                    vortex_panic!("Unexpected type after roundtrip for {name}: {read_back:?}")
1096                }
1097            }
1098        }
1099
1100        // Signed integers must also recover their original widths.
1101        let signed_cases = vec![
1102            (
1103                "i8",
1104                ScalarValue::Primitive(PValue::I8(-128)),
1105                DType::Primitive(PType::I8, Nullability::Nullable),
1106                -128i64,
1107            ),
1108            (
1109                "i16",
1110                ScalarValue::Primitive(PValue::I16(-32768)),
1111                DType::Primitive(PType::I16, Nullability::Nullable),
1112                -32768i64,
1113            ),
1114            (
1115                "i32",
1116                ScalarValue::Primitive(PValue::I32(-2147483648)),
1117                DType::Primitive(PType::I32, Nullability::Nullable),
1118                -2147483648i64,
1119            ),
1120        ];
1121
1122        for (name, value, dtype, expected) in signed_cases {
1123            let pb_value = ScalarValue::to_proto(Some(&value));
1124            let read_back = ScalarValue::from_proto(&pb_value, &dtype, &session()).unwrap();
1125
1126            match read_back.as_ref() {
1127                Some(ScalarValue::Primitive(pv)) => {
1128                    assert_eq!(pv.ptype(), dtype.as_ptype(), "{name} primitive width");
1129                    let v = match pv {
1130                        PValue::I8(v) => *v as i64,
1131                        PValue::I16(v) => *v as i64,
1132                        PValue::I32(v) => *v as i64,
1133                        PValue::I64(v) => *v,
1134                        _ => {
1135                            vortex_panic!("Unexpected primitive type for {name}: {pv:?}")
1136                        }
1137                    };
1138                    assert_eq!(
1139                        v, expected,
1140                        "ScalarValue {name} value not preserved: expected {expected}, got {v}"
1141                    );
1142                }
1143                _ => {
1144                    vortex_panic!("Unexpected type after roundtrip for {name}: {read_back:?}")
1145                }
1146            }
1147        }
1148    }
1149
1150    // Backwards compatibility: signed integer stats could previously be serialized as unsigned.
1151    // Therefore, we allow casting between signed and unsigned integers of the same bit width.
1152    #[test]
1153    fn test_backcompat_signed_integer_deserialized_as_unsigned() {
1154        let v = ScalarValue::Primitive(PValue::I64(0));
1155        assert_eq!(
1156            Scalar::from_proto_value(
1157                &pb::ScalarValue::from(&v),
1158                &DType::Primitive(PType::U64, Nullability::Nullable),
1159                &session()
1160            )
1161            .unwrap(),
1162            Scalar::primitive(0u64, Nullability::Nullable)
1163        );
1164    }
1165
1166    // Backwards compatibility: unsigned integer stats could previously be serialized as signed.
1167    // Therefore, we allow casting between signed and unsigned integers of the same bit width.
1168    #[test]
1169    fn test_backcompat_unsigned_integer_deserialized_as_signed() {
1170        let v = ScalarValue::Primitive(PValue::U64(0));
1171        assert_eq!(
1172            Scalar::from_proto_value(
1173                &pb::ScalarValue::from(&v),
1174                &DType::Primitive(PType::I64, Nullability::Nullable),
1175                &session()
1176            )
1177            .unwrap(),
1178            Scalar::primitive(0i64, Nullability::Nullable)
1179        );
1180    }
1181}