1use num_traits::ToBytes;
7use num_traits::ToPrimitive;
8use prost::Message;
9use vortex_buffer::BufferString;
10use vortex_buffer::ByteBuffer;
11use vortex_error::VortexExpect;
12use vortex_error::VortexResult;
13use vortex_error::vortex_bail;
14use vortex_error::vortex_ensure;
15use vortex_error::vortex_ensure_eq;
16use vortex_error::vortex_err;
17use vortex_session::VortexSession;
18
19use crate::dtype::DType;
20use crate::dtype::PType;
21use crate::dtype::half::f16;
22use crate::dtype::i256;
23use crate::proto::scalar as pb;
24use crate::proto::scalar::ListValue;
25use crate::proto::scalar::UnionValue as PbUnionValue;
26use crate::proto::scalar::scalar_value::Kind;
27use crate::scalar::DecimalValue;
28use crate::scalar::PValue;
29use crate::scalar::Scalar;
30use crate::scalar::ScalarValue;
31use crate::scalar::UnionValue;
32
33impl From<&Scalar> for pb::Scalar {
38 fn from(value: &Scalar) -> Self {
39 pb::Scalar {
40 dtype: Some(
41 (value.dtype())
42 .try_into()
43 .vortex_expect("Failed to convert DType to proto"),
44 ),
45 value: Some(Box::new(ScalarValue::to_proto(value.value()))),
46 }
47 }
48}
49
50impl ScalarValue {
51 pub fn to_proto(this: Option<&Self>) -> pb::ScalarValue {
61 match this {
62 None => pb::ScalarValue {
63 kind: Some(Kind::NullValue(0)),
64 },
65 Some(this) => pb::ScalarValue::from(this),
66 }
67 }
68
69 pub fn to_proto_bytes<B: Default + bytes::BufMut>(value: Option<&ScalarValue>) -> B {
71 let proto = Self::to_proto(value);
72 let mut buf = B::default();
73 proto
74 .encode(&mut buf)
75 .vortex_expect("Failed to encode scalar value");
76 buf
77 }
78}
79
80impl From<&ScalarValue> for pb::ScalarValue {
81 fn from(value: &ScalarValue) -> Self {
82 match value {
83 ScalarValue::Bool(v) => pb::ScalarValue {
84 kind: Some(Kind::BoolValue(*v)),
85 },
86 ScalarValue::Primitive(v) => pb::ScalarValue::from(v),
87 ScalarValue::Decimal(v) => {
88 let inner_value = match v {
89 DecimalValue::I8(v) => v.to_le_bytes().to_vec(),
90 DecimalValue::I16(v) => v.to_le_bytes().to_vec(),
91 DecimalValue::I32(v) => v.to_le_bytes().to_vec(),
92 DecimalValue::I64(v) => v.to_le_bytes().to_vec(),
93 DecimalValue::I128(v128) => v128.to_le_bytes().to_vec(),
94 DecimalValue::I256(v256) => v256.to_le_bytes().to_vec(),
95 };
96
97 pb::ScalarValue {
98 kind: Some(Kind::BytesValue(inner_value)),
99 }
100 }
101 ScalarValue::Utf8(v) => pb::ScalarValue {
102 kind: Some(Kind::StringValue(v.to_string())),
103 },
104 ScalarValue::Binary(v) => pb::ScalarValue {
105 kind: Some(Kind::BytesValue(v.to_vec())),
106 },
107 ScalarValue::Tuple(v) => {
108 let mut values = Vec::with_capacity(v.len());
109 for elem in v.iter() {
110 values.push(ScalarValue::to_proto(elem.as_ref()));
111 }
112 pb::ScalarValue {
113 kind: Some(Kind::ListValue(ListValue { values })),
114 }
115 }
116 ScalarValue::Union(v) => pb::ScalarValue {
117 kind: Some(Kind::UnionValue(Box::new(PbUnionValue {
118 type_id: u32::from(v.type_id()),
119 value: Some(Box::new(ScalarValue::to_proto(v.child_value()))),
120 }))),
121 },
122 ScalarValue::Variant(v) => pb::ScalarValue {
123 kind: Some(Kind::VariantValue(Box::new(pb::Scalar::from(v.as_ref())))),
124 },
125 }
126 }
127}
128
129impl From<&PValue> for pb::ScalarValue {
130 fn from(value: &PValue) -> Self {
131 match value {
132 PValue::I8(v) => pb::ScalarValue {
133 kind: Some(Kind::Int64Value(*v as i64)),
134 },
135 PValue::I16(v) => pb::ScalarValue {
136 kind: Some(Kind::Int64Value(*v as i64)),
137 },
138 PValue::I32(v) => pb::ScalarValue {
139 kind: Some(Kind::Int64Value(*v as i64)),
140 },
141 PValue::I64(v) => pb::ScalarValue {
142 kind: Some(Kind::Int64Value(*v)),
143 },
144 PValue::U8(v) => pb::ScalarValue {
145 kind: Some(Kind::Uint64Value(*v as u64)),
146 },
147 PValue::U16(v) => pb::ScalarValue {
148 kind: Some(Kind::Uint64Value(*v as u64)),
149 },
150 PValue::U32(v) => pb::ScalarValue {
151 kind: Some(Kind::Uint64Value(*v as u64)),
152 },
153 PValue::U64(v) => pb::ScalarValue {
154 kind: Some(Kind::Uint64Value(*v)),
155 },
156 PValue::F16(v) => pb::ScalarValue {
157 kind: Some(Kind::F16Value(v.to_bits() as u64)),
158 },
159 PValue::F32(v) => pb::ScalarValue {
160 kind: Some(Kind::F32Value(*v)),
161 },
162 PValue::F64(v) => pb::ScalarValue {
163 kind: Some(Kind::F64Value(*v)),
164 },
165 }
166 }
167}
168
169impl Scalar {
174 pub fn from_proto_value(
183 value: &pb::ScalarValue,
184 dtype: &DType,
185 session: &VortexSession,
186 ) -> VortexResult<Self> {
187 let scalar_value = ScalarValue::from_proto(value, dtype, session)?;
188
189 Scalar::try_new(dtype.clone(), scalar_value)
190 }
191
192 pub fn from_proto(value: &pb::Scalar, session: &VortexSession) -> VortexResult<Self> {
198 let dtype = DType::from_proto(
199 value
200 .dtype
201 .as_ref()
202 .ok_or_else(|| vortex_err!("Scalar missing dtype"))?,
203 session,
204 )?;
205
206 let pb_scalar_value: &pb::ScalarValue = value
207 .value
208 .as_ref()
209 .ok_or_else(|| vortex_err!("Scalar missing value"))?;
210
211 let value: Option<ScalarValue> = ScalarValue::from_proto(pb_scalar_value, &dtype, session)?;
212
213 Scalar::try_new(dtype, value)
214 }
215}
216
217impl ScalarValue {
218 pub fn from_proto_bytes(
227 bytes: &[u8],
228 dtype: &DType,
229 session: &VortexSession,
230 ) -> VortexResult<Option<Self>> {
231 let proto = pb::ScalarValue::decode(bytes)?;
232 Self::from_proto(&proto, dtype, session)
233 }
234
235 pub fn from_proto(
244 value: &pb::ScalarValue,
245 dtype: &DType,
246 session: &VortexSession,
247 ) -> VortexResult<Option<Self>> {
248 let kind = value
249 .kind
250 .as_ref()
251 .ok_or_else(|| vortex_err!("Scalar value missing kind"))?;
252
253 let dtype = match dtype {
255 DType::Extension(ext) => ext.storage_dtype(),
256 _ => dtype,
257 };
258
259 Ok(match kind {
260 Kind::NullValue(_) => None,
261 Kind::BoolValue(v) => Some(bool_from_proto(*v, dtype)?),
262 Kind::Int64Value(v) => Some(int64_from_proto(*v, dtype)?),
263 Kind::Uint64Value(v) => Some(uint64_from_proto(*v, dtype)?),
264 Kind::F16Value(v) => Some(f16_from_proto(*v, dtype)?),
265 Kind::F32Value(v) => Some(f32_from_proto(*v, dtype)?),
266 Kind::F64Value(v) => Some(f64_from_proto(*v, dtype)?),
267 Kind::StringValue(s) => Some(string_from_proto(s, dtype)?),
268 Kind::BytesValue(b) => Some(bytes_from_proto(b, dtype)?),
269 Kind::ListValue(v) => Some(list_from_proto(v, dtype, session)?),
270 Kind::UnionValue(v) => Some(union_from_proto(v, dtype, session)?),
271 Kind::VariantValue(v) => match dtype {
272 DType::Variant(_) => Some(ScalarValue::Variant(Box::new(Scalar::from_proto(
273 v, session,
274 )?))),
275 _ => vortex_bail!("expected non-Variant scalar proto for dtype {dtype}"),
276 },
277 })
278 }
279}
280
281fn bool_from_proto(v: bool, dtype: &DType) -> VortexResult<ScalarValue> {
283 vortex_ensure!(
284 dtype.is_boolean(),
285 "expected Bool dtype for BoolValue, got {dtype}"
286 );
287
288 Ok(ScalarValue::Bool(v))
289}
290
291fn int64_from_proto(v: i64, dtype: &DType) -> VortexResult<ScalarValue> {
296 vortex_ensure!(
297 dtype.is_primitive(),
298 "expected Primitive dtype for Int64Value, got {dtype}"
299 );
300
301 let pvalue = match dtype.as_ptype() {
302 PType::I8 => v.to_i8().map(PValue::I8),
303 PType::I16 => v.to_i16().map(PValue::I16),
304 PType::I32 => v.to_i32().map(PValue::I32),
305 PType::I64 => Some(PValue::I64(v)),
306 PType::U8 => v.to_u8().map(PValue::U8),
309 PType::U16 => v.to_u16().map(PValue::U16),
310 PType::U32 => v.to_u32().map(PValue::U32),
311 PType::U64 => v.to_u64().map(PValue::U64),
312 ftype @ (PType::F16 | PType::F32 | PType::F64) => vortex_bail!(
313 "expected signed integer ptype for serialized Int64Value, got float {ftype}"
314 ),
315 }
316 .ok_or_else(|| vortex_err!("Int64 value {v} out of range for dtype {dtype}"))?;
317
318 Ok(ScalarValue::Primitive(pvalue))
319}
320
321fn uint64_from_proto(v: u64, dtype: &DType) -> VortexResult<ScalarValue> {
327 vortex_ensure!(
328 dtype.is_primitive(),
329 "expected Primitive dtype for Uint64Value, got {dtype}"
330 );
331
332 let pvalue = match dtype.as_ptype() {
333 PType::U8 => v.to_u8().map(PValue::U8),
334 PType::U16 => v.to_u16().map(PValue::U16),
335 PType::U32 => v.to_u32().map(PValue::U32),
336 PType::U64 => Some(PValue::U64(v)),
337 PType::I8 => v.to_i8().map(PValue::I8),
340 PType::I16 => v.to_i16().map(PValue::I16),
341 PType::I32 => v.to_i32().map(PValue::I32),
342 PType::I64 => v.to_i64().map(PValue::I64),
343 PType::F16 => v.to_u16().map(f16::from_bits).map(PValue::F16),
345 ftype @ (PType::F32 | PType::F64) => {
346 vortex_bail!("expected unsigned integer ptype for serialized Uint64Value, got {ftype}")
347 }
348 }
349 .ok_or_else(|| vortex_err!("Uint64 value {v} out of range for dtype {dtype}"))?;
350
351 Ok(ScalarValue::Primitive(pvalue))
352}
353
354fn f16_from_proto(v: u64, dtype: &DType) -> VortexResult<ScalarValue> {
356 vortex_ensure!(
357 matches!(dtype, DType::Primitive(PType::F16, _)),
358 "expected F16 dtype for F16Value, got {dtype}"
359 );
360
361 let bits = u16::try_from(v)
362 .map_err(|_| vortex_err!("f16 bitwise representation has more than 16 bits: {v}"))?;
363
364 Ok(ScalarValue::Primitive(PValue::F16(f16::from_bits(bits))))
365}
366
367fn f32_from_proto(v: f32, dtype: &DType) -> VortexResult<ScalarValue> {
369 vortex_ensure!(
370 matches!(dtype, DType::Primitive(PType::F32, _)),
371 "expected F32 dtype for F32Value, got {dtype}"
372 );
373
374 Ok(ScalarValue::Primitive(PValue::F32(v)))
375}
376
377fn f64_from_proto(v: f64, dtype: &DType) -> VortexResult<ScalarValue> {
379 vortex_ensure!(
380 matches!(dtype, DType::Primitive(PType::F64, _)),
381 "expected F64 dtype for F64Value, got {dtype}"
382 );
383
384 Ok(ScalarValue::Primitive(PValue::F64(v)))
385}
386
387fn string_from_proto(s: &str, dtype: &DType) -> VortexResult<ScalarValue> {
390 match dtype {
391 DType::Utf8(_) => Ok(ScalarValue::Utf8(BufferString::from(s))),
392 DType::Binary(_) => Ok(ScalarValue::Binary(ByteBuffer::copy_from(s.as_bytes()))),
393 _ => vortex_bail!("expected Utf8 or Binary dtype for StringValue, got {dtype}"),
394 }
395}
396
397fn bytes_from_proto(bytes: &[u8], dtype: &DType) -> VortexResult<ScalarValue> {
402 match dtype {
403 DType::Utf8(_) => Ok(ScalarValue::Utf8(BufferString::try_from(bytes)?)),
404 DType::Binary(_) => Ok(ScalarValue::Binary(ByteBuffer::copy_from(bytes))),
405 DType::Decimal(..) => Ok(ScalarValue::Decimal(match bytes.len() {
407 1 => DecimalValue::I8(bytes[0] as i8),
408 2 => DecimalValue::I16(i16::from_le_bytes(
409 bytes
410 .try_into()
411 .ok()
412 .vortex_expect("Buffer has invalid number of bytes"),
413 )),
414 4 => DecimalValue::I32(i32::from_le_bytes(
415 bytes
416 .try_into()
417 .ok()
418 .vortex_expect("Buffer has invalid number of bytes"),
419 )),
420 8 => DecimalValue::I64(i64::from_le_bytes(
421 bytes
422 .try_into()
423 .ok()
424 .vortex_expect("Buffer has invalid number of bytes"),
425 )),
426 16 => DecimalValue::I128(i128::from_le_bytes(
427 bytes
428 .try_into()
429 .ok()
430 .vortex_expect("Buffer has invalid number of bytes"),
431 )),
432 32 => DecimalValue::I256(i256::from_le_bytes(
433 bytes
434 .try_into()
435 .ok()
436 .vortex_expect("Buffer has invalid number of bytes"),
437 )),
438 l => vortex_bail!("invalid decimal byte length: {l}"),
439 })),
440 _ => vortex_bail!("expected Utf8, Binary, or Decimal dtype for BytesValue, got {dtype}"),
441 }
442}
443
444fn list_from_proto(
446 v: &ListValue,
447 dtype: &DType,
448 session: &VortexSession,
449) -> VortexResult<ScalarValue> {
450 let values = match dtype {
451 DType::List(element_dtype, _) | DType::FixedSizeList(element_dtype, ..) => v
452 .values
453 .iter()
454 .map(|elem| ScalarValue::from_proto(elem, element_dtype.as_ref(), session))
455 .collect::<VortexResult<Vec<_>>>()?,
456 DType::Struct(fields, _) => {
457 vortex_ensure_eq!(
458 v.values.len(),
459 fields.nfields(),
460 "ListValue has the wrong number of struct fields"
461 );
462
463 v.values
464 .iter()
465 .zip(fields.fields())
466 .map(|(value, field_dtype)| ScalarValue::from_proto(value, &field_dtype, session))
467 .collect::<VortexResult<Vec<_>>>()?
468 }
469 DType::Map(map, _) => {
470 let entry_dtype = map.entries_dtype();
471 v.values
472 .iter()
473 .map(|entry| ScalarValue::from_proto(entry, &entry_dtype, session))
474 .collect::<VortexResult<Vec<_>>>()?
475 }
476 _ => vortex_bail!("expected a tuple-backed dtype for ListValue, got {dtype}"),
477 };
478 Ok(ScalarValue::Tuple(values))
479}
480
481fn union_from_proto(
483 value: &PbUnionValue,
484 dtype: &DType,
485 session: &VortexSession,
486) -> VortexResult<ScalarValue> {
487 let DType::Union(variants, _) = dtype else {
488 vortex_bail!("expected Union dtype for UnionValue, got {dtype}");
489 };
490
491 let type_id = u8::try_from(value.type_id)
492 .map_err(|_| vortex_err!("union type ID {} is outside the u8 range", value.type_id))?;
493
494 let child_index = variants.tag_to_child_index(type_id).ok_or_else(|| {
495 vortex_err!(
496 "union type ID {type_id} is not present in {:?}",
497 variants.type_ids()
498 )
499 })?;
500
501 let child_dtype = variants
502 .variant_by_index(child_index)
503 .ok_or_else(|| vortex_err!("union type ID {type_id} resolved out of bounds"))?;
504
505 let child_proto = value
506 .value
507 .as_deref()
508 .ok_or_else(|| vortex_err!("UnionValue missing child value"))?;
509
510 let child_value = ScalarValue::from_proto(child_proto, &child_dtype, session)?;
511 Scalar::validate(&child_dtype, child_value.as_ref()).map_err(|error| {
512 error.with_context(format!(
513 "union type ID {type_id} has invalid child for dtype {child_dtype}"
514 ))
515 })?;
516
517 Ok(ScalarValue::Union(UnionValue::new(type_id, child_value)))
518}
519
520#[cfg(test)]
521mod tests {
522 use std::f32;
523 use std::f64;
524 use std::sync::Arc;
525
526 use vortex_buffer::BufferString;
527 use vortex_error::vortex_panic;
528 use vortex_session::VortexSession;
529
530 use super::*;
531 use crate::dtype::DType;
532 use crate::dtype::DecimalDType;
533 use crate::dtype::Nullability;
534 use crate::dtype::PType;
535 use crate::dtype::UnionVariants;
536 use crate::dtype::half::f16;
537 use crate::proto::scalar as pb;
538 use crate::scalar::DecimalValue;
539 use crate::scalar::Scalar;
540 use crate::scalar::ScalarValue;
541
542 fn session() -> VortexSession {
543 VortexSession::empty()
544 }
545
546 fn round_trip(scalar: Scalar) {
547 let decoded = Scalar::from_proto(&pb::Scalar::from(&scalar), &session()).unwrap();
548 assert_eq!(decoded.dtype(), scalar.dtype());
549 assert_eq!(decoded, scalar);
550 }
551
552 #[test]
553 fn test_null() {
554 round_trip(Scalar::null(DType::Null));
555 }
556
557 #[test]
558 fn test_bool() {
559 round_trip(Scalar::new(
560 DType::Bool(Nullability::Nullable),
561 Some(ScalarValue::Bool(true)),
562 ));
563 }
564
565 #[test]
566 fn test_primitive() {
567 round_trip(Scalar::new(
568 DType::Primitive(PType::I32, Nullability::Nullable),
569 Some(ScalarValue::Primitive(42i32.into())),
570 ));
571 }
572
573 #[test]
574 fn test_buffer() {
575 round_trip(Scalar::new(
576 DType::Binary(Nullability::Nullable),
577 Some(ScalarValue::Binary(vec![1, 2, 3].into())),
578 ));
579 }
580
581 #[test]
582 fn test_buffer_string() {
583 round_trip(Scalar::new(
584 DType::Utf8(Nullability::Nullable),
585 Some(ScalarValue::Utf8(BufferString::from("hello".to_string()))),
586 ));
587 }
588
589 #[test]
590 fn test_list() {
591 round_trip(Scalar::new(
592 DType::List(
593 Arc::new(DType::Primitive(PType::I32, Nullability::Nullable)),
594 Nullability::Nullable,
595 ),
596 Some(ScalarValue::Tuple(vec![
597 Some(ScalarValue::Primitive(42i32.into())),
598 Some(ScalarValue::Primitive(43i32.into())),
599 ])),
600 ));
601 }
602
603 #[test]
604 fn test_map() {
605 let dtype = DType::map(
606 DType::Primitive(PType::I32, Nullability::NonNullable),
607 DType::Utf8(Nullability::Nullable),
608 true,
609 Nullability::Nullable,
610 )
611 .unwrap();
612 round_trip(
613 Scalar::try_map(
614 dtype.clone(),
615 [
616 (
617 Scalar::primitive(1i32, Nullability::NonNullable),
618 Scalar::utf8("one", Nullability::Nullable),
619 ),
620 (
621 Scalar::primitive(2i32, Nullability::NonNullable),
622 Scalar::null(DType::Utf8(Nullability::Nullable)),
623 ),
624 ],
625 )
626 .unwrap(),
627 );
628 round_trip(Scalar::null(dtype));
629 }
630
631 #[test]
632 fn test_f16() {
633 round_trip(Scalar::primitive(
634 f16::from_f32(0.42),
635 Nullability::Nullable,
636 ));
637 }
638
639 #[test]
640 fn test_i8() {
641 round_trip(Scalar::new(
642 DType::Primitive(PType::I8, Nullability::Nullable),
643 Some(ScalarValue::Primitive(i8::MIN.into())),
644 ));
645
646 round_trip(Scalar::new(
647 DType::Primitive(PType::I8, Nullability::Nullable),
648 Some(ScalarValue::Primitive(0i8.into())),
649 ));
650
651 round_trip(Scalar::new(
652 DType::Primitive(PType::I8, Nullability::Nullable),
653 Some(ScalarValue::Primitive(i8::MAX.into())),
654 ));
655 }
656
657 #[test]
658 fn test_decimal_i32_roundtrip() {
659 round_trip(Scalar::decimal(
661 DecimalValue::I32(123_456),
662 DecimalDType::new(10, 2),
663 Nullability::NonNullable,
664 ));
665 }
666
667 #[test]
668 fn test_decimal_i128_roundtrip() {
669 round_trip(Scalar::decimal(
671 DecimalValue::I128(99_999_999_999_999_999_999),
672 DecimalDType::new(38, 6),
673 Nullability::Nullable,
674 ));
675 }
676
677 #[test]
678 fn test_decimal_null_roundtrip() {
679 round_trip(Scalar::null(DType::Decimal(
680 DecimalDType::new(10, 2),
681 Nullability::Nullable,
682 )));
683 }
684
685 #[test]
686 fn test_scalar_value_serde_roundtrip_binary() {
687 round_trip(Scalar::binary(
688 ByteBuffer::copy_from(b"hello"),
689 Nullability::NonNullable,
690 ));
691 }
692
693 #[test]
694 fn test_scalar_value_serde_roundtrip_utf8() {
695 round_trip(Scalar::utf8("hello", Nullability::NonNullable));
696 }
697
698 #[test]
699 fn test_variant_scalar_roundtrip() {
700 let nums = Scalar::list(
701 Arc::new(DType::Variant(Nullability::NonNullable)),
702 vec![
703 Scalar::variant(Scalar::primitive(-7_i16, Nullability::NonNullable)),
704 Scalar::variant(Scalar::primitive(42_u32, Nullability::NonNullable)),
705 Scalar::variant(Scalar::decimal(
706 DecimalValue::I128(123_456_789),
707 DecimalDType::new(18, 0),
708 Nullability::NonNullable,
709 )),
710 ],
711 Nullability::NonNullable,
712 );
713
714 let nested = Scalar::list(
715 Arc::new(DType::Variant(Nullability::NonNullable)),
716 vec![
717 Scalar::variant(Scalar::from(true)),
718 Scalar::variant(nums),
719 Scalar::variant(Scalar::binary(
720 ByteBuffer::copy_from(b"abc"),
721 Nullability::NonNullable,
722 )),
723 Scalar::variant(Scalar::null(DType::Null)),
724 ],
725 Nullability::NonNullable,
726 );
727
728 round_trip(Scalar::variant(nested));
729 }
730
731 #[test]
732 fn test_variant_scalar_proto_preserves_scalar_null_vs_variant_null() {
733 let scalar_null = Scalar::null(DType::Variant(Nullability::Nullable));
734 let variant_null = Scalar::variant(Scalar::null(DType::Null));
735
736 let scalar_null_pb = pb::Scalar::from(&scalar_null);
737 let variant_null_pb = pb::Scalar::from(&variant_null);
738
739 assert_ne!(scalar_null_pb, variant_null_pb);
740 assert_eq!(
741 Scalar::from_proto(&scalar_null_pb, &session()).unwrap(),
742 scalar_null,
743 );
744 assert_eq!(
745 Scalar::from_proto(&variant_null_pb, &session()).unwrap(),
746 variant_null,
747 );
748 }
749
750 #[test]
751 fn test_union_scalar_roundtrip() -> VortexResult<()> {
752 let variants = UnionVariants::try_new(
753 ["int", "string"].into(),
754 vec![
755 DType::Primitive(PType::I32, Nullability::Nullable),
756 DType::Utf8(Nullability::NonNullable),
757 ],
758 vec![5, 9],
759 )?;
760
761 round_trip(Scalar::union(
762 variants.clone(),
763 5,
764 Scalar::primitive(42_i32, Nullability::Nullable),
765 Nullability::Nullable,
766 )?);
767
768 let inner_null = Scalar::union(
769 variants.clone(),
770 5,
771 Scalar::null(DType::Primitive(PType::I32, Nullability::Nullable)),
772 Nullability::Nullable,
773 )?;
774 let inner_null_proto = pb::Scalar::from(&inner_null);
775
776 assert!(matches!(
777 inner_null_proto
778 .value
779 .as_deref()
780 .and_then(|value| value.kind.as_ref()),
781 Some(Kind::UnionValue(union_value))
782 if matches!(
783 union_value
784 .value
785 .as_deref()
786 .and_then(|value| value.kind.as_ref()),
787 Some(Kind::NullValue(_))
788 )
789 ));
790 let outer_null = Scalar::null(DType::Union(variants, Nullability::Nullable));
791 let outer_null_proto = pb::Scalar::from(&outer_null);
792 assert!(matches!(
793 outer_null_proto
794 .value
795 .as_deref()
796 .and_then(|value| value.kind.as_ref()),
797 Some(Kind::NullValue(_))
798 ));
799
800 assert_ne!(inner_null_proto, outer_null_proto);
801 round_trip(inner_null);
802 round_trip(outer_null);
803
804 let struct_dtype = DType::struct_(
805 [
806 (
807 "number",
808 DType::Primitive(PType::I32, Nullability::NonNullable),
809 ),
810 ("label", DType::Utf8(Nullability::Nullable)),
811 ],
812 Nullability::NonNullable,
813 );
814 let struct_scalar = Scalar::struct_(
815 struct_dtype.clone(),
816 [
817 Scalar::primitive(42_i32, Nullability::NonNullable),
818 Scalar::utf8("answer", Nullability::Nullable),
819 ],
820 );
821 let struct_variants =
822 UnionVariants::try_new(["record"].into(), vec![struct_dtype], vec![13])?;
823 round_trip(Scalar::union(
824 struct_variants,
825 13,
826 struct_scalar,
827 Nullability::NonNullable,
828 )?);
829
830 Ok(())
831 }
832
833 #[test]
834 fn test_union_proto_rejects_malformed_values() -> VortexResult<()> {
835 let variants = UnionVariants::try_new(
836 ["int"].into(),
837 vec![DType::Primitive(PType::I32, Nullability::Nullable)],
838 vec![5],
839 )?;
840 let dtype = DType::Union(variants, Nullability::NonNullable);
841
842 let unknown_tag = pb::ScalarValue {
843 kind: Some(Kind::UnionValue(Box::new(PbUnionValue {
844 type_id: 7,
845 value: Some(Box::new(ScalarValue::to_proto(
846 Scalar::primitive(42_i32, Nullability::Nullable).value(),
847 ))),
848 }))),
849 };
850
851 assert!(ScalarValue::from_proto(&unknown_tag, &dtype, &session()).is_err());
852
853 let missing_child = pb::ScalarValue {
854 kind: Some(Kind::UnionValue(Box::new(PbUnionValue {
855 type_id: 5,
856 value: None,
857 }))),
858 };
859
860 let err = ScalarValue::from_proto(&missing_child, &dtype, &session())
861 .unwrap_err()
862 .to_string();
863 assert!(err.contains("UnionValue missing child value"), "{err}");
864
865 let wrong_child_value = pb::ScalarValue {
866 kind: Some(Kind::UnionValue(Box::new(PbUnionValue {
867 type_id: 5,
868 value: Some(Box::new(ScalarValue::to_proto(
869 Scalar::utf8("wrong", Nullability::NonNullable).value(),
870 ))),
871 }))),
872 };
873
874 let err = ScalarValue::from_proto(&wrong_child_value, &dtype, &session())
875 .unwrap_err()
876 .to_string();
877 assert!(
878 err.contains("expected Utf8 or Binary dtype for StringValue, got i32?"),
879 "{err}"
880 );
881
882 Ok(())
883 }
884
885 #[test]
886 fn test_backcompat_f16_serialized_as_u64() {
887 let f16_value = f16::from_f32(0.42);
903 let f16_bits_as_u64 = f16_value.to_bits() as u64; let pb_scalar_value = pb::ScalarValue {
906 kind: Some(Kind::Uint64Value(f16_bits_as_u64)),
907 };
908
909 let scalar_value = ScalarValue::from_proto(
911 &pb_scalar_value,
912 &DType::Primitive(PType::U64, Nullability::NonNullable),
913 &session(),
914 )
915 .unwrap();
916 assert_eq!(
917 scalar_value.as_ref().map(|v| v.as_primitive()),
918 Some(&PValue::U64(14008u64)),
919 );
920
921 let scalar_value_f16 = ScalarValue::from_proto(
923 &pb_scalar_value,
924 &DType::Primitive(PType::F16, Nullability::Nullable),
925 &session(),
926 )
927 .unwrap();
928
929 let scalar = Scalar::new(
930 DType::Primitive(PType::F16, Nullability::Nullable),
931 scalar_value_f16,
932 );
933
934 assert_eq!(
935 scalar.as_primitive().pvalue().unwrap(),
936 PValue::F16(f16::from_f32(0.42)),
937 "Uint64Value should be correctly interpreted as f16 when dtype is F16"
938 );
939 }
940
941 #[test]
942 fn test_scalar_value_direct_roundtrip_f16() {
943 let f16_values = vec![
945 f16::from_f32(0.0),
946 f16::from_f32(1.0),
947 f16::from_f32(-1.0),
948 f16::from_f32(0.42),
949 f16::from_f32(5.722046e-6),
950 f16::from_f32(f32::consts::PI),
951 f16::INFINITY,
952 f16::NEG_INFINITY,
953 f16::NAN,
954 ];
955
956 for f16_val in f16_values {
957 let scalar_value = ScalarValue::Primitive(PValue::F16(f16_val));
958 let pb_value = ScalarValue::to_proto(Some(&scalar_value));
959 let read_back = ScalarValue::from_proto(
960 &pb_value,
961 &DType::Primitive(PType::F16, Nullability::NonNullable),
962 &session(),
963 )
964 .unwrap();
965
966 match (&scalar_value, read_back.as_ref()) {
967 (
968 ScalarValue::Primitive(PValue::F16(original)),
969 Some(ScalarValue::Primitive(PValue::F16(roundtripped))),
970 ) => {
971 if original.is_nan() && roundtripped.is_nan() {
972 continue;
974 }
975 assert_eq!(
976 original, roundtripped,
977 "F16 value {original:?} did not roundtrip correctly"
978 );
979 }
980 _ => {
981 vortex_panic!(
982 "Expected f16 primitive values, got {scalar_value:?} and {read_back:?}"
983 )
984 }
985 }
986 }
987 }
988
989 #[test]
990 fn test_scalar_value_direct_roundtrip_preserves_values() {
991 let exact_roundtrip_cases: Vec<(&str, Option<ScalarValue>, DType)> = vec![
996 ("null", None, DType::Null),
997 (
998 "bool_true",
999 Some(ScalarValue::Bool(true)),
1000 DType::Bool(Nullability::Nullable),
1001 ),
1002 (
1003 "bool_false",
1004 Some(ScalarValue::Bool(false)),
1005 DType::Bool(Nullability::Nullable),
1006 ),
1007 (
1008 "u64",
1009 Some(ScalarValue::Primitive(PValue::U64(18446744073709551615))),
1010 DType::Primitive(PType::U64, Nullability::Nullable),
1011 ),
1012 (
1013 "i64",
1014 Some(ScalarValue::Primitive(PValue::I64(-9223372036854775808))),
1015 DType::Primitive(PType::I64, Nullability::Nullable),
1016 ),
1017 (
1018 "f32",
1019 Some(ScalarValue::Primitive(PValue::F32(f32::consts::E))),
1020 DType::Primitive(PType::F32, Nullability::Nullable),
1021 ),
1022 (
1023 "f64",
1024 Some(ScalarValue::Primitive(PValue::F64(f64::consts::PI))),
1025 DType::Primitive(PType::F64, Nullability::Nullable),
1026 ),
1027 (
1028 "string",
1029 Some(ScalarValue::Utf8(BufferString::from("test"))),
1030 DType::Utf8(Nullability::Nullable),
1031 ),
1032 (
1033 "bytes",
1034 Some(ScalarValue::Binary(vec![1, 2, 3, 4, 5].into())),
1035 DType::Binary(Nullability::Nullable),
1036 ),
1037 ];
1038
1039 for (name, value, dtype) in exact_roundtrip_cases {
1040 let pb_value = ScalarValue::to_proto(value.as_ref());
1041 let read_back = ScalarValue::from_proto(&pb_value, &dtype, &session()).unwrap();
1042
1043 let original_debug = format!("{value:?}");
1044 let roundtrip_debug = format!("{read_back:?}");
1045 assert_eq!(
1046 original_debug, roundtrip_debug,
1047 "ScalarValue {name} did not roundtrip exactly"
1048 );
1049 }
1050
1051 let unsigned_cases = vec![
1053 (
1054 "u8",
1055 ScalarValue::Primitive(PValue::U8(255)),
1056 DType::Primitive(PType::U8, Nullability::Nullable),
1057 255u64,
1058 ),
1059 (
1060 "u16",
1061 ScalarValue::Primitive(PValue::U16(65535)),
1062 DType::Primitive(PType::U16, Nullability::Nullable),
1063 65535u64,
1064 ),
1065 (
1066 "u32",
1067 ScalarValue::Primitive(PValue::U32(4294967295)),
1068 DType::Primitive(PType::U32, Nullability::Nullable),
1069 4294967295u64,
1070 ),
1071 ];
1072
1073 for (name, value, dtype, expected) in unsigned_cases {
1074 let pb_value = ScalarValue::to_proto(Some(&value));
1075 let read_back = ScalarValue::from_proto(&pb_value, &dtype, &session()).unwrap();
1076
1077 match read_back.as_ref() {
1078 Some(ScalarValue::Primitive(pv)) => {
1079 assert_eq!(pv.ptype(), dtype.as_ptype(), "{name} primitive width");
1080 let v = match pv {
1081 PValue::U8(v) => *v as u64,
1082 PValue::U16(v) => *v as u64,
1083 PValue::U32(v) => *v as u64,
1084 PValue::U64(v) => *v,
1085 _ => {
1086 vortex_panic!("Unexpected primitive type for {name}: {pv:?}")
1087 }
1088 };
1089 assert_eq!(
1090 v, expected,
1091 "ScalarValue {name} value not preserved: expected {expected}, got {v}"
1092 );
1093 }
1094 _ => {
1095 vortex_panic!("Unexpected type after roundtrip for {name}: {read_back:?}")
1096 }
1097 }
1098 }
1099
1100 let signed_cases = vec![
1102 (
1103 "i8",
1104 ScalarValue::Primitive(PValue::I8(-128)),
1105 DType::Primitive(PType::I8, Nullability::Nullable),
1106 -128i64,
1107 ),
1108 (
1109 "i16",
1110 ScalarValue::Primitive(PValue::I16(-32768)),
1111 DType::Primitive(PType::I16, Nullability::Nullable),
1112 -32768i64,
1113 ),
1114 (
1115 "i32",
1116 ScalarValue::Primitive(PValue::I32(-2147483648)),
1117 DType::Primitive(PType::I32, Nullability::Nullable),
1118 -2147483648i64,
1119 ),
1120 ];
1121
1122 for (name, value, dtype, expected) in signed_cases {
1123 let pb_value = ScalarValue::to_proto(Some(&value));
1124 let read_back = ScalarValue::from_proto(&pb_value, &dtype, &session()).unwrap();
1125
1126 match read_back.as_ref() {
1127 Some(ScalarValue::Primitive(pv)) => {
1128 assert_eq!(pv.ptype(), dtype.as_ptype(), "{name} primitive width");
1129 let v = match pv {
1130 PValue::I8(v) => *v as i64,
1131 PValue::I16(v) => *v as i64,
1132 PValue::I32(v) => *v as i64,
1133 PValue::I64(v) => *v,
1134 _ => {
1135 vortex_panic!("Unexpected primitive type for {name}: {pv:?}")
1136 }
1137 };
1138 assert_eq!(
1139 v, expected,
1140 "ScalarValue {name} value not preserved: expected {expected}, got {v}"
1141 );
1142 }
1143 _ => {
1144 vortex_panic!("Unexpected type after roundtrip for {name}: {read_back:?}")
1145 }
1146 }
1147 }
1148 }
1149
1150 #[test]
1153 fn test_backcompat_signed_integer_deserialized_as_unsigned() {
1154 let v = ScalarValue::Primitive(PValue::I64(0));
1155 assert_eq!(
1156 Scalar::from_proto_value(
1157 &pb::ScalarValue::from(&v),
1158 &DType::Primitive(PType::U64, Nullability::Nullable),
1159 &session()
1160 )
1161 .unwrap(),
1162 Scalar::primitive(0u64, Nullability::Nullable)
1163 );
1164 }
1165
1166 #[test]
1169 fn test_backcompat_unsigned_integer_deserialized_as_signed() {
1170 let v = ScalarValue::Primitive(PValue::U64(0));
1171 assert_eq!(
1172 Scalar::from_proto_value(
1173 &pb::ScalarValue::from(&v),
1174 &DType::Primitive(PType::I64, Nullability::Nullable),
1175 &session()
1176 )
1177 .unwrap(),
1178 Scalar::primitive(0i64, Nullability::Nullable)
1179 );
1180 }
1181}