1use std::fmt::Display;
5use std::fmt::Formatter;
6
7use num_traits::AsPrimitive;
8use vortex_array::arrays::PrimitiveArray;
9use vortex_buffer::BufferAllocatorRef;
10use vortex_buffer::ByteBuffer;
11use vortex_error::VortexExpect;
12use vortex_error::VortexResult;
13use vortex_error::vortex_bail;
14use vortex_error::vortex_ensure;
15use vortex_error::vortex_ensure_eq;
16use vortex_error::vortex_err;
17
18use crate::ArrayRef;
19use crate::ArraySlots;
20use crate::ExecutionCtx;
21use crate::aggregate_fn::NumericalAggregateOpts;
22use crate::aggregate_fn::fns::min_max::min_max;
23use crate::array::Array;
24use crate::array::ArrayParts;
25use crate::array::TypedArrayRef;
26use crate::array::child_to_validity;
27use crate::array::validity_to_child;
28use crate::array_slots;
29use crate::arrays::VarBin;
30use crate::arrays::varbin::builder::VarBinBuilder;
31use crate::buffer::BufferHandle;
32use crate::dtype::DType;
33use crate::dtype::Nullability;
34use crate::dtype::OffsetBuilderPType;
35use crate::match_each_integer_ptype;
36use crate::validity::Validity;
37
38fn is_char_boundary_at(bytes: &[u8], index: usize) -> bool {
40 index == bytes.len() || bytes[index] & 0b1100_0000 != 0b1000_0000
42}
43
44pub(crate) fn offsets_tile_utf8<O: AsPrimitive<usize> + PartialOrd>(
51 offsets: &[O],
52 bytes: &[u8],
53) -> bool {
54 let (Some(first), Some(last)) = (offsets.first(), offsets.last()) else {
55 return false;
56 };
57 let (first, last): (usize, usize) = (first.as_(), last.as_());
58
59 first <= last
60 && last <= bytes.len()
61 && offsets.windows(2).all(|pair| pair[0] <= pair[1])
62 && simdutf8::basic::from_utf8(&bytes[first..last]).is_ok()
63 && offsets
64 .iter()
65 .all(|&offset| is_char_boundary_at(bytes, offset.as_()))
66}
67
68#[array_slots(VarBin)]
69pub struct VarBinSlots {
70 #[slot(0)]
72 pub offsets: ArrayRef,
73 #[slot(1)]
75 pub validity: Option<ArrayRef>,
76}
77
78#[derive(Clone, Debug)]
79pub struct VarBinData {
80 pub(super) bytes: BufferHandle,
81}
82
83impl Display for VarBinData {
84 fn fmt(&self, _f: &mut Formatter<'_>) -> std::fmt::Result {
85 Ok(())
86 }
87}
88
89pub struct VarBinDataParts {
90 pub dtype: DType,
91 pub bytes: BufferHandle,
92 pub offsets: ArrayRef,
93 pub validity: Validity,
94}
95
96impl VarBinData {
97 pub fn build(
104 offsets: ArrayRef,
105 bytes: ByteBuffer,
106 dtype: DType,
107 validity: Validity,
108 ctx: &mut ExecutionCtx,
109 ) -> Self {
110 Self::try_build(offsets, bytes, dtype, validity, ctx).vortex_expect("VarBinArray new")
111 }
112
113 pub fn build_from_handle(
120 offset: ArrayRef,
121 bytes: BufferHandle,
122 dtype: DType,
123 validity: Validity,
124 ctx: &mut ExecutionCtx,
125 ) -> Self {
126 Self::try_build_from_handle(offset, bytes, dtype, validity, ctx)
127 .vortex_expect("VarBinArray new")
128 }
129
130 pub(crate) fn make_slots(offsets: ArrayRef, validity: &Validity, len: usize) -> ArraySlots {
131 VarBinSlots {
132 offsets,
133 validity: validity_to_child(validity, len),
134 }
135 .into_slots()
136 }
137
138 pub fn try_build(
147 offsets: ArrayRef,
148 bytes: ByteBuffer,
149 dtype: DType,
150 validity: Validity,
151 ctx: &mut ExecutionCtx,
152 ) -> VortexResult<Self> {
153 let bytes = BufferHandle::new_host(bytes);
154 Self::validate(&offsets, &bytes, &dtype, &validity, ctx)?;
155
156 Ok(unsafe { Self::new_unchecked_from_handle(bytes) })
158 }
159
160 pub fn try_build_from_handle(
170 offsets: ArrayRef,
171 bytes: BufferHandle,
172 dtype: DType,
173 validity: Validity,
174 ctx: &mut ExecutionCtx,
175 ) -> VortexResult<Self> {
176 Self::validate(&offsets, &bytes, &dtype, &validity, ctx)?;
177
178 Ok(unsafe { Self::new_unchecked_from_handle(bytes) })
180 }
181
182 pub unsafe fn new_unchecked(bytes: ByteBuffer) -> Self {
211 unsafe { Self::new_unchecked_from_handle(BufferHandle::new_host(bytes)) }
214 }
215
216 pub unsafe fn new_unchecked_from_handle(bytes: BufferHandle) -> Self {
223 Self { bytes }
224 }
225
226 pub fn validate(
230 offsets: &ArrayRef,
231 bytes: &BufferHandle,
232 dtype: &DType,
233 validity: &Validity,
234 ctx: &mut ExecutionCtx,
235 ) -> VortexResult<()> {
236 vortex_ensure!(
238 offsets.dtype().is_int() && !offsets.dtype().is_nullable(),
239 "expected type: non nullable int but instead got {}",
240 offsets.dtype()
241 );
242
243 vortex_ensure!(
245 matches!(dtype, DType::Binary(_) | DType::Utf8(_)),
246 "expected type: utf8 or binary but instead got {}",
247 dtype
248 );
249
250 vortex_ensure!(
252 dtype.is_nullable() != matches!(validity, Validity::NonNullable),
253 "incorrect validity {:?} for dtype {}",
254 validity,
255 dtype
256 );
257
258 vortex_ensure!(
260 !offsets.is_empty(),
261 "Offsets must have at least one element"
262 );
263
264 if let Some(validity_len) = validity.maybe_len() {
266 vortex_ensure_eq!(
267 validity_len,
268 offsets.len() - 1,
269 "Validity length doesn't match array length",
270 );
271 }
272
273 if offsets.is_host()
276 && bytes.is_on_host()
277 && matches!(dtype, DType::Utf8(_))
278 && let Some(bytes) = bytes.as_host_opt()
279 {
280 Self::validate_utf8(offsets, bytes.as_ref(), validity, ctx)?;
281 } else if offsets.is_host() {
282 Self::validate_offsets(offsets, bytes.len(), ctx)?;
283 }
284
285 Ok(())
286 }
287
288 fn validate_offsets(
290 offsets: &ArrayRef,
291 bytes_len: usize,
292 ctx: &mut ExecutionCtx,
293 ) -> VortexResult<()> {
294 let Some(is_sorted) = offsets.statistics().compute_is_sorted(ctx) else {
296 vortex_bail!("offsets must report is_sorted statistic");
297 };
298 vortex_ensure!(is_sorted, "offsets must be sorted");
299
300 let Some(min_max) = min_max(offsets, ctx, NumericalAggregateOpts::default())? else {
303 vortex_bail!(
304 "offsets array with encoding {} must support min_max compute function",
305 offsets.encoding_id()
306 );
307 };
308
309 match_each_integer_ptype!(offsets.dtype().as_ptype(), |P| {
310 #[allow(clippy::absurd_extreme_comparisons, unused_comparisons)]
311 {
312 let max = min_max
313 .max
314 .as_primitive()
315 .as_::<P>()
316 .vortex_expect("offsets type must fit offsets values");
317 let min = min_max
318 .min
319 .as_primitive()
320 .as_::<P>()
321 .vortex_expect("offsets type must fit offsets values");
322
323 vortex_ensure!(
324 min >= 0,
325 "offsets minimum {min} outside valid range [0, {max}]"
326 );
327
328 vortex_ensure!(
330 P::try_from(bytes_len).ok().is_none_or(|len| max <= len),
331 "Max offset {max} is beyond the length of the bytes buffer {bytes_len}"
332 );
333 }
334 });
335
336 Ok(())
337 }
338
339 fn validate_utf8(
341 offsets: &ArrayRef,
342 bytes: &[u8],
343 validity: &Validity,
344 ctx: &mut ExecutionCtx,
345 ) -> VortexResult<()> {
346 let validate_at = |i: usize, start: usize, end: usize| -> VortexResult<()> {
347 let string_bytes = bytes.get(start..end).ok_or_else(|| {
348 vortex_err!(
349 "offsets {start}..{end} at index {i} are out of order or out of bounds for bytes of length {}",
350 bytes.len()
351 )
352 })?;
353 simdutf8::basic::from_utf8(string_bytes).map_err(|_| {
354 #[expect(clippy::unwrap_used)]
355 let err = simdutf8::compat::from_utf8(string_bytes).unwrap_err();
357 vortex_err!("invalid utf-8: {err} at index {i}")
358 })?;
359 Ok(())
360 };
361
362 let primitive_offsets = offsets.clone().execute::<PrimitiveArray>(ctx)?;
364
365 let mask = match validity {
369 Validity::Array(_) => {
370 Some(validity.execute_mask(primitive_offsets.len().saturating_sub(1), ctx)?)
371 }
372 _ => None,
373 };
374 let all_invalid = validity.definitely_all_null();
375
376 match_each_integer_ptype!(primitive_offsets.dtype().as_ptype(), |O| {
377 let offsets_slice = primitive_offsets.as_slice::<O>();
378
379 let last_offset: usize = offsets_slice[offsets_slice.len() - 1].as_();
380 vortex_ensure!(
381 last_offset <= bytes.len(),
382 "Last offset {} exceeds bytes length {}",
383 last_offset,
384 bytes.len()
385 );
386
387 if offsets_tile_utf8(offsets_slice, bytes) {
388 return Ok(());
389 }
390
391 Self::validate_offsets(offsets, bytes.len(), ctx)?;
394
395 for (i, (start, end)) in offsets_slice
397 .windows(2)
398 .map(|o| (o[0].as_(), o[1].as_()))
399 .enumerate()
400 {
401 let valid = mask.as_ref().map_or(!all_invalid, |mask| mask.value(i));
402 if valid {
403 validate_at(i, start, end)?;
404 }
405 }
406 });
407 Ok(())
408 }
409
410 #[inline]
418 pub fn bytes(&self) -> &ByteBuffer {
419 self.bytes.as_host()
420 }
421
422 #[inline]
424 pub fn bytes_handle(&self) -> &BufferHandle {
425 &self.bytes
426 }
427}
428
429pub trait VarBinArrayExt: VarBinArraySlotsExt {
430 fn dtype_parts(&self) -> (bool, Nullability) {
431 match self.dtype() {
432 DType::Utf8(nullability) => (true, *nullability),
433 DType::Binary(nullability) => (false, *nullability),
434 _ => unreachable!("VarBinArrayExt requires a utf8 or binary dtype"),
435 }
436 }
437
438 fn is_utf8(&self) -> bool {
439 self.dtype_parts().0
440 }
441
442 fn nullability(&self) -> Nullability {
443 self.dtype_parts().1
444 }
445
446 fn varbin_validity(&self) -> Validity {
447 child_to_validity(
448 self.slots()[VarBinSlots::VALIDITY].as_ref(),
449 self.nullability(),
450 )
451 }
452
453 fn offset_at(&self, index: usize, ctx: &mut ExecutionCtx) -> usize {
454 assert!(
455 index <= self.len(),
456 "Index {index} out of bounds 0..={}",
457 self.len()
458 );
459
460 (&self
461 .offsets()
462 .execute_scalar(index, ctx)
463 .vortex_expect("offsets must support execute_scalar"))
464 .try_into()
465 .vortex_expect("Failed to convert offset to usize")
466 }
467
468 fn bytes_at(&self, index: usize, ctx: &mut ExecutionCtx) -> ByteBuffer {
469 let start = self.offset_at(index, ctx);
470 let end = self.offset_at(index + 1, ctx);
471 self.bytes().slice(start..end)
472 }
473
474 fn sliced_bytes(&self, ctx: &mut ExecutionCtx) -> ByteBuffer {
475 let first_offset: usize = self.offset_at(0, ctx);
476 let last_offset = self.offset_at(self.len(), ctx);
477 self.bytes().slice(first_offset..last_offset)
478 }
479}
480impl<T: TypedArrayRef<VarBin>> VarBinArrayExt for T {}
481
482impl Array<VarBin> {
484 pub fn from_vec<T: AsRef<[u8]>>(vec: Vec<T>, dtype: DType) -> Self {
485 let size: usize = vec.iter().map(|v| v.as_ref().len()).sum();
486 if size < u32::MAX as usize {
487 Self::from_vec_sized::<u32, T>(vec, dtype)
488 } else {
489 Self::from_vec_sized::<u64, T>(vec, dtype)
490 }
491 }
492
493 #[expect(
494 clippy::same_name_method,
495 reason = "intentionally named from_iter like Iterator::from_iter"
496 )]
497 pub fn from_iter<T: AsRef<[u8]>, I: IntoIterator<Item = Option<T>>>(
498 iter: I,
499 dtype: DType,
500 ) -> Self {
501 let iter = iter.into_iter();
502 let mut builder = VarBinBuilder::<u32>::with_capacity_in(
503 dtype,
504 iter.size_hint().0,
505 BufferAllocatorRef::static_ref(),
506 );
507 for v in iter {
508 builder.append(v.as_ref().map(|o| o.as_ref()));
509 }
510 builder.finish_into_varbin()
511 }
512
513 pub fn from_iter_nonnull<T: AsRef<[u8]>, I: IntoIterator<Item = T>>(
514 iter: I,
515 dtype: DType,
516 ) -> Self {
517 let iter = iter.into_iter();
518 let mut builder = VarBinBuilder::<u32>::with_capacity_in(
519 dtype,
520 iter.size_hint().0,
521 BufferAllocatorRef::static_ref(),
522 );
523 for v in iter {
524 builder.append_value(v);
525 }
526 builder.finish_into_varbin()
527 }
528
529 fn from_vec_sized<O, T>(vec: Vec<T>, dtype: DType) -> Self
530 where
531 O: OffsetBuilderPType,
532 T: AsRef<[u8]>,
533 {
534 let mut builder = VarBinBuilder::<O>::with_capacity_in(
535 dtype,
536 vec.len(),
537 BufferAllocatorRef::static_ref(),
538 );
539 for v in vec {
540 builder.append_value(v.as_ref());
541 }
542 builder.finish_into_varbin()
543 }
544
545 pub fn from_strs(value: Vec<&str>) -> Self {
547 Self::from_vec(value, DType::Utf8(Nullability::NonNullable))
548 }
549
550 pub fn from_nullable_strs(value: Vec<Option<&str>>) -> Self {
552 Self::from_iter(value, DType::Utf8(Nullability::Nullable))
553 }
554
555 pub fn from_bytes(value: Vec<&[u8]>) -> Self {
557 Self::from_vec(value, DType::Binary(Nullability::NonNullable))
558 }
559
560 pub fn from_nullable_bytes(value: Vec<Option<&[u8]>>) -> Self {
562 Self::from_iter(value, DType::Binary(Nullability::Nullable))
563 }
564
565 pub fn into_data_parts(self) -> VarBinDataParts {
566 let dtype = self.dtype().clone();
567 let validity = self.varbin_validity();
568 let offsets = self.offsets().clone();
569 let data = self.into_data();
570 VarBinDataParts {
571 dtype,
572 bytes: data.bytes,
573 offsets,
574 validity,
575 }
576 }
577}
578
579impl Array<VarBin> {
580 pub fn new(
582 offsets: ArrayRef,
583 bytes: ByteBuffer,
584 dtype: DType,
585 validity: Validity,
586 ctx: &mut ExecutionCtx,
587 ) -> Self {
588 let len = offsets.len().saturating_sub(1);
589 let slots = VarBinData::make_slots(offsets, &validity, len);
590 let data = VarBinData::build(
591 slots[VarBinSlots::OFFSETS]
592 .as_ref()
593 .vortex_expect("VarBinArray offsets slot")
594 .clone(),
595 bytes,
596 dtype.clone(),
597 validity,
598 ctx,
599 );
600 unsafe { Array::from_parts_unchecked(ArrayParts::new(VarBin, dtype, len, data, slots)) }
601 }
602
603 pub unsafe fn new_unchecked(
609 offsets: ArrayRef,
610 bytes: ByteBuffer,
611 dtype: DType,
612 validity: Validity,
613 ) -> Self {
614 let len = offsets.len().saturating_sub(1);
615 let slots = VarBinData::make_slots(offsets, &validity, len);
616 let data = unsafe { VarBinData::new_unchecked(bytes) };
617 unsafe { Array::from_parts_unchecked(ArrayParts::new(VarBin, dtype, len, data, slots)) }
618 }
619
620 pub unsafe fn new_unchecked_from_handle(
626 offsets: ArrayRef,
627 bytes: BufferHandle,
628 dtype: DType,
629 validity: Validity,
630 ) -> Self {
631 let len = offsets.len().saturating_sub(1);
632 let slots = VarBinData::make_slots(offsets, &validity, len);
633 let data = unsafe { VarBinData::new_unchecked_from_handle(bytes) };
634 unsafe { Array::from_parts_unchecked(ArrayParts::new(VarBin, dtype, len, data, slots)) }
635 }
636
637 pub fn try_new(
640 offsets: ArrayRef,
641 bytes: ByteBuffer,
642 dtype: DType,
643 validity: Validity,
644 ctx: &mut ExecutionCtx,
645 ) -> VortexResult<Self> {
646 let len = offsets.len() - 1;
647 let bytes = BufferHandle::new_host(bytes);
648 VarBinData::validate(&offsets, &bytes, &dtype, &validity, ctx)?;
649 let slots = VarBinData::make_slots(offsets, &validity, len);
650 let data = unsafe { VarBinData::new_unchecked_from_handle(bytes) };
652 Ok(
653 unsafe {
654 Array::from_parts_unchecked(ArrayParts::new(VarBin, dtype, len, data, slots))
655 },
656 )
657 }
658}
659
660impl From<Vec<&[u8]>> for Array<VarBin> {
661 fn from(value: Vec<&[u8]>) -> Self {
662 Self::from_vec(value, DType::Binary(Nullability::NonNullable))
663 }
664}
665
666impl From<Vec<Vec<u8>>> for Array<VarBin> {
667 fn from(value: Vec<Vec<u8>>) -> Self {
668 Self::from_vec(value, DType::Binary(Nullability::NonNullable))
669 }
670}
671
672impl From<Vec<String>> for Array<VarBin> {
673 fn from(value: Vec<String>) -> Self {
674 Self::from_vec(value, DType::Utf8(Nullability::NonNullable))
675 }
676}
677
678impl From<Vec<&str>> for Array<VarBin> {
679 fn from(value: Vec<&str>) -> Self {
680 Self::from_vec(value, DType::Utf8(Nullability::NonNullable))
681 }
682}
683
684impl From<Vec<Option<&[u8]>>> for Array<VarBin> {
685 fn from(value: Vec<Option<&[u8]>>) -> Self {
686 Self::from_iter(value, DType::Binary(Nullability::Nullable))
687 }
688}
689
690impl From<Vec<Option<Vec<u8>>>> for Array<VarBin> {
691 fn from(value: Vec<Option<Vec<u8>>>) -> Self {
692 Self::from_iter(value, DType::Binary(Nullability::Nullable))
693 }
694}
695
696impl From<Vec<Option<String>>> for Array<VarBin> {
697 fn from(value: Vec<Option<String>>) -> Self {
698 Self::from_iter(value, DType::Utf8(Nullability::Nullable))
699 }
700}
701
702impl From<Vec<Option<&str>>> for Array<VarBin> {
703 fn from(value: Vec<Option<&str>>) -> Self {
704 Self::from_iter(value, DType::Utf8(Nullability::Nullable))
705 }
706}
707
708impl<'a> FromIterator<Option<&'a [u8]>> for Array<VarBin> {
709 fn from_iter<T: IntoIterator<Item = Option<&'a [u8]>>>(iter: T) -> Self {
710 Self::from_iter(iter, DType::Binary(Nullability::Nullable))
711 }
712}
713
714impl FromIterator<Option<Vec<u8>>> for Array<VarBin> {
715 fn from_iter<T: IntoIterator<Item = Option<Vec<u8>>>>(iter: T) -> Self {
716 Self::from_iter(iter, DType::Binary(Nullability::Nullable))
717 }
718}
719
720impl FromIterator<Option<String>> for Array<VarBin> {
721 fn from_iter<T: IntoIterator<Item = Option<String>>>(iter: T) -> Self {
722 Self::from_iter(iter, DType::Utf8(Nullability::Nullable))
723 }
724}
725
726impl<'a> FromIterator<Option<&'a str>> for Array<VarBin> {
727 fn from_iter<T: IntoIterator<Item = Option<&'a str>>>(iter: T) -> Self {
728 Self::from_iter(iter, DType::Utf8(Nullability::Nullable))
729 }
730}