1use std::any::Any;
55use std::fmt;
56use std::sync::Arc;
57
58use crate::array::binary::GenericBinaryArray;
59use crate::array::iter::{ArrayAccessor, ArrayIter};
60use crate::array::offset::OffsetSizeTrait;
61use crate::array::{Array, ArrayRef, LARGE_UTF8_TYPE, UTF8_TYPE, debug_array};
62use crate::buffer::{Bitmap, Buffer, ScalarBuffer};
63use crate::datatype::DataType;
64use crate::error::{DataError, Result};
65use crate::sealed::Sealed;
66
67#[derive(Clone)]
72pub struct GenericStringArray<O: OffsetSizeTrait> {
73 inner: GenericBinaryArray<O>,
75}
76
77pub type StringArray = GenericStringArray<i32>;
79pub type LargeStringArray = GenericStringArray<i64>;
81
82fn validate_utf8<O: OffsetSizeTrait>(offsets: &[O], values: &[u8]) -> Result<()> {
92 let (Some(first), Some(last)) = (offsets.first(), offsets.last()) else {
93 return Ok(());
94 };
95 let start = first.to_usize().unwrap_or(0);
96 let end = last.to_usize().unwrap_or(0);
97 let Some(region) = values.get(start..end) else {
98 return Err(DataError::OffsetOutOfBounds {
99 index: offsets.len().saturating_sub(1),
100 offset: end,
101 values_len: values.len(),
102 });
103 };
104 let text = std::str::from_utf8(region).map_err(|error| DataError::InvalidUtf8 {
105 valid_up_to: start + error.valid_up_to(),
106 })?;
107 for (index, offset) in offsets.iter().enumerate() {
108 let absolute = offset.to_usize().unwrap_or(0);
109 let relative = absolute.saturating_sub(start);
110 if !text.is_char_boundary(relative) {
111 return Err(DataError::OffsetNotCharBoundary {
112 index,
113 offset: absolute,
114 });
115 }
116 }
117 Ok(())
118}
119
120impl<O: OffsetSizeTrait> GenericStringArray<O> {
121 pub fn try_new(
131 offsets: ScalarBuffer<O>,
132 values: Buffer,
133 validity: Option<Bitmap>,
134 ) -> Result<Self> {
135 let inner = GenericBinaryArray::try_new(offsets, values, validity)?;
136 validate_utf8(inner.value_offsets(), inner.value_data().as_slice())?;
137 Ok(Self { inner })
138 }
139
140 #[must_use]
155 pub unsafe fn new_unchecked(
156 offsets: ScalarBuffer<O>,
157 values: Buffer,
158 validity: Option<Bitmap>,
159 ) -> Self {
160 Self::from_parts(offsets, values, validity)
161 }
162
163 pub(crate) fn from_parts(
166 offsets: ScalarBuffer<O>,
167 values: Buffer,
168 validity: Option<Bitmap>,
169 ) -> Self {
170 debug_assert!(validate_utf8(offsets.as_slice(), values.as_slice()).is_ok());
171 Self {
172 inner: GenericBinaryArray::from_parts(offsets, values, validity),
173 }
174 }
175
176 #[must_use]
186 pub fn from_values<V: AsRef<str>>(values: impl IntoIterator<Item = V>) -> Self {
187 Self {
188 inner: GenericBinaryArray::from_values(values.into_iter().map(|value| BytesOf(value))),
189 }
190 }
191
192 #[must_use]
194 pub fn from_opt_iter<V: AsRef<str>>(values: impl IntoIterator<Item = Option<V>>) -> Self {
195 Self {
196 inner: GenericBinaryArray::from_opt_iter(
197 values.into_iter().map(|value| value.map(BytesOf)),
198 ),
199 }
200 }
201
202 #[must_use]
204 pub fn new_null(len: usize) -> Self {
205 Self {
206 inner: GenericBinaryArray::new_null(len),
207 }
208 }
209
210 #[inline]
212 #[must_use]
213 pub fn value_offsets(&self) -> &[O] {
214 self.inner.value_offsets()
215 }
216
217 #[inline]
229 #[must_use]
230 pub const fn offsets_buffer(&self) -> &ScalarBuffer<O> {
231 self.inner.offsets_buffer()
232 }
233
234 #[inline]
236 #[must_use]
237 pub const fn value_data(&self) -> &Buffer {
238 self.inner.value_data()
239 }
240
241 #[inline]
246 #[must_use]
247 pub fn value_bytes(&self, index: usize) -> Option<&[u8]> {
248 self.inner.value(index)
249 }
250
251 #[inline]
256 #[must_use]
257 pub fn value(&self, index: usize) -> Option<&str> {
258 std::str::from_utf8(self.inner.value(index)?).ok()
259 }
260
261 #[inline]
264 #[must_use]
265 pub fn get(&self, index: usize) -> Option<&str> {
266 std::str::from_utf8(self.inner.get(index)?).ok()
267 }
268
269 #[inline]
271 #[must_use]
272 pub fn value_length(&self, index: usize) -> Option<usize> {
273 self.inner.value_length(index)
274 }
275
276 #[must_use]
278 pub fn total_value_bytes(&self) -> usize {
279 self.inner.total_value_bytes()
280 }
281
282 #[inline]
284 pub fn iter(&self) -> ArrayIter<&Self> {
285 ArrayIter::new(self)
286 }
287
288 #[must_use]
295 pub fn slice(&self, offset: usize, len: usize) -> Self {
296 Self {
297 inner: self.inner.slice(offset, len),
298 }
299 }
300
301 pub fn try_slice(&self, offset: usize, len: usize) -> Result<Self> {
307 Ok(Self {
308 inner: self.inner.try_slice(offset, len)?,
309 })
310 }
311
312 pub fn with_validity(&self, validity: Option<Bitmap>) -> Result<Self> {
318 Ok(Self {
319 inner: self.inner.with_validity(validity)?,
320 })
321 }
322
323 #[must_use]
327 pub fn into_binary(self) -> GenericBinaryArray<O> {
328 self.inner
329 }
330
331 #[inline]
333 #[must_use]
334 pub const fn as_binary(&self) -> &GenericBinaryArray<O> {
335 &self.inner
336 }
337}
338
339struct BytesOf<V: AsRef<str>>(V);
341
342impl<V: AsRef<str>> AsRef<[u8]> for BytesOf<V> {
343 #[inline]
344 fn as_ref(&self) -> &[u8] {
345 self.0.as_ref().as_bytes()
346 }
347}
348
349impl<O: OffsetSizeTrait> TryFrom<GenericBinaryArray<O>> for GenericStringArray<O> {
350 type Error = DataError;
351
352 fn try_from(inner: GenericBinaryArray<O>) -> Result<Self> {
354 validate_utf8(inner.value_offsets(), inner.value_data().as_slice())?;
355 Ok(Self { inner })
356 }
357}
358
359impl<O: OffsetSizeTrait> Sealed for GenericStringArray<O> {}
360
361impl<O: OffsetSizeTrait> Array for GenericStringArray<O> {
362 fn as_any(&self) -> &dyn Any {
363 self
364 }
365
366 fn data_type(&self) -> &DataType {
367 if O::IS_LARGE {
368 &LARGE_UTF8_TYPE
369 } else {
370 &UTF8_TYPE
371 }
372 }
373
374 fn len(&self) -> usize {
375 self.inner.len()
376 }
377
378 fn validity(&self) -> Option<&Bitmap> {
379 self.inner.validity()
380 }
381
382 fn slice(&self, offset: usize, len: usize) -> ArrayRef {
383 Arc::new(Self::slice(self, offset, len))
384 }
385
386 fn buffer_memory_size(&self) -> usize {
387 self.inner.buffer_memory_size()
388 }
389
390 fn equals(&self, other: &dyn Array) -> bool {
391 other
392 .as_any()
393 .downcast_ref::<Self>()
394 .is_some_and(|other| self.inner.equals(&other.inner))
395 }
396}
397
398impl<'a, O: OffsetSizeTrait> ArrayAccessor for &'a GenericStringArray<O> {
399 type Item = &'a str;
400
401 #[inline]
402 fn accessor_len(&self) -> usize {
403 self.inner.len()
404 }
405
406 #[inline]
407 fn accessor_get(&self, index: usize) -> Option<&'a str> {
408 let array: &'a GenericStringArray<O> = self;
409 array.get(index)
410 }
411}
412
413impl<O: OffsetSizeTrait> PartialEq for GenericStringArray<O> {
414 #[inline]
415 fn eq(&self, other: &Self) -> bool {
416 self.inner == other.inner
417 }
418}
419
420impl<O: OffsetSizeTrait> Eq for GenericStringArray<O> {}
421
422impl<O: OffsetSizeTrait, V: AsRef<str>> FromIterator<Option<V>> for GenericStringArray<O> {
423 fn from_iter<I: IntoIterator<Item = Option<V>>>(iter: I) -> Self {
424 Self::from_opt_iter(iter)
425 }
426}
427
428impl<'a, O: OffsetSizeTrait> IntoIterator for &'a GenericStringArray<O> {
429 type Item = Option<&'a str>;
430 type IntoIter = ArrayIter<&'a GenericStringArray<O>>;
431
432 fn into_iter(self) -> Self::IntoIter {
433 self.iter()
434 }
435}
436
437impl<O: OffsetSizeTrait> fmt::Debug for GenericStringArray<O> {
438 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
439 debug_array(
440 f,
441 if O::IS_LARGE {
442 "LargeStringArray"
443 } else {
444 "StringArray"
445 },
446 self.data_type(),
447 self.len(),
448 self.null_count(),
449 self.iter(),
450 )
451 }
452}
453
454#[cfg(test)]
455mod tests {
456 #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
457
458 use super::*;
459
460 #[test]
461 fn empty_array() {
462 let array = StringArray::from_values([] as [&str; 0]);
463 assert!(array.is_empty());
464 assert_eq!(array.null_count(), 0);
465 assert_eq!(array.get(0), None);
466 assert_eq!(array.iter().count(), 0);
467 assert_eq!(array.data_type(), &DataType::Utf8);
468 assert_eq!(array.total_value_bytes(), 0);
469 }
470
471 #[test]
472 fn no_null_array() {
473 let array = StringArray::from_values(["lidar", "", "camera"]);
474 assert_eq!(array.len(), 3);
475 assert_eq!(array.null_count(), 0);
476 assert!(array.validity().is_none());
477 assert_eq!(array.get(0), Some("lidar"));
478 assert_eq!(array.get(1), Some(""));
479 assert_eq!(array.get(2), Some("camera"));
480 assert_eq!(array.get(3), None);
481 assert_eq!(array.value_length(0), Some(5));
482 assert_eq!(array.value_bytes(2), Some(&b"camera"[..]));
483 assert_eq!(array.value_data().as_slice(), b"lidarcamera");
484 }
485
486 #[test]
487 fn all_null_array() {
488 let array = LargeStringArray::new_null(3);
489 assert_eq!(array.len(), 3);
490 assert_eq!(array.null_count(), 3);
491 assert_eq!(array.data_type(), &DataType::LargeUtf8);
492 for index in 0..3 {
493 assert!(array.is_null(index));
494 assert_eq!(array.get(index), None);
495 assert_eq!(array.value(index), Some(""));
496 }
497 }
498
499 #[test]
500 fn mixed_nulls() {
501 let array = StringArray::from_opt_iter([Some("a"), None, Some("ccc"), None]);
502 assert_eq!(array.len(), 4);
503 assert_eq!(array.null_count(), 2);
504 assert_eq!(
505 array.iter().collect::<Vec<_>>(),
506 vec![Some("a"), None, Some("ccc"), None]
507 );
508 assert_eq!(array.total_value_bytes(), 4);
509 }
510
511 #[test]
512 fn multibyte_text_round_trips() {
513 let values = ["日本語", "é", "🤖", "mixed 日本 text"];
514 let array = StringArray::from_values(values);
515 for (index, expected) in values.iter().enumerate() {
516 assert_eq!(array.get(index), Some(*expected));
517 assert_eq!(array.value_length(index), Some(expected.len()));
518 }
519 assert_eq!(array.get(2).map(str::chars).map(Iterator::count), Some(1));
520 }
521
522 #[test]
523 fn valid_region_with_a_split_code_point_is_rejected() {
524 let err = StringArray::try_new(
527 ScalarBuffer::from_slice(&[0, 1, 2]),
528 Buffer::from_slice("é".as_bytes()),
529 None,
530 )
531 .unwrap_err();
532 assert_eq!(
533 err,
534 DataError::OffsetNotCharBoundary {
535 index: 1,
536 offset: 1
537 }
538 );
539
540 let emoji = "🤖".as_bytes();
542 for split in 1..4 {
543 let err = StringArray::try_new(
544 ScalarBuffer::from_slice(&[0, split, 4]),
545 Buffer::from_slice(emoji),
546 None,
547 )
548 .unwrap_err();
549 assert!(
550 matches!(err, DataError::OffsetNotCharBoundary { .. }),
551 "split {split} accepted"
552 );
553 }
554 }
555
556 #[test]
557 fn invalid_bytes_are_rejected() {
558 let err = StringArray::try_new(
559 ScalarBuffer::from_slice(&[0, 2]),
560 Buffer::from_slice(&[0xff, 0xfe]),
561 None,
562 )
563 .unwrap_err();
564 assert_eq!(err, DataError::InvalidUtf8 { valid_up_to: 0 });
565
566 let err = StringArray::try_new(
567 ScalarBuffer::from_slice(&[0, 4]),
568 Buffer::from_slice(b"ok\xff\xfe"),
569 None,
570 )
571 .unwrap_err();
572 assert_eq!(err, DataError::InvalidUtf8 { valid_up_to: 2 });
573
574 let mut truncated = "日".as_bytes().to_vec();
576 truncated.pop();
577 assert!(matches!(
578 StringArray::try_new(
579 ScalarBuffer::from_slice(&[0, 2]),
580 Buffer::from_slice(&truncated),
581 None
582 ),
583 Err(DataError::InvalidUtf8 { .. })
584 ));
585 }
586
587 #[test]
588 fn validation_reports_offsets_relative_to_the_buffer() {
589 let values = Buffer::from_slice(b"good\xffbad");
592 let err =
593 StringArray::try_new(ScalarBuffer::from_slice(&[4, 8]), values, None).unwrap_err();
594 assert_eq!(err, DataError::InvalidUtf8 { valid_up_to: 4 });
595 }
596
597 #[test]
598 fn offsets_are_still_validated() {
599 let values = Buffer::from_slice(b"abc");
600 assert!(matches!(
601 StringArray::try_new(ScalarBuffer::from_slice(&[0, 9]), values.clone(), None),
602 Err(DataError::OffsetOutOfBounds { .. })
603 ));
604 assert!(matches!(
605 StringArray::try_new(ScalarBuffer::from_slice(&[0, 3, 1]), values, None),
606 Err(DataError::NonMonotonicOffsets { .. })
607 ));
608 }
609
610 #[test]
611 fn slicing_preserves_the_utf8_invariant() {
612 let array = StringArray::from_values(["日本語", "é", "plain", "🤖"]);
613 for offset in 0..4 {
614 for len in 0..=(4 - offset) {
615 let window = array.slice(offset, len);
616 assert_eq!(window.len(), len);
617 for index in 0..len {
618 assert_eq!(window.get(index), array.get(offset + index));
619 }
620 assert!(
622 StringArray::try_new(
623 ScalarBuffer::from_slice(window.value_offsets()),
624 window.value_data().clone(),
625 None,
626 )
627 .is_ok(),
628 "offset {offset} len {len}"
629 );
630 }
631 }
632 }
633
634 #[test]
635 fn slicing_clamps() {
636 let array = StringArray::from_values(["a", "b", "c"]);
637 assert_eq!(array.slice(2, 99).len(), 1);
638 assert_eq!(array.slice(99, 1).len(), 0);
639 assert_eq!(array.try_slice(1, 2).unwrap().len(), 2);
640 assert!(array.try_slice(2, 2).is_err());
641 }
642
643 #[test]
644 fn unchecked_constructor_matches_the_checked_one() {
645 let offsets = ScalarBuffer::from_slice(&[0i32, 3, 6]);
646 let values = Buffer::from_slice(b"abcdef");
647 let checked = StringArray::try_new(offsets.clone(), values.clone(), None).unwrap();
648 let unchecked = unsafe { StringArray::new_unchecked(offsets, values, None) };
650 assert_eq!(checked, unchecked);
651 assert_eq!(unchecked.get(1), Some("def"));
652 }
653
654 #[test]
655 fn binary_conversions_are_free() {
656 let text = StringArray::from_opt_iter([Some("a"), None, Some("bb")]);
657 let bytes = text.clone().into_binary();
658 assert_eq!(bytes.get(2), Some(&b"bb"[..]));
659 assert_eq!(bytes.null_count(), 1);
660 let back = StringArray::try_from(bytes).unwrap();
661 assert_eq!(back, text);
662 assert_eq!(text.as_binary().len(), 3);
663
664 let invalid = crate::array::BinaryArray::from_values([&[0xffu8, 0xfe][..]]);
665 assert!(StringArray::try_from(invalid).is_err());
666 }
667
668 #[test]
669 fn both_offset_widths_work() {
670 let small = StringArray::from_values(["x"]);
671 let large = LargeStringArray::from_values(["x"]);
672 assert_eq!(small.data_type(), &DataType::Utf8);
673 assert_eq!(large.data_type(), &DataType::LargeUtf8);
674 assert_eq!(small.get(0), large.get(0));
675 let small_ref: ArrayRef = Arc::new(small);
676 let large_ref: ArrayRef = Arc::new(large);
677 assert_ne!(small_ref, large_ref, "different data types");
678 }
679
680 #[test]
681 fn equality_is_by_logical_content() {
682 let a = StringArray::from_opt_iter([Some("aa"), None, Some("cc")]);
683 let padded = StringArray::from_opt_iter([Some("zz"), Some("aa"), None, Some("cc")]);
684 assert_eq!(a, padded.slice(1, 3));
685 assert_ne!(a, StringArray::from_values(["aa", "cc"]));
686
687 let text_ref: ArrayRef = Arc::new(a.clone());
688 let bytes_ref: ArrayRef = Arc::new(a.into_binary());
689 assert_ne!(text_ref, bytes_ref, "Utf8 never equals Binary");
690 }
691
692 #[test]
693 fn with_validity_replaces_the_bitmap() {
694 let array = StringArray::from_values(["a", "b"]);
695 let masked = array
696 .with_validity(Some([true, false].into_iter().collect()))
697 .unwrap();
698 assert_eq!(masked.null_count(), 1);
699 assert_eq!(masked.get(1), None);
700 assert_eq!(masked.value(1), Some("b"), "the bytes are still there");
701 assert!(array.with_validity(Some(Bitmap::new_set(9))).is_err());
702 }
703
704 #[test]
705 fn collects_from_iterators() {
706 let collected: StringArray = [Some("a".to_owned()), None].into_iter().collect();
707 assert_eq!(collected.len(), 2);
708 assert_eq!(collected.null_count(), 1);
709 let owned = StringArray::from_values(vec!["one".to_owned(), "two".to_owned()]);
710 assert_eq!(owned.get(0), Some("one"));
711 assert_eq!((&owned).into_iter().count(), 2);
712 }
713
714 #[test]
715 fn debug_output_names_the_offset_width() {
716 let small = StringArray::from_opt_iter([Some("a"), None]);
717 let rendered = format!("{small:?}");
718 assert!(rendered.starts_with("StringArray"), "{rendered}");
719 assert!(rendered.contains("nulls=1"), "{rendered}");
720 assert!(
721 format!("{:?}", LargeStringArray::from_values(["a"])).starts_with("LargeStringArray")
722 );
723 }
724
725 #[test]
726 fn empty_offsets_pass_validation_vacuously() {
727 assert!(validate_utf8(&[] as &[i32], b"anything").is_ok());
728 assert!(validate_utf8(&[0i32], b"").is_ok());
729 }
730
731 #[test]
732 fn large_text_column() {
733 let values: Vec<String> = (0..5_000).map(|i| format!("node-{i}-日本語")).collect();
734 let array = LargeStringArray::from_values(&values);
735 assert_eq!(array.len(), 5_000);
736 assert_eq!(array.get(4_999), Some(values[4_999].as_str()));
737 let window = array.slice(2_500, 100);
738 assert_eq!(window.get(0), Some(values[2_500].as_str()));
739 }
740}