Skip to main content

astrs_data/array/
string.rs

1//! [`GenericStringArray`] — the variable-length UTF-8 column, in its `Utf8`
2//! (32-bit offsets) and `LargeUtf8` (64-bit offsets) forms.
3//!
4//! The layout is identical to [`crate::array::GenericBinaryArray`] — this type
5//! *is* one, wrapped — plus one invariant: **every slot is valid UTF-8**.
6//!
7//! # Validating untrusted bytes
8//!
9//! A payload arriving over the wire is untrusted, and two independent things
10//! can be wrong with it:
11//!
12//! 1. the concatenated value region is not valid UTF-8; or
13//! 2. the region *is* valid UTF-8, but an offset lands in the middle of a
14//!    multi-byte code point, so an individual slot is not.
15//!
16//! Checking only the first is the classic bug: `"é"` is `[0xC3, 0xA9]`, so
17//! offsets `[0, 1, 2]` split a perfectly valid two-byte region into two
18//! invalid slots. [`GenericStringArray::try_new`] checks both — a single
19//! [`str::from_utf8`] pass over the whole region, then
20//! [`str::is_char_boundary`] on every offset.
21//!
22//! ```
23//! use astrs_data::array::{Array, StringArray};
24//! use astrs_data::{Buffer, DataError, ScalarBuffer};
25//!
26//! // Valid region, invalid split.
27//! let err = StringArray::try_new(
28//!     ScalarBuffer::from_slice(&[0, 1, 2]),
29//!     Buffer::from_slice("é".as_bytes()),
30//!     None,
31//! )
32//! .unwrap_err();
33//! assert!(matches!(err, DataError::OffsetNotCharBoundary { index: 1, offset: 1 }));
34//!
35//! // The same region, split correctly.
36//! let ok = StringArray::try_new(
37//!     ScalarBuffer::from_slice(&[0, 2]),
38//!     Buffer::from_slice("é".as_bytes()),
39//!     None,
40//! )?;
41//! assert_eq!(ok.get(0), Some("é"));
42//! # Ok::<(), astrs_data::DataError>(())
43//! ```
44//!
45//! # Access cost
46//!
47//! Value access returns `&str` through a checked [`str::from_utf8`] rather
48//! than an unchecked reinterpretation. That keeps `unsafe` confined to the two
49//! buffer modules ([`crate::buffer::aligned`] and [`crate::buffer::scalar`]),
50//! and the check is a vectorised scan over one — usually short — value. Hot
51//! paths that do not need `&str` can read the raw bytes through
52//! [`GenericStringArray::value_bytes`] and skip it entirely.
53
54use std::any::Any;
55use std::fmt;
56use std::sync::Arc;
57
58use crate::array::binary::GenericBinaryArray;
59use crate::array::iter::{ArrayAccessor, ArrayIter};
60use crate::array::offset::OffsetSizeTrait;
61use crate::array::{Array, ArrayRef, LARGE_UTF8_TYPE, UTF8_TYPE, debug_array};
62use crate::buffer::{Bitmap, Buffer, ScalarBuffer};
63use crate::datatype::DataType;
64use crate::error::{DataError, Result};
65use crate::sealed::Sealed;
66
67/// A column of variable-length UTF-8 strings.
68///
69/// Use the [`StringArray`] and [`LargeStringArray`] aliases rather than naming
70/// the offset parameter directly.
71#[derive(Clone)]
72pub struct GenericStringArray<O: OffsetSizeTrait> {
73    /// The byte layout. Every slot is guaranteed to be valid UTF-8.
74    inner: GenericBinaryArray<O>,
75}
76
77/// `Utf8` column — variable-length UTF-8 with 32-bit offsets.
78pub type StringArray = GenericStringArray<i32>;
79/// `LargeUtf8` column — variable-length UTF-8 with 64-bit offsets.
80pub type LargeStringArray = GenericStringArray<i64>;
81
82/// Checks that every slot the offsets carve out of `values` is valid UTF-8.
83///
84/// Assumes the offsets themselves have already passed
85/// [`crate::array::validate_offsets`].
86///
87/// # Errors
88///
89/// * [`DataError::InvalidUtf8`] — the covered region is not valid UTF-8.
90/// * [`DataError::OffsetNotCharBoundary`] — an offset splits a code point.
91fn validate_utf8<O: OffsetSizeTrait>(offsets: &[O], values: &[u8]) -> Result<()> {
92    let (Some(first), Some(last)) = (offsets.first(), offsets.last()) else {
93        return Ok(());
94    };
95    let start = first.to_usize().unwrap_or(0);
96    let end = last.to_usize().unwrap_or(0);
97    let Some(region) = values.get(start..end) else {
98        return Err(DataError::OffsetOutOfBounds {
99            index: offsets.len().saturating_sub(1),
100            offset: end,
101            values_len: values.len(),
102        });
103    };
104    let text = std::str::from_utf8(region).map_err(|error| DataError::InvalidUtf8 {
105        valid_up_to: start + error.valid_up_to(),
106    })?;
107    for (index, offset) in offsets.iter().enumerate() {
108        let absolute = offset.to_usize().unwrap_or(0);
109        let relative = absolute.saturating_sub(start);
110        if !text.is_char_boundary(relative) {
111            return Err(DataError::OffsetNotCharBoundary {
112                index,
113                offset: absolute,
114            });
115        }
116    }
117    Ok(())
118}
119
120impl<O: OffsetSizeTrait> GenericStringArray<O> {
121    /// Builds an array from its three buffers, validating offsets *and* UTF-8.
122    ///
123    /// This is the constructor for untrusted bytes.
124    ///
125    /// # Errors
126    ///
127    /// * The offset errors listed on [`crate::array::validate_offsets`].
128    /// * [`DataError::InvalidUtf8`] and [`DataError::OffsetNotCharBoundary`].
129    /// * [`DataError::ValidityLengthMismatch`] when the bitmap length is wrong.
130    pub fn try_new(
131        offsets: ScalarBuffer<O>,
132        values: Buffer,
133        validity: Option<Bitmap>,
134    ) -> Result<Self> {
135        let inner = GenericBinaryArray::try_new(offsets, values, validity)?;
136        validate_utf8(inner.value_offsets(), inner.value_data().as_slice())?;
137        Ok(Self { inner })
138    }
139
140    /// Builds an array from its three buffers **without validating**.
141    ///
142    /// # Safety
143    ///
144    /// This constructor is `unsafe` by contract, not by memory model: the
145    /// accessors stay memory-safe whatever the bytes say, returning `None` for
146    /// a slot that is not valid UTF-8. Skipping validation turns a decode
147    /// error into a silent `None`, so only use it on the trusted zero-copy
148    /// path — bytes this crate produced, or bytes a caller has already
149    /// validated.
150    ///
151    /// Callers must guarantee the offsets satisfy
152    /// [`crate::array::validate_offsets`] and that every slot they carve out is
153    /// valid UTF-8.
154    #[must_use]
155    pub unsafe fn new_unchecked(
156        offsets: ScalarBuffer<O>,
157        values: Buffer,
158        validity: Option<Bitmap>,
159    ) -> Self {
160        Self::from_parts(offsets, values, validity)
161    }
162
163    /// Assembles the parts without revalidating. See
164    /// [`GenericBinaryArray::from_parts`].
165    pub(crate) fn from_parts(
166        offsets: ScalarBuffer<O>,
167        values: Buffer,
168        validity: Option<Bitmap>,
169    ) -> Self {
170        debug_assert!(validate_utf8(offsets.as_slice(), values.as_slice()).is_ok());
171        Self {
172            inner: GenericBinaryArray::from_parts(offsets, values, validity),
173        }
174    }
175
176    /// Builds an array with no nulls.
177    ///
178    /// ```
179    /// use astrs_data::array::{Array, StringArray};
180    ///
181    /// let names = StringArray::from_values(["lidar", "camera"]);
182    /// assert_eq!(names.len(), 2);
183    /// assert_eq!(names.get(1), Some("camera"));
184    /// ```
185    #[must_use]
186    pub fn from_values<V: AsRef<str>>(values: impl IntoIterator<Item = V>) -> Self {
187        Self {
188            inner: GenericBinaryArray::from_values(values.into_iter().map(|value| BytesOf(value))),
189        }
190    }
191
192    /// Builds an array from optional strings.
193    #[must_use]
194    pub fn from_opt_iter<V: AsRef<str>>(values: impl IntoIterator<Item = Option<V>>) -> Self {
195        Self {
196            inner: GenericBinaryArray::from_opt_iter(
197                values.into_iter().map(|value| value.map(BytesOf)),
198            ),
199        }
200    }
201
202    /// An all-null array of `len` slots.
203    #[must_use]
204    pub fn new_null(len: usize) -> Self {
205        Self {
206            inner: GenericBinaryArray::new_null(len),
207        }
208    }
209
210    /// The offset buffer, `len + 1` entries.
211    #[inline]
212    #[must_use]
213    pub fn value_offsets(&self) -> &[O] {
214        self.inner.value_offsets()
215    }
216
217    /// The offset buffer as a shareable typed window.
218    ///
219    /// See [`GenericBinaryArray::offsets_buffer`]: the IPC encoder clones this
220    /// window instead of copying the offsets out of a slice.
221    ///
222    /// ```
223    /// use astrs_data::array::StringArray;
224    ///
225    /// let array = StringArray::from_values(["ab", "c"]);
226    /// assert_eq!(array.offsets_buffer().as_slice(), &[0, 2, 3]);
227    /// ```
228    #[inline]
229    #[must_use]
230    pub const fn offsets_buffer(&self) -> &ScalarBuffer<O> {
231        self.inner.offsets_buffer()
232    }
233
234    /// The flat value region every slot points into.
235    #[inline]
236    #[must_use]
237    pub const fn value_data(&self) -> &Buffer {
238        self.inner.value_data()
239    }
240
241    /// The raw slot bytes at `index`, ignoring validity and skipping the UTF-8
242    /// check.
243    ///
244    /// The cheapest accessor: no validation at all.
245    #[inline]
246    #[must_use]
247    pub fn value_bytes(&self, index: usize) -> Option<&[u8]> {
248        self.inner.value(index)
249    }
250
251    /// The raw slot text at `index`, ignoring validity.
252    ///
253    /// Returns `None` when `index` is out of range (or, on the
254    /// [`Self::new_unchecked`] path only, when the slot is not valid UTF-8).
255    #[inline]
256    #[must_use]
257    pub fn value(&self, index: usize) -> Option<&str> {
258        std::str::from_utf8(self.inner.value(index)?).ok()
259    }
260
261    /// The logical value at `index`: `None` when the slot is null or out of
262    /// range.
263    #[inline]
264    #[must_use]
265    pub fn get(&self, index: usize) -> Option<&str> {
266        std::str::from_utf8(self.inner.get(index)?).ok()
267    }
268
269    /// Byte length of slot `index` — *not* the character count.
270    #[inline]
271    #[must_use]
272    pub fn value_length(&self, index: usize) -> Option<usize> {
273        self.inner.value_length(index)
274    }
275
276    /// Total bytes the values occupy, across the whole array.
277    #[must_use]
278    pub fn total_value_bytes(&self) -> usize {
279        self.inner.total_value_bytes()
280    }
281
282    /// Iterates over the logical values, `None` for nulls.
283    #[inline]
284    pub fn iter(&self) -> ArrayIter<&Self> {
285        ArrayIter::new(self)
286    }
287
288    /// A zero-copy sub-range, clamped to the array (the crate-wide slicing
289    /// convention).
290    ///
291    /// Slicing preserves the UTF-8 invariant: it narrows the offset buffer
292    /// without changing any offset, and every retained offset was already
293    /// checked to be a character boundary.
294    #[must_use]
295    pub fn slice(&self, offset: usize, len: usize) -> Self {
296        Self {
297            inner: self.inner.slice(offset, len),
298        }
299    }
300
301    /// Checked [`GenericStringArray::slice`].
302    ///
303    /// # Errors
304    ///
305    /// [`DataError::SliceOutOfBounds`] when the window leaves the array.
306    pub fn try_slice(&self, offset: usize, len: usize) -> Result<Self> {
307        Ok(Self {
308            inner: self.inner.try_slice(offset, len)?,
309        })
310    }
311
312    /// Returns a copy with a different validity bitmap.
313    ///
314    /// # Errors
315    ///
316    /// [`DataError::ValidityLengthMismatch`] when the bitmap length is wrong.
317    pub fn with_validity(&self, validity: Option<Bitmap>) -> Result<Self> {
318        Ok(Self {
319            inner: self.inner.with_validity(validity)?,
320        })
321    }
322
323    /// Reinterprets the column as bytes, dropping the UTF-8 guarantee.
324    ///
325    /// Free: the two types share a representation.
326    #[must_use]
327    pub fn into_binary(self) -> GenericBinaryArray<O> {
328        self.inner
329    }
330
331    /// The byte view of this column.
332    #[inline]
333    #[must_use]
334    pub const fn as_binary(&self) -> &GenericBinaryArray<O> {
335        &self.inner
336    }
337}
338
339/// Adapts an `AsRef<str>` to the `AsRef<[u8]>` the binary builder wants.
340struct BytesOf<V: AsRef<str>>(V);
341
342impl<V: AsRef<str>> AsRef<[u8]> for BytesOf<V> {
343    #[inline]
344    fn as_ref(&self) -> &[u8] {
345        self.0.as_ref().as_bytes()
346    }
347}
348
349impl<O: OffsetSizeTrait> TryFrom<GenericBinaryArray<O>> for GenericStringArray<O> {
350    type Error = DataError;
351
352    /// Validates the bytes, then reinterprets them as text without copying.
353    fn try_from(inner: GenericBinaryArray<O>) -> Result<Self> {
354        validate_utf8(inner.value_offsets(), inner.value_data().as_slice())?;
355        Ok(Self { inner })
356    }
357}
358
359impl<O: OffsetSizeTrait> Sealed for GenericStringArray<O> {}
360
361impl<O: OffsetSizeTrait> Array for GenericStringArray<O> {
362    fn as_any(&self) -> &dyn Any {
363        self
364    }
365
366    fn data_type(&self) -> &DataType {
367        if O::IS_LARGE {
368            &LARGE_UTF8_TYPE
369        } else {
370            &UTF8_TYPE
371        }
372    }
373
374    fn len(&self) -> usize {
375        self.inner.len()
376    }
377
378    fn validity(&self) -> Option<&Bitmap> {
379        self.inner.validity()
380    }
381
382    fn slice(&self, offset: usize, len: usize) -> ArrayRef {
383        Arc::new(Self::slice(self, offset, len))
384    }
385
386    fn buffer_memory_size(&self) -> usize {
387        self.inner.buffer_memory_size()
388    }
389
390    fn equals(&self, other: &dyn Array) -> bool {
391        other
392            .as_any()
393            .downcast_ref::<Self>()
394            .is_some_and(|other| self.inner.equals(&other.inner))
395    }
396}
397
398impl<'a, O: OffsetSizeTrait> ArrayAccessor for &'a GenericStringArray<O> {
399    type Item = &'a str;
400
401    #[inline]
402    fn accessor_len(&self) -> usize {
403        self.inner.len()
404    }
405
406    #[inline]
407    fn accessor_get(&self, index: usize) -> Option<&'a str> {
408        let array: &'a GenericStringArray<O> = self;
409        array.get(index)
410    }
411}
412
413impl<O: OffsetSizeTrait> PartialEq for GenericStringArray<O> {
414    #[inline]
415    fn eq(&self, other: &Self) -> bool {
416        self.inner == other.inner
417    }
418}
419
420impl<O: OffsetSizeTrait> Eq for GenericStringArray<O> {}
421
422impl<O: OffsetSizeTrait, V: AsRef<str>> FromIterator<Option<V>> for GenericStringArray<O> {
423    fn from_iter<I: IntoIterator<Item = Option<V>>>(iter: I) -> Self {
424        Self::from_opt_iter(iter)
425    }
426}
427
428impl<'a, O: OffsetSizeTrait> IntoIterator for &'a GenericStringArray<O> {
429    type Item = Option<&'a str>;
430    type IntoIter = ArrayIter<&'a GenericStringArray<O>>;
431
432    fn into_iter(self) -> Self::IntoIter {
433        self.iter()
434    }
435}
436
437impl<O: OffsetSizeTrait> fmt::Debug for GenericStringArray<O> {
438    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
439        debug_array(
440            f,
441            if O::IS_LARGE {
442                "LargeStringArray"
443            } else {
444                "StringArray"
445            },
446            self.data_type(),
447            self.len(),
448            self.null_count(),
449            self.iter(),
450        )
451    }
452}
453
454#[cfg(test)]
455mod tests {
456    #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
457
458    use super::*;
459
460    #[test]
461    fn empty_array() {
462        let array = StringArray::from_values([] as [&str; 0]);
463        assert!(array.is_empty());
464        assert_eq!(array.null_count(), 0);
465        assert_eq!(array.get(0), None);
466        assert_eq!(array.iter().count(), 0);
467        assert_eq!(array.data_type(), &DataType::Utf8);
468        assert_eq!(array.total_value_bytes(), 0);
469    }
470
471    #[test]
472    fn no_null_array() {
473        let array = StringArray::from_values(["lidar", "", "camera"]);
474        assert_eq!(array.len(), 3);
475        assert_eq!(array.null_count(), 0);
476        assert!(array.validity().is_none());
477        assert_eq!(array.get(0), Some("lidar"));
478        assert_eq!(array.get(1), Some(""));
479        assert_eq!(array.get(2), Some("camera"));
480        assert_eq!(array.get(3), None);
481        assert_eq!(array.value_length(0), Some(5));
482        assert_eq!(array.value_bytes(2), Some(&b"camera"[..]));
483        assert_eq!(array.value_data().as_slice(), b"lidarcamera");
484    }
485
486    #[test]
487    fn all_null_array() {
488        let array = LargeStringArray::new_null(3);
489        assert_eq!(array.len(), 3);
490        assert_eq!(array.null_count(), 3);
491        assert_eq!(array.data_type(), &DataType::LargeUtf8);
492        for index in 0..3 {
493            assert!(array.is_null(index));
494            assert_eq!(array.get(index), None);
495            assert_eq!(array.value(index), Some(""));
496        }
497    }
498
499    #[test]
500    fn mixed_nulls() {
501        let array = StringArray::from_opt_iter([Some("a"), None, Some("ccc"), None]);
502        assert_eq!(array.len(), 4);
503        assert_eq!(array.null_count(), 2);
504        assert_eq!(
505            array.iter().collect::<Vec<_>>(),
506            vec![Some("a"), None, Some("ccc"), None]
507        );
508        assert_eq!(array.total_value_bytes(), 4);
509    }
510
511    #[test]
512    fn multibyte_text_round_trips() {
513        let values = ["日本語", "é", "🤖", "mixed 日本 text"];
514        let array = StringArray::from_values(values);
515        for (index, expected) in values.iter().enumerate() {
516            assert_eq!(array.get(index), Some(*expected));
517            assert_eq!(array.value_length(index), Some(expected.len()));
518        }
519        assert_eq!(array.get(2).map(str::chars).map(Iterator::count), Some(1));
520    }
521
522    #[test]
523    fn valid_region_with_a_split_code_point_is_rejected() {
524        // "é" is two bytes; an offset at 1 lands mid-code-point even though
525        // the whole region is valid UTF-8.
526        let err = StringArray::try_new(
527            ScalarBuffer::from_slice(&[0, 1, 2]),
528            Buffer::from_slice("é".as_bytes()),
529            None,
530        )
531        .unwrap_err();
532        assert_eq!(
533            err,
534            DataError::OffsetNotCharBoundary {
535                index: 1,
536                offset: 1
537            }
538        );
539
540        // A four-byte emoji split three ways.
541        let emoji = "🤖".as_bytes();
542        for split in 1..4 {
543            let err = StringArray::try_new(
544                ScalarBuffer::from_slice(&[0, split, 4]),
545                Buffer::from_slice(emoji),
546                None,
547            )
548            .unwrap_err();
549            assert!(
550                matches!(err, DataError::OffsetNotCharBoundary { .. }),
551                "split {split} accepted"
552            );
553        }
554    }
555
556    #[test]
557    fn invalid_bytes_are_rejected() {
558        let err = StringArray::try_new(
559            ScalarBuffer::from_slice(&[0, 2]),
560            Buffer::from_slice(&[0xff, 0xfe]),
561            None,
562        )
563        .unwrap_err();
564        assert_eq!(err, DataError::InvalidUtf8 { valid_up_to: 0 });
565
566        let err = StringArray::try_new(
567            ScalarBuffer::from_slice(&[0, 4]),
568            Buffer::from_slice(b"ok\xff\xfe"),
569            None,
570        )
571        .unwrap_err();
572        assert_eq!(err, DataError::InvalidUtf8 { valid_up_to: 2 });
573
574        // A truncated multi-byte sequence at the very end.
575        let mut truncated = "日".as_bytes().to_vec();
576        truncated.pop();
577        assert!(matches!(
578            StringArray::try_new(
579                ScalarBuffer::from_slice(&[0, 2]),
580                Buffer::from_slice(&truncated),
581                None
582            ),
583            Err(DataError::InvalidUtf8 { .. })
584        ));
585    }
586
587    #[test]
588    fn validation_reports_offsets_relative_to_the_buffer() {
589        // A sliced array's region starts partway into the value buffer; the
590        // reported `valid_up_to` must be an absolute buffer offset.
591        let values = Buffer::from_slice(b"good\xffbad");
592        let err =
593            StringArray::try_new(ScalarBuffer::from_slice(&[4, 8]), values, None).unwrap_err();
594        assert_eq!(err, DataError::InvalidUtf8 { valid_up_to: 4 });
595    }
596
597    #[test]
598    fn offsets_are_still_validated() {
599        let values = Buffer::from_slice(b"abc");
600        assert!(matches!(
601            StringArray::try_new(ScalarBuffer::from_slice(&[0, 9]), values.clone(), None),
602            Err(DataError::OffsetOutOfBounds { .. })
603        ));
604        assert!(matches!(
605            StringArray::try_new(ScalarBuffer::from_slice(&[0, 3, 1]), values, None),
606            Err(DataError::NonMonotonicOffsets { .. })
607        ));
608    }
609
610    #[test]
611    fn slicing_preserves_the_utf8_invariant() {
612        let array = StringArray::from_values(["日本語", "é", "plain", "🤖"]);
613        for offset in 0..4 {
614            for len in 0..=(4 - offset) {
615                let window = array.slice(offset, len);
616                assert_eq!(window.len(), len);
617                for index in 0..len {
618                    assert_eq!(window.get(index), array.get(offset + index));
619                }
620                // Rebuilding from the sliced buffers must pass validation.
621                assert!(
622                    StringArray::try_new(
623                        ScalarBuffer::from_slice(window.value_offsets()),
624                        window.value_data().clone(),
625                        None,
626                    )
627                    .is_ok(),
628                    "offset {offset} len {len}"
629                );
630            }
631        }
632    }
633
634    #[test]
635    fn slicing_clamps() {
636        let array = StringArray::from_values(["a", "b", "c"]);
637        assert_eq!(array.slice(2, 99).len(), 1);
638        assert_eq!(array.slice(99, 1).len(), 0);
639        assert_eq!(array.try_slice(1, 2).unwrap().len(), 2);
640        assert!(array.try_slice(2, 2).is_err());
641    }
642
643    #[test]
644    fn unchecked_constructor_matches_the_checked_one() {
645        let offsets = ScalarBuffer::from_slice(&[0i32, 3, 6]);
646        let values = Buffer::from_slice(b"abcdef");
647        let checked = StringArray::try_new(offsets.clone(), values.clone(), None).unwrap();
648        // SAFETY: the same buffers just passed validation above.
649        let unchecked = unsafe { StringArray::new_unchecked(offsets, values, None) };
650        assert_eq!(checked, unchecked);
651        assert_eq!(unchecked.get(1), Some("def"));
652    }
653
654    #[test]
655    fn binary_conversions_are_free() {
656        let text = StringArray::from_opt_iter([Some("a"), None, Some("bb")]);
657        let bytes = text.clone().into_binary();
658        assert_eq!(bytes.get(2), Some(&b"bb"[..]));
659        assert_eq!(bytes.null_count(), 1);
660        let back = StringArray::try_from(bytes).unwrap();
661        assert_eq!(back, text);
662        assert_eq!(text.as_binary().len(), 3);
663
664        let invalid = crate::array::BinaryArray::from_values([&[0xffu8, 0xfe][..]]);
665        assert!(StringArray::try_from(invalid).is_err());
666    }
667
668    #[test]
669    fn both_offset_widths_work() {
670        let small = StringArray::from_values(["x"]);
671        let large = LargeStringArray::from_values(["x"]);
672        assert_eq!(small.data_type(), &DataType::Utf8);
673        assert_eq!(large.data_type(), &DataType::LargeUtf8);
674        assert_eq!(small.get(0), large.get(0));
675        let small_ref: ArrayRef = Arc::new(small);
676        let large_ref: ArrayRef = Arc::new(large);
677        assert_ne!(small_ref, large_ref, "different data types");
678    }
679
680    #[test]
681    fn equality_is_by_logical_content() {
682        let a = StringArray::from_opt_iter([Some("aa"), None, Some("cc")]);
683        let padded = StringArray::from_opt_iter([Some("zz"), Some("aa"), None, Some("cc")]);
684        assert_eq!(a, padded.slice(1, 3));
685        assert_ne!(a, StringArray::from_values(["aa", "cc"]));
686
687        let text_ref: ArrayRef = Arc::new(a.clone());
688        let bytes_ref: ArrayRef = Arc::new(a.into_binary());
689        assert_ne!(text_ref, bytes_ref, "Utf8 never equals Binary");
690    }
691
692    #[test]
693    fn with_validity_replaces_the_bitmap() {
694        let array = StringArray::from_values(["a", "b"]);
695        let masked = array
696            .with_validity(Some([true, false].into_iter().collect()))
697            .unwrap();
698        assert_eq!(masked.null_count(), 1);
699        assert_eq!(masked.get(1), None);
700        assert_eq!(masked.value(1), Some("b"), "the bytes are still there");
701        assert!(array.with_validity(Some(Bitmap::new_set(9))).is_err());
702    }
703
704    #[test]
705    fn collects_from_iterators() {
706        let collected: StringArray = [Some("a".to_owned()), None].into_iter().collect();
707        assert_eq!(collected.len(), 2);
708        assert_eq!(collected.null_count(), 1);
709        let owned = StringArray::from_values(vec!["one".to_owned(), "two".to_owned()]);
710        assert_eq!(owned.get(0), Some("one"));
711        assert_eq!((&owned).into_iter().count(), 2);
712    }
713
714    #[test]
715    fn debug_output_names_the_offset_width() {
716        let small = StringArray::from_opt_iter([Some("a"), None]);
717        let rendered = format!("{small:?}");
718        assert!(rendered.starts_with("StringArray"), "{rendered}");
719        assert!(rendered.contains("nulls=1"), "{rendered}");
720        assert!(
721            format!("{:?}", LargeStringArray::from_values(["a"])).starts_with("LargeStringArray")
722        );
723    }
724
725    #[test]
726    fn empty_offsets_pass_validation_vacuously() {
727        assert!(validate_utf8(&[] as &[i32], b"anything").is_ok());
728        assert!(validate_utf8(&[0i32], b"").is_ok());
729    }
730
731    #[test]
732    fn large_text_column() {
733        let values: Vec<String> = (0..5_000).map(|i| format!("node-{i}-日本語")).collect();
734        let array = LargeStringArray::from_values(&values);
735        assert_eq!(array.len(), 5_000);
736        assert_eq!(array.get(4_999), Some(values[4_999].as_str()));
737        let window = array.slice(2_500, 100);
738        assert_eq!(window.get(0), Some(values[2_500].as_str()));
739    }
740}