Skip to main content

arrow_buffer/buffer/
scalar.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5use crate::alloc::Deallocation;
6use crate::buffer::Buffer;
7use crate::native::ArrowNativeType;
8use crate::{BufferBuilder, MutableBuffer, OffsetBuffer};
9use std::fmt::Formatter;
10use std::marker::PhantomData;
11use std::ops::Deref;
12
13/// A strongly-typed [`Buffer`] supporting zero-copy cloning and slicing
14///
15/// The easiest way to think about `ScalarBuffer<T>` is being equivalent to a `Arc<Vec<T>>`,
16/// with the following differences:
17///
18/// - slicing and cloning is O(1).
19/// - support for external allocated memory (e.g. via FFI).
20///
21/// See [`Buffer`] for more low-level memory management details.
22///
23/// # Example: Convert to/from Vec (without copies)
24///
25/// (See [`Buffer::from_vec`] and [`Buffer::into_vec`] for a lower level API)
26/// ```
27/// # use arrow_buffer::ScalarBuffer;
28/// // Zero-copy conversion from Vec
29/// let buffer = ScalarBuffer::from(vec![1, 2, 3]);
30/// assert_eq!(&buffer, &[1, 2, 3]);
31/// // convert the buffer back to Vec without copy assuming:
32/// // 1. the inner buffer is not sliced
33/// // 2. the inner buffer uses standard allocation
34/// // 3. there are no other references to the inner buffer
35/// let vec: Vec<i32> = buffer.into();
36/// assert_eq!(&vec, &[1, 2, 3]);
37/// ```
38///
39/// # Example: Zero copy slicing
40/// ```
41/// # use arrow_buffer::ScalarBuffer;
42/// let buffer = ScalarBuffer::from(vec![1, 2, 3]);
43/// assert_eq!(&buffer, &[1, 2, 3]);
44/// // Zero-copy slicing
45/// let sliced = buffer.slice(1, 2);
46/// assert_eq!(&sliced, &[2, 3]);
47/// // Original buffer is unchanged
48/// assert_eq!(&buffer, &[1, 2, 3]);
49/// // converting the sliced buffer back to Vec incurs a copy
50/// let vec: Vec<i32> = sliced.into();
51/// ```
52#[derive(Clone, Default)]
53pub struct ScalarBuffer<T: ArrowNativeType> {
54    /// Underlying data buffer
55    buffer: Buffer,
56    phantom: PhantomData<T>,
57}
58
59impl<T: ArrowNativeType> std::fmt::Debug for ScalarBuffer<T> {
60    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
61        f.debug_tuple("ScalarBuffer").field(&self.as_ref()).finish()
62    }
63}
64
65impl<T: ArrowNativeType> ScalarBuffer<T> {
66    /// Create a new [`ScalarBuffer`] from a [`Buffer`], and an `offset`
67    /// and `length` in units of `T`
68    ///
69    /// # Panics
70    ///
71    /// This method will panic if
72    ///
73    /// * `offset` or `len` would result in overflow
74    /// * `buffer` is not aligned to a multiple of `std::mem::align_of::<T>`
75    /// * `bytes` is not large enough for the requested slice
76    pub fn new(buffer: Buffer, offset: usize, len: usize) -> Self {
77        let size = std::mem::size_of::<T>();
78        let byte_offset = offset.checked_mul(size).expect("offset overflow");
79        let byte_len = len.checked_mul(size).expect("length overflow");
80        buffer.slice_with_length(byte_offset, byte_len).into()
81    }
82
83    /// Unsafe function to create a new [`ScalarBuffer`] from a [`Buffer`].
84    /// Only use for testing purpose.
85    ///
86    /// # Safety
87    ///
88    /// This function is unsafe because it does not check if the `buffer` is aligned
89    pub unsafe fn new_unchecked(buffer: Buffer) -> Self {
90        Self {
91            buffer,
92            phantom: Default::default(),
93        }
94    }
95
96    /// Free up unused memory.
97    pub fn shrink_to_fit(&mut self) {
98        self.buffer.shrink_to_fit();
99    }
100
101    /// Returns a zero-copy slice of this buffer with length `len` and starting at `offset`
102    ///
103    /// # Panics
104    ///
105    /// Panics for the same reasons as [`Self::new`]
106    pub fn slice(&self, offset: usize, len: usize) -> Self {
107        Self::new(self.buffer.clone(), offset, len)
108    }
109
110    /// Returns the inner [`Buffer`]
111    pub fn inner(&self) -> &Buffer {
112        &self.buffer
113    }
114
115    /// Returns the inner [`Buffer`], consuming self
116    pub fn into_inner(self) -> Buffer {
117        self.buffer
118    }
119
120    /// Claim memory used by this buffer in the provided memory pool.
121    ///
122    /// See [`Buffer::claim`] for details.
123    #[cfg(feature = "pool")]
124    pub fn claim(&self, pool: &dyn crate::MemoryPool) {
125        self.buffer.claim(pool);
126    }
127
128    /// Returns true if this [`ScalarBuffer`] is equal to `other`, using pointer comparisons
129    /// to determine buffer equality. This is cheaper than `PartialEq::eq` but may
130    /// return false when the arrays are logically equal
131    #[inline]
132    pub fn ptr_eq(&self, other: &Self) -> bool {
133        self.buffer.ptr_eq(&other.buffer)
134    }
135
136    /// Returns the number of elements in the buffer
137    pub fn len(&self) -> usize {
138        self.buffer.len() / std::mem::size_of::<T>()
139    }
140
141    /// Returns if the buffer is empty
142    pub fn is_empty(&self) -> bool {
143        self.len() == 0
144    }
145}
146
147impl<T: ArrowNativeType> Deref for ScalarBuffer<T> {
148    type Target = [T];
149
150    #[inline]
151    fn deref(&self) -> &Self::Target {
152        // SAFETY: Verified alignment in From<Buffer>
153        unsafe {
154            std::slice::from_raw_parts(
155                self.buffer.as_ptr().cast::<T>(),
156                self.buffer.len() / std::mem::size_of::<T>(),
157            )
158        }
159    }
160}
161
162impl<T: ArrowNativeType> AsRef<[T]> for ScalarBuffer<T> {
163    #[inline]
164    fn as_ref(&self) -> &[T] {
165        self
166    }
167}
168
169impl<T: ArrowNativeType> From<MutableBuffer> for ScalarBuffer<T> {
170    fn from(value: MutableBuffer) -> Self {
171        Buffer::from(value).into()
172    }
173}
174
175impl<T: ArrowNativeType> From<Buffer> for ScalarBuffer<T> {
176    fn from(buffer: Buffer) -> Self {
177        let align = std::mem::align_of::<T>();
178        let is_aligned = buffer.as_ptr().align_offset(align) == 0;
179
180        match buffer.deallocation() {
181            Deallocation::Standard(_) => assert!(
182                is_aligned,
183                "Memory pointer is not aligned with the specified scalar type"
184            ),
185            Deallocation::Custom(_, _) => assert!(
186                is_aligned,
187                "Memory pointer from external source (e.g, FFI) is not aligned with the specified scalar type. Before importing buffer through FFI, please make sure the allocation is aligned."
188            ),
189        }
190
191        Self {
192            buffer,
193            phantom: Default::default(),
194        }
195    }
196}
197
198impl<T: ArrowNativeType> From<OffsetBuffer<T>> for ScalarBuffer<T> {
199    fn from(value: OffsetBuffer<T>) -> Self {
200        value.into_inner()
201    }
202}
203
204impl<T: ArrowNativeType> From<Vec<T>> for ScalarBuffer<T> {
205    fn from(value: Vec<T>) -> Self {
206        Self {
207            buffer: Buffer::from_vec(value),
208            phantom: Default::default(),
209        }
210    }
211}
212
213impl<T: ArrowNativeType> From<ScalarBuffer<T>> for Vec<T> {
214    fn from(value: ScalarBuffer<T>) -> Self {
215        value
216            .buffer
217            .into_vec()
218            .unwrap_or_else(|buffer| buffer.typed_data::<T>().into())
219    }
220}
221
222impl<T: ArrowNativeType> From<BufferBuilder<T>> for ScalarBuffer<T> {
223    fn from(mut value: BufferBuilder<T>) -> Self {
224        let len = value.len();
225        Self::new(value.finish(), 0, len)
226    }
227}
228
229impl<T: ArrowNativeType> FromIterator<T> for ScalarBuffer<T> {
230    #[inline]
231    fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> Self {
232        iter.into_iter().collect::<Vec<_>>().into()
233    }
234}
235
236impl<'a, T: ArrowNativeType> IntoIterator for &'a ScalarBuffer<T> {
237    type Item = &'a T;
238    type IntoIter = std::slice::Iter<'a, T>;
239
240    fn into_iter(self) -> Self::IntoIter {
241        self.as_ref().iter()
242    }
243}
244
245impl<T: ArrowNativeType, S: AsRef<[T]> + ?Sized> PartialEq<S> for ScalarBuffer<T> {
246    fn eq(&self, other: &S) -> bool {
247        self.as_ref().eq(other.as_ref())
248    }
249}
250
251impl<T: ArrowNativeType, const N: usize> PartialEq<ScalarBuffer<T>> for [T; N] {
252    fn eq(&self, other: &ScalarBuffer<T>) -> bool {
253        self.as_ref().eq(other.as_ref())
254    }
255}
256
257impl<T: ArrowNativeType> PartialEq<ScalarBuffer<T>> for [T] {
258    fn eq(&self, other: &ScalarBuffer<T>) -> bool {
259        self.as_ref().eq(other.as_ref())
260    }
261}
262
263impl<T: ArrowNativeType> PartialEq<ScalarBuffer<T>> for Vec<T> {
264    fn eq(&self, other: &ScalarBuffer<T>) -> bool {
265        self.as_slice().eq(other.as_ref())
266    }
267}
268
269/// If T implements Eq, then so does ScalarBuffer.
270impl<T: ArrowNativeType + Eq> Eq for ScalarBuffer<T> {}
271
272#[cfg(test)]
273mod tests {
274    use std::{ptr::NonNull, sync::Arc};
275
276    use super::*;
277
278    #[test]
279    fn test_basic() {
280        let expected = [0_i32, 1, 2];
281        let buffer = Buffer::from_iter(expected.iter().copied());
282        let typed = ScalarBuffer::<i32>::new(buffer.clone(), 0, 3);
283        assert_eq!(*typed, expected);
284
285        let typed = ScalarBuffer::<i32>::new(buffer.clone(), 1, 2);
286        assert_eq!(*typed, expected[1..]);
287
288        let typed = ScalarBuffer::<i32>::new(buffer.clone(), 1, 0);
289        assert!(typed.is_empty());
290
291        let typed = ScalarBuffer::<i32>::new(buffer, 3, 0);
292        assert!(typed.is_empty());
293    }
294
295    #[test]
296    fn test_debug() {
297        let buffer = ScalarBuffer::from(vec![1, 2, 3]);
298        assert_eq!(format!("{buffer:?}"), "ScalarBuffer([1, 2, 3])");
299    }
300
301    #[test]
302    #[should_panic(expected = "Memory pointer is not aligned with the specified scalar type")]
303    fn test_unaligned() {
304        let expected = [0_i32, 1, 2];
305        let buffer = Buffer::from_iter(expected.iter().copied());
306        let buffer = buffer.slice(1);
307        ScalarBuffer::<i32>::new(buffer, 0, 2);
308    }
309
310    #[test]
311    #[should_panic(expected = "the offset of the new Buffer cannot exceed the existing length")]
312    fn test_length_out_of_bounds() {
313        let buffer = Buffer::from_iter([0_i32, 1, 2]);
314        ScalarBuffer::<i32>::new(buffer, 1, 3);
315    }
316
317    #[test]
318    #[should_panic(expected = "the offset of the new Buffer cannot exceed the existing length")]
319    fn test_offset_out_of_bounds() {
320        let buffer = Buffer::from_iter([0_i32, 1, 2]);
321        ScalarBuffer::<i32>::new(buffer, 4, 0);
322    }
323
324    #[test]
325    #[should_panic(expected = "offset overflow")]
326    fn test_length_overflow() {
327        let buffer = Buffer::from_iter([0_i32, 1, 2]);
328        ScalarBuffer::<i32>::new(buffer, usize::MAX, 1);
329    }
330
331    #[test]
332    #[should_panic(expected = "offset overflow")]
333    fn test_start_overflow() {
334        let buffer = Buffer::from_iter([0_i32, 1, 2]);
335        ScalarBuffer::<i32>::new(buffer, usize::MAX / 4 + 1, 0);
336    }
337
338    #[test]
339    #[should_panic(expected = "length overflow")]
340    fn test_end_overflow() {
341        let buffer = Buffer::from_iter([0_i32, 1, 2]);
342        ScalarBuffer::<i32>::new(buffer, 0, usize::MAX / 4 + 1);
343    }
344
345    #[test]
346    fn convert_from_buffer_builder() {
347        let input = vec![1, 2, 3, 4];
348        let buffer_builder = BufferBuilder::from(input.clone());
349        let scalar_buffer = ScalarBuffer::from(buffer_builder);
350        assert_eq!(scalar_buffer.as_ref(), input);
351    }
352
353    #[test]
354    fn into_vec() {
355        let input = vec![1u8, 2, 3, 4];
356
357        // No copy
358        let input_buffer = Buffer::from_vec(input.clone());
359        let input_ptr = input_buffer.as_ptr();
360        let input_len = input_buffer.len();
361        let scalar_buffer = ScalarBuffer::<u8>::new(input_buffer, 0, input_len);
362        let vec = Vec::from(scalar_buffer);
363        assert_eq!(vec.as_slice(), input.as_slice());
364        assert_eq!(vec.as_ptr(), input_ptr);
365
366        // Custom allocation - makes a copy
367        let mut input_clone = input.clone();
368        let input_ptr = NonNull::new(input_clone.as_mut_ptr()).unwrap();
369        let dealloc = Arc::new(());
370        let buffer =
371            unsafe { Buffer::from_custom_allocation(input_ptr, input_clone.len(), dealloc as _) };
372        let scalar_buffer = ScalarBuffer::<u8>::new(buffer, 0, input.len());
373        let vec = Vec::from(scalar_buffer);
374        assert_eq!(vec, input.as_slice());
375        assert_ne!(vec.as_ptr(), input_ptr.as_ptr());
376
377        // Offset - makes a copy
378        let input_buffer = Buffer::from_vec(input.clone());
379        let input_ptr = input_buffer.as_ptr();
380        let input_len = input_buffer.len();
381        let scalar_buffer = ScalarBuffer::<u8>::new(input_buffer, 1, input_len - 1);
382        let vec = Vec::from(scalar_buffer);
383        assert_eq!(vec.as_slice(), &input[1..]);
384        assert_ne!(vec.as_ptr(), input_ptr);
385
386        // Inner buffer Arc ref count != 0 - makes a copy
387        let buffer = Buffer::from_slice_ref(input.as_slice());
388        let scalar_buffer = ScalarBuffer::<u8>::new(buffer, 0, input.len());
389        let vec = Vec::from(scalar_buffer);
390        assert_eq!(vec, input.as_slice());
391        assert_ne!(vec.as_ptr(), input.as_ptr());
392    }
393
394    #[test]
395    fn scalar_buffer_impl_eq() {
396        fn are_equal<T: Eq>(a: &T, b: &T) -> bool {
397            a.eq(b)
398        }
399
400        assert!(
401            are_equal(
402                &ScalarBuffer::<i16>::from(vec![23]),
403                &ScalarBuffer::<i16>::from(vec![23])
404            ),
405            "ScalarBuffer should implement Eq if the inner type does"
406        );
407    }
408}