vortex_buffer/
lib.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4#![deny(missing_docs)]
5// cudarc HostSlice has len and is_empty methods that duplicate BufferMut methods.
6#![allow(clippy::same_name_method)]
7
8//! A library for working with custom aligned buffers of sized values.
9//!
10//! The `vortex-buffer` crate is built around `bytes::Bytes` and therefore supports zero-copy
11//! cloning and slicing, but differs in that it can define and maintain a custom alignment.
12//!
13//! * `Buffer<T>` and `BufferMut<T>` provide immutable and mutable wrappers around `bytes::Bytes`
14//!   and `bytes::BytesMut` respectively.
15//! * `ByteBuffer` and `ByteBufferMut` are type aliases for `u8` buffers.
16//! * `BufferString` is a wrapper around a `ByteBuffer` that enforces utf-8 encoding.
17//! * `ConstBuffer<T, const A: usize>` provides similar functionality to `Buffer<T>` except with a
18//!   compile-time alignment of `A`.
19//! * `buffer!` and `buffer_mut!` macros with the same syntax as the builtin `vec!` macro for
20//!   inline construction of buffers.
21//! * `BitBuffer` and `BitBufferMut` provide packed bitsets that can be used to store boolean values.
22//!
23//! You can think of `BufferMut<T>` as similar to a `Vec<T>`, except that any operation that may
24//! cause a re-allocation, e.g. extend, will ensure the new allocation maintains the buffer's
25//! defined alignment.
26//!
27//! For example, it's possible to incrementally build a `Buffer<T>` with a 4KB alignment.
28//! ```
29//! use vortex_buffer::{Alignment, BufferMut};
30//!
31//! let mut buf = BufferMut::<i32>::empty_aligned(Alignment::new(4096));
32//! buf.extend(0i32..1_000);
33//! assert_eq!(buf.as_ptr().align_offset(4096), 0)
34//! ```
35//!
36//! ## Comparison
37//!
38//! | Implementation                   | Zero-copy | Custom Alignment | Typed    |
39//! | -------------------------------- | --------- | ---------------- | -------- |
40//! | `vortex_buffer::Buffer<T>`       | ✔️        | ✔️               | ✔️       |
41//! | `arrow_buffer::ScalarBuffer<T> ` | ✔️        | ❌️️️               | ✔️       |
42//! | `bytes::Bytes`                   | ✔️        | ❌️️️               | ❌️️️       |
43//! | `Vec<T>`                         | ❌️        | ❌️️               | ✔️       |
44//!
45//! ## Features
46//!
47//! The `arrow` feature can be enabled to provide conversion functions to/from Arrow Rust buffers,
48//! including `arrow_buffer::Buffer`, `arrow_buffer::ScalarBuffer<T>`, and
49//! `arrow_buffer::OffsetBuffer`.
50
51pub use alignment::*;
52pub use bit::*;
53pub use buffer::*;
54pub use buffer_mut::*;
55pub use bytes::*;
56pub use r#const::*;
57pub use string::*;
58use vortex_error::VortexResult;
59use vortex_error::vortex_bail;
60
61mod alignment;
62#[cfg(feature = "arrow")]
63mod arrow;
64mod bit;
65mod buffer;
66mod buffer_mut;
67mod bytes;
68mod r#const;
69#[cfg(gpu_unstable)]
70mod cuda;
71mod debug;
72mod macros;
73#[cfg(feature = "memmap2")]
74mod memmap2;
75#[cfg(feature = "serde")]
76mod serde;
77mod string;
78mod trusted_len;
79
80/// An immutable buffer of u8.
81pub type ByteBuffer = Buffer<u8>;
82
83/// A mutable buffer of u8.
84pub type ByteBufferMut = BufferMut<u8>;
85
86/// A const-aligned buffer of u8.
87pub type ConstByteBuffer<const A: usize> = ConstBuffer<u8, A>;
88
89#[derive(Debug, Clone)]
90/// A buffer can be either on the CPU or on an attached device (e.g. GPU).
91/// The Device implementation will come later.
92pub enum BufferHandle {
93    /// On the host/cpu.
94    Buffer(ByteBuffer),
95    /// On the device.
96    // TODO: impl this.
97    DeviceBuffer,
98}
99
100impl BufferHandle {
101    /// Fetches the cpu buffer and fails otherwise.
102    pub fn bytes(&self) -> &ByteBuffer {
103        match self {
104            BufferHandle::Buffer(b) => b,
105            BufferHandle::DeviceBuffer => todo!(),
106        }
107    }
108
109    /// Fetches the cpu buffer and fails otherwise.
110    pub fn into_bytes(self) -> ByteBuffer {
111        match self {
112            BufferHandle::Buffer(b) => b,
113            BufferHandle::DeviceBuffer => todo!(),
114        }
115    }
116
117    /// Attempts to convert this handle into a CPU ByteBuffer.
118    /// Returns an error if the buffer is on a device.
119    pub fn try_to_bytes(self) -> VortexResult<ByteBuffer> {
120        match self {
121            BufferHandle::Buffer(b) => Ok(b),
122            BufferHandle::DeviceBuffer => vortex_bail!("cannot move device_buffer to buffer"),
123        }
124    }
125}
vortex_buffer/lib.rs

vortex_buffer/
lib.rs
