Struct LiquidByteViewArray 

pub struct LiquidByteViewArray<B: FsstBacking> { /* private fields */ }

An array that stores strings using the FSST format with compact offsets:

• Dictionary keys with 2-byte keys stored in memory
• Compact offsets with variable-size residuals (1, 2, or 4 bytes) stored in memory
• Per-value prefix keys (7-byte prefix + len) stored in memory
• FSST buffer can be stored in memory or on disk

Initialization

The recommended way to create a LiquidByteViewArray is using the from_*_array constructors which build a compact (offset + prefix key) representation directly from Arrow inputs.

let liquid_array = LiquidByteViewArray::from_string_array(&input, compressor);

Data access flow:

1. Use dictionary key to index into compact offsets buffer
2. Reconstruct actual offset from linear regression (predicted + residual)
3. Use prefix keys for quick comparisons to avoid decompression when possible
4. Decompress bytes from FSST buffer to get the full value when needed

Implementations

impl LiquidByteViewArray<FsstArray>

pub fn compare_with(&self, needle: &[u8], op: &ByteViewOperator) -> BooleanArray

Compare with prefix optimization and fallback to Arrow operations

impl<B: FsstBacking> LiquidByteViewArray<B>

pub fn prefix_compare_counts( &self, needle: &[u8], op: &Comparison, ) -> (usize, usize, usize)

Return (selected_rows, ambiguous_rows, unique_rows) based on prefix-only comparison.

impl<B: FsstBacking> LiquidByteViewArray<B>

pub fn from_string_view_array( array: &StringViewArray, compressor: Arc<Compressor>, ) -> LiquidByteViewArray<FsstArray>

Create a LiquidByteViewArray from an Arrow StringViewArray

pub fn from_binary_view_array( array: &BinaryViewArray, compressor: Arc<Compressor>, ) -> LiquidByteViewArray<FsstArray>

Create a LiquidByteViewArray from an Arrow BinaryViewArray

pub fn from_string_array( array: &StringArray, compressor: Arc<Compressor>, ) -> LiquidByteViewArray<FsstArray>

Create a LiquidByteViewArray from an Arrow StringArray

pub fn from_binary_array( array: &BinaryArray, compressor: Arc<Compressor>, ) -> LiquidByteViewArray<FsstArray>

Create a LiquidByteViewArray from an Arrow BinaryArray

pub fn train_from_string_view( array: &StringViewArray, ) -> (Arc<Compressor>, LiquidByteViewArray<FsstArray>)

Train a compressor from an Arrow StringViewArray

pub fn train_from_binary_view( array: &BinaryViewArray, ) -> (Arc<Compressor>, LiquidByteViewArray<FsstArray>)

Train a compressor from an Arrow BinaryViewArray

pub fn train_from_arrow<T: ByteArrayType>( array: &GenericByteArray<T>, ) -> (Arc<Compressor>, LiquidByteViewArray<FsstArray>)

Train a compressor from an Arrow ByteArray.

pub unsafe fn from_unique_dict_array( array: &DictionaryArray<UInt16Type>, compressor: Arc<Compressor>, ) -> LiquidByteViewArray<FsstArray>

Only used when the dictionary is read from a trusted parquet reader, which reads a trusted parquet file, written by a trusted writer.

Safety

The caller must ensure that the values in the dictionary are unique.

pub fn train_from_arrow_dict( array: &DictionaryArray<UInt16Type>, ) -> (Arc<Compressor>, LiquidByteViewArray<FsstArray>)

Train a compressor from an Arrow DictionaryArray.

pub fn train_compressor<'a, T: ArrayAccessor<Item = &'a str>>( array: ArrayIter<T>, ) -> Arc<Compressor>

Train a compressor from an iterator of strings

pub fn train_compressor_bytes<'a, T: ArrayAccessor<Item = &'a [u8]>>( array: ArrayIter<T>, ) -> Arc<Compressor>

Train a compressor from an iterator of byte arrays

impl LiquidByteViewArray<FsstArray>

pub fn from_bytes( bytes: Bytes, compressor: Arc<Compressor>, ) -> LiquidByteViewArray<FsstArray>

Deserialize a LiquidByteViewArray from bytes.

impl<B: FsstBacking> LiquidByteViewArray<B>

pub fn nulls(&self) -> Option<&NullBuffer>

Get the nulls buffer

pub fn get_detailed_memory_usage(&self) -> ByteViewArrayMemoryUsage

Get detailed memory usage of the byte view array

pub fn len(&self) -> usize

Get the length of the array

pub fn is_empty(&self) -> bool

Is the array empty?

impl LiquidByteViewArray<FsstArray>

pub fn to_dict_arrow(&self) -> DictionaryArray<UInt16Type>

Convert to Arrow DictionaryArray

pub fn to_arrow_array(&self) -> ArrayRef

Convert to Arrow array with original type

pub fn is_fsst_buffer_on_disk(&self) -> bool

Check if the FSST buffer is currently stored on disk

impl LiquidByteViewArray<DiskBuffer>

pub fn is_fsst_buffer_on_disk(&self) -> bool

Check if the FSST buffer is currently stored on disk

pub async fn to_dict_arrow(&self) -> DictionaryArray<UInt16Type>

Convert to Arrow DictionaryArray

pub async fn to_arrow_array(&self) -> ArrayRef

Convert to Arrow array with original type

Trait Implementations

impl<B: Clone + FsstBacking> Clone for LiquidByteViewArray<B>

fn clone(&self) -> LiquidByteViewArray<B>

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl<B: FsstBacking> Debug for LiquidByteViewArray<B>

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl LiquidArray for LiquidByteViewArray<FsstArray>

fn as_any(&self) -> &dyn Any

Get the underlying any type.

fn get_array_memory_size(&self) -> usize

Get the memory size of the Liquid array.

fn len(&self) -> usize

Get the length of the Liquid array.

fn to_arrow_array(&self) -> ArrayRef

Convert the Liquid array to an Arrow array.

fn to_best_arrow_array(&self) -> ArrayRef

Convert the Liquid array to an Arrow array. Except that it will pick the best encoding for the arrow array. Meaning that it may not obey the data type of the original arrow array.

fn try_eval_predicate( &self, expr: &Arc<dyn PhysicalExpr>, filter: &BooleanBuffer, ) -> Option<BooleanArray>

Try to evaluate a predicate on the Liquid array with a filter. Returns None if the predicate is not supported.

fn to_bytes(&self) -> Vec<u8>

Serialize the Liquid array to a byte array.

fn original_arrow_data_type(&self) -> DataType

Get the original arrow data type of the Liquid array.

fn data_type(&self) -> LiquidDataType

Get the logical data type of the Liquid array.

fn squeeze( &self, io: Arc<dyn SqueezeIoHandler>, squeeze_hint: Option<&CacheExpression>, ) -> Option<(LiquidSqueezedArrayRef, Bytes)>

Squeeze the Liquid array to a LiquidHybridArrayRef and a bytes::Bytes. Return None if the Liquid array cannot be squeezed.

fn filter(&self, selection: &BooleanBuffer) -> ArrayRef

Filter the Liquid array with a boolean array and return an arrow array.

fn is_empty(&self) -> bool

Check if the Liquid array is empty.

impl LiquidSqueezedArray for LiquidByteViewArray<DiskBuffer>

fn as_any(&self) -> &dyn Any

Get the underlying any type.

fn get_array_memory_size(&self) -> usize

Get the memory size of the Liquid array.

fn len(&self) -> usize

Get the length of the Liquid array.

fn is_empty(&self) -> bool

Check if the Liquid array is empty.

fn to_arrow_array<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = ArrayRef> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Convert the Liquid array to an Arrow array.

fn data_type(&self) -> LiquidDataType

Get the logical data type of the Liquid array.

fn filter<'life0, 'life1, 'async_trait>( &'life0 self, selection: &'life1 BooleanBuffer, ) -> Pin<Box<dyn Future<Output = ArrayRef> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Filter the Liquid array with a boolean array and return an arrow array.

fn try_eval_predicate<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, expr: &'life1 Arc<dyn PhysicalExpr>, filter: &'life2 BooleanBuffer, ) -> Pin<Box<dyn Future<Output = Option<BooleanArray>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait,

Try to evaluate a predicate on the Liquid array with a filter. Returns Ok(None) if the predicate is not supported.

Note that the filter is a boolean buffer, not a boolean array, i.e., filter can’t be nullable. The returned boolean mask is nullable if the the original array is nullable.

fn original_arrow_data_type(&self) -> DataType

Get the original arrow data type of the Liquid squeezed array.

fn to_best_arrow_array<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = ArrayRef> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Convert the Liquid array to an Arrow array. Except that it will pick the best encoding for the arrow array. Meaning that it may not obey the data type of the original arrow array.

fn disk_backing(&self) -> SqueezedBacking

Describe how the squeezed array persists its backing bytes on disk.