vortex_btrblocks/schemes/binary/
varbin.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4//! Offset-based storage for binary arrays.
5//!
6//! Canonical binary arrays are [`VarBinViewArray`](vortex_array::arrays::VarBinViewArray), which
7//! spends a fixed 16 bytes per element on an opaque views buffer that no scheme can compress.
8//! Re-encoding as [`VarBinArray`] replaces that buffer with an offsets child array, which the
9//! cascading compressor can then compress with the ordinary integer schemes. For fixed-width
10//! values the offsets are a constant-stride sequence and collapse to nothing.
11
12use vortex_array::ArrayId;
13use vortex_array::ArrayRef;
14use vortex_array::Canonical;
15use vortex_array::ExecutionCtx;
16use vortex_array::IntoArray;
17use vortex_array::VTable;
18use vortex_array::arrays::PrimitiveArray;
19use vortex_array::arrays::VarBin;
20use vortex_array::arrays::VarBinArray;
21use vortex_array::arrays::primitive::PrimitiveArrayExt;
22use vortex_array::arrays::varbin::VarBinArraySlotsExt;
23use vortex_array::builders::VarBinBuilder;
24use vortex_compressor::scheme::CompressionEstimate;
25use vortex_compressor::scheme::DeferredEstimate;
26use vortex_compressor::scheme::SchemeExt;
27use vortex_error::VortexResult;
28
29use crate::ArrayAndStats;
30use crate::CascadingCompressor;
31use crate::CompressorContext;
32use crate::Scheme;
33
34/// Offset-based (rather than view-based) storage for binary arrays.
35#[derive(Debug, Copy, Clone, PartialEq, Eq)]
36pub struct VarBinScheme;
37
38impl Scheme for VarBinScheme {
39    fn scheme_name(&self) -> &'static str {
40        "vortex.binary.varbin"
41    }
42
43    fn matches(&self, canonical: &Canonical) -> bool {
44        canonical.dtype().is_binary()
45    }
46
47    fn produced_encodings(&self) -> Vec<ArrayId> {
48        vec![VarBin.id()]
49    }
50
51    fn num_children(&self) -> usize {
52        1
53    }
54
55    fn expected_compression_ratio(
56        &self,
57        _data: &ArrayAndStats,
58        _compress_ctx: CompressorContext,
59        _exec_ctx: &mut ExecutionCtx,
60    ) -> CompressionEstimate {
61        CompressionEstimate::Deferred(DeferredEstimate::Sample)
62    }
63
64    fn compress(
65        &self,
66        compressor: &CascadingCompressor,
67        data: &ArrayAndStats,
68        compress_ctx: CompressorContext,
69        exec_ctx: &mut ExecutionCtx,
70    ) -> VortexResult<ArrayRef> {
71        // `append_to_builder` resolves the views slice and data buffers once and appends
72        // borrowed slices into a single pre-sized allocation. Iterating the array per element
73        // instead would clone a buffer handle and allocate for every value.
74        let array = data.array();
75        let mut builder = VarBinBuilder::<u64>::with_capacity_in(
76            array.dtype().clone(),
77            array.len(),
78            exec_ctx.allocator(),
79        );
80        array.append_to_builder(&mut builder, exec_ctx)?;
81        let varbin = builder.finish_into_varbin();
82
83        let offsets = varbin
84            .offsets()
85            .clone()
86            .execute::<PrimitiveArray>(exec_ctx)?
87            .narrow(exec_ctx)?
88            .into_array();
89        let compressed_offsets =
90            compressor.compress_child(&offsets, &compress_ctx, self.id(), 0, exec_ctx)?;
91
92        Ok(VarBinArray::try_new(
93            compressed_offsets,
94            varbin.bytes().clone(),
95            varbin.dtype().clone(),
96            varbin.validity()?,
97        )?
98        .into_array())
99    }
100}
vortex_btrblocks/schemes/binary/varbin.rs

vortex_btrblocks/schemes/binary/
varbin.rs
