fastpfor 0.9.0

FastPFOR lib with C++ Rust wrapper and pure Rust implementation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96

use cxx::UniquePtr;

use crate::FastPForResult;
use crate::codec::default_max_decoded_len;
use crate::cpp::ffi;
use crate::helpers::AsUsize;

/// Pass-through to C++ `encodeArray` / `decodeArray`. No extra header is added.
///
/// Block-based C++ codecs (`FastPFor`, `PFor`, etc.) store the original data length
/// in their own wire format. Byte-oriented codecs (`VariableByte`) rely on the
/// caller passing the encoded stream length to decode, which we have via `input.len()`.
pub fn encode32_to_vec_ffi(
    codec: &UniquePtr<ffi::IntegerCODEC>,
    input: &[u32],
    out: &mut Vec<u32>,
) -> FastPForResult<()> {
    let capacity = input.len() * 2 + 1024;
    let start = out.len();
    out.resize(start + capacity, 0);
    let n = ffi::codec_encode32(codec, input, &mut out[start..])?;
    // SAFETY: It is better to panic than to have UB
    assert!(
        n <= capacity,
        "C++ codec encoded more than the allocated capacity"
    );
    out.truncate(start + n);
    Ok(())
}

pub fn decode32_anylen_ffi(
    codec: &UniquePtr<ffi::IntegerCODEC>,
    input: &[u32],
    out: &mut Vec<u32>,
    expected_len: Option<u32>,
) -> FastPForResult<()> {
    let max = default_max_decoded_len(input.len());
    let capacity = if let Some(n) = expected_len {
        n.is_valid_expected(max)?
    } else {
        // C++ decodeArray needs output buffer capacity. Block codecs read count from stream;
        // variable-byte decodes until input is consumed. Simple9/16 pack up to 28 values/word.
        max
    };
    let start = out.len();
    if !input.is_empty() {
        // Simple16/Simple9/Simple8b unpack functions always write a fixed number of
        // values (up to 28) per word regardless of how many values remain. When the
        // last word is decoded the function can write up to 27 elements past `nvalue`.
        // Allocate extra padding so those writes land in owned memory rather than
        // corrupting adjacent heap allocations.  The excess elements are discarded by
        // the truncate below.
        const DECODE_OVERFLOW_PADDING: usize = 32;
        out.resize(start + capacity + DECODE_OVERFLOW_PADDING, 0);
        let n = ffi::codec_decode32(codec, input, &mut out[start..])?;
        // SAFETY: It is better to panic than to have UB
        assert!(
            n < capacity + DECODE_OVERFLOW_PADDING,
            "C++ codec decoded more than the allocated capacity + padding"
        );
        out.truncate(start + n);
    }
    if let Some(n) = expected_len {
        (out.len() - start).is_decoded_mismatch(n)?;
    }
    Ok(())
}

pub fn encode64_to_vec_ffi(
    codec: &UniquePtr<ffi::IntegerCODEC>,
    input: &[u64],
    out: &mut Vec<u32>,
) -> FastPForResult<()> {
    let capacity = input.len() * 3 + 1024;
    let start = out.len();
    out.resize(start + capacity, 0);
    let n = ffi::codec_encode64(codec, input, &mut out[start..])?;
    out.truncate(start + n);
    Ok(())
}

pub fn decode64_to_vec_ffi(
    codec: &UniquePtr<ffi::IntegerCODEC>,
    input: &[u32],
    out: &mut Vec<u64>,
) -> FastPForResult<()> {
    if !input.is_empty() {
        // C++ decodeArray needs output buffer. Variable-byte can pack multiple values per word.
        let capacity = input.len().saturating_mul(4);
        let start = out.len();
        out.resize(start + capacity, 0);
        let n = ffi::codec_decode64(codec, input, &mut out[start..])?;
        out.truncate(start + n);
    }
    Ok(())
}