1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
//! Dictionary building for float columns.
use crate::codecs::float::FloatValue;
use crate::encoder::stream::float_cost::{ENCODING_BYTE, raw_stored_bytes, varint_len};
use crate::errors::MltResult;
/// A float column's distinct values and one code per element.
pub(crate) struct FloatDict<T> {
/// Distinct values in first-appearance order, so the result does not depend on hash iteration order.
pub(crate) values: Vec<T>,
/// One index into `values` per input element.
pub(crate) codes: Vec<u32>,
}
impl<T: FloatValue> FloatDict<T> {
/// Build a dictionary for `values`, or [`None`] when it would not be smaller.
/// Both layouts are costed as stored bytes, headers included, assuming nothing about transport compression.
pub(crate) fn worth_building(values: &[T]) -> Option<Self> {
let dict = Self::build(values).ok()?;
(dict.stored_bytes() < raw_stored_bytes::<T>(values.len())).then_some(dict)
}
/// The dictionary of the distinct values, however many of them there are.
/// A caller that pinned the encoding wants it even when every element is distinct.
pub(crate) fn build(values: &[T]) -> MltResult<Self> {
let mut seen = std::collections::HashMap::new();
let mut dict = Self {
values: Vec::new(),
codes: Vec::with_capacity(values.len()),
};
for &value in values {
let next = u32::try_from(dict.values.len())?;
let code = *seen.entry(value.key()).or_insert_with(|| {
dict.values.push(value);
next
});
dict.codes.push(code);
}
Ok(dict)
}
/// Bytes the codes and dictionary streams occupy, headers included.
/// The codes count is implied by context, while the dictionary's always needs its own varint.
pub(crate) fn stored_bytes(&self) -> usize {
let codes: usize = self.codes.iter().copied().map(varint_len).sum();
let values = size_of_val(self.values.as_slice());
let codes_stream = ENCODING_BYTE + varint_len(codes) + codes;
let dict_stream =
ENCODING_BYTE + varint_len(self.values.len()) + varint_len(values) + values;
codes_stream + dict_stream
}
}