q_compress 0.11.5

Good compression for numerical sequences and time series
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78

use crate::bit_reader::BitReader;
use crate::data_types::{NumberLike, UnsignedLike};
use crate::gcd_utils::GcdOperator;
use crate::num_decompressor::NumDecompressor;
use crate::prefix::PrefixDecompressionInfo;
use crate::Prefix;

pub fn use_run_len<T: NumberLike>(prefixes: &[Prefix<T>]) -> bool {
  prefixes.iter().any(|p| p.run_len_jumpstart.is_some())
}

fn unchecked_decompress_offset<U: UnsignedLike, GcdOp: GcdOperator<U>>(
  reader: &mut BitReader,
  unsigneds: &mut Vec<U>,
  p: PrefixDecompressionInfo<U>,
) {
  let mut offset = reader.unchecked_read_diff(p.k);
  if offset < p.min_unambiguous_k_bit_offset && reader.unchecked_read_one() {
    offset |= p.most_significant;
  }
  let unsigned = p.lower_unsigned + GcdOp::get_diff(offset, p.gcd);
  unsigneds.push(unsigned);
}

pub trait RunLenOperator {
  fn unchecked_decompress_offsets<U: UnsignedLike, GcdOp: GcdOperator<U>>(
    num_decompressor: &mut NumDecompressor<U>,
    reader: &mut BitReader,
    unsigneds: &mut Vec<U>,
    p: PrefixDecompressionInfo<U>,
    batch_size: usize,
  );
}

pub struct GeneralRunLenOp;

impl RunLenOperator for GeneralRunLenOp {
  fn unchecked_decompress_offsets<U: UnsignedLike, GcdOp: GcdOperator<U>>(
    num_decompressor: &mut NumDecompressor<U>,
    reader: &mut BitReader,
    unsigneds: &mut Vec<U>,
    p: PrefixDecompressionInfo<U>,
    batch_size: usize,
  ) {
    match p.run_len_jumpstart {
      None => unchecked_decompress_offset::<U, GcdOp>(reader, unsigneds, p),
      // we stored the number of occurrences minus 1 because we knew it's at least 1
      Some(jumpstart) => {
        let full_reps = reader.unchecked_read_varint(jumpstart) + 1;
        let reps =
          num_decompressor.unchecked_limit_reps(p, full_reps, batch_size - unsigneds.len());
        if p.k == 0 {
          for _ in 0..reps {
            unsigneds.push(p.lower_unsigned);
          }
        } else {
          for _ in 0..reps {
            unchecked_decompress_offset::<U, GcdOp>(reader, unsigneds, p);
          }
        }
      }
    };
  }
}

pub struct TrivialRunLenOp;

impl RunLenOperator for TrivialRunLenOp {
  fn unchecked_decompress_offsets<U: UnsignedLike, GcdOp: GcdOperator<U>>(
    _num_decompressor: &mut NumDecompressor<U>,
    reader: &mut BitReader,
    unsigneds: &mut Vec<U>,
    p: PrefixDecompressionInfo<U>,
    _batch_size: usize,
  ) {
    unchecked_decompress_offset::<U, GcdOp>(reader, unsigneds, p)
  }
}