parallel_processor/buckets/readers/
compressed_binary_reader.rs

1use crate::buckets::readers::generic_binary_reader::{ChunkDecoder, GenericChunkedBinaryReader};
2use crate::buckets::writers::compressed_binary_writer::COMPRESSED_BUCKET_MAGIC;
3use crate::memory_fs::file::reader::FileReader;
4use crate::utils::vec_reader::VecReader;
5
6pub struct CompressedStreamDecoder;
7
8const DEFAULT_BUFFER_SIZE: usize = 1024 * 1024;
9
10impl ChunkDecoder for CompressedStreamDecoder {
11    const MAGIC_HEADER: &'static [u8; 16] = COMPRESSED_BUCKET_MAGIC;
12    type ReaderType = VecReader<lz4::Decoder<FileReader>>;
13
14    fn decode_stream(reader: FileReader, _size: u64) -> Self::ReaderType {
15        VecReader::new(DEFAULT_BUFFER_SIZE, lz4::Decoder::new(reader).unwrap())
16    }
17
18    fn dispose_stream(stream: Self::ReaderType) -> FileReader {
19        let (file, result) = stream.into_inner().finish();
20        result.unwrap();
21        file
22    }
23}
24
25pub type CompressedBinaryReader = GenericChunkedBinaryReader<CompressedStreamDecoder>;