structured_zstd/encoding/mod.rs
1//! Structures and utilities used for compressing/encoding data into the Zstd format.
2
3pub(crate) mod block_header;
4pub(crate) mod blocks;
5pub(crate) mod frame_header;
6pub(crate) mod match_generator;
7pub(crate) mod util;
8
9mod frame_compressor;
10mod levels;
11mod streaming_encoder;
12pub use frame_compressor::FrameCompressor;
13pub use match_generator::MatchGeneratorDriver;
14pub use streaming_encoder::StreamingEncoder;
15
16use crate::io::{Read, Write};
17use alloc::vec::Vec;
18
19/// Convenience function to compress some source into a target without reusing any resources of the compressor
20/// ```rust
21/// use structured_zstd::encoding::{compress, CompressionLevel};
22/// let data: &[u8] = &[0,0,0,0,0,0,0,0,0,0,0,0];
23/// let mut target = Vec::new();
24/// compress(data, &mut target, CompressionLevel::Fastest);
25/// ```
26pub fn compress<R: Read, W: Write>(source: R, target: W, level: CompressionLevel) {
27 let mut frame_enc = FrameCompressor::new(level);
28 frame_enc.set_source(source);
29 frame_enc.set_drain(target);
30 frame_enc.compress();
31}
32
33/// Convenience function to compress some source into a Vec without reusing any resources of the compressor
34/// ```rust
35/// use structured_zstd::encoding::{compress_to_vec, CompressionLevel};
36/// let data: &[u8] = &[0,0,0,0,0,0,0,0,0,0,0,0];
37/// let compressed = compress_to_vec(data, CompressionLevel::Fastest);
38/// ```
39pub fn compress_to_vec<R: Read>(source: R, level: CompressionLevel) -> Vec<u8> {
40 let mut vec = Vec::new();
41 compress(source, &mut vec, level);
42 vec
43}
44
45/// The compression mode used impacts the speed of compression,
46/// and resulting compression ratios. Faster compression will result
47/// in worse compression ratios, and vice versa.
48#[derive(Copy, Clone)]
49pub enum CompressionLevel {
50 /// This level does not compress the data at all, and simply wraps
51 /// it in a Zstandard frame.
52 Uncompressed,
53 /// This level is roughly equivalent to Zstd compression level 1
54 Fastest,
55 /// This level uses the crate's dedicated `dfast`-style matcher to
56 /// target a better speed/ratio tradeoff than [`CompressionLevel::Fastest`].
57 ///
58 /// It represents this crate's "default" compression setting and may
59 /// evolve in future versions as the implementation moves closer to
60 /// reference zstd level 3 behavior.
61 Default,
62 /// This level is roughly equivalent to Zstd level 7.
63 ///
64 /// UNIMPLEMENTED
65 Better,
66 /// This level is roughly equivalent to Zstd level 11.
67 ///
68 /// UNIMPLEMENTED
69 Best,
70}
71
72/// Trait used by the encoder that users can use to extend the matching facilities with their own algorithm
73/// making their own tradeoffs between runtime, memory usage and compression ratio
74///
75/// This trait operates on buffers that represent the chunks of data the matching algorithm wants to work on.
76/// Each one of these buffers is referred to as a *space*. One or more of these buffers represent the window
77/// the decoder will need to decode the data again.
78///
79/// This library asks the Matcher for a new buffer using `get_next_space` to allow reusing of allocated buffers when they are no longer part of the
80/// window of data that is being used for matching.
81///
82/// The library fills the buffer with data that is to be compressed and commits them back to the matcher using `commit_space`.
83///
84/// Then it will either call `start_matching` or, if the space is deemed not worth compressing, `skip_matching` is called.
85///
86/// This is repeated until no more data is left to be compressed.
87pub trait Matcher {
88 /// Get a space where we can put data to be matched on. Will be encoded as one block. The maximum allowed size is 128 kB.
89 fn get_next_space(&mut self) -> alloc::vec::Vec<u8>;
90 /// Get a reference to the last commited space
91 fn get_last_space(&mut self) -> &[u8];
92 /// Commit a space to the matcher so it can be matched against
93 fn commit_space(&mut self, space: alloc::vec::Vec<u8>);
94 /// Just process the data in the last commited space for future matching
95 fn skip_matching(&mut self);
96 /// Process the data in the last commited space for future matching AND generate matches for the data
97 fn start_matching(&mut self, handle_sequence: impl for<'a> FnMut(Sequence<'a>));
98 /// Reset this matcher so it can be used for the next new frame
99 fn reset(&mut self, level: CompressionLevel);
100 /// Prime matcher state with dictionary history before compressing the next frame.
101 /// Default implementation is a no-op for custom matchers that do not support this.
102 fn prime_with_dictionary(&mut self, _dict_content: &[u8], _offset_hist: [u32; 3]) {}
103 /// Returns whether this matcher can consume dictionary priming state and produce
104 /// dictionary-dependent sequences. Defaults to `false` for custom matchers.
105 fn supports_dictionary_priming(&self) -> bool {
106 false
107 }
108 /// The size of the window the decoder will need to execute all sequences produced by this matcher.
109 ///
110 /// Must return a positive (non-zero) value; returning 0 causes
111 /// [`StreamingEncoder`] to reject the first write with an invalid-input error
112 /// (`InvalidInput` with `std`, `Other` with `no_std`).
113 ///
114 /// Must remain stable for the lifetime of a frame.
115 /// It may change only after `reset()` is called for the next frame
116 /// (for example because the compression level changed).
117 fn window_size(&self) -> u64;
118}
119
120#[derive(PartialEq, Eq, Debug)]
121/// Sequences that a [`Matcher`] can produce
122pub enum Sequence<'data> {
123 /// Is encoded as a sequence for the decoder sequence execution.
124 ///
125 /// First the literals will be copied to the decoded data,
126 /// then `match_len` bytes are copied from `offset` bytes back in the decoded data
127 Triple {
128 literals: &'data [u8],
129 offset: usize,
130 match_len: usize,
131 },
132 /// This is returned as the last sequence in a block
133 ///
134 /// These literals will just be copied at the end of the sequence execution by the decoder
135 Literals { literals: &'data [u8] },
136}