mopenzl/lib.rs
1//! Safe Rust [OpenZL] codecs, independently selected with Cargo features.
2//!
3//! OpenZL is Meta's format-aware compression framework: a frame records the
4//! graph of codecs that produced it, so one universal decoder reads every
5//! frame. This crate ports the framework's core to Rust, starting with what
6//! `ZL_compress` and `ZL_decompress` do for serial data:
7//!
8//! - **Byte-identical encoder.** For the same input and global parameters
9//! (level, format version, checksums), [`Compressor`] writes exactly the
10//! frame of OpenZL v0.3.0's `ZL_CCtx_compress` with its default graph, or,
11//! with [`Chunking::Fixed`], of its serial segmenter. Inputs below ten bytes
12//! are stored, larger ones go through Zstandard (the safe-Rust `mzstd`,
13//! byte-identical to the zstd 1.5.7 OpenZL vendors), and any chunk that
14//! would grow is stored instead.
15//! - **Caller-scheduled parallel compression** through `compressor::parallel`
16//! (without `no_std`): one compatible OpenZL frame from independent segments.
17//! Byte identity with the reference serial encoder is not guaranteed.
18//! Streaming retains compressed chunks until the total input size is known.
19//! - **Decoder** for frames of format versions 21 to 27 built from those
20//! codecs, with both checksums verified, bounded input and output, and
21//! precise errors for frames that need codecs not ported yet.
22//! - **No `unsafe`.** Kernels use [`fearless_simd`] with the instruction set
23//! detected once per codec ([`Backend`]); every backend writes the same
24//! bytes.
25//! - **Sans-I/O sessions** that work only with the buffers you pass in, plus
26//! `std::io` adapters.
27//! - **Indexed resource containers** behind the opt-in `framed` feature:
28//! independent graphs per resource, typed outputs, metadata, native sessions
29//! and whole-resource seek. Each resource contains one ordinary OpenZL frame;
30//! the outer container uses mopenzl's own format. See `framing` when enabled.
31//!
32//! `compression` and `decompression` are enabled by default. Disable default
33//! features and select either codec alongside `std` or `no_std`:
34//!
35//! ```toml
36//! mopenzl = { version = "0.2", default-features = false, features = ["std", "decompression"] }
37//! ```
38//!
39//! # Choose your API
40//!
41//! | I/O shape | Compression | Decompression |
42//! | --- | --- | --- |
43//! | Return a new `Vec<u8>` | [`Compressor::compress`] | [`Decompressor::decompress`] |
44//! | Append to an existing Vec | [`Compressor::compress_into`] | [`Decompressor::decompress_into`] |
45//! | Write into a caller-owned slice | [`Compressor::compress_to_slice`] | [`Decompressor::decompress_to_slice`] |
46//! | Write into uninitialized memory | [`Compressor::compress_to_uninit`] | [`Decompressor::decompress_to_uninit`] |
47//! | Choose the graph / codec nodes | [`Compressor::builder`] / [`ParallelCompressor::builder`] → `with_graph` | — (the frame carries its own) |
48//! | Pull output through `std::io::Read` | `Compressor::reader` | `Decompressor::reader` |
49//! | Push input through `std::io::Write` | `Compressor::writer` | `Decompressor::writer` |
50//! | Drive input/output incrementally | [`Compressor::start`] → [`EncoderSession`] | [`Decompressor::start`] → [`DecoderSession`] |
51//! | Same, into uninitialized memory | [`EncoderSession::process_uninit`] | [`DecoderSession::process_uninit`] |
52//! | Same, with the session owning the codec | [`Compressor::into_session`] → [`EncoderSessionOwned`] | [`Decompressor::into_session`] → [`DecoderSessionOwned`] |
53//! | Split one known input across your threads | [`prepare_slice`][parallel-prepare-slice] / [`prepare_source`][parallel-prepare-source] → [`ScopedParallelBatch`] | — |
54//! | Stream chunked input across your threads | [`start`][parallel-start] → [`ParallelEncoderSession`] + [`ParallelTaskQueue`] | — |
55//! | Same, into uninitialized memory | [`ParallelEncoderSession::process_uninit`] | — |
56//! | Same, with the session owning the compressor | [`into_session`][parallel-into-session] → [`ParallelEncoderSessionOwned`] | — |
57//! | Pull parallel output through `std::io::Read` | [`reader`][parallel-reader] → [`ParallelEncoderReader`] + [`ParallelTaskQueue`] | — |
58//! | Push input through `std::io::Write` across your threads | [`writer`][parallel-writer] → [`ParallelEncoderWriter`] + [`ParallelTaskQueue`] | — |
59//!
60//! [`ParallelCompressor`] lives in `compressor::parallel` and is unavailable with
61//! `no_std`. The caller runs its tasks; streaming buffers compressed chunks until
62//! finish, when the total input size is known.
63//!
64//! ```rust
65//! use mopenzl::{Compressor, Decompressor, EncoderConfig, Level};
66//!
67//! let mut encoder = Compressor::new(EncoderConfig::from(Level::try_from(3)?))?;
68//! let mut decoder = Decompressor::default();
69//! let mut frame = Vec::new();
70//! for input in [b"first payload".as_slice(), b"second payload".as_slice()] {
71//! frame.clear(); // Keep the allocation; compress_into appends.
72//! encoder.compress_into(input, &mut frame)?;
73//! assert_eq!(decoder.decompress(&frame)?, input);
74//! }
75//! # Ok::<(), Box<dyn std::error::Error>>(())
76//! ```
77//!
78//! # Streaming and chunks
79//!
80//! A frame header records the frame's total size, and the reference decoder
81//! rejects frames that leave it unknown. Declare the size with
82//! [`InputSize::Exact`] when you know it: a session then writes the header at
83//! once and holds at most one chunk of input. With [`InputSize::Unknown`] it
84//! holds the whole input until the frame ends. [`Chunking::Fixed`] bounds a
85//! chunk, so with a declared size both the encoder and the decoder work in
86//! bounded memory; the default, [`Chunking::Whole`], matches `ZL_compress`.
87//!
88//! ```rust
89//! # #[cfg(not(feature = "no_std"))]
90//! # {
91//! use mopenzl::{ChunkSize, Chunking, Compressor, Decompressor, EncoderConfig, InputSize};
92//! use std::io::{Read, Write};
93//!
94//! let payload = b"streamed payload ".repeat(10_000);
95//! let config = EncoderConfig::default().with_chunking(Chunking::Fixed(ChunkSize::MIN));
96//! let mut encoder = Compressor::new(config)?;
97//! let mut writer = encoder.writer(Vec::new(), InputSize::Exact(payload.len() as u64).into());
98//! writer.write_all(&payload)?;
99//! let frame = writer.finish().map_err(mopenzl::io::FinishError::into_error)?;
100//!
101//! let mut decoder = Decompressor::default();
102//! let mut content = Vec::new();
103//! decoder.reader(&frame[..]).read_to_end(&mut content)?;
104//! assert_eq!(content, payload);
105//! # }
106//! # Ok::<(), Box<dyn std::error::Error>>(())
107//! ```
108//!
109//! # Untrusted input
110//!
111//! Limits are unlimited by default; set [`DecodeLimits`] for untrusted
112//! frames. A frame declaring more output than allowed fails before any chunk
113//! is decoded.
114//!
115//! ```rust
116//! use mopenzl::{DecodeLimits, DecoderConfig, Decompressor};
117//!
118//! let limits = DecodeLimits::default()
119//! .with_max_input_bytes(Some(1 << 20))
120//! .with_max_output_bytes(Some(8 << 20));
121//! let mut decoder = Decompressor::new(limits.into());
122//! assert!(decoder.decompress(b"not a frame").is_err());
123//! ```
124//!
125//! # Select codecs
126//!
127//! Features are additive; the defaults enable everything.
128//!
129//! | Feature | Enables |
130//! | --- | --- |
131//! | `compression` | [`Compressor`] and its sessions |
132//! | `decompression` | [`Decompressor`] and its sessions |
133//! | `std` | the `std::io` adapters, `std::error::Error` and runtime SIMD detection |
134//! | `no_std` | alloc-only builds with the compile-time SIMD baseline |
135//! | `zstd` | the Zstandard codec (`mzstd`), the default graph's backend |
136//!
137//! Every OpenZL codec has its own feature, all enabled by default. Without a
138//! codec's feature the encoder never writes it (chunks that would use it are
139//! stored, so frames stop matching the reference's default graph) and the
140//! decoder reports frames using it as [`Unsupported`].
141//!
142//! ```toml
143//! mopenzl = { version = "0.2", default-features = false, features = ["std", "decompression", "zstd"] }
144//! ```
145//!
146//! [OpenZL]: https://github.com/facebook/openzl
147#![cfg_attr(
148 all(feature = "compression", not(feature = "no_std")),
149 doc = r#"
150[`ParallelCompressor::builder`]: crate::compressor::parallel::ParallelCompressor::builder
151[`ParallelCompressor`]: crate::compressor::parallel::ParallelCompressor
152[parallel-prepare-slice]: crate::compressor::parallel::ParallelCompressor::prepare_slice
153[parallel-prepare-source]: crate::compressor::parallel::ParallelCompressor::prepare_source
154[`ScopedParallelBatch`]: crate::compressor::parallel::ScopedParallelBatch
155[parallel-start]: crate::compressor::parallel::ParallelCompressor::start
156[`ParallelEncoderSession`]: crate::compressor::parallel::ParallelEncoderSession
157[`ParallelEncoderSession::process_uninit`]: crate::compressor::parallel::ParallelEncoderSession::process_uninit
158[`ParallelTaskQueue`]: crate::compressor::parallel::ParallelTaskQueue
159[parallel-into-session]: crate::compressor::parallel::ParallelCompressor::into_session
160[`ParallelEncoderSessionOwned`]: crate::compressor::parallel::ParallelEncoderSessionOwned
161[parallel-reader]: crate::compressor::parallel::ParallelCompressor::reader
162[`ParallelEncoderReader`]: crate::compressor::parallel::ParallelEncoderReader
163[parallel-writer]: crate::compressor::parallel::ParallelCompressor::writer
164[`ParallelEncoderWriter`]: crate::compressor::parallel::ParallelEncoderWriter
165"#
166)]
167#![cfg_attr(
168 not(all(feature = "compression", not(feature = "no_std"))),
169 doc = r#"
170[`ParallelCompressor::builder`]: #select-codecs
171[`ParallelCompressor`]: #select-codecs
172[parallel-prepare-slice]: #select-codecs
173[parallel-prepare-source]: #select-codecs
174[`ScopedParallelBatch`]: #select-codecs
175[parallel-start]: #select-codecs
176[`ParallelEncoderSession`]: #select-codecs
177[`ParallelEncoderSession::process_uninit`]: #select-codecs
178[`ParallelTaskQueue`]: #select-codecs
179[parallel-into-session]: #select-codecs
180[`ParallelEncoderSessionOwned`]: #select-codecs
181[parallel-reader]: #select-codecs
182[`ParallelEncoderReader`]: #select-codecs
183[parallel-writer]: #select-codecs
184[`ParallelEncoderWriter`]: #select-codecs
185"#
186)]
187#![cfg_attr(
188 feature = "compression",
189 doc = r#"
190[`Chunking::Fixed`]: crate::Chunking::Fixed
191[`Chunking::Whole`]: crate::Chunking::Whole
192[`Compressor::compress_into`]: crate::Compressor::compress_into
193[`Compressor::compress_to_slice`]: crate::Compressor::compress_to_slice
194[`Compressor::compress_to_uninit`]: crate::Compressor::compress_to_uninit
195[`Compressor::compress`]: crate::Compressor::compress
196[`Compressor::into_session`]: crate::Compressor::into_session
197[`Compressor::start`]: crate::Compressor::start
198[`Compressor::builder`]: crate::Compressor::builder
199[`Compressor`]: crate::Compressor
200[`EncoderSession::process_uninit`]: crate::EncoderSession::process_uninit
201[`EncoderSessionOwned`]: crate::EncoderSessionOwned
202[`EncoderSession`]: crate::EncoderSession
203[`InputSize::Exact`]: crate::InputSize::Exact
204[`InputSize::Unknown`]: crate::InputSize::Unknown
205"#
206)]
207#![cfg_attr(
208 not(feature = "compression"),
209 doc = r#"
210[`Chunking::Fixed`]: #select-codecs
211[`Chunking::Whole`]: #select-codecs
212[`Compressor::compress_into`]: #select-codecs
213[`Compressor::compress_to_slice`]: #select-codecs
214[`Compressor::compress_to_uninit`]: #select-codecs
215[`Compressor::compress`]: #select-codecs
216[`Compressor::into_session`]: #select-codecs
217[`Compressor::start`]: #select-codecs
218[`Compressor::builder`]: #select-codecs
219[`Compressor`]: #select-codecs
220[`EncoderSession::process_uninit`]: #select-codecs
221[`EncoderSessionOwned`]: #select-codecs
222[`EncoderSession`]: #select-codecs
223[`InputSize::Exact`]: #select-codecs
224[`InputSize::Unknown`]: #select-codecs
225"#
226)]
227#![cfg_attr(
228 feature = "decompression",
229 doc = r#"
230[`DecodeLimits`]: crate::DecodeLimits
231[`DecoderSession::process_uninit`]: crate::DecoderSession::process_uninit
232[`DecoderSessionOwned`]: crate::DecoderSessionOwned
233[`DecoderSession`]: crate::DecoderSession
234[`Decompressor::decompress_into`]: crate::Decompressor::decompress_into
235[`Decompressor::decompress_to_slice`]: crate::Decompressor::decompress_to_slice
236[`Decompressor::decompress_to_uninit`]: crate::Decompressor::decompress_to_uninit
237[`Decompressor::decompress`]: crate::Decompressor::decompress
238[`Decompressor::into_session`]: crate::Decompressor::into_session
239[`Decompressor::start`]: crate::Decompressor::start
240[`Decompressor`]: crate::Decompressor
241"#
242)]
243#![cfg_attr(
244 not(feature = "decompression"),
245 doc = r#"
246[`DecodeLimits`]: #select-codecs
247[`DecoderSession::process_uninit`]: #select-codecs
248[`DecoderSessionOwned`]: #select-codecs
249[`DecoderSession`]: #select-codecs
250[`Decompressor::decompress_into`]: #select-codecs
251[`Decompressor::decompress_to_slice`]: #select-codecs
252[`Decompressor::decompress_to_uninit`]: #select-codecs
253[`Decompressor::decompress`]: #select-codecs
254[`Decompressor::into_session`]: #select-codecs
255[`Decompressor::start`]: #select-codecs
256[`Decompressor`]: #select-codecs
257"#
258)]
259#![cfg_attr(
260 not(feature = "decompression"),
261 doc = "[`Unsupported`]: #select-codecs"
262)]
263#![cfg_attr(
264 not(any(feature = "compression", feature = "decompression")),
265 doc = "[`Backend`]: #select-codecs\n\n[`fearless_simd`]: https://docs.rs/fearless_simd"
266)]
267#![cfg_attr(not(test), forbid(unsafe_code))]
268#![cfg_attr(feature = "no_std", no_std)]
269#![deny(missing_docs)]
270#![deny(missing_debug_implementations)]
271#![deny(rustdoc::broken_intra_doc_links)]
272
273#[cfg(any(feature = "compression", feature = "decompression"))]
274extern crate alloc;
275
276#[cfg(test)]
277extern crate std;
278
279#[cfg(any(feature = "compression", feature = "decompression"))]
280mod backend;
281#[cfg(any(feature = "compression", feature = "decompression"))]
282mod data_type;
283#[cfg(any(feature = "compression", feature = "decompression"))]
284mod format;
285#[cfg(any(feature = "compression", feature = "decompression"))]
286mod retention;
287#[cfg(any(feature = "compression", feature = "decompression"))]
288mod shared;
289#[cfg(any(feature = "compression", feature = "decompression"))]
290pub use backend::Backend;
291#[cfg(any(feature = "compression", feature = "decompression"))]
292pub use data_type::DataType;
293#[cfg(any(feature = "compression", feature = "decompression"))]
294pub use format::{FormatVersion, UnsupportedFormatVersion};
295#[cfg(any(feature = "compression", feature = "decompression"))]
296pub use retention::RetentionPolicy;
297
298#[cfg(all(
299 any(feature = "compression", feature = "decompression"),
300 not(feature = "no_std")
301))]
302mod finish_error;
303#[cfg(all(
304 any(feature = "compression", feature = "decompression"),
305 not(feature = "no_std")
306))]
307pub mod io;
308
309#[cfg(all(
310 feature = "custom-codec",
311 any(feature = "compression", feature = "decompression")
312))]
313pub mod custom_codec;
314#[cfg(any(feature = "compression", feature = "decompression"))]
315pub mod dict;
316#[cfg(feature = "compression")]
317pub mod graph;
318#[cfg(feature = "compression")]
319pub mod input;
320
321#[cfg(all(
322 feature = "framed",
323 any(feature = "compression", feature = "decompression")
324))]
325pub mod framing;
326
327#[cfg(feature = "compression")]
328pub mod compressor;
329#[cfg(feature = "compression")]
330pub use compressor::{
331 ChunkSize, Chunking, Compressor, CompressorBuilder, ConfigError, EncodeError, EncoderConfig,
332 EncoderSession, EncoderSessionOwned, EncoderStatus, InputSize, Level, Operation, Parameter,
333 Progress, SizeOverflow, StreamConfig,
334};
335
336#[cfg(feature = "decompression")]
337pub mod decompressor;
338#[cfg(feature = "decompression")]
339pub use decompressor::{
340 DecodeError, DecodeFailure, DecodeLimits, DecodeOperation, DecodeProgress, DecoderConfig,
341 DecoderSession, DecoderSessionOwned, DecoderStatus, Decompressor, DecompressorBuilder,
342 FrameInfo, FrameMode, InvalidDataKind, TypedOutput, Unsupported,
343};