1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
//! Utilities and representations for a frame header.
use crate::common::MAGIC_NUM;
use crate::encoding::util::{find_fcs_field_size, find_min_size, write_minified_val};
use alloc::vec::Vec;
/// A header for a single Zstandard frame.
///
/// <https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#frame_header>
///
/// The containing module is `pub(crate)`, so this struct is not
/// externally constructible. The `Default` derive is kept as a
/// convenience for tests and for internal callers that only need
/// to override specific fields.
#[derive(Debug, Default)]
pub struct FrameHeader {
/// Optionally, the original (uncompressed) size of the data within the frame in bytes.
/// If not present, `window_size` must be set.
pub frame_content_size: Option<u64>,
/// If set to true, data must be regenerated within a single
/// continuous memory segment.
pub single_segment: bool,
/// If set to true, a 32 bit content checksum will be present
/// at the end of the frame.
pub content_checksum: bool,
/// If a dictionary ID is provided, the ID of that dictionary.
pub dictionary_id: Option<u64>,
/// The minimum memory buffer required to compress a frame. If not present,
/// `single_segment` will be set to true. If present, this value must be greater than 1KB
/// and less than 3.75TB. Encoders should not generate a frame that requires a window size larger than
/// 8mb.
pub window_size: Option<u64>,
/// If true, the 4-byte magic number prefix is omitted from the
/// serialized output. The caller MUST know out-of-band that the
/// stream is magicless and use a magicless-aware decoder.
/// Upstream zstd parity: `ZSTD_f_zstd1_magicless` (see `ZSTD_d_format`).
pub magicless: bool,
}
impl FrameHeader {
/// Writes the serialized frame header into the provided buffer.
///
/// The returned header *does include* a frame header descriptor.
pub fn serialize(self, output: &mut Vec<u8>) {
vprintln!("Serializing frame with header: {self:?}");
// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#frame_header
// Magic Number (omitted in magicless mode — `ZSTD_f_zstd1_magicless`):
if !self.magicless {
output.extend_from_slice(&MAGIC_NUM.to_le_bytes());
}
// `Frame_Header_Descriptor`:
output.push(self.descriptor());
// `Window_Descriptor
// TODO: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#window_descriptor
if !self.single_segment
&& let Some(window_size) = self.window_size
{
let log = window_size.next_power_of_two().ilog2();
let exponent = if log > 10 { log - 10 } else { 1 } as u8;
output.push(exponent << 3);
}
if let Some(id) = self.dictionary_id {
write_minified_val(id, output);
}
if let Some(frame_content_size) = self.frame_content_size {
let field_size = find_fcs_field_size(frame_content_size, self.single_segment);
write_fcs(frame_content_size, field_size, output);
}
}
/// Generate a serialized frame header descriptor for the frame header.
///
/// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#frame_header_descriptor
fn descriptor(&self) -> u8 {
// A frame header starts with a frame header descriptor.
// It describes what other fields are present
// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#frame_header_descriptor
// Built with plain bit arithmetic (LSB-first field order below); a
// `BitWriter` here allocated a fresh `Vec` for the single byte on
// every frame header.
// Bits 0-1, `Dictionary_ID_flag`: size class of the
// `Dictionary_ID` field (0 = absent).
let dict_id_flag: u8 = match self.dictionary_id.map(find_min_size) {
None => 0,
Some(1) => 1,
Some(2) => 2,
Some(4) => 3,
_ => panic!(),
};
// Bit 2, `Content_Checksum_flag`.
let checksum_flag = u8::from(self.content_checksum);
// Bits 3-4: `Reserved_bit` + `Unused_bit`, both zero.
// Bit 5, `Single_Segment_flag`:
// If this flag is set, data must be regenerated within a single continuous memory segment,
// and the `Frame_Content_Size` field must be present in the header.
// If this flag is not set, the `Window_Descriptor` field must be present in the frame header.
if self.single_segment {
assert!(
self.frame_content_size.is_some(),
"if the `single_segment` flag is set to true, then a frame content size must be provided"
);
} else {
assert!(
self.window_size.is_some(),
"if the `single_segment` flag is set to false, then a window size must be provided"
);
}
let single_segment_flag = u8::from(self.single_segment);
// Bits 6-7, `Frame_Content_Size_flag`: size class of the FCS field.
// If the `Single_Segment_flag` is set and this value is zero,
// the size of the FCS field is 1 byte; otherwise the FCS field is
// omitted when the flag is zero.
// | Value | Size of field (Bytes)
// | 0 | 0 or 1
// | 1 | 2
// | 2 | 4
// | 3 | 8
let fcs_flag: u8 = match self.frame_content_size {
Some(frame_content_size) => {
match find_fcs_field_size(frame_content_size, self.single_segment) {
1 => 0,
2 => 1,
4 => 2,
8 => 3,
_ => unreachable!(),
}
}
None => 0,
};
dict_id_flag | (checksum_flag << 2) | (single_segment_flag << 5) | (fcs_flag << 6)
}
}
/// Serialize a `Frame_Content_Size` value into `field_size` bytes.
///
/// For 1, 4, and 8-byte fields the value is stored directly.
/// For 2-byte fields an offset of 256 is subtracted before encoding.
///
/// <https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#frame_content_size>
fn write_fcs(val: u64, field_size: usize, output: &mut Vec<u8>) {
debug_assert!(matches!(field_size, 1 | 2 | 4 | 8));
debug_assert!(field_size != 2 || val >= 256);
let adjusted = match field_size {
2 => val - 256,
1 | 4 | 8 => val,
_ => unreachable!("invalid Frame_Content_Size field size: {field_size}"),
};
output.extend_from_slice(&adjusted.to_le_bytes()[..field_size]);
}
#[cfg(test)]
mod tests;