zesven 3.2.0

A pure Rust implementation of the 7z archive format
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276

//! Large entries are compressed straight into the sink.
//!
//! Archiving a file used to cost as much memory as the file was long, because
//! every entry was read into memory before being compressed. Entries past a
//! size threshold now go through the codec as they are read.
//!
//! That is a second way of producing a folder, with its own way of arriving at
//! the packed size and the checksum, so these cases check that what it writes
//! decodes, describes itself correctly, and leaves the entries around it in the
//! order they were added - including the cases it declines and hands back to
//! the buffered path.

#![cfg(feature = "lzma2")]

use std::io::Cursor;

use zesven::codec::CodecMethod;
use zesven::read::Archive;
use zesven::write::{WriteOptions, Writer};
use zesven::{ArchivePath, WriteFilter};

/// Just past the threshold at which the streaming path takes over.
///
/// No larger than it needs to be: every case here compresses this much, and
/// the feature matrix runs the suite once per feature combination.
const LARGE: usize = 68 * 1024 * 1024;

/// The fastest level, since these cases exercise the path rather than the
/// codec. It runs an order of magnitude quicker than the default.
fn fast() -> WriteOptions {
    WriteOptions::new().level(1).unwrap()
}

/// Data with matches at varying distances, so the dictionary matters.
fn payload(len: usize) -> Vec<u8> {
    let mut data = Vec::with_capacity(len);
    let mut state = 0x243F_6A88_85A3_08D3u64;
    while data.len() < len {
        state ^= state << 13;
        state ^= state >> 7;
        state ^= state << 17;
        data.extend_from_slice(b"a large entry should not have to fit in memory to be archived ");
        data.extend_from_slice(&state.to_le_bytes());
    }
    data.truncate(len);
    data
}

/// Writes one entry and returns the archive.
fn archive_one(options: WriteOptions, data: &[u8]) -> Vec<u8> {
    let mut writer = Writer::create(Cursor::new(Vec::new()))
        .unwrap()
        .options(options);
    writer
        .add_bytes(ArchivePath::new("big.bin").unwrap(), data)
        .unwrap();
    writer.finish_into_inner().unwrap().1.into_inner()
}

/// A large entry must decode to exactly what went in.
#[test]
fn test_large_entry_round_trips() {
    let data = payload(LARGE);
    let bytes = archive_one(fast(), &data);

    let mut opened = Archive::open(Cursor::new(bytes)).unwrap();
    assert_eq!(opened.extract_to_vec("big.bin").unwrap(), data);
}

/// A streamed entry must describe itself the way a buffered one does.
///
/// The two paths record a folder from different places - the buffered one
/// measures the compressed buffer it holds, the streaming one counts bytes as
/// they leave - so the sizes each writes into the header can disagree without
/// either failing to decode. The archives themselves cannot be compared
/// directly, since nothing can force the same data down both paths, so this
/// checks the figures a reader sees.
#[test]
fn test_a_streamed_entry_reports_its_sizes() {
    let data = payload(LARGE);
    let streamed = archive_one(fast(), &data);

    let mut opened = Archive::open(Cursor::new(streamed)).unwrap();
    let entries = opened.entries().to_vec();
    assert_eq!(entries.len(), 1);
    assert_eq!(entries[0].size, data.len() as u64);
    assert_eq!(opened.extract_to_vec("big.bin").unwrap(), data);
}

/// A stored entry must stream too, and decode unchanged.
#[test]
fn test_large_stored_entry_round_trips() {
    let data = payload(LARGE);
    let bytes = archive_one(fast().method(CodecMethod::Copy), &data);

    let mut opened = Archive::open(Cursor::new(bytes)).unwrap();
    assert_eq!(opened.extract_to_vec("big.bin").unwrap(), data);
}

/// LZMA1 has its own encoder and its own properties; both must survive.
#[cfg(feature = "lzma")]
#[test]
fn test_large_lzma1_entry_round_trips() {
    let data = payload(LARGE);
    let bytes = archive_one(fast().method(CodecMethod::Lzma), &data);

    let mut opened = Archive::open(Cursor::new(bytes)).unwrap();
    assert_eq!(opened.extract_to_vec("big.bin").unwrap(), data);
}

/// Cases the streaming path declines must still work, and still be correct.
///
/// A filter sits ahead of the codec and encryption behind it, so neither can be
/// applied to bytes that have already left for the sink. Those keep the
/// buffered path, and this checks the writer really does fall back rather than
/// dropping the filter or the encryption on the floor.
#[test]
fn test_large_entries_with_filters_fall_back_and_stay_correct() {
    let data = payload(LARGE);

    let filtered = archive_one(fast().filter(WriteFilter::delta(4)), &data);
    let mut opened = Archive::open(Cursor::new(filtered)).unwrap();
    assert_eq!(opened.extract_to_vec("big.bin").unwrap(), data);
}

/// Encrypted large entries fall back too, and must decrypt.
#[cfg(feature = "aes")]
#[test]
fn test_large_encrypted_entry_round_trips() {
    use zesven::crypto::{NoncePolicy, Password};

    let data = payload(LARGE);
    let options = fast()
        .password("correct horse battery staple")
        .nonce_policy(NoncePolicy::random_with_params(4, 8))
        .encrypt_data(true);

    let bytes = archive_one(options, &data);
    let mut opened = Archive::open_with_password(
        Cursor::new(bytes),
        Password::new("correct horse battery staple"),
    )
    .unwrap();
    assert_eq!(opened.extract_to_vec("big.bin").unwrap(), data);
}

/// A large entry among small ones must not disturb their order.
///
/// The streaming path writes immediately while the buffered one gathers
/// entries, so a large entry has to push whatever is waiting to the sink first.
#[test]
fn test_a_large_entry_keeps_the_order_of_small_ones() {
    let small = payload(64 * 1024);
    let large = payload(LARGE);

    let mut writer = Writer::create(Cursor::new(Vec::new()))
        .unwrap()
        .options(fast());
    writer
        .add_bytes(ArchivePath::new("01-small.bin").unwrap(), &small)
        .unwrap();
    writer
        .add_bytes(ArchivePath::new("02-large.bin").unwrap(), &large)
        .unwrap();
    writer
        .add_bytes(ArchivePath::new("03-small.bin").unwrap(), &small)
        .unwrap();
    let bytes = writer.finish_into_inner().unwrap().1.into_inner();

    let mut opened = Archive::open(Cursor::new(bytes)).unwrap();
    let paths: Vec<String> = opened
        .entries()
        .iter()
        .map(|e| e.path.as_str().to_string())
        .collect();
    assert_eq!(paths, vec!["01-small.bin", "02-large.bin", "03-small.bin"]);

    assert_eq!(opened.extract_to_vec("01-small.bin").unwrap(), small);
    assert_eq!(opened.extract_to_vec("02-large.bin").unwrap(), large);
    assert_eq!(opened.extract_to_vec("03-small.bin").unwrap(), small);
}

/// A source that claims a large size and yields nothing must not corrupt the
/// archive.
///
/// What an entry claims decides nothing - the path it takes is settled by
/// reading it - so a source that reports eighty megabytes and delivers none is
/// simply an empty entry. It carries no stream at all, and an encoder built
/// before that was known would have put its framing into the sink belonging to
/// no folder, moving every folder written after it.
#[test]
fn test_an_empty_source_claiming_to_be_large_is_harmless() {
    use zesven::write::EntryMeta;

    let data = payload(64 * 1024);

    let mut writer = Writer::create(Cursor::new(Vec::new()))
        .unwrap()
        .options(fast());
    // Claims 80 MB, delivers none.
    writer
        .add_stream(
            ArchivePath::new("empty.bin").unwrap(),
            &mut Cursor::new(Vec::new()),
            EntryMeta::file(80 * 1024 * 1024),
        )
        .unwrap();
    writer
        .add_bytes(ArchivePath::new("after.bin").unwrap(), &data)
        .unwrap();
    let bytes = writer.finish_into_inner().unwrap().1.into_inner();

    let mut opened = Archive::open(Cursor::new(bytes)).unwrap();
    let paths: Vec<String> = opened
        .entries()
        .iter()
        .map(|e| e.path.as_str().to_string())
        .collect();
    assert_eq!(paths, vec!["empty.bin", "after.bin"]);
    assert!(opened.extract_to_vec("empty.bin").unwrap().is_empty());
    assert_eq!(opened.extract_to_vec("after.bin").unwrap(), data);
}

/// Data that does not compress, so an archive of it is as large as its input.
fn incompressible(len: usize) -> Vec<u8> {
    let mut data = Vec::with_capacity(len);
    let mut state = 0x2545_F491_4F6C_DD1Du64;
    while data.len() < len {
        state ^= state << 13;
        state ^= state >> 7;
        state ^= state << 17;
        data.extend_from_slice(&state.to_le_bytes());
    }
    data.truncate(len);
    data
}

/// Streaming must work into a multivolume sink, whose position spans files.
#[test]
fn test_large_entry_across_volumes() {
    use zesven::VolumeConfig;

    let dir = tempfile::TempDir::new().unwrap();
    // Incompressible, so the archive really does outgrow a volume: compressible
    // data at this size fits in one, and then the case proves nothing.
    let data = incompressible(LARGE);

    let config = VolumeConfig::new(dir.path().join("big.7z"), 17 * 1024 * 1024);
    let mut writer = Writer::create_multivolume(config).unwrap().options(fast());
    writer
        .add_bytes(ArchivePath::new("big.bin").unwrap(), &data)
        .unwrap();
    let result = writer.finish().unwrap();

    // The point is that the entry spans volumes: a sink whose position is not a
    // single file's offset is exactly what the streaming path's byte counting
    // has to cope with. `volume_count >= 1` would have been true of any archive
    // at all, including one that never crossed a boundary.
    assert!(
        result.volume_count > 1,
        "the entry fitted in one volume, so this proves nothing: {} volume(s)",
        result.volume_count,
    );
    assert!(
        result
            .volume_sizes
            .iter()
            .take(result.volume_count as usize - 1)
            .all(|&size| size == 17 * 1024 * 1024),
        "every volume but the last must be full: {:?}",
        result.volume_sizes,
    );

    let mut opened = Archive::open_path(dir.path().join("big.7z.001")).unwrap();
    assert_eq!(opened.extract_to_vec("big.bin").unwrap(), data);
}