gemachain_runtime/
hardened_unpack.rs

1use gemachain_sdk::genesis_config::{DEFAULT_GENESIS_ARCHIVE, DEFAULT_GENESIS_FILE};
2use {
3    bzip2::bufread::BzDecoder,
4    log::*,
5    rand::{thread_rng, Rng},
6    gemachain_sdk::genesis_config::GenesisConfig,
7    std::{
8        collections::HashMap,
9        fs::{self, File},
10        io::{BufReader, Read},
11        path::{
12            Component::{self, CurDir, Normal},
13            Path, PathBuf,
14        },
15        time::Instant,
16    },
17    tar::{
18        Archive,
19        EntryType::{Directory, GNUSparse, Regular},
20    },
21    thiserror::Error,
22};
23
24#[derive(Error, Debug)]
25pub enum UnpackError {
26    #[error("IO error: {0}")]
27    Io(#[from] std::io::Error),
28    #[error("Archive error: {0}")]
29    Archive(String),
30}
31
32pub type Result<T> = std::result::Result<T, UnpackError>;
33
34// 64 TiB; some safe margin to the max 128 TiB in amd64 linux userspace VmSize
35// (ref: https://unix.stackexchange.com/a/386555/364236)
36// note that this is directly related to the mmaped data size
37// so protect against insane value
38// This is the file size including holes for sparse files
39const MAX_SNAPSHOT_ARCHIVE_UNPACKED_APPARENT_SIZE: u64 = 64 * 1024 * 1024 * 1024 * 1024;
40
41// 4 TiB;
42// This is the actually consumed disk usage for sparse files
43const MAX_SNAPSHOT_ARCHIVE_UNPACKED_ACTUAL_SIZE: u64 = 4 * 1024 * 1024 * 1024 * 1024;
44
45const MAX_SNAPSHOT_ARCHIVE_UNPACKED_COUNT: u64 = 5_000_000;
46pub const MAX_GENESIS_ARCHIVE_UNPACKED_SIZE: u64 = 10 * 1024 * 1024; // 10 MiB
47const MAX_GENESIS_ARCHIVE_UNPACKED_COUNT: u64 = 100;
48
49fn checked_total_size_sum(total_size: u64, entry_size: u64, limit_size: u64) -> Result<u64> {
50    trace!(
51        "checked_total_size_sum: {} + {} < {}",
52        total_size,
53        entry_size,
54        limit_size,
55    );
56    let total_size = total_size.saturating_add(entry_size);
57    if total_size > limit_size {
58        return Err(UnpackError::Archive(format!(
59            "too large archive: {} than limit: {}",
60            total_size, limit_size,
61        )));
62    }
63    Ok(total_size)
64}
65
66fn checked_total_count_increment(total_count: u64, limit_count: u64) -> Result<u64> {
67    let total_count = total_count + 1;
68    if total_count > limit_count {
69        return Err(UnpackError::Archive(format!(
70            "too many files in snapshot: {:?}",
71            total_count
72        )));
73    }
74    Ok(total_count)
75}
76
77fn check_unpack_result(unpack_result: bool, path: String) -> Result<()> {
78    if !unpack_result {
79        return Err(UnpackError::Archive(format!(
80            "failed to unpack: {:?}",
81            path
82        )));
83    }
84    Ok(())
85}
86
87pub enum UnpackPath<'a> {
88    Valid(&'a Path),
89    Ignore,
90    Invalid,
91}
92
93fn unpack_archive<'a, A: Read, C>(
94    archive: &mut Archive<A>,
95    apparent_limit_size: u64,
96    actual_limit_size: u64,
97    limit_count: u64,
98    mut entry_checker: C,
99) -> Result<()>
100where
101    C: FnMut(&[&str], tar::EntryType) -> UnpackPath<'a>,
102{
103    let mut apparent_total_size: u64 = 0;
104    let mut actual_total_size: u64 = 0;
105    let mut total_count: u64 = 0;
106
107    let mut total_entries = 0;
108    let mut last_log_update = Instant::now();
109    for entry in archive.entries()? {
110        let mut entry = entry?;
111        let path = entry.path()?;
112        let path_str = path.display().to_string();
113
114        // Although the `tar` crate safely skips at the actual unpacking, fail
115        // first by ourselves when there are odd paths like including `..` or /
116        // for our clearer pattern matching reasoning:
117        //   https://docs.rs/tar/0.4.26/src/tar/entry.rs.html#371
118        let parts = path.components().map(|p| match p {
119            CurDir => Some("."),
120            Normal(c) => c.to_str(),
121            _ => None, // Prefix (for Windows) and RootDir are forbidden
122        });
123
124        // Reject old-style BSD directory entries that aren't explicitly tagged as directories
125        let legacy_dir_entry =
126            entry.header().as_ustar().is_none() && entry.path_bytes().ends_with(b"/");
127        let kind = entry.header().entry_type();
128        let reject_legacy_dir_entry = legacy_dir_entry && (kind != Directory);
129
130        if parts.clone().any(|p| p.is_none()) || reject_legacy_dir_entry {
131            return Err(UnpackError::Archive(format!(
132                "invalid path found: {:?}",
133                path_str
134            )));
135        }
136
137        let parts: Vec<_> = parts.map(|p| p.unwrap()).collect();
138        let unpack_dir = match entry_checker(parts.as_slice(), kind) {
139            UnpackPath::Invalid => {
140                return Err(UnpackError::Archive(format!(
141                    "extra entry found: {:?} {:?}",
142                    path_str,
143                    entry.header().entry_type(),
144                )));
145            }
146            UnpackPath::Ignore => {
147                continue;
148            }
149            UnpackPath::Valid(unpack_dir) => unpack_dir,
150        };
151
152        apparent_total_size = checked_total_size_sum(
153            apparent_total_size,
154            entry.header().size()?,
155            apparent_limit_size,
156        )?;
157        actual_total_size = checked_total_size_sum(
158            actual_total_size,
159            entry.header().entry_size()?,
160            actual_limit_size,
161        )?;
162        total_count = checked_total_count_increment(total_count, limit_count)?;
163
164        let target = sanitize_path(&entry.path()?, unpack_dir)?; // ? handles file system errors
165        if target.is_none() {
166            continue; // skip it
167        }
168        let target = target.unwrap();
169
170        let unpack = entry.unpack(target);
171        check_unpack_result(unpack.map(|_unpack| true)?, path_str)?;
172
173        // Sanitize permissions.
174        let mode = match entry.header().entry_type() {
175            GNUSparse | Regular => 0o644,
176            _ => 0o755,
177        };
178        set_perms(&unpack_dir.join(entry.path()?), mode)?;
179
180        total_entries += 1;
181        let now = Instant::now();
182        if now.duration_since(last_log_update).as_secs() >= 10 {
183            info!("unpacked {} entries so far...", total_entries);
184            last_log_update = now;
185        }
186    }
187    info!("unpacked {} entries total", total_entries);
188
189    return Ok(());
190
191    #[cfg(unix)]
192    fn set_perms(dst: &Path, mode: u32) -> std::io::Result<()> {
193        use std::os::unix::fs::PermissionsExt;
194
195        let perm = fs::Permissions::from_mode(mode as _);
196        fs::set_permissions(dst, perm)
197    }
198
199    #[cfg(windows)]
200    fn set_perms(dst: &Path, _mode: u32) -> std::io::Result<()> {
201        let mut perm = fs::metadata(dst)?.permissions();
202        perm.set_readonly(false);
203        fs::set_permissions(dst, perm)
204    }
205}
206
207// return Err on file system error
208// return Some(path) if path is good
209// return None if we should skip this file
210fn sanitize_path(entry_path: &Path, dst: &Path) -> Result<Option<PathBuf>> {
211    // We cannot call unpack_in because it errors if we try to use 2 account paths.
212    // So, this code is borrowed from unpack_in
213    // ref: https://docs.rs/tar/*/tar/struct.Entry.html#method.unpack_in
214    let mut file_dst = dst.to_path_buf();
215    const SKIP: Result<Option<PathBuf>> = Ok(None);
216    {
217        let path = entry_path;
218        for part in path.components() {
219            match part {
220                // Leading '/' characters, root paths, and '.'
221                // components are just ignored and treated as "empty
222                // components"
223                Component::Prefix(..) | Component::RootDir | Component::CurDir => continue,
224
225                // If any part of the filename is '..', then skip over
226                // unpacking the file to prevent directory traversal
227                // security issues.  See, e.g.: CVE-2001-1267,
228                // CVE-2002-0399, CVE-2005-1918, CVE-2007-4131
229                Component::ParentDir => return SKIP,
230
231                Component::Normal(part) => file_dst.push(part),
232            }
233        }
234    }
235
236    // Skip cases where only slashes or '.' parts were seen, because
237    // this is effectively an empty filename.
238    if *dst == *file_dst {
239        return SKIP;
240    }
241
242    // Skip entries without a parent (i.e. outside of FS root)
243    let parent = match file_dst.parent() {
244        Some(p) => p,
245        None => return SKIP,
246    };
247
248    fs::create_dir_all(parent)?;
249
250    // Here we are different than untar_in. The code for tar::unpack_in internally calling unpack is a little different.
251    // ignore return value here
252    validate_inside_dst(dst, parent)?;
253    let target = parent.join(entry_path.file_name().unwrap());
254
255    Ok(Some(target))
256}
257
258// copied from:
259// https://github.com/alexcrichton/tar-rs/blob/d90a02f582c03dfa0fd11c78d608d0974625ae5d/src/entry.rs#L781
260fn validate_inside_dst(dst: &Path, file_dst: &Path) -> Result<PathBuf> {
261    // Abort if target (canonical) parent is outside of `dst`
262    let canon_parent = file_dst.canonicalize().map_err(|err| {
263        UnpackError::Archive(format!(
264            "{} while canonicalizing {}",
265            err,
266            file_dst.display()
267        ))
268    })?;
269    let canon_target = dst.canonicalize().map_err(|err| {
270        UnpackError::Archive(format!("{} while canonicalizing {}", err, dst.display()))
271    })?;
272    if !canon_parent.starts_with(&canon_target) {
273        return Err(UnpackError::Archive(format!(
274            "trying to unpack outside of destination path: {}",
275            canon_target.display()
276        )));
277    }
278    Ok(canon_target)
279}
280
281/// Map from AppendVec file name to unpacked file system location
282pub type UnpackedAppendVecMap = HashMap<String, PathBuf>;
283
284// select/choose only 'index' out of each # of 'divisions' of total items.
285pub struct ParallelSelector {
286    pub index: usize,
287    pub divisions: usize,
288}
289
290impl ParallelSelector {
291    pub fn select_index(&self, index: usize) -> bool {
292        index % self.divisions == self.index
293    }
294}
295
296pub fn unpack_snapshot<A: Read>(
297    archive: &mut Archive<A>,
298    ledger_dir: &Path,
299    account_paths: &[PathBuf],
300    parallel_selector: Option<ParallelSelector>,
301) -> Result<UnpackedAppendVecMap> {
302    assert!(!account_paths.is_empty());
303    let mut unpacked_append_vec_map = UnpackedAppendVecMap::new();
304    let mut i = 0;
305
306    unpack_archive(
307        archive,
308        MAX_SNAPSHOT_ARCHIVE_UNPACKED_APPARENT_SIZE,
309        MAX_SNAPSHOT_ARCHIVE_UNPACKED_ACTUAL_SIZE,
310        MAX_SNAPSHOT_ARCHIVE_UNPACKED_COUNT,
311        |parts, kind| {
312            if is_valid_snapshot_archive_entry(parts, kind) {
313                i += 1;
314                match &parallel_selector {
315                    Some(parallel_selector) => {
316                        if !parallel_selector.select_index(i - 1) {
317                            return UnpackPath::Ignore;
318                        }
319                    }
320                    None => {}
321                };
322                if let ["accounts", file] = parts {
323                    // Randomly distribute the accounts files about the available `account_paths`,
324                    let path_index = thread_rng().gen_range(0, account_paths.len());
325                    match account_paths.get(path_index).map(|path_buf| {
326                        unpacked_append_vec_map
327                            .insert(file.to_string(), path_buf.join("accounts").join(file));
328                        path_buf.as_path()
329                    }) {
330                        Some(path) => UnpackPath::Valid(path),
331                        None => UnpackPath::Invalid,
332                    }
333                } else {
334                    UnpackPath::Valid(ledger_dir)
335                }
336            } else {
337                UnpackPath::Invalid
338            }
339        },
340    )
341    .map(|_| unpacked_append_vec_map)
342}
343
344fn all_digits(v: &str) -> bool {
345    if v.is_empty() {
346        return false;
347    }
348    for x in v.chars() {
349        if !x.is_numeric() {
350            return false;
351        }
352    }
353    true
354}
355
356fn like_storage(v: &str) -> bool {
357    let mut periods = 0;
358    let mut saw_numbers = false;
359    for x in v.chars() {
360        if !x.is_numeric() {
361            if x == '.' {
362                if periods > 0 || !saw_numbers {
363                    return false;
364                }
365                saw_numbers = false;
366                periods += 1;
367            } else {
368                return false;
369            }
370        } else {
371            saw_numbers = true;
372        }
373    }
374    saw_numbers && periods == 1
375}
376
377fn is_valid_snapshot_archive_entry(parts: &[&str], kind: tar::EntryType) -> bool {
378    match (parts, kind) {
379        (["version"], Regular) => true,
380        (["accounts"], Directory) => true,
381        (["accounts", file], GNUSparse) if like_storage(file) => true,
382        (["accounts", file], Regular) if like_storage(file) => true,
383        (["snapshots"], Directory) => true,
384        (["snapshots", "status_cache"], GNUSparse) => true,
385        (["snapshots", "status_cache"], Regular) => true,
386        (["snapshots", dir, file], GNUSparse) if all_digits(dir) && all_digits(file) => true,
387        (["snapshots", dir, file], Regular) if all_digits(dir) && all_digits(file) => true,
388        (["snapshots", dir], Directory) if all_digits(dir) => true,
389        _ => false,
390    }
391}
392
393pub fn open_genesis_config(
394    ledger_path: &Path,
395    max_genesis_archive_unpacked_size: u64,
396) -> GenesisConfig {
397    GenesisConfig::load(ledger_path).unwrap_or_else(|load_err| {
398        let genesis_package = ledger_path.join(DEFAULT_GENESIS_ARCHIVE);
399        unpack_genesis_archive(
400            &genesis_package,
401            ledger_path,
402            max_genesis_archive_unpacked_size,
403        )
404        .unwrap_or_else(|unpack_err| {
405            warn!(
406                "Failed to open ledger genesis_config at {:?}: {}, {}",
407                ledger_path, load_err, unpack_err,
408            );
409            std::process::exit(1);
410        });
411
412        // loading must succeed at this moment
413        GenesisConfig::load(ledger_path).unwrap()
414    })
415}
416
417pub fn unpack_genesis_archive(
418    archive_filename: &Path,
419    destination_dir: &Path,
420    max_genesis_archive_unpacked_size: u64,
421) -> std::result::Result<(), UnpackError> {
422    info!("Extracting {:?}...", archive_filename);
423    let extract_start = Instant::now();
424
425    fs::create_dir_all(destination_dir)?;
426    let tar_bz2 = File::open(&archive_filename)?;
427    let tar = BzDecoder::new(BufReader::new(tar_bz2));
428    let mut archive = Archive::new(tar);
429    unpack_genesis(
430        &mut archive,
431        destination_dir,
432        max_genesis_archive_unpacked_size,
433    )?;
434    info!(
435        "Extracted {:?} in {:?}",
436        archive_filename,
437        Instant::now().duration_since(extract_start)
438    );
439    Ok(())
440}
441
442fn unpack_genesis<A: Read>(
443    archive: &mut Archive<A>,
444    unpack_dir: &Path,
445    max_genesis_archive_unpacked_size: u64,
446) -> Result<()> {
447    unpack_archive(
448        archive,
449        max_genesis_archive_unpacked_size,
450        max_genesis_archive_unpacked_size,
451        MAX_GENESIS_ARCHIVE_UNPACKED_COUNT,
452        |p, k| {
453            if is_valid_genesis_archive_entry(p, k) {
454                UnpackPath::Valid(unpack_dir)
455            } else {
456                UnpackPath::Invalid
457            }
458        },
459    )
460}
461
462fn is_valid_genesis_archive_entry(parts: &[&str], kind: tar::EntryType) -> bool {
463    trace!("validating: {:?} {:?}", parts, kind);
464    #[allow(clippy::match_like_matches_macro)]
465    match (parts, kind) {
466        ([DEFAULT_GENESIS_FILE], GNUSparse) => true,
467        ([DEFAULT_GENESIS_FILE], Regular) => true,
468        (["rocksdb"], Directory) => true,
469        (["rocksdb", _], GNUSparse) => true,
470        (["rocksdb", _], Regular) => true,
471        _ => false,
472    }
473}
474
475#[cfg(test)]
476mod tests {
477    use super::*;
478    use assert_matches::assert_matches;
479    use tar::{Builder, Header};
480
481    #[test]
482    fn test_archive_is_valid_entry() {
483        assert!(is_valid_snapshot_archive_entry(
484            &["snapshots"],
485            tar::EntryType::Directory
486        ));
487        assert!(!is_valid_snapshot_archive_entry(
488            &["snapshots", ""],
489            tar::EntryType::Directory
490        ));
491        assert!(is_valid_snapshot_archive_entry(
492            &["snapshots", "3"],
493            tar::EntryType::Directory
494        ));
495        assert!(is_valid_snapshot_archive_entry(
496            &["snapshots", "3", "3"],
497            tar::EntryType::Regular
498        ));
499        assert!(is_valid_snapshot_archive_entry(
500            &["version"],
501            tar::EntryType::Regular
502        ));
503        assert!(is_valid_snapshot_archive_entry(
504            &["accounts"],
505            tar::EntryType::Directory
506        ));
507        assert!(!is_valid_snapshot_archive_entry(
508            &["accounts", ""],
509            tar::EntryType::Regular
510        ));
511
512        assert!(!is_valid_snapshot_archive_entry(
513            &["snapshots"],
514            tar::EntryType::Regular
515        ));
516        assert!(!is_valid_snapshot_archive_entry(
517            &["snapshots", "x0"],
518            tar::EntryType::Directory
519        ));
520        assert!(!is_valid_snapshot_archive_entry(
521            &["snapshots", "0x"],
522            tar::EntryType::Directory
523        ));
524        assert!(!is_valid_snapshot_archive_entry(
525            &["snapshots", "0", "aa"],
526            tar::EntryType::Regular
527        ));
528        assert!(!is_valid_snapshot_archive_entry(
529            &["aaaa"],
530            tar::EntryType::Regular
531        ));
532    }
533
534    #[test]
535    fn test_valid_snapshot_accounts() {
536        gemachain_logger::setup();
537        assert!(is_valid_snapshot_archive_entry(
538            &["accounts", "0.0"],
539            tar::EntryType::Regular
540        ));
541        assert!(is_valid_snapshot_archive_entry(
542            &["accounts", "01829.077"],
543            tar::EntryType::Regular
544        ));
545
546        assert!(!is_valid_snapshot_archive_entry(
547            &["accounts", "1.2.34"],
548            tar::EntryType::Regular
549        ));
550        assert!(!is_valid_snapshot_archive_entry(
551            &["accounts", "12."],
552            tar::EntryType::Regular
553        ));
554        assert!(!is_valid_snapshot_archive_entry(
555            &["accounts", ".12"],
556            tar::EntryType::Regular
557        ));
558        assert!(!is_valid_snapshot_archive_entry(
559            &["accounts", "0x0"],
560            tar::EntryType::Regular
561        ));
562        assert!(!is_valid_snapshot_archive_entry(
563            &["accounts", "abc"],
564            tar::EntryType::Regular
565        ));
566        assert!(!is_valid_snapshot_archive_entry(
567            &["accounts", "232323"],
568            tar::EntryType::Regular
569        ));
570    }
571
572    #[test]
573    fn test_archive_is_valid_archive_entry() {
574        assert!(is_valid_genesis_archive_entry(
575            &["genesis.bin"],
576            tar::EntryType::Regular
577        ));
578        assert!(is_valid_genesis_archive_entry(
579            &["genesis.bin"],
580            tar::EntryType::GNUSparse,
581        ));
582        assert!(is_valid_genesis_archive_entry(
583            &["rocksdb"],
584            tar::EntryType::Directory
585        ));
586        assert!(is_valid_genesis_archive_entry(
587            &["rocksdb", "foo"],
588            tar::EntryType::Regular
589        ));
590        assert!(is_valid_genesis_archive_entry(
591            &["rocksdb", "foo"],
592            tar::EntryType::GNUSparse,
593        ));
594
595        assert!(!is_valid_genesis_archive_entry(
596            &["aaaa"],
597            tar::EntryType::Regular
598        ));
599        assert!(!is_valid_genesis_archive_entry(
600            &["aaaa"],
601            tar::EntryType::GNUSparse,
602        ));
603        assert!(!is_valid_genesis_archive_entry(
604            &["rocksdb"],
605            tar::EntryType::Regular
606        ));
607        assert!(!is_valid_genesis_archive_entry(
608            &["rocksdb"],
609            tar::EntryType::GNUSparse,
610        ));
611        assert!(!is_valid_genesis_archive_entry(
612            &["rocksdb", "foo"],
613            tar::EntryType::Directory,
614        ));
615        assert!(!is_valid_genesis_archive_entry(
616            &["rocksdb", "foo", "bar"],
617            tar::EntryType::Directory,
618        ));
619        assert!(!is_valid_genesis_archive_entry(
620            &["rocksdb", "foo", "bar"],
621            tar::EntryType::Regular
622        ));
623        assert!(!is_valid_genesis_archive_entry(
624            &["rocksdb", "foo", "bar"],
625            tar::EntryType::GNUSparse
626        ));
627    }
628
629    fn with_finalize_and_unpack<C>(archive: tar::Builder<Vec<u8>>, checker: C) -> Result<()>
630    where
631        C: Fn(&mut Archive<BufReader<&[u8]>>, &Path) -> Result<()>,
632    {
633        let data = archive.into_inner().unwrap();
634        let reader = BufReader::new(&data[..]);
635        let mut archive: Archive<std::io::BufReader<&[u8]>> = Archive::new(reader);
636        let temp_dir = tempfile::TempDir::new().unwrap();
637
638        checker(&mut archive, temp_dir.path())?;
639        // Check that there is no bad permissions preventing deletion.
640        let result = temp_dir.close();
641        assert_matches!(result, Ok(()));
642        Ok(())
643    }
644
645    fn finalize_and_unpack_snapshot(archive: tar::Builder<Vec<u8>>) -> Result<()> {
646        with_finalize_and_unpack(archive, |a, b| {
647            unpack_snapshot(a, b, &[PathBuf::new()], None).map(|_| ())
648        })
649    }
650
651    fn finalize_and_unpack_genesis(archive: tar::Builder<Vec<u8>>) -> Result<()> {
652        with_finalize_and_unpack(archive, |a, b| {
653            unpack_genesis(a, b, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE)
654        })
655    }
656
657    #[test]
658    fn test_archive_unpack_snapshot_ok() {
659        let mut header = Header::new_gnu();
660        header.set_path("version").unwrap();
661        header.set_size(4);
662        header.set_cksum();
663
664        let data: &[u8] = &[1, 2, 3, 4];
665
666        let mut archive = Builder::new(Vec::new());
667        archive.append(&header, data).unwrap();
668
669        let result = finalize_and_unpack_snapshot(archive);
670        assert_matches!(result, Ok(()));
671    }
672
673    #[test]
674    fn test_archive_unpack_genesis_ok() {
675        let mut header = Header::new_gnu();
676        header.set_path("genesis.bin").unwrap();
677        header.set_size(4);
678        header.set_cksum();
679
680        let data: &[u8] = &[1, 2, 3, 4];
681
682        let mut archive = Builder::new(Vec::new());
683        archive.append(&header, data).unwrap();
684
685        let result = finalize_and_unpack_genesis(archive);
686        assert_matches!(result, Ok(()));
687    }
688
689    #[test]
690    fn test_archive_unpack_genesis_bad_perms() {
691        let mut archive = Builder::new(Vec::new());
692
693        let mut header = Header::new_gnu();
694        header.set_path("rocksdb").unwrap();
695        header.set_entry_type(Directory);
696        header.set_size(0);
697        header.set_cksum();
698        let data: &[u8] = &[];
699        archive.append(&header, data).unwrap();
700
701        let mut header = Header::new_gnu();
702        header.set_path("rocksdb/test").unwrap();
703        header.set_size(4);
704        header.set_cksum();
705        let data: &[u8] = &[1, 2, 3, 4];
706        archive.append(&header, data).unwrap();
707
708        // Removing all permissions makes it harder to delete this directory
709        // or work with files inside it.
710        let mut header = Header::new_gnu();
711        header.set_path("rocksdb").unwrap();
712        header.set_entry_type(Directory);
713        header.set_mode(0o000);
714        header.set_size(0);
715        header.set_cksum();
716        let data: &[u8] = &[];
717        archive.append(&header, data).unwrap();
718
719        let result = finalize_and_unpack_genesis(archive);
720        assert_matches!(result, Ok(()));
721    }
722
723    #[test]
724    fn test_archive_unpack_genesis_bad_rocksdb_subdir() {
725        let mut archive = Builder::new(Vec::new());
726
727        let mut header = Header::new_gnu();
728        header.set_path("rocksdb").unwrap();
729        header.set_entry_type(Directory);
730        header.set_size(0);
731        header.set_cksum();
732        let data: &[u8] = &[];
733        archive.append(&header, data).unwrap();
734
735        // tar-rs treats following entry as a Directory to support old tar formats.
736        let mut header = Header::new_gnu();
737        header.set_path("rocksdb/test/").unwrap();
738        header.set_entry_type(Regular);
739        header.set_size(0);
740        header.set_cksum();
741        let data: &[u8] = &[];
742        archive.append(&header, data).unwrap();
743
744        let result = finalize_and_unpack_genesis(archive);
745        assert_matches!(result, Err(UnpackError::Archive(ref message)) if message == "invalid path found: \"rocksdb/test/\"");
746    }
747
748    #[test]
749    fn test_archive_unpack_snapshot_invalid_path() {
750        let mut header = Header::new_gnu();
751        // bypass the sanitization of the .set_path()
752        for (p, c) in header
753            .as_old_mut()
754            .name
755            .iter_mut()
756            .zip(b"foo/../../../dangerous".iter().chain(Some(&0)))
757        {
758            *p = *c;
759        }
760        header.set_size(4);
761        header.set_cksum();
762
763        let data: &[u8] = &[1, 2, 3, 4];
764
765        let mut archive = Builder::new(Vec::new());
766        archive.append(&header, data).unwrap();
767        let result = finalize_and_unpack_snapshot(archive);
768        assert_matches!(result, Err(UnpackError::Archive(ref message)) if message == "invalid path found: \"foo/../../../dangerous\"");
769    }
770
771    fn with_archive_unpack_snapshot_invalid_path(path: &str) -> Result<()> {
772        let mut header = Header::new_gnu();
773        // bypass the sanitization of the .set_path()
774        for (p, c) in header
775            .as_old_mut()
776            .name
777            .iter_mut()
778            .zip(path.as_bytes().iter().chain(Some(&0)))
779        {
780            *p = *c;
781        }
782        header.set_size(4);
783        header.set_cksum();
784
785        let data: &[u8] = &[1, 2, 3, 4];
786
787        let mut archive = Builder::new(Vec::new());
788        archive.append(&header, data).unwrap();
789        with_finalize_and_unpack(archive, |unpacking_archive, path| {
790            for entry in unpacking_archive.entries()? {
791                if !entry?.unpack_in(path)? {
792                    return Err(UnpackError::Archive("failed!".to_string()));
793                } else if !path.join(path).exists() {
794                    return Err(UnpackError::Archive("not existing!".to_string()));
795                }
796            }
797            Ok(())
798        })
799    }
800
801    #[test]
802    fn test_archive_unpack_itself() {
803        assert_matches!(
804            with_archive_unpack_snapshot_invalid_path("ryoqun/work"),
805            Ok(())
806        );
807        // Absolute paths are neutralized as relative
808        assert_matches!(
809            with_archive_unpack_snapshot_invalid_path("/etc/passwd"),
810            Ok(())
811        );
812        assert_matches!(with_archive_unpack_snapshot_invalid_path("../../../dangerous"), Err(UnpackError::Archive(ref message)) if message == "failed!");
813    }
814
815    #[test]
816    fn test_archive_unpack_snapshot_invalid_entry() {
817        let mut header = Header::new_gnu();
818        header.set_path("foo").unwrap();
819        header.set_size(4);
820        header.set_cksum();
821
822        let data: &[u8] = &[1, 2, 3, 4];
823
824        let mut archive = Builder::new(Vec::new());
825        archive.append(&header, data).unwrap();
826        let result = finalize_and_unpack_snapshot(archive);
827        assert_matches!(result, Err(UnpackError::Archive(ref message)) if message == "extra entry found: \"foo\" Regular");
828    }
829
830    #[test]
831    fn test_archive_unpack_snapshot_too_large() {
832        let mut header = Header::new_gnu();
833        header.set_path("version").unwrap();
834        header.set_size(1024 * 1024 * 1024 * 1024 * 1024);
835        header.set_cksum();
836
837        let data: &[u8] = &[1, 2, 3, 4];
838
839        let mut archive = Builder::new(Vec::new());
840        archive.append(&header, data).unwrap();
841        let result = finalize_and_unpack_snapshot(archive);
842        assert_matches!(
843            result,
844            Err(UnpackError::Archive(ref message))
845                if message == &format!(
846                    "too large archive: 1125899906842624 than limit: {}", MAX_SNAPSHOT_ARCHIVE_UNPACKED_APPARENT_SIZE
847                )
848        );
849    }
850
851    #[test]
852    fn test_archive_unpack_snapshot_bad_unpack() {
853        let result = check_unpack_result(false, "abc".to_string());
854        assert_matches!(result, Err(UnpackError::Archive(ref message)) if message == "failed to unpack: \"abc\"");
855    }
856
857    #[test]
858    fn test_archive_checked_total_size_sum() {
859        let result = checked_total_size_sum(500, 500, MAX_SNAPSHOT_ARCHIVE_UNPACKED_ACTUAL_SIZE);
860        assert_matches!(result, Ok(1000));
861
862        let result = checked_total_size_sum(
863            u64::max_value() - 2,
864            2,
865            MAX_SNAPSHOT_ARCHIVE_UNPACKED_ACTUAL_SIZE,
866        );
867        assert_matches!(
868            result,
869            Err(UnpackError::Archive(ref message))
870                if message == &format!(
871                    "too large archive: 18446744073709551615 than limit: {}", MAX_SNAPSHOT_ARCHIVE_UNPACKED_ACTUAL_SIZE
872                )
873        );
874    }
875
876    #[test]
877    fn test_archive_checked_total_size_count() {
878        let result = checked_total_count_increment(101, MAX_SNAPSHOT_ARCHIVE_UNPACKED_COUNT);
879        assert_matches!(result, Ok(102));
880
881        let result =
882            checked_total_count_increment(999_999_999_999, MAX_SNAPSHOT_ARCHIVE_UNPACKED_COUNT);
883        assert_matches!(
884            result,
885            Err(UnpackError::Archive(ref message))
886                if message == "too many files in snapshot: 1000000000000"
887        );
888    }
889}
gemachain_runtime/hardened_unpack.rs

gemachain_runtime/
hardened_unpack.rs
