apt-mirror-check 1.0.0

Check errors for apt mirror
Documentation
use crate::bad_action::BadAction;
use crate::{file_attr, packages, release};
use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
use rayon::prelude::*;
use std::collections::HashMap;
use std::iter::zip;
use std::path::{Path, PathBuf};

#[derive(Debug, thiserror::Error)]
pub enum RepositoryError {
    #[error("Failed to parse Release file: {0}")]
    Release(#[from] release::ReleaseError),

    #[error("Failed to parse Packages file: {0}")]
    Package(#[from] packages::PackageError),

    #[error("Missing Release and InRelease for repository {0}")]
    MissingRelease(PathBuf),
}

/// Packages and sub Release files' attrs is store in top-level Release files
fn get_distribution_meta_files(
    path: &Path,
) -> Result<Vec<file_attr::RelativeFileAttr>, RepositoryError> {
    let release_path = path.join("Release");
    if release_path.exists() {
        return Ok(release::parse_release_file(release_path)?);
    } else {
        let inrelease_path = path.join("InRelease");
        if inrelease_path.exists() {
            return Ok(release::parse_inrelease_file(inrelease_path)?);
        }
    }

    Err(RepositoryError::MissingRelease(path.to_path_buf()))
}

#[derive(Default)]
struct Packages {
    none: Option<PathBuf>,
    gz: Option<PathBuf>,
    xz: Option<PathBuf>,
}

impl Packages {
    fn into_preferred(self) -> Option<PathBuf> {
        match (self.none, self.gz, self.xz) {
            (Some(path), _, _) => Some(path),
            (_, Some(path), _) => Some(path),
            (_, _, Some(path)) => Some(path),
            _ => None,
        }
    }
}

pub fn check_distribution(repository_dir: &Path, dist_dir: &Path, bad_action: &BadAction) {
    log::info!(dist_dir:?; "start checking distribution");

    // note: relative paths in Release file are relative to distribution directory not repository directory
    let meta_files: Vec<_> = match get_distribution_meta_files(dist_dir) {
        Ok(meta_attrs) => meta_attrs
            .into_iter()
            .map(|attr| attr.into_absolute(dist_dir))
            .filter(|attr| attr.path.exists())
            .collect(),
        Err(err) => {
            log::error!(repository_dir:?, err:err; "fail to read Release/InRelease");
            return;
        }
    };

    log::info!(count=meta_files.len(); "found meta files");

    let multi = MultiProgress::new();
    let pb_meta = multi.add(ProgressBar::new(meta_files.len() as u64));
    pb_meta.set_style(
        ProgressStyle::default_bar()
            .template("{spinner} checking meta files: [{bar}] {pos}/{len}")
            .expect("Failed to set style"),
    );

    let meta_check_results: Vec<_> = meta_files
        .par_iter()
        .map(|attr| {
            let ok = attr.check();
            pb_meta.inc(1);
            ok
        })
        .collect();

    pb_meta.finish_and_clear();

    let mut packages: HashMap<_, Packages> = HashMap::new();
    for (ok, attr) in zip(meta_check_results, meta_files) {
        if !ok {
            bad_action.on_bad_file(&attr.path)
        } else {
            // collect correct Packages
            if let Some(stem) = attr.path.file_stem()
                && stem == "Packages"
            {
                // Packages/Packages.gz/Packages.xz
                let key = attr.path.parent().unwrap().to_path_buf();
                match attr.path.extension() {
                    None => {
                        packages.entry(key).or_default().none = Some(attr.path);
                    }
                    Some(ext) => match ext.to_string_lossy().as_ref() {
                        "gz" => {
                            packages.entry(key).or_default().gz = Some(attr.path);
                        }
                        "xz" => {
                            packages.entry(key).or_default().xz = Some(attr.path);
                        }
                        _ => {
                            println!(
                                "[BUG] unknown Packages file extension: {}",
                                attr.path.display()
                            );
                        }
                    },
                }
            }
        }
    }

    log::info!(count=packages.len(); "found Packages files");

    let pb_parse = multi.add(ProgressBar::new(packages.len() as u64));
    pb_parse.set_style(
        ProgressStyle::default_bar()
            .template("{spinner} parsing Packages files: [{bar}] {pos}/{len}")
            .expect("Failed to set style"),
    );

    let files: Vec<_> = packages
        .into_values()
        .enumerate()
        .filter_map(|(pos, packages)| packages.into_preferred().map(|path| (pos, path)))
        .filter_map(|(pos, path)| {
            let attrs = packages::parse_packages_file(&path).ok();
            pb_parse.set_position(pos as u64);
            attrs
        })
        .flatten()
        .map(|attr| attr.into_absolute(repository_dir))
        .filter(|attr| attr.path.exists())
        .collect();

    pb_parse.finish_and_clear();

    log::info!(count=files.len(); "found files (with attrs)");

    let pb_files = multi.add(ProgressBar::new(files.len() as u64));
    pb_files.set_style(
        ProgressStyle::default_bar()
            .template("{spinner} checking deb files: [{bar}] {pos}/{len}")
            .expect("Failed to set style"),
    );

    let file_check_results: Vec<_> = files
        .par_iter()
        .map(|attr| {
            let ok = attr.check();
            pb_files.inc(1);
            ok
        })
        .collect();

    pb_files.finish_and_clear();

    for (file, ok) in zip(files, file_check_results) {
        if !ok {
            bad_action.on_bad_file(&file.path);
        }
    }
}