parqkit 0.1.0

A fast CLI and Rust library for inspecting, converting, and merging Parquet files.
Documentation
use crate::error::ParqkitError;
use crate::Result;
use std::collections::BTreeSet;
use std::fs;
use std::io::ErrorKind;
use std::path::{Path, PathBuf};

const MAX_GLOB_FILES: usize = 10_000;

/// Ordered, validated Parquet input paths.
///
/// Explicit repeated inputs are preserved. Glob matches are sorted and
/// deduplicated according to [`Dataset::from_inputs`].
#[derive(Clone, Debug)]
pub struct Dataset {
    paths: Vec<PathBuf>,
}

#[derive(Clone, Debug)]
pub(crate) struct InputFile {
    path: PathBuf,
}

impl InputFile {
    pub(crate) fn from_input(input: PathBuf) -> Result<Self> {
        let paths = paths_from_input(&input)?;
        match paths.as_slice() {
            [path] => Ok(Self {
                path: path.to_path_buf(),
            }),
            [] => Err(ParqkitError::NoInputFiles),
            _ => Err(ParqkitError::TooManyInputFiles { count: paths.len() }),
        }
    }

    pub(crate) fn path(&self) -> &Path {
        &self.path
    }
}

impl Dataset {
    /// Validate paths, expand glob patterns, and construct a dataset.
    ///
    /// Returns [`ParqkitError::NoInputFiles`] when `inputs` is empty,
    /// [`ParqkitError::NoFilesMatched`] for an unmatched pattern, and a typed
    /// path error when an input is missing or is a directory.
    pub fn from_inputs(inputs: Vec<PathBuf>) -> Result<Self> {
        if inputs.is_empty() {
            return Err(ParqkitError::NoInputFiles);
        }

        let mut paths = Vec::new();
        let mut seen_paths = BTreeSet::new();
        let mut seen_from_globs = BTreeSet::new();

        for input in inputs {
            if is_glob_pattern(&input)? {
                let matches = glob_matches(&input)?;
                push_glob_matches(&matches, &mut paths, &mut seen_paths, &mut seen_from_globs);
            } else {
                validate_file_path(&input)?;
                if !seen_from_globs.contains(&input) {
                    seen_paths.insert(input.clone());
                    paths.push(input);
                }
            }
        }

        if paths.is_empty() {
            return Err(ParqkitError::NoInputFiles);
        }

        Ok(Self { paths })
    }

    /// Iterate over source paths in dataset order.
    pub fn paths(&self) -> impl ExactSizeIterator<Item = &Path> {
        self.paths.iter().map(PathBuf::as_path)
    }

    /// Return whether the dataset contains more than one source occurrence.
    pub fn is_multi_source(&self) -> bool {
        self.paths.len() > 1
    }
}

fn paths_from_input(input: &Path) -> Result<Vec<PathBuf>> {
    if is_glob_pattern(input)? {
        glob_matches(input)
    } else {
        validate_file_path(input)?;
        Ok(vec![input.to_path_buf()])
    }
}

fn glob_matches(input: &Path) -> Result<Vec<PathBuf>> {
    let pattern = input.to_string_lossy().into_owned();
    let mut matches = Vec::new();

    for entry in
        glob::glob(&pattern).map_err(|error| ParqkitError::invalid_glob_pattern(&pattern, error))?
    {
        let path = entry.map_err(|error| ParqkitError::from_read(error.path(), error.error()))?;
        validate_file_path(&path)?;
        matches.push(path);

        if matches.len() > MAX_GLOB_FILES {
            return Err(ParqkitError::TooManyFilesMatched {
                pattern,
                max_matches: MAX_GLOB_FILES,
            });
        }
    }

    if matches.is_empty() {
        return Err(ParqkitError::NoFilesMatched { pattern });
    }

    matches.sort();
    Ok(matches)
}

fn push_glob_matches(
    matches: &[PathBuf],
    paths: &mut Vec<PathBuf>,
    seen_paths: &mut BTreeSet<PathBuf>,
    seen_from_globs: &mut BTreeSet<PathBuf>,
) {
    for path in matches {
        if seen_paths.insert(path.clone()) {
            seen_from_globs.insert(path.clone());
            paths.push(path.clone());
        }
    }
}

fn is_glob_pattern(path: &Path) -> Result<bool> {
    match path.try_exists() {
        Ok(true) => return Ok(false),
        Ok(false) => {}
        Err(error) => return Err(ParqkitError::from_read(path, error)),
    }
    let path = path.to_string_lossy();
    Ok(path.contains('*') || path.contains('?') || path.contains('['))
}

fn validate_file_path(path: &Path) -> Result<()> {
    let metadata = fs::metadata(path).map_err(|error| {
        if error.kind() == ErrorKind::NotFound {
            ParqkitError::file_not_found(path)
        } else {
            ParqkitError::from_read(path, error)
        }
    })?;
    if metadata.is_dir() {
        return Err(ParqkitError::is_directory(path));
    }

    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::fs;
    use std::sync::atomic::{AtomicU64, Ordering};
    use std::time::{SystemTime, UNIX_EPOCH};

    static TEMP_DIR_COUNTER: AtomicU64 = AtomicU64::new(0);

    fn temp_dir() -> Result<PathBuf> {
        let unique = SystemTime::now()
            .duration_since(UNIX_EPOCH)
            .map_err(ParqkitError::output_error)?
            .as_nanos();
        let counter = TEMP_DIR_COUNTER.fetch_add(1, Ordering::Relaxed);
        let dir = std::env::temp_dir().join(format!("parqkit_dataset_{unique}_{counter}"));
        fs::create_dir_all(&dir)?;
        Ok(dir)
    }

    #[test]
    fn deduplicates_overlapping_explicit_and_glob_inputs() -> Result<()> {
        let dir = temp_dir()?;
        let file = dir.join("sample.parquet");
        fs::write(&file, b"PAR1")?;
        let glob = dir.join("*.parquet");

        let dataset = Dataset::from_inputs(vec![file.clone(), glob])?;
        let paths = dataset.paths().collect::<Vec<_>>();

        assert_eq!(paths, vec![file.as_path()]);

        fs::remove_file(file)?;
        fs::remove_dir(dir)?;
        Ok(())
    }

    #[test]
    fn deduplicates_overlapping_glob_and_explicit_inputs() -> Result<()> {
        let dir = temp_dir()?;
        let file = dir.join("sample.parquet");
        fs::write(&file, b"PAR1")?;
        let glob = dir.join("*.parquet");

        let dataset = Dataset::from_inputs(vec![glob, file.clone()])?;
        let paths = dataset.paths().collect::<Vec<_>>();

        assert_eq!(paths, vec![file.as_path()]);

        fs::remove_file(file)?;
        fs::remove_dir(dir)?;
        Ok(())
    }

    #[test]
    fn preserves_repeated_explicit_inputs() -> Result<()> {
        let dir = temp_dir()?;
        let file = dir.join("sample.parquet");
        fs::write(&file, b"PAR1")?;

        let dataset = Dataset::from_inputs(vec![file.clone(), file.clone()])?;
        let paths = dataset.paths().collect::<Vec<_>>();

        assert_eq!(paths, vec![file.as_path(), file.as_path()]);

        fs::remove_file(file)?;
        fs::remove_dir(dir)?;
        Ok(())
    }

    #[test]
    fn input_file_rejects_multi_match_glob() -> Result<()> {
        let dir = temp_dir()?;
        let first = dir.join("a.parquet");
        let second = dir.join("b.parquet");
        fs::write(&first, b"PAR1")?;
        fs::write(&second, b"PAR1")?;
        let glob = dir.join("*.parquet");

        let Err(error) = InputFile::from_input(glob) else {
            return Err(ParqkitError::output_error("multi-match glob should fail"));
        };

        assert!(matches!(
            error,
            ParqkitError::TooManyInputFiles { count: 2 }
        ));

        fs::remove_file(first)?;
        fs::remove_file(second)?;
        fs::remove_dir(dir)?;
        Ok(())
    }
}