use crate::files_source::constants::{
CSV_SUFFIX, GZIP_SUFFIX, JSON_SUFFIXES, PARQUET_SUFFIX, ZSTD_SUFFIX,
};
use crate::files_source::models::{Codec, Compression, FilesEncoding, SourceFormat};
fn format_suffixes(format: SourceFormat) -> &'static [&'static str] {
match format {
SourceFormat::Parquet => std::slice::from_ref(&PARQUET_SUFFIX),
SourceFormat::Ndjson | SourceFormat::Json => JSON_SUFFIXES,
SourceFormat::Csv => std::slice::from_ref(&CSV_SUFFIX),
}
}
pub(crate) fn file_codec(key: &str, encoding: &FilesEncoding) -> Option<Codec> {
let lower = key.to_ascii_lowercase();
let (stem, suffix_codec) = match (
lower.strip_suffix(GZIP_SUFFIX),
lower.strip_suffix(ZSTD_SUFFIX),
) {
(Some(stem), _) => (stem, Some(Codec::Gzip)),
(None, Some(stem)) => (stem, Some(Codec::Zstd)),
(None, None) => (lower.as_str(), None),
};
let codec = match (encoding.format, encoding.compression, suffix_codec) {
(SourceFormat::Parquet, _, Some(_)) => return None,
(SourceFormat::Parquet, _, None) => Codec::Plain,
(_, Compression::Auto, suffix) => suffix.unwrap_or(Codec::Plain),
(_, Compression::None, None) => Codec::Plain,
(_, Compression::Gzip, None | Some(Codec::Gzip)) => Codec::Gzip,
(_, Compression::Zstd, None | Some(Codec::Zstd)) => Codec::Zstd,
(_, Compression::None | Compression::Gzip | Compression::Zstd, Some(_)) => return None,
};
format_suffixes(encoding.format)
.iter()
.any(|suffix| stem.ends_with(suffix))
.then_some(codec)
}
pub(crate) fn file_names(encoding: &FilesEncoding) -> String {
let compressed: &[&str] = match (encoding.format, encoding.compression) {
(SourceFormat::Parquet, _) | (_, Compression::None) => &[""],
(_, Compression::Auto) => &["", GZIP_SUFFIX, ZSTD_SUFFIX],
(_, Compression::Gzip) => &["", GZIP_SUFFIX],
(_, Compression::Zstd) => &["", ZSTD_SUFFIX],
};
let mut names: Vec<String> = Vec::new();
for suffix in format_suffixes(encoding.format) {
for compression in compressed {
names.push(format!("{suffix}{compression}"));
}
}
names.join(", ")
}