use crate::{CollectError, Datatype, MetaDatatype, ParseError, Partition, Query};
use std::{collections::HashMap, path::PathBuf};
#[derive(Clone, Debug)]
pub struct FileOutput {
pub output_dir: std::path::PathBuf,
pub prefix: String,
pub suffix: Option<String>,
pub subdirs: Vec<SubDir>,
pub overwrite: bool,
pub format: FileFormat,
pub row_group_size: Option<usize>,
pub parquet_statistics: bool,
pub parquet_compression: polars::prelude::ParquetCompression,
}
#[derive(Clone, Debug)]
pub enum SubDir {
Datatype,
Network,
Custom(String),
}
impl FileOutput {
pub fn get_paths(
&self,
query: &Query,
partition: &Partition,
meta_datatypes: Option<Vec<MetaDatatype>>,
) -> Result<HashMap<Datatype, PathBuf>, CollectError> {
let mut paths = HashMap::new();
let meta_datatypes = if let Some(meta_datatypes) = meta_datatypes {
meta_datatypes
} else {
query.datatypes.clone()
};
for meta_datatype in meta_datatypes.iter() {
for datatype in meta_datatype.datatypes().into_iter() {
paths.insert(datatype, self.get_path(query, partition, datatype)?);
}
}
Ok(paths)
}
pub fn get_path(
&self,
query: &Query,
partition: &Partition,
datatype: Datatype,
) -> Result<PathBuf, CollectError> {
let filename = if let Some(suffix) = self.suffix.clone() {
format!(
"{}__{}__{}__{}.{}",
self.prefix.clone(),
datatype.name(),
suffix,
partition.label(&query.partitioned_by)?,
self.format.as_str(),
)
} else {
format!(
"{}__{}__{}.{}",
self.prefix.clone(),
datatype.name(),
partition.label(&query.partitioned_by)?,
self.format.as_str(),
)
};
let filename = std::path::Path::new(&filename).to_path_buf();
let mut output_dir = std::path::Path::new(&self.output_dir).to_path_buf();
for subdir in self.subdirs.iter() {
let subdir_str: String = match subdir {
SubDir::Network => self.prefix.clone(),
SubDir::Datatype => match &self.suffix {
Some(suffix) => datatype.name() + "__" + suffix.as_str(),
None => datatype.name(),
},
SubDir::Custom(subdir_str) => subdir_str.to_string(),
};
output_dir = output_dir.join(std::path::Path::new(&subdir_str));
}
std::fs::create_dir_all(output_dir.clone())
.map_err(|_| ParseError::ParseError("could not create dir".to_string()))?;
Ok(output_dir.join(filename))
}
}
#[derive(Clone, Eq, PartialEq, Debug)]
pub enum FileFormat {
Parquet,
Csv,
Json,
}
impl FileFormat {
pub fn as_str(&self) -> &'static str {
match *self {
FileFormat::Parquet => "parquet",
FileFormat::Csv => "csv",
FileFormat::Json => "json",
}
}
}
#[derive(Clone, Eq, PartialEq, Debug)]
pub enum ColumnEncoding {
Binary,
Hex,
}
impl ColumnEncoding {
pub fn as_str(&self) -> &'static str {
match *self {
ColumnEncoding::Binary => "binary",
ColumnEncoding::Hex => "hex",
}
}
}