#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum FileFormat {
Parquet,
Json,
Csv,
}
impl std::fmt::Display for FileFormat {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str(match self {
Self::Parquet => "parquet",
Self::Json => "json",
Self::Csv => "csv",
})
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Store {
ObjectStore(String),
Local(String),
InternalStage(String),
}
impl Store {
pub fn is_within(&self, other: &Store) -> bool {
match (self, other) {
(Self::ObjectStore(path), Self::ObjectStore(prefix))
| (Self::Local(path), Self::Local(prefix))
| (Self::InternalStage(path), Self::InternalStage(prefix)) => {
path.starts_with(prefix.as_str())
}
_ => false,
}
}
}
impl std::fmt::Display for Store {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::ObjectStore(path) | Self::Local(path) => formatter.write_str(path),
Self::InternalStage(name) => write!(formatter, "internal stage {name}"),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FileSet {
pub store: Store,
pub format: FileFormat,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ReaderOffer {
pub arrow: bool,
pub existing_files: Option<FileSet>,
pub unload_to: Vec<Store>,
pub estimated_bytes: Option<u64>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct WriterAccepts {
pub arrow: bool,
pub pulls_from: Vec<Store>,
pub pull_formats: Vec<FileFormat>,
pub staging: Vec<Store>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum ReadVia {
#[default]
Auto,
Query,
Unload,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum StagePreference {
#[default]
Auto,
External,
Internal,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub struct RouteOverrides {
pub read_via: ReadVia,
pub stage: StagePreference,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Step {
ReadArrow,
Unload(Store),
UseFiles(FileSet),
FileReader(FileFormat),
ParquetWriter(Store),
DestinationPulls(FileSet),
DestinationTakesArrow,
}
impl std::fmt::Display for Step {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::ReadArrow => formatter.write_str("read as Arrow"),
Self::Unload(store) => write!(formatter, "unload Parquet to {store}"),
Self::UseFiles(files) => {
write!(formatter, "use {} files in {}", files.format, files.store)
}
Self::FileReader(format) => write!(formatter, "read {format} into Arrow"),
Self::ParquetWriter(store) => write!(formatter, "write Parquet to {store}"),
Self::DestinationPulls(files) => {
write!(
formatter,
"destination loads {} files from {}",
files.format, files.store
)
}
Self::DestinationTakesArrow => formatter.write_str("destination takes Arrow"),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Route {
pub steps: Vec<Step>,
pub reason: String,
}