use polars::prelude::*;
use super::ResolvedPartition;
use crate::error::TesseraError;
const EXPIRY_MARKERS: [&str; 5] = [
"403",
"expired",
"accessdenied",
"access denied",
"forbidden",
];
pub fn build_lazyframe(
parts: &[ResolvedPartition],
columns: Option<&[&str]>,
) -> Result<LazyFrame, TesseraError> {
let multi = parts.len() > 1;
let objects: usize = parts.iter().map(|(_, urls)| urls.len()).sum();
let mut frames = Vec::with_capacity(objects);
for (partition, urls) in parts {
for url in urls {
let mut lazy = LazyFrame::scan_parquet(url.as_str().into(), ScanArgsParquet::default())
.map_err(|err| TesseraError::Network(err.to_string()))?;
if let Some(columns) = columns {
lazy = lazy.select(
columns
.iter()
.map(|column| col(*column))
.collect::<Vec<_>>(),
);
}
if multi {
lazy = lazy.with_columns(vec![
lit(partition.coin.as_str()).alias("coin"),
lit(partition.month.as_str()).alias("month"),
]);
}
frames.push(lazy);
}
}
if let [single] = &frames[..] {
return Ok(single.clone());
}
concat(
frames,
UnionArgs {
rechunk: true,
parallel: true,
to_supertypes: true,
..Default::default()
},
)
.map_err(|err| TesseraError::Network(err.to_string()))
}
pub fn collect(lazy: LazyFrame) -> Result<DataFrame, TesseraError> {
lazy.collect().map_err(|err| {
let message = err.to_string().to_lowercase();
if EXPIRY_MARKERS.iter().any(|marker| message.contains(marker)) {
TesseraError::PresignExpired
} else {
TesseraError::Network(err.to_string())
}
})
}