use crate::snowflake_source::_helpers::folders::{
destination_folder, remove_folder, unload_folder, unloaded_keys,
};
use crate::snowflake_source::_helpers::unload::{copy_statement, rows_unloaded};
use crate::snowflake_source::_helpers::unload_query::unload_query;
use crate::snowflake_source::errors::SnowflakeSourceError;
use crate::snowflake_source::models::{SnowflakeReader, UnloadRequest, UnloadedFiles};
use crate::sql_api::main::execute_sql::execute_sql;
pub async fn unload_files<T, E: From<SnowflakeSourceError>>(
reader: &SnowflakeReader,
request: &UnloadRequest<'_>,
load: impl AsyncFnOnce(&UnloadedFiles) -> Result<T, E>,
) -> Result<T, E> {
let query = unload_query(request).map_err(SnowflakeSourceError::Invalid)?;
let run = uuid::Uuid::new_v4().simple().to_string();
let folder = unload_folder(reader, &run).await?;
let (target, copy) = copy_statement(&folder, &query);
let loaded = async {
let copied = execute_sql(&reader.sql, ©)
.await
.map_err(SnowflakeSourceError::from)?;
let rows = rows_unloaded(&copied)?;
let (location, downloads) = destination_folder(&folder)?;
let keys = match rows {
0 => Vec::new(),
_ => unloaded_keys(reader, &folder, downloads.as_ref()).await?,
};
let files = UnloadedFiles {
folder: location,
keys,
rows,
};
let loaded = load(&files).await;
if let Some(local) = downloads {
let path = local.path().display().to_string();
if let Err(error) = local.close() {
tracing::warn!(folder = %path, %error, "cannot remove the downloaded files");
}
}
loaded
}
.await;
if let Err(error) = remove_folder(reader, &folder).await {
tracing::warn!(folder = %target, %error, "cannot remove the unloaded files");
}
loaded
}