use arrow_array::RecordBatch;
use crate::snowflake_source::_helpers::convert::output_schema;
use crate::snowflake_source::_helpers::folders::{remove_folder, unload_folder, unloaded_files};
use crate::snowflake_source::_helpers::unload::{copy_statement, read_unloaded, rows_unloaded};
use crate::snowflake_source::_helpers::unload_query::unload_query;
use crate::snowflake_source::errors::SnowflakeSourceError;
use crate::snowflake_source::models::{SnowflakeReader, UnloadRequest};
use crate::sql_api::main::execute_sql::execute_sql;
pub async fn unload_rows(
reader: &SnowflakeReader,
request: &UnloadRequest<'_>,
mut on_batch: impl AsyncFnMut(RecordBatch) -> Result<(), String>,
) -> Result<u64, SnowflakeSourceError> {
let query = unload_query(request).map_err(SnowflakeSourceError::Invalid)?;
let schema = output_schema(request.columns).map_err(SnowflakeSourceError::Invalid)?;
let run = uuid::Uuid::new_v4().simple().to_string();
let folder = unload_folder(reader, &run).await?;
let (target, copy) = copy_statement(&folder, &query);
let read = async {
let unloaded = rows_unloaded(&execute_sql(&reader.sql, ©).await?)?;
if unloaded == 0 {
return Ok(0);
}
let files = unloaded_files(reader, &folder).await?;
let read =
read_unloaded(reader, &files, (&schema, request.batch_rows), &mut on_batch).await?;
match read == unloaded {
true => Ok(read),
false => Err(SnowflakeSourceError::Invalid(format!(
"COPY INTO unloaded {unloaded} rows to {target}, but its files hold {read}"
))),
}
}
.await;
if let Err(error) = remove_folder(reader, &folder).await {
tracing::warn!(folder = %target, %error, "cannot remove the unloaded files");
}
read
}