use arrow_array::RecordBatch;
use crate::adapters::contract::models::WriteTarget;
use crate::adapters::contract::types::TableWriter;
use crate::adapters::databricks::_helpers::stage::folders::{batch_folder, load_folder};
use crate::adapters::databricks::_helpers::write::copy_select::{copy_statement, readable_batch};
use crate::adapters::databricks::constants::CONCURRENT_UPLOADS;
use crate::adapters::databricks::main::session::_run_statements::run_statements;
use crate::adapters::databricks::main::stage::_remove_folder::remove_folder;
use crate::adapters::databricks::models::DatabricksWriter;
use crate::databricks_api::main::put_file::put_file;
use crate::loader::errors::LoaderError;
use crate::parquet_parts::main::finish_part::finish_part;
use crate::parquet_parts::main::write_part::write_part;
use crate::parquet_parts::models::PartWriter;
impl TableWriter for DatabricksWriter {
fn writes_directly(&self) -> bool {
true
}
async fn write_batch(
&mut self,
target: &WriteTarget<'_>,
batch: RecordBatch,
) -> Result<(), LoaderError> {
let parts = std::mem::take(&mut self.parts);
let (parts, files) = encode(move || write_part(parts, &readable_batch(batch)?)).await?;
self.parts = parts;
for body in files {
self.put_file(target, body).await?;
}
Ok(())
}
fn staged_bytes(&self) -> Option<usize> {
Some(self.parts.bytes())
}
async fn finish_files(&mut self, target: &WriteTarget<'_>) -> Result<(), LoaderError> {
let parts = std::mem::take(&mut self.parts);
let (_, file) = encode(move || finish_part(parts)).await?;
if let Some(body) = file {
self.put_file(target, body).await?;
}
Ok(())
}
async fn load_batch(&mut self, target: &WriteTarget<'_>) -> Result<(), LoaderError> {
self.settle_uploads().await?;
let folder = batch_folder(&self.destination.volume, target);
let copied = run_statements(&self.destination, &[copy_statement(target, &folder)]).await;
self.remove_files(target).await;
copied?;
Ok(())
}
fn next_batch(&mut self) {
self.staged.clear();
self.parts = PartWriter::default();
}
async fn settle_rows(&mut self) -> Result<(), LoaderError> {
Ok(())
}
fn reopen_rows(&mut self) {}
async fn rollback_batch(&mut self) {}
async fn remove_batch_files(&mut self, target: &WriteTarget<'_>) {
while self.uploads.join_next().await.is_some() {}
if !self.staged.is_empty() {
self.remove_files(target).await;
}
}
}
async fn encode<T: Send + 'static>(
work: impl FnOnce() -> Result<T, String> + Send + 'static,
) -> Result<T, LoaderError> {
tokio::task::spawn_blocking(work)
.await
.map_err(|error| {
LoaderError::Invalid(format!("the Parquet encoding thread failed: {error}"))
})?
.map_err(LoaderError::Invalid)
}
impl DatabricksWriter {
async fn put_file(
&mut self,
target: &WriteTarget<'_>,
body: Vec<u8>,
) -> Result<(), LoaderError> {
let name = format!("part-{:05}.parquet", self.staged.len() + 1);
let path = format!("{}/{name}", batch_folder(&self.destination.volume, target));
self.staged.push(name);
while self.uploads.len() >= CONCURRENT_UPLOADS {
self.join_upload().await?;
}
let api = self.destination.api.clone();
self.uploads.spawn(async move {
put_file(&api, &path, bytes::Bytes::from(body)).await?;
Ok(())
});
Ok(())
}
async fn join_upload(&mut self) -> Result<(), LoaderError> {
match self.uploads.join_next().await {
Some(joined) => joined.map_err(|error| {
LoaderError::Invalid(format!("a staged file's upload failed: {error}"))
})?,
None => Ok(()),
}
}
async fn settle_uploads(&mut self) -> Result<(), LoaderError> {
while !self.uploads.is_empty() {
self.join_upload().await?;
}
Ok(())
}
async fn remove_files(&mut self, target: &WriteTarget<'_>) {
let volume = &self.destination.volume;
let removed = remove_folder(&self.destination.api, &batch_folder(volume, target)).await;
if let Err(error) = removed {
tracing::warn!(%error, load_id = %target.load_id, "cannot remove a batch's staged files; a later load removes them after a day");
return;
}
let load = load_folder(volume, target.load_id);
if let Err(error) = crate::databricks_api::main::delete_path::delete_path(
&self.destination.api,
&load,
true,
)
.await
{
tracing::debug!(%error, "the load's folder still holds files");
}
}
}