use std::collections::BTreeMap;
use crate::batching::models::Batch;
use crate::config::models::FileFormat;
use crate::databricks_api::main::delete_path::delete_path;
use crate::databricks_api::main::execute_statement::execute_statement;
use crate::databricks_api::main::list_directory::list_directory;
use crate::databricks_api::main::put_file::put_file;
use crate::delivery::_helpers::stage::{Part, by_topic, retry, rows_digest, split_parts};
use crate::delivery::_helpers::stage_paths::{
FolderValues, expired_dates, render_folder, s3_target,
};
use crate::delivery::constants::PRUNE_LOOKBACK;
use crate::delivery::errors::DeliveryError;
use crate::delivery::models::{CopyDelivery, DeliveryReport, StagedBatch, StagedFile};
use crate::file_format::main::databricks_copy::databricks_copy;
use crate::file_format::main::encode_file::encode_file;
use crate::file_format::main::file_extension::file_extension;
use crate::http::errors::ApiError;
use crate::naming::main::quoted_table::quoted_table;
use crate::stage_api::main::put_object::put_object;
use crate::stage_api::models::{ObjectUpload, PutOutcome};
use crate::time_text::main::now_ms::now_ms;
use crate::time_text::main::utc_date::utc_date;
impl CopyDelivery {
pub(crate) async fn deliver(&mut self, batch: &Batch) -> Result<DeliveryReport, DeliveryError> {
let parts = split_parts(batch, self.s3.is_some() && by_topic(&self.config));
let mut staged = self.staged_batch(batch, &parts);
let mut uncompressed_bytes = 0;
let mut sent_bytes = 0;
for (index, part) in parts.iter().enumerate() {
let lines: Vec<&str> = part.rows.iter().map(|row| row.line.as_str()).collect();
let (body, plain) =
encode_file(FileFormat::Parquet, &self.config.envelope.columns, &lines)
.map_err(DeliveryError::Fatal)?;
uncompressed_bytes += plain;
sent_bytes += body.len() as u64;
if staged.files[index].uploaded {
continue;
}
let key = staged.files[index].key.clone();
let uploaded = self.upload(&key, body).await;
if let Err(error) = uploaded {
self.staged = Some(staged);
return Err(error);
}
staged.files[index].uploaded = true;
}
self.staged = Some(staged.clone());
let keys: Vec<&str> = staged.files.iter().map(|file| file.key.as_str()).collect();
self.load(&keys).await?;
self.staged = None;
Ok(DeliveryReport {
rows: batch.rows.len() as u64,
uncompressed_bytes,
sent_bytes,
requests: keys.len() as u64,
skipped_rows: 0,
confirmed: batch.last_offsets(),
})
}
pub(crate) fn staged_batch(&self, batch: &Batch, parts: &[Part<'_>]) -> StagedBatch {
let digest = rows_digest(&batch.rows.iter().collect::<Vec<_>>());
if let Some(staged) = self
.staged
.as_ref()
.filter(|staged| staged.digest == digest)
{
return staged.clone();
}
let now = now_ms();
let files: Vec<StagedFile> = parts
.iter()
.map(|part| StagedFile {
key: self.file_key(part, now),
uploaded: false,
})
.collect();
StagedBatch { digest, files }
}
fn file_key(&self, part: &Part<'_>, now: i64) -> String {
let file = format!(
"{}_{}_{}.{}",
self.stream.name,
self.run,
rows_digest(&part.rows),
file_extension(FileFormat::Parquet)
);
let folder = match (&self.s3, self.config.staging.s3.as_ref()) {
(Some(_), Some(s3)) => render_folder(
&s3.path,
&FolderValues {
topic: part.topic,
table: &self.stream.table,
stream: &self.stream.name,
now_ms: now,
},
),
_ => format!("{}/", utc_date(now)),
};
format!("{folder}{file}")
}
async fn upload(&mut self, key: &str, body: Vec<u8>) -> Result<(), DeliveryError> {
let _slot = self.in_flight.upload().await;
let Some(target) = &self.s3 else {
return put_file(&self.api, &self.volume_path(key), body.into())
.await
.map_err(|error| retry("volume upload", &error));
};
let headers: BTreeMap<String, String> = BTreeMap::new();
let upload = ObjectUpload {
target,
key,
headers: &headers,
};
match put_object(&self.http, upload, body)
.await
.map_err(|error| retry("upload", &error))?
{
PutOutcome::Uploaded => Ok(()),
PutOutcome::CredentialsExpired => {
self.s3 = Some(s3_target(&self.config).map_err(DeliveryError::Fatal)?);
Err(DeliveryError::Retry(
"the S3 credentials expired; reloaded them from the environment".to_owned(),
))
}
}
}
async fn load(&self, keys: &[&str]) -> Result<(), DeliveryError> {
let _slot = self.in_flight.load().await;
let mut folders: BTreeMap<&str, Vec<&str>> = BTreeMap::new();
for key in keys {
let (folder, file) = key.rsplit_once('/').unwrap_or(("", key));
folders.entry(folder).or_default().push(file);
}
let table = quoted_table(&self.config, &self.stream.table);
for (folder, files) in folders {
let location = match &self.s3 {
Some(target) => format!("s3://{}/{}{folder}/", target.bucket, target.prefix),
None => format!("{}/", self.volume_path(folder)),
};
let statement =
databricks_copy(&table, &self.config.envelope.columns, &location, &files);
execute_statement(&self.api, &statement)
.await
.map_err(copy_error)?;
}
Ok(())
}
fn volume_path(&self, key: &str) -> String {
format!(
"/Volumes/{}/{}/{}/{key}",
self.config.destination.database(),
self.config.destination.schema(),
self.stream.stage
)
}
pub(crate) async fn prune(&self) -> Result<u64, DeliveryError> {
if self.s3.is_some() {
return Ok(0);
}
let mut removed = 0;
for date in expired_dates(now_ms(), self.config.staging.retain, PRUNE_LOOKBACK) {
let folder = self.volume_path(&date);
let entries = list_directory(&self.api, &folder)
.await
.map_err(|error| retry("volume cleanup", &error))?;
let Some(entries) = entries else {
continue;
};
for entry in entries.iter().filter(|entry| !entry.is_directory) {
delete_path(&self.api, &entry.path, false)
.await
.map_err(|error| retry("volume cleanup", &error))?;
removed += 1;
}
delete_path(&self.api, &folder, true)
.await
.map_err(|error| retry("volume cleanup", &error))?;
}
Ok(removed)
}
}
fn copy_error(error: ApiError) -> DeliveryError {
match error {
ApiError::Auth(_) => DeliveryError::Fatal(format!("COPY INTO failed: {error}")),
error => DeliveryError::Retry(format!("COPY INTO failed: {error}")),
}
}