use databricks_zerobus_ingest_sdk::{
GrpcCode, JsonString, ZerobusError, ZerobusSdk, ZerobusStream,
};
use serde_json::{Map, Value};
use crate::batching::models::{Batch, BatchRow};
use crate::config::models::{ColumnConfig, OversizedPolicy};
use crate::delivery::errors::DeliveryError;
use crate::delivery::models::{DeliveryReport, ZerobusDelivery, ZerobusLink};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum ZerobusKind {
Timestamp,
Date,
Variant,
AsIs,
}
impl ZerobusDelivery {
pub(crate) async fn deliver(&mut self, batch: &Batch) -> Result<DeliveryReport, DeliveryError> {
let kinds: Vec<ZerobusKind> = self.columns.iter().map(zerobus_kind).collect();
let mut records: Vec<(u64, String)> = Vec::with_capacity(batch.rows.len());
let mut skipped_rows = 0;
for row in &batch.rows {
let record = zerobus_record(&self.columns, &kinds, &row.line).map_err(|error| {
DeliveryError::Fatal(format!(
"{}/{} offset {}: {error}",
row.topic, row.partition, row.offset
))
})?;
match record.len() as u64 > self.max_request_bytes {
false => records.push((record.len() as u64, record)),
true if self.oversized == OversizedPolicy::Skip => {
tracing::warn!(topic = %row.topic, partition = row.partition, offset = row.offset, bytes = record.len(), "skipping a row too large for one Zerobus Ingest request");
skipped_rows += 1;
}
true => {
return Err(DeliveryError::Fatal(oversized_message(
row,
record.len(),
self.max_request_bytes,
)));
}
}
}
let sent_bytes: u64 = records.iter().map(|(bytes, _)| bytes).sum();
let requests = chunk_records(records, self.max_request_bytes);
let count = requests.len() as u64;
let sent = self.send(requests).await;
if let Err(error) = sent {
self.link = ZerobusLink::default();
return Err(error);
}
Ok(DeliveryReport {
rows: batch.rows.len() as u64 - skipped_rows,
uncompressed_bytes: sent_bytes,
sent_bytes,
requests: count,
skipped_rows,
confirmed: batch.last_offsets(),
})
}
async fn send(&mut self, requests: Vec<Vec<String>>) -> Result<(), DeliveryError> {
let stream = self.stream().await?;
for request in requests {
stream
.ingest_records_offset(request.into_iter().map(JsonString))
.await
.map_err(|error| zerobus_error("ingest", &error))?;
}
stream
.flush()
.await
.map_err(|error| zerobus_error("flush", &error))
}
async fn stream(&mut self) -> Result<&ZerobusStream, DeliveryError> {
if self.link.sdk.is_none() {
let sdk = ZerobusSdk::builder()
.endpoint(self.settings.endpoint.clone())
.unity_catalog_url(self.settings.workspace_url.clone())
.build()
.map_err(|error| zerobus_error("connect", &error))?;
self.link.sdk = Some(sdk);
}
if self.link.stream.is_none()
&& let Some(sdk) = &self.link.sdk
{
let stream = sdk
.stream_builder()
.table(self.table.clone())
.oauth(
self.settings.client_id.clone(),
self.settings.client_secret.expose().to_owned(),
)
.json()
.build()
.await
.map_err(|error| open_error(&self.table, &self.settings.client_id, &error))?;
self.link.stream = Some(stream);
}
self.link
.stream
.as_ref()
.ok_or_else(|| DeliveryError::Retry("no Zerobus Ingest stream".to_owned()))
}
}
pub(crate) fn zerobus_kind(column: &ColumnConfig) -> ZerobusKind {
let upper = column.sql_type.trim().to_ascii_uppercase();
match upper.split(['(', '<', ' ']).next().unwrap_or_default() {
"TIMESTAMP" | "TIMESTAMP_NTZ" | "TIMESTAMP_LTZ" => ZerobusKind::Timestamp,
"DATE" => ZerobusKind::Date,
"VARIANT" => ZerobusKind::Variant,
_ => ZerobusKind::AsIs,
}
}
pub(crate) fn zerobus_record(
columns: &[ColumnConfig],
kinds: &[ZerobusKind],
line: &str,
) -> Result<String, String> {
let mut row: Map<String, Value> =
serde_json::from_str(line).map_err(|error| format!("row is not a JSON object: {error}"))?;
for (column, kind) in columns.iter().zip(kinds) {
let Some(value) = row.get_mut(&column.name) else {
continue;
};
if value.is_null() {
continue;
}
*value = match (kind, value.as_str()) {
(ZerobusKind::Timestamp, Some(text)) => match instant_micros(text) {
Some(micros) => Value::from(micros),
None => return Err(unreadable(&column.name, text, "timestamp")),
},
(ZerobusKind::Date, Some(text)) => match epoch_days(text) {
Some(days) => Value::from(days),
None => return Err(unreadable(&column.name, text, "date")),
},
(ZerobusKind::Variant, _) => Value::from(value.to_string()),
_ => continue,
};
}
Ok(Value::Object(row).to_string())
}
fn instant_micros(text: &str) -> Option<i64> {
let text = serde_json::from_str::<String>(text).unwrap_or_else(|_| text.to_owned());
if let Ok(instant) = chrono::DateTime::parse_from_rfc3339(&text) {
return Some(instant.timestamp_micros());
}
for format in ["%Y-%m-%dT%H:%M:%S%.f", "%Y-%m-%d %H:%M:%S%.f"] {
match chrono::NaiveDateTime::parse_from_str(&text, format) {
Ok(naive) => return Some(naive.and_utc().timestamp_micros()),
Err(_) => continue,
}
}
None
}
fn epoch_days(text: &str) -> Option<i64> {
let unquoted = serde_json::from_str::<String>(text).unwrap_or_else(|_| text.to_owned());
match chrono::NaiveDate::parse_from_str(&unquoted, "%Y-%m-%d") {
Ok(date) => Some((date - chrono::NaiveDate::default()).num_days()),
Err(_) => instant_micros(text).map(|micros| micros.div_euclid(86_400_000_000)),
}
}
fn unreadable(column: &str, text: &str, kind: &str) -> String {
format!(
"column {column} holds {text:?}, which Zerobus Ingest cannot take as a {kind}; fix the producer, or type the column STRING to keep such values as text"
)
}
pub(crate) fn chunk_records(records: Vec<(u64, String)>, limit: u64) -> Vec<Vec<String>> {
let mut requests: Vec<Vec<String>> = Vec::new();
let mut size = 0;
for (bytes, record) in records {
match requests.last_mut() {
Some(request) if size + bytes <= limit => request.push(record),
_ => {
requests.push(vec![record]);
size = 0;
}
}
size += bytes;
}
requests
}
fn oversized_message(row: &BatchRow, bytes: usize, limit: u64) -> String {
format!(
"row {}/{}@{} is {bytes} bytes, over the {limit}-byte request limit; set oversized = \"skip\" to drop such rows",
row.topic, row.partition, row.offset
)
}
pub(crate) fn open_error(table: &str, client_id: &str, error: &ZerobusError) -> DeliveryError {
let refused = match error {
ZerobusError::InvalidUCTokenError(message) => {
(message.starts_with("Client error (401)") || message.starts_with("Client error (403)"))
&& !message.contains("invalid_client")
}
ZerobusError::CreateStreamError(status) => {
matches!(
status.code(),
GrpcCode::Unauthenticated | GrpcCode::PermissionDenied
)
}
_ => false,
};
match refused {
true => DeliveryError::Retry(format!(
"Zerobus Ingest open stream for {table} was refused: {error}; if this lasts, the service principal {client_id} needs USE CATALOG, USE SCHEMA, and MODIFY and SELECT on the table, granted explicitly"
)),
false => zerobus_error("open stream", error),
}
}
fn zerobus_error(action: &str, error: &ZerobusError) -> DeliveryError {
match error.is_retryable() {
true => DeliveryError::Retry(format!("Zerobus Ingest {action} failed: {error}")),
false => DeliveryError::Fatal(format!("Zerobus Ingest {action} failed: {error}")),
}
}