use crate::auth::models::{Authenticator, Secret};
use crate::batching::models::{Batch, Partition};
use crate::config::models::{ColumnConfig, Config, LoadErrorPolicy, OversizedPolicy};
use crate::databricks_api::main::execute_statement::execute_statement;
use crate::databricks_api::models::DatabricksClient;
use crate::delivery::_helpers::stage_paths::s3_target;
use crate::delivery::constants::{STAGE_CONFIRM_POLL, STREAMING_CONFIRM_POLL};
use crate::delivery::errors::DeliveryError;
use crate::http::errors::ApiError;
use crate::http::models::InFlight;
use crate::naming::models::Stream;
use crate::planning::models::LoadPath;
use crate::sql_api::main::execute_sql::execute_sql;
use crate::sql_api::models::{SqlClient, SqlRows};
use crate::stage_api::models::{DriverSession, ObjectTarget, StageInfo};
use crate::streaming_api::models::StreamingClient;
use std::collections::{BTreeMap, VecDeque};
use std::sync::Arc;
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct DeliveryReport {
pub rows: u64,
pub uncompressed_bytes: u64,
pub sent_bytes: u64,
pub requests: u64,
pub skipped_rows: u64,
pub confirmed: BTreeMap<Partition, i64>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct Confirmation {
pub confirmed: BTreeMap<Partition, i64>,
pub error_rows: u64,
pub skipped_files: u64,
pub deferred_error: Option<DeliveryError>,
}
#[derive(Debug, Clone)]
pub struct StageClients {
pub http: reqwest::Client,
pub auth: Arc<Authenticator>,
pub sql: SqlClient,
pub driver: Arc<DriverSession>,
pub account_url: String,
pub in_flight: InFlight,
}
#[derive(Debug, Clone)]
pub enum Warehouse {
Snowflake(SnowflakeClients),
Databricks(DatabricksClients),
}
#[derive(Debug, Clone)]
pub struct SnowflakeClients {
pub auth: Arc<Authenticator>,
pub sql: SqlClient,
pub streaming: Arc<StreamingClient>,
pub driver: Arc<DriverSession>,
pub account_url: String,
}
#[derive(Debug, Clone)]
pub struct DatabricksClients {
pub api: DatabricksClient,
pub zerobus: Option<ZerobusSettings>,
}
#[derive(Debug, Clone)]
pub struct ZerobusSettings {
pub endpoint: String,
pub workspace_url: String,
pub client_id: String,
pub client_secret: Secret,
}
impl Warehouse {
pub async fn execute(&self, statement: &str) -> Result<SqlRows, ApiError> {
match self {
Self::Snowflake(snowflake) => execute_sql(&snowflake.sql, statement).await,
Self::Databricks(databricks) => execute_statement(&databricks.api, statement).await,
}
}
pub fn uses_key_pair(&self) -> bool {
match self {
Self::Snowflake(snowflake) => snowflake.auth.uses_key_pair(),
Self::Databricks(_) => false,
}
}
}
#[derive(Debug)]
pub struct StreamingDelivery {
pub(crate) client: Arc<StreamingClient>,
pub(crate) stream: Stream,
pub(crate) max_request_bytes: u64,
pub(crate) oversized: OversizedPolicy,
pub(crate) continuations: BTreeMap<Partition, String>,
pub(crate) pending: BTreeMap<Partition, i64>,
pub(crate) error_rows: BTreeMap<String, i64>,
}
#[derive(Debug)]
pub struct StageDelivery {
pub(crate) clients: StageClients,
pub(crate) config: Arc<Config>,
pub(crate) stream: Stream,
pub(crate) path: LoadPath,
pub(crate) stage_info: Option<StageInfo>,
pub(crate) s3: Option<ObjectTarget>,
pub(crate) run: String,
pub(crate) staged: Option<StagedBatch>,
pub(crate) pending: PendingLoads,
pub(crate) begin_mark: Option<String>,
pub(crate) final_checks: bool,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct PendingLoads {
pub batches: VecDeque<PendingBatch>,
pub policy: LoadErrorPolicy,
pub timing: CheckTiming,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct CheckTiming {
pub check_after: std::time::Duration,
pub recheck_every: std::time::Duration,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PendingBatch {
pub files: Vec<PendingFile>,
pub offsets: BTreeMap<Partition, i64>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PendingFile {
pub key: String,
pub first_sent_ms: i64,
pub resent: bool,
pub received_before: bool,
pub next_check_ms: i64,
pub last_check_ms: Option<i64>,
pub failed_checks: u32,
pub warned_no_history: bool,
pub state: FileState,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum FileState {
Waiting,
InProgress,
Loaded,
Failed(String),
Skipped,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct Settled {
pub confirmed: BTreeMap<Partition, i64>,
pub skipped: Vec<(String, String)>,
pub halted: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct StagedBatch {
pub digest: String,
pub files: Vec<StagedFile>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct StagedFile {
pub key: String,
pub uploaded: bool,
}
#[derive(Debug)]
pub struct ZerobusDelivery {
pub(crate) settings: ZerobusSettings,
pub(crate) table: String,
pub(crate) columns: Vec<ColumnConfig>,
pub(crate) max_request_bytes: u64,
pub(crate) oversized: OversizedPolicy,
pub(crate) link: ZerobusLink,
}
#[derive(Default)]
pub struct ZerobusLink {
pub(crate) sdk: Option<databricks_zerobus_ingest_sdk::ZerobusSdk>,
pub(crate) stream: Option<databricks_zerobus_ingest_sdk::ZerobusStream>,
}
impl std::fmt::Debug for ZerobusLink {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter
.debug_struct("ZerobusLink")
.field("open", &self.stream.is_some())
.finish()
}
}
#[derive(Debug)]
pub struct CopyDelivery {
pub(crate) api: DatabricksClient,
pub(crate) http: reqwest::Client,
pub(crate) in_flight: InFlight,
pub(crate) config: Arc<Config>,
pub(crate) stream: Stream,
pub(crate) s3: Option<ObjectTarget>,
pub(crate) run: String,
pub(crate) staged: Option<StagedBatch>,
}
#[derive(Debug)]
pub enum Delivery {
Streaming(Box<StreamingDelivery>),
Stage(Box<StageDelivery>),
Zerobus(Box<ZerobusDelivery>),
Copy(Box<CopyDelivery>),
}
impl Delivery {
pub fn streaming(
client: Arc<StreamingClient>,
stream: Stream,
max_request_bytes: u64,
oversized: OversizedPolicy,
) -> Self {
Self::Streaming(Box::new(StreamingDelivery::new(
client,
stream,
max_request_bytes,
oversized,
)))
}
pub fn stage(
clients: StageClients,
config: Arc<Config>,
stream: Stream,
path: LoadPath,
) -> Result<Self, DeliveryError> {
StageDelivery::new(clients, config, stream, path).map(|stage| Self::Stage(Box::new(stage)))
}
pub fn zerobus(
settings: ZerobusSettings,
config: &Config,
stream: &Stream,
oversized: OversizedPolicy,
) -> Self {
Self::Zerobus(Box::new(ZerobusDelivery {
settings,
table: format!(
"{}.{}.{}",
config.destination.database(),
config.destination.schema(),
stream.table
),
columns: config.envelope.columns.clone(),
max_request_bytes: config.batching.max_request_bytes.0,
oversized,
link: ZerobusLink::default(),
}))
}
pub fn copy(
inputs: &DeliveryInputs,
api: &DatabricksClient,
stream: &Stream,
) -> Result<Self, DeliveryError> {
let s3 = match inputs.path {
LoadPath::S3Copy => Some(s3_target(&inputs.config).map_err(DeliveryError::Fatal)?),
_ => None,
};
Ok(Self::Copy(Box::new(CopyDelivery {
api: api.clone(),
http: inputs.http.clone(),
in_flight: inputs.in_flight.clone(),
config: inputs.config.clone(),
stream: stream.clone(),
s3,
run: uuid::Uuid::new_v4().simple().to_string(),
staged: None,
})))
}
pub fn resumes_from_warehouse(&self) -> bool {
matches!(self, Self::Streaming(_))
}
pub async fn open(&mut self, partition: &Partition) -> Result<Option<i64>, DeliveryError> {
match self {
Self::Streaming(streaming) => streaming.open(partition).await,
Self::Stage(_) | Self::Zerobus(_) | Self::Copy(_) => Ok(None),
}
}
pub fn is_open(&self, partition: &Partition) -> bool {
match self {
Self::Streaming(streaming) => streaming.is_open(partition),
Self::Stage(_) | Self::Zerobus(_) | Self::Copy(_) => true,
}
}
pub async fn deliver(&mut self, batch: &Batch) -> Result<DeliveryReport, DeliveryError> {
match self {
Self::Streaming(streaming) => streaming.deliver(batch).await,
Self::Stage(stage) => stage.deliver(batch).await,
Self::Zerobus(zerobus) => zerobus.deliver(batch).await,
Self::Copy(copy) => copy.deliver(batch).await,
}
}
pub fn has_pending(&self) -> bool {
match self {
Self::Streaming(streaming) => streaming.has_pending(),
Self::Stage(stage) => !stage.pending.is_empty(),
Self::Zerobus(_) | Self::Copy(_) => false,
}
}
pub fn confirm_interval(&self) -> std::time::Duration {
match self {
Self::Streaming(_) => STREAMING_CONFIRM_POLL,
Self::Stage(_) | Self::Zerobus(_) | Self::Copy(_) => STAGE_CONFIRM_POLL,
}
}
pub fn is_full(&self) -> bool {
match self {
Self::Streaming(_) | Self::Zerobus(_) | Self::Copy(_) => false,
Self::Stage(stage) => stage.is_full(),
}
}
pub fn pending_files(&self) -> usize {
match self {
Self::Streaming(_) | Self::Zerobus(_) | Self::Copy(_) => 0,
Self::Stage(stage) => stage.pending.file_count(),
}
}
pub fn oldest_pending_ms(&self) -> Option<i64> {
match self {
Self::Streaming(_) | Self::Zerobus(_) | Self::Copy(_) => None,
Self::Stage(stage) => stage.pending.oldest_sent_ms(),
}
}
pub async fn poll(&mut self) -> Result<Confirmation, DeliveryError> {
match self {
Self::Streaming(streaming) => streaming.poll().await,
Self::Stage(stage) => stage.poll().await,
Self::Zerobus(_) | Self::Copy(_) => Ok(Confirmation::default()),
}
}
pub fn check_until_stopped(&mut self) {
if let Self::Stage(stage) = self {
stage.final_checks = true;
}
}
pub fn reset(&mut self) {
match self {
Self::Streaming(streaming) => streaming.reset(),
Self::Stage(stage) => stage.pending.clear(),
Self::Zerobus(zerobus) => zerobus.link = ZerobusLink::default(),
Self::Copy(copy) => copy.staged = None,
}
}
pub fn forget(&mut self, partitions: &[Partition]) {
match self {
Self::Streaming(streaming) => streaming.forget(partitions),
Self::Stage(stage) => stage.pending.forget(partitions),
Self::Zerobus(_) | Self::Copy(_) => {}
}
}
pub async fn prune(&self) -> Result<u64, DeliveryError> {
match self {
Self::Streaming(_) | Self::Zerobus(_) => Ok(0),
Self::Stage(stage) => stage.prune().await,
Self::Copy(copy) => copy.prune().await,
}
}
}
#[derive(Debug, Clone)]
pub struct DeliveryInputs {
pub http: reqwest::Client,
pub in_flight: InFlight,
pub warehouse: Warehouse,
pub config: Arc<Config>,
pub path: LoadPath,
pub oversized: OversizedPolicy,
}