use crate::accounting::WriteReport;
use crate::capabilities::Capabilities;
use crate::check::{CheckContext, CheckReport};
use crate::cleanup::SeenKeys;
use crate::dlq::BatchAtomicity;
use crate::drift::SchemaEvolution;
use crate::error::FaucetError;
use crate::idempotency::SinkGuarantee;
use crate::io::{CallContext, CallSlot};
use crate::local_outputs::LocalOutput;
use crate::observability::RoundtripRecorder;
use crate::rollback::{RollbackOptions, RollbackOutcome};
use crate::traits::{RowOutcome, Sink};
use crate::write_mode::WriteMode;
use async_trait::async_trait;
use serde_json::Value;
use std::collections::BTreeMap;
use std::sync::Arc;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum RowPolicy {
WholePage,
PerRow,
}
#[derive(Debug)]
#[non_exhaustive]
pub struct PageWrite {
pub report: WriteReport,
pub failed: Vec<(usize, FaucetError)>,
}
impl PageWrite {
pub fn accepted(n: usize) -> Self {
Self::reported(WriteReport::accepted(n))
}
pub fn reported(report: WriteReport) -> Self {
Self {
report,
failed: Vec::new(),
}
}
pub fn with_failures(mut self, failed: Vec<(usize, FaucetError)>) -> Self {
self.failed = failed;
self
}
pub fn from_outcomes(outcomes: Vec<RowOutcome>) -> Self {
let mut failed = Vec::new();
let mut ok = 0usize;
for (i, o) in outcomes.into_iter().enumerate() {
match o {
Ok(()) => ok += 1,
Err(e) => failed.push((i, e)),
}
}
Self::accepted(ok).with_failures(failed)
}
}
#[async_trait]
pub trait StagingDriver: Send + Sync {
fn active(&self) -> bool;
async fn stage(&self, calls: &CallSlot) -> Result<(), FaucetError>;
async fn publish(&self, calls: &CallSlot) -> Result<(), FaucetError>;
async fn discard(&self, calls: &CallSlot) -> Result<(), FaucetError>;
async fn staged(&self, calls: &CallSlot) -> Result<bool, FaucetError>;
fn reports_staged(&self) -> bool {
true
}
}
#[async_trait]
pub trait CommitTokenDriver: Send + Sync {
async fn last_token(
&self,
calls: &CallSlot,
scope: &str,
) -> Result<Option<String>, FaucetError>;
async fn write_with_token(
&self,
calls: &CallSlot,
rows: &[Value],
scope: &str,
token: &str,
) -> Result<WriteReport, FaucetError>;
}
#[async_trait]
pub trait CleanupDriver: Send + Sync {
async fn delete_absent(
&self,
calls: &CallSlot,
scope: &BTreeMap<String, Value>,
seen: &SeenKeys,
) -> Result<u64, FaucetError>;
}
#[async_trait]
pub trait SchemaDriver: Send + Sync {
async fn current_schema(&self, calls: &CallSlot) -> Result<Option<Value>, FaucetError>;
fn evolvable(&self) -> bool {
false
}
async fn evolve(
&self,
calls: &CallSlot,
evolution: &SchemaEvolution,
) -> Result<(), FaucetError> {
let _ = (calls, evolution);
Err(FaucetError::Sink(
"schema evolution is not supported".into(),
))
}
}
#[async_trait]
pub trait RollbackDriver: Send + Sync {
fn enabled(&self) -> bool {
true
}
async fn rollback_run(
&self,
calls: &CallSlot,
run_id: &str,
opts: &RollbackOptions,
) -> Result<RollbackOutcome, FaucetError>;
async fn forget_run(&self, calls: &CallSlot, run_id: &str) -> Result<(), FaucetError>;
async fn rewind_commit_token(
&self,
calls: &CallSlot,
scope: &str,
token: Option<&str>,
) -> Result<(), FaucetError>;
}
#[cfg(feature = "arrow")]
#[async_trait]
pub trait ColumnarDriver: Send + Sync {
async fn write_columnar(
&self,
calls: &CallSlot,
batch: &arrow::array::RecordBatch,
) -> Result<usize, FaucetError>;
}
#[async_trait]
pub trait SinkDriver: Send + Sync {
fn connector_name(&self) -> &'static str;
fn config_schema(&self) -> Value {
serde_json::json!({"type": "object", "properties": {}})
}
fn dataset_uri(&self) -> String {
format!("{}://unknown", self.connector_name())
}
fn call_context(&self) -> CallContext {
CallContext::new().without_timeout()
}
async fn check(
&self,
calls: &CallSlot,
ctx: &CheckContext,
) -> Result<CheckReport, FaucetError> {
let _ = (calls, ctx);
Ok(CheckReport::not_implemented())
}
async fn write(
&self,
calls: &CallSlot,
rows: &[Value],
policy: RowPolicy,
) -> Result<PageWrite, FaucetError>;
async fn flush(&self, calls: &CallSlot) -> Result<(), FaucetError> {
let _ = calls;
Ok(())
}
fn write_modes(&self) -> &'static [WriteMode] {
&[WriteMode::Append]
}
fn dedups_by_key(&self) -> bool {
false
}
fn write_is_replay_safe(&self) -> bool {
self.dedups_by_key()
}
fn batch_atomicity(&self) -> BatchAtomicity {
BatchAtomicity::BestEffort
}
fn staged_load(&self) -> bool {
false
}
fn capabilities(&self, derived: Capabilities) -> Capabilities {
derived
}
fn sink_guarantee(&self, derived: SinkGuarantee) -> SinkGuarantee {
derived
}
fn staging(&self) -> Option<&dyn StagingDriver> {
None
}
fn commit_tokens(&self) -> Option<&dyn CommitTokenDriver> {
None
}
fn cleanup(&self) -> Option<&dyn CleanupDriver> {
None
}
fn schema(&self) -> Option<&dyn SchemaDriver> {
None
}
fn rollback(&self) -> Option<&dyn RollbackDriver> {
None
}
#[cfg(feature = "arrow")]
fn columnar(&self) -> Option<&dyn ColumnarDriver> {
None
}
fn readback_source(&self) -> Option<(String, Value)> {
None
}
async fn complete_run(&self, calls: &CallSlot) -> Result<(), FaucetError> {
let _ = calls;
Ok(())
}
async fn local_outputs(&self) -> Vec<LocalOutput> {
Vec::new()
}
fn set_call_context(&self, ctx: CallContext) {
let _ = ctx;
}
fn set_roundtrip_recorder(&self, recorder: Arc<RoundtripRecorder>) {
let _ = recorder;
}
}
pub struct DriverSink<D> {
driver: D,
calls: CallSlot,
}
impl<D: SinkDriver> DriverSink<D> {
pub fn new(driver: D) -> Self {
let sink = Self::new_unchecked(driver);
if let Err(e) = sink.check_declarations() {
debug_assert!(false, "{e}");
tracing::error!(error = %e, "sink driver declarations disagree with its primitives");
}
sink
}
#[doc(hidden)]
pub fn new_unchecked(driver: D) -> Self {
let calls = CallSlot::new(driver.call_context());
Self { driver, calls }
}
pub fn driver(&self) -> &D {
&self.driver
}
pub fn driver_mut(&mut self) -> &mut D {
&mut self.driver
}
pub fn calls(&self) -> &CallSlot {
&self.calls
}
pub fn into_driver(self) -> D {
self.driver
}
pub fn check_declarations(&self) -> Result<(), FaucetError> {
let modes = self.driver.write_modes();
let name = self.driver.connector_name();
let staging = self.driver.staging();
if modes.contains(&WriteMode::Overwrite) != staging.is_some() {
return Err(FaucetError::Config(format!(
"sink driver '{name}': write mode overwrite is declared {} but a staging driver \
is {}",
if modes.contains(&WriteMode::Overwrite) {
"on"
} else {
"off"
},
if staging.is_some() {
"present"
} else {
"absent"
}
)));
}
if self.driver.dedups_by_key()
&& !modes
.iter()
.any(|m| matches!(m, WriteMode::Upsert | WriteMode::Delete))
{
return Err(FaucetError::Config(format!(
"sink driver '{name}': it dedups by key but declares no keyed write mode"
)));
}
Ok(())
}
async fn whole_page(&self, records: &[Value]) -> Result<WriteReport, FaucetError> {
let w = self
.driver
.write(&self.calls, records, RowPolicy::WholePage)
.await?;
if let Some((_, e)) = w.failed.into_iter().min_by_key(|(i, _)| *i) {
return Err(e);
}
w.report.check(records.len())?;
Ok(w.report)
}
}
#[async_trait]
impl<D: SinkDriver> Sink for DriverSink<D> {
async fn write_batch(&self, records: &[Value]) -> Result<usize, FaucetError> {
Ok(self.whole_page(records).await?.accepted)
}
async fn write_batch_reported(&self, records: &[Value]) -> Result<WriteReport, FaucetError> {
self.whole_page(records).await
}
async fn write_batch_partial(&self, records: &[Value]) -> Result<Vec<RowOutcome>, FaucetError> {
let w = self
.driver
.write(&self.calls, records, RowPolicy::PerRow)
.await?;
let mut outcomes: Vec<RowOutcome> = records.iter().map(|_| Ok(())).collect();
let failed = w.failed.len();
for (i, e) in w.failed {
match outcomes.get_mut(i) {
Some(slot) => *slot = Err(e),
None => {
return Err(FaucetError::Sink(format!(
"sink driver '{}' reported row {i} failed in a page of {}",
self.driver.connector_name(),
records.len()
)));
}
}
}
w.report.check(records.len().saturating_sub(failed))?;
Ok(outcomes)
}
async fn flush(&self) -> Result<(), FaucetError> {
self.driver.flush(&self.calls).await
}
fn batch_atomicity(&self) -> BatchAtomicity {
self.driver.batch_atomicity()
}
#[cfg(feature = "arrow")]
fn supports_columnar(&self) -> bool {
self.driver.columnar().is_some()
}
#[cfg(feature = "arrow")]
async fn write_batch_columnar(
&self,
batch: &arrow::array::RecordBatch,
) -> Result<usize, FaucetError> {
match self.driver.columnar() {
Some(c) => c.write_columnar(&self.calls, batch).await,
None => {
let rows = crate::columnar::record_batch_to_values(batch)?;
self.write_batch(&rows).await
}
}
}
fn supports_idempotent_writes(&self) -> bool {
self.driver.commit_tokens().is_some()
}
fn capabilities(&self) -> Capabilities {
self.driver.capabilities(Capabilities::of_sink(self))
}
fn sink_guarantee(&self) -> SinkGuarantee {
let derived = if self.supports_idempotent_writes() {
SinkGuarantee::AtomicWatermark
} else if self.driver.write_modes().contains(&WriteMode::Upsert) {
SinkGuarantee::KeyedUpsert
} else {
SinkGuarantee::AtLeastOnce
};
self.driver.sink_guarantee(derived)
}
fn write_batch_is_replay_safe(&self) -> bool {
self.driver.write_is_replay_safe()
}
fn dedups_by_key(&self) -> bool {
self.driver.dedups_by_key()
}
fn supported_write_modes(&self) -> &'static [WriteMode] {
self.driver.write_modes()
}
async fn current_schema(&self) -> Result<Option<Value>, FaucetError> {
match self.driver.schema() {
Some(s) => s.current_schema(&self.calls).await,
None => Ok(None),
}
}
fn supports_schema_evolution(&self) -> bool {
self.driver.schema().is_some_and(SchemaDriver::evolvable)
}
async fn evolve_schema(&self, evolution: &SchemaEvolution) -> Result<(), FaucetError> {
match self.driver.schema().filter(|s| s.evolvable()) {
Some(s) => s.evolve(&self.calls, evolution).await,
None => Err(FaucetError::Sink(format!(
"sink '{}' does not support schema evolution",
self.driver.connector_name()
))),
}
}
fn supports_cleanup(&self) -> bool {
self.driver.cleanup().is_some()
}
fn supports_staged_load(&self) -> bool {
self.driver.staged_load()
}
async fn cleanup_scope(
&self,
scope: &BTreeMap<String, Value>,
seen: &SeenKeys,
) -> Result<u64, FaucetError> {
match self.driver.cleanup() {
Some(c) => c.delete_absent(&self.calls, scope, seen).await,
None => Err(FaucetError::Sink(format!(
"sink '{}' does not support scoped cleanup",
self.driver.connector_name()
))),
}
}
#[allow(deprecated)] async fn write_batch_idempotent(
&self,
records: &[Value],
scope: &str,
token: &str,
) -> Result<usize, FaucetError> {
Ok(self
.write_batch_idempotent_reported(records, scope, token)
.await?
.accepted)
}
async fn write_batch_idempotent_reported(
&self,
records: &[Value],
scope: &str,
token: &str,
) -> Result<WriteReport, FaucetError> {
match self.driver.commit_tokens() {
Some(c) => c.write_with_token(&self.calls, records, scope, token).await,
None => self.whole_page(records).await,
}
}
async fn last_committed_token(&self, scope: &str) -> Result<Option<String>, FaucetError> {
match self.driver.commit_tokens() {
Some(c) => c.last_token(&self.calls, scope).await,
None => Ok(None),
}
}
fn is_overwrite(&self) -> bool {
self.driver.staging().is_some_and(StagingDriver::active)
}
async fn begin_overwrite(&self) -> Result<(), FaucetError> {
self.staging("write_mode: overwrite")?
.stage(&self.calls)
.await
}
async fn commit_overwrite(&self) -> Result<(), FaucetError> {
self.staging("write_mode: overwrite")?
.publish(&self.calls)
.await
}
async fn abort_overwrite(&self) -> Result<(), FaucetError> {
match self.driver.staging() {
Some(s) => s.discard(&self.calls).await,
None => Ok(()),
}
}
async fn overwrite_staging_exists(&self) -> Result<Option<bool>, FaucetError> {
match self.driver.staging() {
Some(s) if s.reports_staged() => s.staged(&self.calls).await.map(Some),
_ => Ok(None),
}
}
fn supports_rollback(&self) -> bool {
self.driver.rollback().is_some_and(RollbackDriver::enabled)
}
async fn rollback_run(
&self,
run_id: &str,
opts: &RollbackOptions,
) -> Result<RollbackOutcome, FaucetError> {
match self.driver.rollback() {
Some(r) => r.rollback_run(&self.calls, run_id, opts).await,
None => Err(FaucetError::Sink(format!(
"sink '{}' does not support rollback",
self.driver.connector_name()
))),
}
}
async fn forget_run(&self, run_id: &str) -> Result<(), FaucetError> {
match self.driver.rollback() {
Some(r) => r.forget_run(&self.calls, run_id).await,
None => Ok(()),
}
}
async fn rewind_commit_token(
&self,
scope: &str,
token: Option<&str>,
) -> Result<(), FaucetError> {
match self.driver.rollback() {
Some(r) => r.rewind_commit_token(&self.calls, scope, token).await,
None => Err(FaucetError::Sink(format!(
"sink '{}' does not support rewinding its commit token",
self.driver.connector_name()
))),
}
}
fn readback_source(&self) -> Option<(String, Value)> {
self.driver.readback_source()
}
async fn complete_run(&self) -> Result<(), FaucetError> {
self.driver.complete_run(&self.calls).await
}
fn config_schema(&self) -> Value {
self.driver.config_schema()
}
fn connector_name(&self) -> &'static str {
self.driver.connector_name()
}
fn set_roundtrip_recorder(&self, recorder: Arc<RoundtripRecorder>) {
self.calls.install_recorder(Arc::clone(&recorder));
self.driver.set_roundtrip_recorder(recorder);
}
fn set_call_context(&self, ctx: CallContext) {
self.calls.install(ctx.clone());
self.driver.set_call_context(ctx);
}
fn dataset_uri(&self) -> String {
self.driver.dataset_uri()
}
async fn local_outputs(&self) -> Vec<LocalOutput> {
self.driver.local_outputs().await
}
async fn check(&self, ctx: &CheckContext) -> Result<CheckReport, FaucetError> {
self.driver.check(&self.calls, ctx).await
}
}
impl<D: SinkDriver> DriverSink<D> {
fn staging(&self, what: &str) -> Result<&dyn StagingDriver, FaucetError> {
self.driver.staging().ok_or_else(|| {
FaucetError::Sink(format!(
"sink '{}' does not support {what}",
self.driver.connector_name()
))
})
}
}