faucet_sink_databricks/
lib.rs

1#![cfg_attr(docsrs, feature(doc_cfg))]
2
3//! # faucet-sink-databricks
4//!
5//! Databricks SQL warehouse sink for the
6//! [`faucet-stream`](https://crates.io/crates/faucet-stream) ecosystem. Loads
7//! pages into a Delta table through a SQL warehouse's Statement Execution API
8//! (plain REST — no JDBC/ODBC driver):
9//!
10//! - **Load paths** — a multi-row `INSERT … FROM VALUES` for small pages, or a
11//!   staged all-string Parquet file (Unity Catalog volume via the Files API,
12//!   or S3 / GCS / ADLS behind the `staging` feature) loaded with `COPY INTO`.
13//!   Every value is cast to the column's declared type on the server.
14//! - **Write modes** — `append`, `upsert` / `delete` via one `MERGE` per
15//!   statement (with `delete_marker`), and `overwrite` via a staging table and
16//!   an atomic `INSERT OVERWRITE` swap.
17//! - **Exactly-once** — a `_faucet_commit_token` watermark table plus a data
18//!   write that is idempotent per page token (`INSERT … REPLACE WHERE` for
19//!   append, `MERGE` for upsert), so a replayed page never duplicates.
20//! - **Schema** — `current_schema` from `information_schema.columns`,
21//!   `ALTER TABLE ADD COLUMNS` / Delta type widening for drift, and table
22//!   auto-create from the first page.
23
24mod config;
25mod sink;
26mod sql;
27mod stage;
28
29pub use config::{
30    DatabricksAuth, DatabricksLoadMethod, DatabricksSinkConfig, DatabricksStagingConfig,
31    STATEMENT_TEXT_LIMIT,
32};
33pub use sink::DatabricksSink;
34
35mod descriptor;
36pub use descriptor::DESCRIPTOR;
faucet_sink_databricks/lib.rs

faucet_sink_databricks/
lib.rs
