faucet_stream/
lib.rs

1#![cfg_attr(docsrs, feature(doc_cfg))]
2
3//! # faucet-stream
4//!
5//! A declarative, config-driven data pipeline with pluggable source and sink
6//! connectors.
7//!
8//! 📖 **Guide, tutorials & cookbook:** <https://pawansikawat.github.io/faucet-stream/>
9//!
10//! ## Feature flags
11//!
12//! | Feature | Description |
13//! |---------|-------------|
14//! | `source-rest` *(default)* | REST API source with pagination, auth, transforms |
15//! | `source-graphql` | GraphQL API source with cursor pagination |
16//! | `source-xml` | XML/SOAP API source with XML-to-JSON conversion |
17//! | `source-grpc` | gRPC source with dynamic protobuf messages |
18//! | `source-postgres` | PostgreSQL query source |
19//! | `source-postgres-cdc` | PostgreSQL CDC source (logical replication) |
20//! | `source-mysql` | MySQL query source |
21//! | `source-mssql` | Microsoft SQL Server query source |
22//! | `source-sqlite` | SQLite query source |
23
24//! | `source-s3` | AWS S3 file source |
25//! | `source-mongodb` | MongoDB query source |
26//! | `source-mongodb-cdc` | MongoDB CDC source (Change Streams) |
27//! | `source-mysql-cdc` | MySQL CDC source (binlog replication) |
28//! | `source-redis` | Redis source (streams, lists, keys) |
29//! | `source-webhook` | Webhook HTTP receiver source |
30//! | `source-websocket` | WebSocket streaming source |
31//! | `source-csv` | CSV file source |
32//! | `source-elasticsearch` | Elasticsearch search/scroll source |
33//! | `source-kafka` | Apache Kafka consumer source |
34//! | `source-kinesis` | AWS Kinesis Data Streams source |
35//! | `source-spanner` | Google Cloud Spanner query source |
36//! | `source-parquet` | Apache Parquet file source (local, glob, S3) |
37//! | `source-delta` | Apache Delta Lake source (local FS or S3/Azure/GCS, time travel) |
38//! | `source-databricks` | Databricks SQL query source (Statement Execution API) |
39//! | `sink-bigquery` | Google BigQuery streaming insert sink |
40//! | `sink-iceberg` | Apache Iceberg sink (append-only, REST/Glue/SQL/HMS catalogs) |
41//! | `sink-postgres` | PostgreSQL sink (jsonb or auto-mapped columns) |
42//! | `sink-jsonl` | JSON Lines file sink |
43//! | `sink-snowflake` | Snowflake SQL REST API sink |
44//! | `sink-mysql` | MySQL sink |
45//! | `sink-mssql` | Microsoft SQL Server sink |
46//! | `sink-sqlite` | SQLite sink |
47
48//! | `sink-s3` | AWS S3 file sink |
49//! | `sink-mongodb` | MongoDB insert sink |
50//! | `sink-redis` | Redis sink (streams, lists, key-value) |
51//! | `sink-csv` | CSV file sink |
52//! | `sink-elasticsearch` | Elasticsearch bulk index sink |
53//! | `sink-http` | HTTP POST sink |
54//! | `sink-kafka` | Apache Kafka producer sink |
55//! | `sink-kinesis` | AWS Kinesis Data Streams sink |
56//! | `sink-spanner` | Google Cloud Spanner mutation sink |
57//! | `sink-parquet` | Apache Parquet file sink (local, S3) |
58//! | `encryption` | AES-256-GCM at-rest sealing for file state-store bookmarks and per-line JSONL/DLQ output |
59//! | `sink-delta` | Apache Delta Lake sink (append-only; local FS or S3/Azure/GCS) |
60//! | `kafka-schema-registry` | Schema Registry support for Kafka connectors |
61//! | `source` | All source connectors |
62//! | `sink` | All sink connectors |
63//! | `full` | Every connector |
64
65// Always re-export core types and traits.
66pub use faucet_core::*;
67
68// Explicit re-exports for the library-side transforms wrapper and observability
69// labels so users can import them via the umbrella path.
70pub use faucet_core::TransformingSource;
71pub use faucet_core::observability::Labels;
72
73// ── Shared auth providers ────────────────────────────────────────────────────
74/// Single-flight OAuth2 / token-endpoint auth providers (enable the `auth`
75/// feature). Share one across connectors via `with_auth_provider` or the CLI
76/// `auth: { ref }` catalog.
77#[cfg(feature = "auth")]
78pub mod auth {
79    pub use faucet_auth::*;
80}
81
82// ── Source connectors ────────────────────────────────────────────────────────
83
84#[cfg(feature = "source-rest")]
85pub mod source {
86    pub mod rest {
87        pub use faucet_source_rest::*;
88    }
89
90    #[cfg(feature = "source-graphql")]
91    pub mod graphql {
92        pub use faucet_source_graphql::*;
93    }
94
95    #[cfg(feature = "source-xml")]
96    pub mod xml {
97        pub use faucet_source_xml::*;
98    }
99
100    #[cfg(feature = "source-grpc")]
101    pub mod grpc {
102        pub use faucet_source_grpc::*;
103    }
104
105    #[cfg(feature = "source-postgres")]
106    pub mod postgres {
107        pub use faucet_source_postgres::*;
108    }
109
110    #[cfg(feature = "source-postgres-cdc")]
111    pub mod postgres_cdc {
112        pub use faucet_source_postgres_cdc::*;
113    }
114
115    #[cfg(feature = "source-mysql")]
116    pub mod mysql {
117        pub use faucet_source_mysql::*;
118    }
119
120    #[cfg(feature = "source-mssql")]
121    pub mod mssql {
122        pub use faucet_source_mssql::*;
123    }
124
125    #[cfg(feature = "source-sqlite")]
126    pub mod sqlite {
127        pub use faucet_source_sqlite::*;
128    }
129
130    #[cfg(feature = "source-s3")]
131    pub mod s3 {
132        pub use faucet_source_s3::*;
133    }
134
135    #[cfg(feature = "source-mongodb")]
136    pub mod mongodb {
137        pub use faucet_source_mongodb::*;
138    }
139
140    #[cfg(feature = "source-mongodb-cdc")]
141    pub mod mongodb_cdc {
142        pub use faucet_source_mongodb_cdc::*;
143    }
144
145    #[cfg(feature = "source-mysql-cdc")]
146    pub mod mysql_cdc {
147        pub use faucet_source_mysql_cdc::*;
148    }
149
150    #[cfg(feature = "source-redis")]
151    pub mod redis {
152        pub use faucet_source_redis::*;
153    }
154
155    #[cfg(feature = "source-webhook")]
156    pub mod webhook {
157        pub use faucet_source_webhook::*;
158    }
159
160    #[cfg(feature = "source-websocket")]
161    pub mod websocket {
162        pub use faucet_source_websocket::*;
163    }
164
165    #[cfg(feature = "source-csv")]
166    pub mod csv {
167        pub use faucet_source_csv::*;
168    }
169
170    #[cfg(feature = "source-elasticsearch")]
171    pub mod elasticsearch {
172        pub use faucet_source_elasticsearch::*;
173    }
174
175    #[cfg(feature = "source-kafka")]
176    pub mod kafka {
177        pub use faucet_source_kafka::*;
178    }
179
180    #[cfg(feature = "source-kinesis")]
181    pub mod kinesis {
182        pub use faucet_source_kinesis::*;
183    }
184
185    #[cfg(feature = "source-bigquery")]
186    pub mod bigquery {
187        pub use faucet_source_bigquery::*;
188    }
189
190    #[cfg(feature = "source-snowflake")]
191    pub mod snowflake {
192        pub use faucet_source_snowflake::*;
193    }
194
195    #[cfg(feature = "source-spanner")]
196    pub mod spanner {
197        pub use faucet_source_spanner::*;
198    }
199
200    #[cfg(feature = "source-parquet")]
201    pub mod parquet {
202        pub use faucet_source_parquet::*;
203    }
204
205    #[cfg(feature = "source-delta")]
206    pub mod delta {
207        pub use faucet_source_delta::*;
208    }
209
210    #[cfg(feature = "source-databricks")]
211    pub mod databricks {
212        pub use faucet_source_databricks::*;
213    }
214
215    #[cfg(feature = "source-gcs")]
216    pub mod gcs {
217        pub use faucet_source_gcs::*;
218    }
219
220    #[cfg(feature = "source-singer")]
221    pub mod singer {
222        pub use faucet_source_singer::*;
223    }
224}
225
226// Source modules available without source-rest (when only other sources are enabled).
227#[cfg(not(feature = "source-rest"))]
228pub mod source {
229    #[cfg(feature = "source-graphql")]
230    pub mod graphql {
231        pub use faucet_source_graphql::*;
232    }
233
234    #[cfg(feature = "source-xml")]
235    pub mod xml {
236        pub use faucet_source_xml::*;
237    }
238
239    #[cfg(feature = "source-grpc")]
240    pub mod grpc {
241        pub use faucet_source_grpc::*;
242    }
243
244    #[cfg(feature = "source-postgres")]
245    pub mod postgres {
246        pub use faucet_source_postgres::*;
247    }
248
249    #[cfg(feature = "source-postgres-cdc")]
250    pub mod postgres_cdc {
251        pub use faucet_source_postgres_cdc::*;
252    }
253
254    #[cfg(feature = "source-mysql")]
255    pub mod mysql {
256        pub use faucet_source_mysql::*;
257    }
258
259    #[cfg(feature = "source-mssql")]
260    pub mod mssql {
261        pub use faucet_source_mssql::*;
262    }
263
264    #[cfg(feature = "source-sqlite")]
265    pub mod sqlite {
266        pub use faucet_source_sqlite::*;
267    }
268
269    #[cfg(feature = "source-s3")]
270    pub mod s3 {
271        pub use faucet_source_s3::*;
272    }
273
274    #[cfg(feature = "source-mongodb")]
275    pub mod mongodb {
276        pub use faucet_source_mongodb::*;
277    }
278
279    #[cfg(feature = "source-mongodb-cdc")]
280    pub mod mongodb_cdc {
281        pub use faucet_source_mongodb_cdc::*;
282    }
283
284    #[cfg(feature = "source-mysql-cdc")]
285    pub mod mysql_cdc {
286        pub use faucet_source_mysql_cdc::*;
287    }
288
289    #[cfg(feature = "source-redis")]
290    pub mod redis {
291        pub use faucet_source_redis::*;
292    }
293
294    #[cfg(feature = "source-webhook")]
295    pub mod webhook {
296        pub use faucet_source_webhook::*;
297    }
298
299    #[cfg(feature = "source-websocket")]
300    pub mod websocket {
301        pub use faucet_source_websocket::*;
302    }
303
304    #[cfg(feature = "source-csv")]
305    pub mod csv {
306        pub use faucet_source_csv::*;
307    }
308
309    #[cfg(feature = "source-elasticsearch")]
310    pub mod elasticsearch {
311        pub use faucet_source_elasticsearch::*;
312    }
313
314    #[cfg(feature = "source-kafka")]
315    pub mod kafka {
316        pub use faucet_source_kafka::*;
317    }
318
319    #[cfg(feature = "source-kinesis")]
320    pub mod kinesis {
321        pub use faucet_source_kinesis::*;
322    }
323
324    #[cfg(feature = "source-bigquery")]
325    pub mod bigquery {
326        pub use faucet_source_bigquery::*;
327    }
328
329    #[cfg(feature = "source-snowflake")]
330    pub mod snowflake {
331        pub use faucet_source_snowflake::*;
332    }
333
334    #[cfg(feature = "source-spanner")]
335    pub mod spanner {
336        pub use faucet_source_spanner::*;
337    }
338
339    #[cfg(feature = "source-parquet")]
340    pub mod parquet {
341        pub use faucet_source_parquet::*;
342    }
343
344    #[cfg(feature = "source-delta")]
345    pub mod delta {
346        pub use faucet_source_delta::*;
347    }
348
349    #[cfg(feature = "source-databricks")]
350    pub mod databricks {
351        pub use faucet_source_databricks::*;
352    }
353
354    #[cfg(feature = "source-gcs")]
355    pub mod gcs {
356        pub use faucet_source_gcs::*;
357    }
358
359    #[cfg(feature = "source-singer")]
360    pub mod singer {
361        pub use faucet_source_singer::*;
362    }
363}
364
365// Backwards-compatible flat re-exports for existing users who depend on
366// `faucet-stream::{RestStream, Auth, ...}` without the `source::rest::` path.
367#[cfg(feature = "source-rest")]
368pub use faucet_source_rest::{
369    Auth, DEFAULT_EXPIRY_RATIO, DEFAULT_TOKEN_ENDPOINT_EXPIRY_RATIO, PaginationStyle,
370    ResponseValidator, RestStream, RestStreamConfig, fetch_oauth2_token, fetch_token_from_endpoint,
371};
372
373// ── Sink connectors ──────────────────────────────────────────────────────────
374
375pub mod sink {
376    #[cfg(feature = "sink-bigquery")]
377    pub mod bigquery {
378        pub use faucet_sink_bigquery::*;
379    }
380
381    #[cfg(feature = "sink-iceberg")]
382    pub mod iceberg {
383        pub use faucet_sink_iceberg::*;
384    }
385
386    #[cfg(feature = "sink-postgres")]
387    pub mod postgres {
388        pub use faucet_sink_postgres::*;
389    }
390
391    #[cfg(feature = "sink-jsonl")]
392    pub mod jsonl {
393        pub use faucet_sink_jsonl::*;
394    }
395
396    #[cfg(feature = "sink-snowflake")]
397    pub mod snowflake {
398        pub use faucet_sink_snowflake::*;
399    }
400
401    #[cfg(feature = "sink-mysql")]
402    pub mod mysql {
403        pub use faucet_sink_mysql::*;
404    }
405
406    #[cfg(feature = "sink-mssql")]
407    pub mod mssql {
408        pub use faucet_sink_mssql::*;
409    }
410
411    #[cfg(feature = "sink-sqlite")]
412    pub mod sqlite {
413        pub use faucet_sink_sqlite::*;
414    }
415
416    #[cfg(feature = "sink-s3")]
417    pub mod s3 {
418        pub use faucet_sink_s3::*;
419    }
420
421    #[cfg(feature = "sink-mongodb")]
422    pub mod mongodb {
423        pub use faucet_sink_mongodb::*;
424    }
425
426    #[cfg(feature = "sink-redis")]
427    pub mod redis {
428        pub use faucet_sink_redis::*;
429    }
430
431    #[cfg(feature = "sink-csv")]
432    pub mod csv {
433        pub use faucet_sink_csv::*;
434    }
435
436    #[cfg(feature = "sink-elasticsearch")]
437    pub mod elasticsearch {
438        pub use faucet_sink_elasticsearch::*;
439    }
440
441    #[cfg(feature = "sink-http")]
442    pub mod http {
443        pub use faucet_sink_http::*;
444    }
445
446    #[cfg(feature = "sink-stdout")]
447    pub mod stdout {
448        pub use faucet_sink_stdout::*;
449    }
450
451    #[cfg(feature = "sink-kafka")]
452    pub mod kafka {
453        pub use faucet_sink_kafka::*;
454    }
455
456    #[cfg(feature = "sink-kinesis")]
457    pub mod kinesis {
458        pub use faucet_sink_kinesis::*;
459    }
460
461    #[cfg(feature = "sink-spanner")]
462    pub mod spanner {
463        pub use faucet_sink_spanner::*;
464    }
465
466    #[cfg(feature = "sink-parquet")]
467    pub mod parquet {
468        pub use faucet_sink_parquet::*;
469    }
470
471    #[cfg(feature = "sink-delta")]
472    pub mod delta {
473        pub use faucet_sink_delta::*;
474    }
475
476    #[cfg(feature = "sink-gcs")]
477    pub mod gcs {
478        pub use faucet_sink_gcs::*;
479    }
480}
481
482// ── GCS common types ─────────────────────────────────────────────────────────
483
484#[cfg(any(feature = "source-gcs", feature = "sink-gcs"))]
485pub mod common_gcs {
486    pub use faucet_common_gcs::*;
487}
488
489// ── Kafka common types ───────────────────────────────────────────────────────
490
491#[cfg(any(feature = "source-kafka", feature = "sink-kafka"))]
492pub mod common_kafka {
493    pub use faucet_common_kafka::*;
494}
495
496/// Shared AWS Kinesis types (credentials enum, client builder), re-exported
497/// for library callers when either Kinesis connector is enabled.
498#[cfg(any(feature = "source-kinesis", feature = "sink-kinesis"))]
499pub mod common_kinesis {
500    pub use faucet_common_kinesis::*;
501}
502
503/// Shared Cloud Spanner types (credentials enum, connection block, value
504/// conversion), re-exported for library callers when either Spanner
505/// connector is enabled.
506#[cfg(any(feature = "source-spanner", feature = "sink-spanner"))]
507pub mod common_spanner {
508    pub use faucet_common_spanner::*;
509}
510
511// ── State-store backends ─────────────────────────────────────────────────────
512
513pub mod state {
514    #[cfg(feature = "state-redis")]
515    pub mod redis {
516        pub use faucet_state_redis::*;
517    }
518
519    #[cfg(feature = "state-postgres")]
520    pub mod postgres {
521        pub use faucet_state_postgres::*;
522    }
523}
524
525// ── Lineage (OpenLineage emission) ───────────────────────────────────────────
526/// OpenLineage event emission for pipeline runs (enable the `lineage` feature;
527/// `lineage-kafka` adds the Kafka transport). The CLI wires this automatically
528/// from a `lineage:` config block; library callers can build a
529/// [`lineage::LineageEmitter`] directly.
530#[cfg(feature = "lineage")]
531pub use faucet_lineage as lineage;
532
533// ── SQL transform (embedded DuckDB) ──────────────────────────────────────────
534/// SQL-as-transform: run DuckDB SQL over each pipeline page (the `batch`
535/// relation). Enable the `transform-sql` feature. The CLI wires this via the
536/// `sql` transform; library callers build [`transform_sql::SqlTransform`] and
537/// attach it with [`TransformingSource`].
538#[cfg(feature = "transform-sql")]
539pub use faucet_transform_sql as transform_sql;
faucet_stream/lib.rs

faucet_stream/
lib.rs
