faucet_stream/
lib.rs

1#![cfg_attr(docsrs, feature(doc_cfg))]
2
3//! # faucet-stream
4//!
5//! A declarative, config-driven data pipeline with pluggable source and sink
6//! connectors.
7//!
8//! 📖 **Guide, tutorials & cookbook:** <https://pawansikawat.github.io/faucet-stream/>
9//!
10//! ## Feature flags
11//!
12//! | Feature | Description |
13//! |---------|-------------|
14//! | `source-rest` *(default)* | REST API source with pagination, auth, transforms |
15//! | `source-graphql` | GraphQL API source with cursor pagination |
16//! | `source-xml` | XML/SOAP API source with XML-to-JSON conversion |
17//! | `source-grpc` | gRPC source with dynamic protobuf messages |
18//! | `source-postgres` | PostgreSQL query source |
19//! | `source-postgres-cdc` | PostgreSQL CDC source (logical replication) |
20//! | `source-mysql` | MySQL query source |
21//! | `source-mssql` | Microsoft SQL Server query source |
22//! | `source-sqlite` | SQLite query source |
23
24//! | `source-s3` | AWS S3 file source |
25//! | `source-mongodb` | MongoDB query source |
26//! | `source-redis` | Redis source (streams, lists, keys) |
27//! | `source-webhook` | Webhook HTTP receiver source |
28//! | `source-websocket` | WebSocket streaming source |
29//! | `source-csv` | CSV file source |
30//! | `source-elasticsearch` | Elasticsearch search/scroll source |
31//! | `source-kafka` | Apache Kafka consumer source |
32//! | `source-parquet` | Apache Parquet file source (local, glob, S3) |
33//! | `sink-bigquery` | Google BigQuery streaming insert sink |
34//! | `sink-postgres` | PostgreSQL sink (jsonb or auto-mapped columns) |
35//! | `sink-jsonl` | JSON Lines file sink |
36//! | `sink-snowflake` | Snowflake SQL REST API sink |
37//! | `sink-mysql` | MySQL sink |
38//! | `sink-mssql` | Microsoft SQL Server sink |
39//! | `sink-sqlite` | SQLite sink |
40
41//! | `sink-s3` | AWS S3 file sink |
42//! | `sink-mongodb` | MongoDB insert sink |
43//! | `sink-redis` | Redis sink (streams, lists, key-value) |
44//! | `sink-csv` | CSV file sink |
45//! | `sink-elasticsearch` | Elasticsearch bulk index sink |
46//! | `sink-http` | HTTP POST sink |
47//! | `sink-kafka` | Apache Kafka producer sink |
48//! | `sink-parquet` | Apache Parquet file sink (local, S3) |
49//! | `kafka-schema-registry` | Schema Registry support for Kafka connectors |
50//! | `source` | All source connectors |
51//! | `sink` | All sink connectors |
52//! | `full` | Every connector |
53
54// Always re-export core types and traits.
55pub use faucet_core::*;
56
57// Explicit re-exports for the library-side transforms wrapper and observability
58// labels so users can import them via the umbrella path.
59pub use faucet_core::TransformingSource;
60pub use faucet_core::observability::Labels;
61
62// ── Shared auth providers ────────────────────────────────────────────────────
63/// Single-flight OAuth2 / token-endpoint auth providers (enable the `auth`
64/// feature). Share one across connectors via `with_auth_provider` or the CLI
65/// `auth: { ref }` catalog.
66#[cfg(feature = "auth")]
67pub mod auth {
68    pub use faucet_auth::*;
69}
70
71// ── Source connectors ────────────────────────────────────────────────────────
72
73#[cfg(feature = "source-rest")]
74pub mod source {
75    pub mod rest {
76        pub use faucet_source_rest::*;
77    }
78
79    #[cfg(feature = "source-graphql")]
80    pub mod graphql {
81        pub use faucet_source_graphql::*;
82    }
83
84    #[cfg(feature = "source-xml")]
85    pub mod xml {
86        pub use faucet_source_xml::*;
87    }
88
89    #[cfg(feature = "source-grpc")]
90    pub mod grpc {
91        pub use faucet_source_grpc::*;
92    }
93
94    #[cfg(feature = "source-postgres")]
95    pub mod postgres {
96        pub use faucet_source_postgres::*;
97    }
98
99    #[cfg(feature = "source-postgres-cdc")]
100    pub mod postgres_cdc {
101        pub use faucet_source_postgres_cdc::*;
102    }
103
104    #[cfg(feature = "source-mysql")]
105    pub mod mysql {
106        pub use faucet_source_mysql::*;
107    }
108
109    #[cfg(feature = "source-mssql")]
110    pub mod mssql {
111        pub use faucet_source_mssql::*;
112    }
113
114    #[cfg(feature = "source-sqlite")]
115    pub mod sqlite {
116        pub use faucet_source_sqlite::*;
117    }
118
119    #[cfg(feature = "source-s3")]
120    pub mod s3 {
121        pub use faucet_source_s3::*;
122    }
123
124    #[cfg(feature = "source-mongodb")]
125    pub mod mongodb {
126        pub use faucet_source_mongodb::*;
127    }
128
129    #[cfg(feature = "source-redis")]
130    pub mod redis {
131        pub use faucet_source_redis::*;
132    }
133
134    #[cfg(feature = "source-webhook")]
135    pub mod webhook {
136        pub use faucet_source_webhook::*;
137    }
138
139    #[cfg(feature = "source-websocket")]
140    pub mod websocket {
141        pub use faucet_source_websocket::*;
142    }
143
144    #[cfg(feature = "source-csv")]
145    pub mod csv {
146        pub use faucet_source_csv::*;
147    }
148
149    #[cfg(feature = "source-elasticsearch")]
150    pub mod elasticsearch {
151        pub use faucet_source_elasticsearch::*;
152    }
153
154    #[cfg(feature = "source-kafka")]
155    pub mod kafka {
156        pub use faucet_source_kafka::*;
157    }
158
159    #[cfg(feature = "source-parquet")]
160    pub mod parquet {
161        pub use faucet_source_parquet::*;
162    }
163
164    #[cfg(feature = "source-gcs")]
165    pub mod gcs {
166        pub use faucet_source_gcs::*;
167    }
168}
169
170// Source modules available without source-rest (when only other sources are enabled).
171#[cfg(not(feature = "source-rest"))]
172pub mod source {
173    #[cfg(feature = "source-graphql")]
174    pub mod graphql {
175        pub use faucet_source_graphql::*;
176    }
177
178    #[cfg(feature = "source-xml")]
179    pub mod xml {
180        pub use faucet_source_xml::*;
181    }
182
183    #[cfg(feature = "source-grpc")]
184    pub mod grpc {
185        pub use faucet_source_grpc::*;
186    }
187
188    #[cfg(feature = "source-postgres")]
189    pub mod postgres {
190        pub use faucet_source_postgres::*;
191    }
192
193    #[cfg(feature = "source-postgres-cdc")]
194    pub mod postgres_cdc {
195        pub use faucet_source_postgres_cdc::*;
196    }
197
198    #[cfg(feature = "source-mysql")]
199    pub mod mysql {
200        pub use faucet_source_mysql::*;
201    }
202
203    #[cfg(feature = "source-mssql")]
204    pub mod mssql {
205        pub use faucet_source_mssql::*;
206    }
207
208    #[cfg(feature = "source-sqlite")]
209    pub mod sqlite {
210        pub use faucet_source_sqlite::*;
211    }
212
213    #[cfg(feature = "source-s3")]
214    pub mod s3 {
215        pub use faucet_source_s3::*;
216    }
217
218    #[cfg(feature = "source-mongodb")]
219    pub mod mongodb {
220        pub use faucet_source_mongodb::*;
221    }
222
223    #[cfg(feature = "source-redis")]
224    pub mod redis {
225        pub use faucet_source_redis::*;
226    }
227
228    #[cfg(feature = "source-webhook")]
229    pub mod webhook {
230        pub use faucet_source_webhook::*;
231    }
232
233    #[cfg(feature = "source-websocket")]
234    pub mod websocket {
235        pub use faucet_source_websocket::*;
236    }
237
238    #[cfg(feature = "source-csv")]
239    pub mod csv {
240        pub use faucet_source_csv::*;
241    }
242
243    #[cfg(feature = "source-elasticsearch")]
244    pub mod elasticsearch {
245        pub use faucet_source_elasticsearch::*;
246    }
247
248    #[cfg(feature = "source-kafka")]
249    pub mod kafka {
250        pub use faucet_source_kafka::*;
251    }
252
253    #[cfg(feature = "source-parquet")]
254    pub mod parquet {
255        pub use faucet_source_parquet::*;
256    }
257
258    #[cfg(feature = "source-gcs")]
259    pub mod gcs {
260        pub use faucet_source_gcs::*;
261    }
262}
263
264// Backwards-compatible flat re-exports for existing users who depend on
265// `faucet-stream::{RestStream, Auth, ...}` without the `source::rest::` path.
266#[cfg(feature = "source-rest")]
267pub use faucet_source_rest::{
268    Auth, DEFAULT_EXPIRY_RATIO, DEFAULT_TOKEN_ENDPOINT_EXPIRY_RATIO, PaginationStyle,
269    ResponseValidator, RestStream, RestStreamConfig, fetch_oauth2_token, fetch_token_from_endpoint,
270};
271
272// ── Sink connectors ──────────────────────────────────────────────────────────
273
274pub mod sink {
275    #[cfg(feature = "sink-bigquery")]
276    pub mod bigquery {
277        pub use faucet_sink_bigquery::*;
278    }
279
280    #[cfg(feature = "sink-postgres")]
281    pub mod postgres {
282        pub use faucet_sink_postgres::*;
283    }
284
285    #[cfg(feature = "sink-jsonl")]
286    pub mod jsonl {
287        pub use faucet_sink_jsonl::*;
288    }
289
290    #[cfg(feature = "sink-snowflake")]
291    pub mod snowflake {
292        pub use faucet_sink_snowflake::*;
293    }
294
295    #[cfg(feature = "sink-mysql")]
296    pub mod mysql {
297        pub use faucet_sink_mysql::*;
298    }
299
300    #[cfg(feature = "sink-mssql")]
301    pub mod mssql {
302        pub use faucet_sink_mssql::*;
303    }
304
305    #[cfg(feature = "sink-sqlite")]
306    pub mod sqlite {
307        pub use faucet_sink_sqlite::*;
308    }
309
310    #[cfg(feature = "sink-s3")]
311    pub mod s3 {
312        pub use faucet_sink_s3::*;
313    }
314
315    #[cfg(feature = "sink-mongodb")]
316    pub mod mongodb {
317        pub use faucet_sink_mongodb::*;
318    }
319
320    #[cfg(feature = "sink-redis")]
321    pub mod redis {
322        pub use faucet_sink_redis::*;
323    }
324
325    #[cfg(feature = "sink-csv")]
326    pub mod csv {
327        pub use faucet_sink_csv::*;
328    }
329
330    #[cfg(feature = "sink-elasticsearch")]
331    pub mod elasticsearch {
332        pub use faucet_sink_elasticsearch::*;
333    }
334
335    #[cfg(feature = "sink-http")]
336    pub mod http {
337        pub use faucet_sink_http::*;
338    }
339
340    #[cfg(feature = "sink-stdout")]
341    pub mod stdout {
342        pub use faucet_sink_stdout::*;
343    }
344
345    #[cfg(feature = "sink-kafka")]
346    pub mod kafka {
347        pub use faucet_sink_kafka::*;
348    }
349
350    #[cfg(feature = "sink-parquet")]
351    pub mod parquet {
352        pub use faucet_sink_parquet::*;
353    }
354
355    #[cfg(feature = "sink-gcs")]
356    pub mod gcs {
357        pub use faucet_sink_gcs::*;
358    }
359}
360
361// ── GCS common types ─────────────────────────────────────────────────────────
362
363#[cfg(any(feature = "source-gcs", feature = "sink-gcs"))]
364pub mod common_gcs {
365    pub use faucet_common_gcs::*;
366}
367
368// ── Kafka common types ───────────────────────────────────────────────────────
369
370#[cfg(any(feature = "source-kafka", feature = "sink-kafka"))]
371pub mod common_kafka {
372    pub use faucet_common_kafka::*;
373}
374
375// ── State-store backends ─────────────────────────────────────────────────────
376
377pub mod state {
378    #[cfg(feature = "state-redis")]
379    pub mod redis {
380        pub use faucet_state_redis::*;
381    }
382
383    #[cfg(feature = "state-postgres")]
384    pub mod postgres {
385        pub use faucet_state_postgres::*;
386    }
387}
faucet_stream/lib.rs

faucet_stream/
lib.rs
