faucet-stream 1.4.0

A declarative, config-driven data pipeline with pluggable source and sink connectors
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

//! AWS S3 → Google BigQuery — full builder showcase for both connectors.
//!
//! S3 source uses prefix scoping, region, JsonLines format, and parallel
//! reads. BigQuery sink shows the inline-credential variant and batch
//! sizing.
//!
//! Run:
//! ```bash
//! cargo run -p faucet-stream --example s3_to_bigquery \
//!     --features "source-s3 sink-bigquery"
//! ```

use faucet_stream::Pipeline;
use faucet_stream::sink::bigquery::{BigQueryCredentials, BigQuerySink, BigQuerySinkConfig};
use faucet_stream::source::s3::{S3FileFormat, S3Source, S3SourceConfig};

#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
    let source = S3Source::new(
        S3SourceConfig::new("my-data-lake")
            .prefix("raw/events/")
            .region("us-east-1")
            .file_format(S3FileFormat::JsonLines)
            .max_objects(usize::MAX)
            .concurrency(16),
    )
    .await?;

    let sink = BigQuerySink::new(
        BigQuerySinkConfig::new(
            "my-gcp-project",
            "raw",
            "events",
            BigQueryCredentials::ServiceAccountKey {
                json: std::env::var("GCP_KEY_JSON")?,
            },
        )
        .with_batch_size(2000),
    )
    .await?;

    let result = Pipeline::new(&source, &sink).run().await?;
    println!(
        "loaded {} records from S3 into BigQuery",
        result.records_written
    );
    Ok(())
}