faucet-cli 1.12.0

Config-driven CLI runner for faucet-stream pipelines (YAML / JSON, Meltano-style)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

# YAML equivalent of `faucet-stream/examples/s3_to_mongodb.rs`.
# S3 (JSON array files) → MongoDB.
version: 1
name: s3_to_mongodb

pipeline:
  source:
    type: s3
    config:
      bucket: my-data-lake
      prefix: snapshots/users/
      region: us-west-2
      file_format: json_array
      max_objects: 500
      concurrency: 8

  sink:
    type: mongodb
    config:
      connection_uri: mongodb://localhost:27017
      database: warehouse
      collection: users
      batch_size: 2000