faucet-cli 1.3.0

Config-driven CLI runner for faucet-stream pipelines (YAML / JSON, Meltano-style)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

# YAML equivalent of `faucet-stream/examples/mongodb_to_postgres.rs`.
# MongoDB (filter, projection, sort, limit) → PostgreSQL JSONB sink.
version: 1
name: mongodb_to_postgres

pipeline:
  source:
    type: mongodb
    config:
      connection_uri: mongodb://localhost:27017
      database: shop
      collection: orders
      filter:
        status: completed
      projection:
        _id: 1
        customer_id: 1
        total: 1
        items: 1
      sort:
        created_at: 1
      limit: 500000
      cursor_batch_size: 1000

  sink:
    type: postgres
    config:
      connection_url: postgres://user:pass@localhost/warehouse
      table_name: orders_mirror
      column_mapping:
        type: jsonb
        column: payload
      batch_size: 1000
      max_connections: 10