faucet-cli 1.10.0

Config-driven CLI runner for faucet-stream pipelines (YAML / JSON, Meltano-style)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

# Amazon Redshift -> JSONL.
#
# Runs a SQL query over the Redshift PostgreSQL wire protocol and writes the
# rows to a local JSON Lines file. Add a `replication:` block to track a
# monotonically increasing column across runs (incremental replication).
#
# Run:
#   faucet run cli/examples/redshift_to_jsonl.yaml
#
# Prereqs:
#   - A reachable Redshift cluster with an `events` table.
version: 1
name: redshift_to_jsonl

pipeline:
  source:
    type: redshift
    config:
      host: my-cluster.abc123.us-east-1.redshift.amazonaws.com
      database: dev
      user: admin
      credentials:
        type: password
        config:
          # Prefer ${env:...} / ${vault:...} in real configs; literal here so the
          # shipped example validates offline.
          password: changeme
      query: SELECT id, email, updated_at FROM events ORDER BY updated_at
      batch_size: 1000

  sink:
    type: jsonl
    config:
      path: ./out/events.jsonl