faucet-cli 1.0.0

Config-driven CLI runner for faucet-stream pipelines (YAML / JSON, Meltano-style)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68

# Demonstrates the three-layer transforms model: a source template carries
# its own input-shape cleanup, the pipeline-level block stamps every row
# with provenance metadata, and matrix rows can append further shaping.
#
#   faucet run cli/examples/rest_to_stdout_transforms.yaml | jq .
#
version: 1
name: rest_transforms_demo

pipeline:
  sources:
    users:
      type: rest
      config:
        base_url: https://jsonplaceholder.typicode.com
        path: /users
        method: GET
        auth: { type: none }
        query_params: {}
        pagination: { type: None }
        max_retries: 3
        retry_backoff: 1
        tolerated_http_errors: []
        replication_method: { type: FullTable }
        primary_keys: ["id"]
        partitions: []
        schema_sample_size: 0
      transforms:
        # Source-shape cleanup: REST's nested address/company objects flatten
        # first, then column names normalise to snake_case.
        - type: flatten
          config: { separator: "__" }
        - type: keys_case
          config: { mode: snake }

  sinks:
    out:
      type: stdout
      config:
        destination: stdout
        format: json_lines
        flush_per_record: true
        max_records: 10

  transforms:
    # Pipeline-wide policy: stamp provenance on every record from every row.
    - type: set
      config:
        values:
          _source: jsonplaceholder
          _ingested_at: "2026-01-01T00:00:00Z"

matrix:
  - id: clean
    source: { ref: users }
    sink: { ref: out }
    # Final transforms = [set, flatten, keys_case]
    # (pipeline first, then source-template)

  - id: clean_pii_redacted
    source: { ref: users }
    sink: { ref: out }
    transforms:
      - type: redact
        config:
          fields: [phone]
          mask: "[redacted]"
    # Final transforms = [set, flatten, keys_case, redact]