1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
# Demonstrates the three-layer transforms model: a source template carries
# its own input-shape cleanup, the pipeline-level block stamps every row
# with provenance metadata, and matrix rows can append further shaping.
#
# faucet run cli/examples/rest_to_stdout_transforms.yaml | jq .
#
version: 1
name: rest_transforms_demo
pipeline:
sources:
users:
type: rest
config:
base_url: https://jsonplaceholder.typicode.com
path: /users
method: GET
auth: { type: none }
query_params: {}
pagination: { type: None }
max_retries: 3
retry_backoff: 1
tolerated_http_errors: []
replication_method: { type: FullTable }
primary_keys: ["id"]
partitions: []
schema_sample_size: 0
transforms:
# Source-shape cleanup: REST's nested address/company objects flatten
# first, then column names normalise to snake_case.
- type: flatten
config: { separator: "__" }
- type: keys_case
config: { mode: snake }
sinks:
out:
type: stdout
config:
destination: stdout
format: json_lines
flush_per_record: true
max_records: 10
transforms:
# Pipeline-wide policy: stamp provenance on every record from every row.
- type: set
config:
values:
_source: jsonplaceholder
_ingested_at: "2026-01-01T00:00:00Z"
matrix:
- id: clean
source: { ref: users }
sink: { ref: out }
# Final transforms = [set, flatten, keys_case]
# (pipeline first, then source-template)
- id: clean_pii_redacted
source: { ref: users }
sink: { ref: out }
transforms:
- type: redact
config:
fields: [phone]
mask: "[redacted]"
# Final transforms = [set, flatten, keys_case, redact]