1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
# Discovery-driven request matrix (#501): fan a stream out over the CROSS
# PRODUCT of value-sets enumerated from live discovery calls at run time —
# "sync this report once per {subsidiary} × {custom-field}".
#
# faucet validate cli/examples/discovery_matrix.yaml
#
# A `discover:` row runs its source once, projects `select` from each record,
# dedups, and publishes the value-set under `as`. A `for_each: [dims]` row then
# runs once per tuple of the cartesian product, with `${dim.alias}` substituted
# into the source URL and sink path. Discovery + fan-out rows share one
# `pipeline.sources` template (the `ref` path) and override only the deltas.
version: 1
name: reports_by_subsidiary_and_field
pipeline:
# One complete REST template both the discovery calls and the report fan-out
# reference — same auth / base_url, different path per row.
sources:
api:
type: rest
config:
method: GET
base_url: https://api.example.com
path: /
auth:
type: bearer
config:
token: ${env:API_TOKEN}
query_params: {}
pagination: { type: None }
replication_method: { type: FullTable }
sink:
type: jsonl
config:
path: ./out/report-${subsidiaries.subsidiary_id}-${fields.field_id}.jsonl
matrix:
# ── Discovery dimensions (run once each, deduped, cached) ──
- id: subsidiaries
discover:
source:
ref: api
config: { path: /subsidiaries, records_path: "$.subsidiaries[*]" }
select: "$.id"
as: subsidiary_id
- id: fields
discover:
source:
ref: api
config: { path: /custom-fields, records_path: "$.fields[*]" }
select: "$.id"
as: field_id
# ── Fan-out: one invocation per (subsidiary × field) tuple ──
- id: trial_balance
for_each: [subsidiaries, fields]
source:
ref: api
config:
path: /reports/trial-balance
records_path: "$.rows[*]"
query_params:
subsidiary_id: "${subsidiaries.subsidiary_id}"
field_id: "${fields.field_id}"