1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
# Chained / two-level discovery + collected list dimensions (#531).
#
# Full CRM custom-object sync, discovered entirely at runtime:
# 1. `types` — discover every object type (standard + custom).
# 2. `props` — for EACH type, discover its property names and COLLECT them
# into one list per type (`collect: true`).
# 3. `records` — for each type, pull its records with the whole property list
# in a single request (`?properties=a,b,c`) — many CRM APIs return
# only default properties unless you enumerate them.
#
# `${types.name}` is the per-tuple type; `${props.name}` renders the collected
# list comma-joined. No bespoke per-type YAML.
#
# faucet validate cli/examples/discovered_custom_objects.yaml
version: 1
name: discovered_custom_objects
pipeline:
sources:
type_schemas:
type: rest
config:
base_url: "https://api.example.com"
path: /crm/v3/schemas
auth: { type: bearer, config: { token: "${env:AUTH_TOKEN}" } }
records_path: "$.results[*]"
type_properties:
type: rest
config:
base_url: "https://api.example.com"
auth: { type: bearer, config: { token: "${env:AUTH_TOKEN}" } }
records_path: "$.results[*]"
type_objects:
type: rest
config:
base_url: "https://api.example.com"
auth: { type: bearer, config: { token: "${env:AUTH_TOKEN}" } }
records_path: "$.results[*]"
sink:
type: file
config: { path: ./out/objects.jsonl }
matrix:
# 1. Enumerate object types.
- id: types
fan_out:
source: { ref: type_schemas }
select: "$.name"
as: name
# 2. Chained discovery: one list of property names PER type (collected).
- id: props
for_each: [types]
fan_out:
source:
ref: type_properties
config: { path: "/crm/v3/properties/${types.name}" }
select: "$.name"
as: name
collect: true
# 3. Data stream: per type, pull records with the whole property list at once.
- id: records
for_each: [types]
source:
ref: type_objects
config:
path: "/crm/v3/objects/${types.name}"
query_params:
properties: "${props.name}"
# One file per object type: every type writes on its own invocation.
sink:
config: { path: "./out/objects-${types.name}.jsonl" }