1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
# Chained / two-level discovery + collected list dimensions (#531).
#
# Full HubSpot custom-object sync, discovered entirely at runtime:
# 1. `types` — discover every object type (standard + custom).
# 2. `props` — for EACH type, discover its property names and COLLECT them
# into one list per type (`collect: true`).
# 3. `records` — for each type, pull its records with the whole property list
# in a single request (`?properties=a,b,c`) — HubSpot returns
# only default properties unless you enumerate them.
#
# `${types.name}` is the per-tuple type; `${props.name}` renders the collected
# list comma-joined. No bespoke per-type YAML.
#
# faucet validate cli/examples/hubspot_custom_objects.yaml
version: 1
name: hubspot_custom_objects
pipeline:
sources:
hs_schemas:
type: rest
config:
base_url: "https://api.hubapi.com"
path: /crm/v3/schemas
auth: { type: bearer, config: { token: "${env:AUTH_TOKEN}" } }
records_path: "$.results[*]"
hs_properties:
type: rest
config:
base_url: "https://api.hubapi.com"
auth: { type: bearer, config: { token: "${env:AUTH_TOKEN}" } }
records_path: "$.results[*]"
hs_objects:
type: rest
config:
base_url: "https://api.hubapi.com"
auth: { type: bearer, config: { token: "${env:AUTH_TOKEN}" } }
records_path: "$.results[*]"
sink:
type: jsonl
config: { path: ./out/hubspot.jsonl }
matrix:
# 1. Enumerate object types.
- id: types
discover:
source: { ref: hs_schemas }
select: "$.name"
as: name
# 2. Chained discovery: one list of property names PER type (collected).
- id: props
for_each: [types]
discover:
source:
ref: hs_properties
config: { path: "/crm/v3/properties/${types.name}" }
select: "$.name"
as: name
collect: true
# 3. Data stream: per type, pull records with the whole property list at once.
- id: records
for_each: [types]
source:
ref: hs_objects
config:
path: "/crm/v3/objects/${types.name}"
query_params:
properties: "${props.name}"