1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
# YAML equivalent of `faucet-stream/examples/dag_users_posts.rs`.
#
# Demonstrates the matrix DAG syntax: a `users` root row produces records,
# and the `posts` child row runs once per user, interpolating `${users.id}`
# into the child's source path. The child's state key is suffixed with the
# parent record's `id` so each per-user fetch resumes independently.
version: 1
name: dag_users_posts
pipeline:
source:
type: rest
config:
base_url: https://api.example.com
method: GET
auth:
type: bearer
config:
token: ${env:API_TOKEN}
headers:
records_path: $.data[*]
pagination:
type: PageNumber
param_name: page
start_page: 1
page_size: 100
page_size_param: per_page
max_pages: 20
timeout: 30
max_retries: 3
retry_backoff: 1
tolerated_http_errors:
replication_method:
type: FullTable
primary_keys:
partitions:
schema_sample_size: 100
sink:
type: jsonl
config:
append: false
pretty: false
matrix:
# Root row: fetch the users list once.
- id: users
source:
config:
path: /v1/users
name: users
query_params:
active: "true"
sink:
config:
path: users.jsonl
# Child row: for each user record produced above, fetch that user's
# posts. `${users.id}` is resolved per parent record at runtime.
- id: posts
parent: users
parent_key: id
source:
config:
path: /v1/users/${users.id}/posts
name: posts
sink:
config:
path: posts-${users.id}.jsonl