1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
# Airtable → JSONL on disk.
#
# Airtable's REST API is a plain bearer-authenticated, offset-token-paginated
# JSON API, so faucet's generic `rest` source covers it end-to-end — no
# dedicated connector crate is needed (issue #414).
#
# How it maps onto the `rest` source:
# - Bearer / personal-access-token auth → auth: { type: bearer }
# - Records live under the `records` array → records_path: "$.records"
# - The next-page token is `offset` in the body, sent back as the `offset`
# query param → pagination: { type: Cursor, next_token_path: offset,
# param_name: offset }. Pagination stops when `offset` is absent.
# - Airtable's 5 req/s per-base rate limit returns HTTP 429 with Retry-After;
# the rest source's built-in retry (max_retries / retry_backoff) honours it.
#
# Required env vars:
# AIRTABLE_TOKEN — a personal access token with data.records:read scope
# AIRTABLE_BASE_ID — the base id, e.g. appXXXXXXXXXXXXXX
#
# faucet run cli/examples/airtable_to_jsonl.yaml
version: 1
name: airtable_to_jsonl
vars:
base_id: ${env:AIRTABLE_BASE_ID}
table: Contacts
pipeline:
source:
type: rest
config:
base_url: https://api.airtable.com
path: /v0/${vars.base_id}/${vars.table}
method: GET
auth:
type: bearer
config:
token: ${env:AIRTABLE_TOKEN}
query_params:
pageSize: "100"
# Optional: restrict to a view, or push a formula filter server-side.
# view: "Grid view"
# filterByFormula: "NOT({Status} = 'Archived')"
records_path: "$.records"
pagination:
type: Cursor
next_token_path: offset
param_name: offset
# Airtable rate-limits at 5 req/s per base (HTTP 429 + Retry-After).
max_retries: 5
retry_backoff: 1
tolerated_http_errors: []
replication_method:
type: FullTable
primary_keys: ["id"]
partitions: []
schema_sample_size: 100
# Each Airtable record is { id, createdTime, fields: {...} }. Flatten collapses
# the nested `fields` object into dotted top-level keys (fields.Name, …) for a
# flatter downstream shape.
transforms:
- type: flatten
sink:
type: jsonl
config:
path: ./out/airtable_contacts.jsonl