faucet-cli 1.2.0

Config-driven CLI runner for faucet-stream pipelines (YAML / JSON, Meltano-style)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

# YAML equivalent of `faucet-stream/examples/graphql_to_bigquery.rs`.
# GraphQL (Relay cursor pagination, custom-header auth) → BigQuery with
# ApplicationDefault credentials.
version: 1
name: graphql_to_bigquery

pipeline:
  source:
    type: graphql
    config:
      endpoint: https://api.example.com/graphql
      query: |
        query Orders($after: String, $first: Int!) {
          orders(first: $first, after: $after) {
            edges { node { id total status createdAt } }
            pageInfo { endCursor hasNextPage }
          }
        }
      variables:
        first: 200
      auth:
        type: custom
        config:
          headers:
            X-Api-Key: ${env:API_KEY}
            X-Tenant: acme
      records_path: $.data.orders.edges[*].node
      pagination:
        has_next_page_path: $.data.orders.pageInfo.hasNextPage
        cursor_path: $.data.orders.pageInfo.endCursor
        cursor_variable: after
        page_size_variable: first
      max_pages: 500
      batch_size: 200

  sink:
    type: bigquery
    config:
      project_id: my-gcp-project
      dataset_id: raw
      table_id: orders
      auth:
        type: application_default
      batch_size: 1000