faucet-cli 1.4.0

Config-driven CLI runner for faucet-stream pipelines (YAML / JSON, Meltano-style)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

# YAML equivalent of `faucet-stream/examples/xml_to_mongodb.rs`.
# XML feed (Bearer auth, offset pagination) → MongoDB.
version: 1
name: xml_to_mongodb

pipeline:
  source:
    type: xml
    config:
      base_url: https://feeds.example.com
      path: /catalog
      method: GET
      auth:
        type: bearer
        config:
          token: ${env:FEED_TOKEN}
      query_params:
        format: xml
      records_element_path: catalog.products.product
      pagination:
        type: Offset
        offset_param: offset
        limit_param: limit
        limit: 250

  sink:
    type: mongodb
    config:
      connection_uri: mongodb://localhost:27017
      database: warehouse
      collection: catalog_items
      batch_size: 1000