faucet-cli 1.3.0

Config-driven CLI runner for faucet-stream pipelines (YAML / JSON, Meltano-style)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

# YAML equivalent of `faucet-stream/examples/xml_to_s3.rs`.
# SOAP-style POST XML → S3 (sharded JSONL with parallel uploads).
#
# Limitation: XmlStreamConfig::headers is `#[serde(skip)]`, so the custom
# Content-Type and SOAPAction headers from the Rust example can't be set
# from YAML today. Use the Rust API when SOAP headers are required.
version: 1
name: xml_to_s3

pipeline:
  source:
    type: xml
    config:
      base_url: https://soap.example.com
      path: /InventoryService
      method: POST
      auth:
        type: basic
        config:
          username: ${env:SOAP_USER}
          password: ${env:SOAP_PASS}
      body: |
        <?xml version="1.0"?>
        <soap:Envelope xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/">
          <soap:Body>
            <GetInventory><Region>us-east</Region></GetInventory>
          </soap:Body>
        </soap:Envelope>
      query_params:
        region: us-east
      records_element_path: soap:Envelope.soap:Body.GetInventoryResponse.Items.Item
      pagination:
        type: PageNumber
        param_name: page
        start_page: 1
        page_size: 500
        page_size_param: size
      max_pages: 50

  sink:
    type: s3
    config:
      bucket: my-data-lake
      prefix: xml/inventory/
      region: us-east-1
      endpoint_url: https://s3.us-east-1.amazonaws.com
      file_extension: .jsonl
      max_records_per_file: 5000
      concurrency: 8