faucet-cli 1.12.0

Config-driven CLI runner for faucet-stream pipelines (YAML / JSON, Meltano-style)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

# CSV → JSONL with the Data Movement Catalog (#279): every run accumulates
# the datasets it touched (identity, schema timeline, volume/freshness) and
# the source→sink lineage edge into a local SQLite store. Browse it with:
#
#   faucet run cli/examples/csv_to_jsonl_with_catalog.yaml
#   faucet catalog datasets --config cli/examples/csv_to_jsonl_with_catalog.yaml
#   faucet catalog show <id> --config cli/examples/csv_to_jsonl_with_catalog.yaml
#   faucet catalog lineage  --config cli/examples/csv_to_jsonl_with_catalog.yaml
#
# Or point `faucet serve --history sqlite:./faucet-catalog.db` at the same
# file and open the web console's Datasets / Lineage views.
#
# Requires a build with the `catalog` feature (included in `--features full`)
# plus `serve-history-sqlite` for the SQLite store.
version: 1
name: csv_to_jsonl_with_catalog

catalog:
  url: sqlite:./faucet-catalog.db
  # How many records to sample per run for schema inference (default 100).
  sample_records: 100

pipeline:
  source:
    type: csv
    config:
      path: ./data/input.csv

  sink:
    type: jsonl
    config:
      path: ./out/records.jsonl