faucet-cli 1.5.0

[package]
name = "faucet-cli"
version = "1.5.0"
edition.workspace = true
rust-version.workspace = true
license.workspace = true
repository.workspace = true
homepage = "https://pawansikawat.github.io/faucet-stream/"

description = "Config-driven CLI runner for faucet-stream pipelines (YAML / JSON, Meltano-style)"
readme = "README.md"
keywords = ["pipeline", "etl", "cli", "meltano", "faucet"]
categories = ["command-line-utilities"]

[[bin]]
name = "faucet"
path = "src/main.rs"

[features]
# Default: every first-party source, sink, and state backend so `cargo install`
# yields a binary that can run any of the published example YAMLs out of the box.
# The two long-running runtime modes are intentionally excluded from this one-shot
# runner: `schedule` (cron scheduler; pulls a cron parser + IANA timezone db) and
# `serve` (HTTP control plane; links a web-server dependency tree and opens a
# network listener). Opt in with `--features schedule` / `--features serve`, or
# take everything with `--features full`.
default = ["observability", "source", "sink", "state", "transforms", "compression", "quality", "contract", "masking"]

# Everything the default build ships, plus the long-running runtime modes:
# `schedule` (cron scheduler) and `serve` (HTTP control plane).
full = ["default", "otel", "schedule", "serve", "serve-ui", "serve-history-postgres", "serve-history-sqlite", "catalog", "lineage", "lineage-kafka", "notify", "transform-sql", "triggers", "triggers-object-store", "triggers-redis", "triggers-kafka", "cli-dev", "encryption", "cli-tui", "delta-s3", "delta-azure", "delta-gcs"]

# Data Movement Catalog (#279): the `catalog:` config block + post-run
# recording (datasets, schema timelines, volume/freshness stats, lineage
# edges), the `/v1/catalog/*` serve endpoints + console views, and the
# `faucet catalog` verb. Implies `serve` (catalog storage rides the
# run-history backends — add `serve-history-sqlite` / `serve-history-postgres`
# for persistence) and `lineage` (record sampling + column-lineage
# derivation). Not in `default`; included in `full`.
catalog = ["serve", "lineage"]

# Data-quality checks (`quality:` config block). `quality-jsonschema` adds the
# `json_schema` record check via a JSON Schema validator. Forwarded to
# `faucet-core`.
quality = ["faucet-core/quality"]
quality-jsonschema = ["quality", "faucet-core/quality-jsonschema"]

# Data contracts (`contract:` config block + `faucet contract` verb, #204):
# a versioned output schema/constraint promise enforced per page. Forwarded
# to `faucet-core`.
contract = ["faucet-core/contract"]

# PII detection + column-level masking (`masking:` config block + `faucet
# masking` verb, #206): classify sensitive fields and redact/hash/tokenize/
# partial-mask them per page, before every sink. Forwarded to `faucet-core`.
masking = ["faucet-core/masking"]

# Encryption at rest (#207): the `encryption:` block on the `file` state store
# (AES-256-GCM bookmark files) and on the jsonl sink (per-line sealing — the
# file-DLQ-at-rest shape), plus decryption support in `faucet dlq
# inspect/replay/discard`. Opt-in; included in `full`.
encryption = ["faucet-core/encryption", "faucet-sink-jsonl?/encryption", "dep:base64"]

source = [
    "source-rest",
    "source-graphql",
    "source-xml",
    "source-grpc",
    "source-postgres",
    "source-postgres-cdc",
    "source-mysql",
    "source-mssql",
    "source-sqlite",
    "source-s3",
    "source-mongodb",
    "source-mongodb-cdc",
    "source-mysql-cdc",
    "source-redis",
    "source-webhook",
    "source-websocket",
    "source-csv",
    "source-elasticsearch",
    "source-kafka",
    "source-kinesis",
    "source-parquet",
    "source-delta",
    "source-databricks",
    "source-gcs",
    "source-bigquery",
    "source-snowflake",
    "source-spanner",
    "source-singer",
]
sink = [
    "sink-bigquery",
    "sink-postgres",
    "sink-jsonl",
    "sink-snowflake",
    "sink-mysql",
    "sink-mssql",
    "sink-sqlite",
    "sink-s3",
    "sink-mongodb",
    "sink-redis",
    "sink-csv",
    "sink-elasticsearch",
    "sink-http",
    "sink-stdout",
    "sink-kafka",
    "sink-kinesis",
    "sink-parquet",
    "sink-delta",
    "sink-gcs",
    "sink-iceberg",
    "sink-spanner",
]
state = ["state-redis", "state-postgres"]

source-rest = ["dep:faucet-source-rest"]
source-graphql = ["dep:faucet-source-graphql"]
source-xml = ["dep:faucet-source-xml"]
source-grpc = ["dep:faucet-source-grpc"]
source-postgres = ["dep:faucet-source-postgres"]
source-postgres-cdc = ["dep:faucet-source-postgres-cdc"]
source-mysql = ["dep:faucet-source-mysql"]
source-mssql = ["dep:faucet-source-mssql"]
source-sqlite = ["dep:faucet-source-sqlite"]
source-s3 = ["dep:faucet-source-s3"]
source-mongodb = ["dep:faucet-source-mongodb"]
source-mongodb-cdc = ["dep:faucet-source-mongodb-cdc"]
source-mysql-cdc = ["dep:faucet-source-mysql-cdc"]
source-redis = ["dep:faucet-source-redis"]
source-webhook = ["dep:faucet-source-webhook"]
source-websocket = ["dep:faucet-source-websocket"]
source-csv = ["dep:faucet-source-csv"]
source-elasticsearch = ["dep:faucet-source-elasticsearch"]
source-kafka = ["dep:faucet-source-kafka"]
source-kinesis = ["dep:faucet-source-kinesis"]
source-parquet = ["dep:faucet-source-parquet"]
source-delta = ["dep:faucet-source-delta"]
source-databricks = ["dep:faucet-source-databricks"]
source-gcs = ["dep:faucet-source-gcs"]
source-bigquery = ["dep:faucet-source-bigquery"]
source-snowflake = ["dep:faucet-source-snowflake"]
source-spanner = ["dep:faucet-source-spanner"]
source-singer = ["dep:faucet-source-singer"]

sink-bigquery = ["dep:faucet-sink-bigquery"]
sink-iceberg = ["dep:faucet-sink-iceberg"]
sink-postgres = ["dep:faucet-sink-postgres"]
sink-jsonl = ["dep:faucet-sink-jsonl"]
sink-snowflake = ["dep:faucet-sink-snowflake"]
sink-mysql = ["dep:faucet-sink-mysql"]
sink-mssql = ["dep:faucet-sink-mssql"]
sink-sqlite = ["dep:faucet-sink-sqlite"]
sink-s3 = ["dep:faucet-sink-s3"]
sink-mongodb = ["dep:faucet-sink-mongodb"]
sink-redis = ["dep:faucet-sink-redis"]
sink-csv = ["dep:faucet-sink-csv"]
sink-elasticsearch = ["dep:faucet-sink-elasticsearch"]
sink-http = ["dep:faucet-sink-http"]
sink-stdout = ["dep:faucet-sink-stdout"]
sink-kafka = ["dep:faucet-sink-kafka"]
sink-kinesis = ["dep:faucet-sink-kinesis"]
sink-spanner = ["dep:faucet-sink-spanner"]
sink-parquet = ["dep:faucet-sink-parquet"]
sink-delta = ["dep:faucet-sink-delta"]
sink-gcs = ["dep:faucet-sink-gcs"]

## Delta Lake cloud object-store backends. Forwarded to whichever delta
## connector is enabled via optional-dep (`?`) syntax, so they don't pull the
## connectors in themselves. Included in `full`; not in `default` (slim).
delta-s3 = ["faucet-source-delta?/s3", "faucet-sink-delta?/s3"]
delta-azure = ["faucet-source-delta?/azure", "faucet-sink-delta?/azure"]
delta-gcs = ["faucet-source-delta?/gcs", "faucet-sink-delta?/gcs"]

kafka-schema-registry = [
    "faucet-source-kafka?/schema-registry",
    "faucet-sink-kafka?/schema-registry",
]

## Enable gzip/zstd compression on every file-shaped connector that has been
## opted in (via its per-connector feature). Forwards through to each
## connector's `compression` feature using optional-dep syntax, so this
## flag does not pull connectors the user has not requested.
compression = [
    "faucet-source-csv?/compression",
    "faucet-source-s3?/compression",
    "faucet-source-gcs?/compression",
    "faucet-sink-jsonl?/compression",
    "faucet-sink-csv?/compression",
    "faucet-sink-s3?/compression",
    "faucet-sink-gcs?/compression",
]

# Secrets-manager interpolation resolvers for the config layer. None in
# defaults — opt in per backend, or `secrets` for all four.
secrets = ["secrets-vault", "secrets-aws-sm", "secrets-gcp-sm", "secrets-azure-kv"]
secrets-vault = ["dep:reqwest"]
secrets-aws-sm = ["dep:aws-sdk-secretsmanager", "dep:aws-config"]
secrets-gcp-sm = ["dep:reqwest", "dep:google-cloud-auth", "dep:base64"]
secrets-azure-kv = ["dep:azure_security_keyvault_secrets", "dep:azure_identity", "dep:azure_core"]

state-redis = ["dep:faucet-state-redis"]
state-postgres = ["dep:faucet-state-postgres"]

# `dep:reqwest` is needed by `lineage_glue::check_transport` (the validate /
# doctor HTTP reachability probe), so the feature-isolation CI build of
# `--features lineage` alone compiles.
lineage = ["dep:faucet-lineage", "dep:reqwest"]
lineage-kafka = ["lineage", "faucet-lineage/transport-kafka"]

# Notification / incident-routing layer (#280): the `notifications:` config
# block + `faucet notify test` + `faucet schema notifications`. Fans pipeline
# lifecycle / health events out to Slack / PagerDuty / a signed webhook.
# CLI-only; not in `default`, included in `full`. `dep:reqwest` for delivery,
# `dep:hmac` + `dep:sha2` for the optional webhook HMAC-SHA256 signature.
notify = ["dep:reqwest", "dep:hmac", "dep:sha2"]

# Enables the `faucet init --interactive` prompt UI (`dialoguer`-driven Select
# prompts for source / sink kind). Off by default so minimal builds don't pull
# in the TTY-prompt dep tree.
cli-interactive = ["dep:inquire"]
# `faucet dev` — filesystem-watch dev loop (`faucet plan` needs no feature and
# ships in default).
cli-dev = ["dep:notify-fs"]
# Live terminal UI for `faucet run --tui` (#203): full-screen ratatui view of
# per-row throughput/errors/DLQ/bookmark age, sampled from the in-process
# Prometheus recorder (implies `observability`). Opt-in; not in `default`.
cli-tui = ["observability", "dep:ratatui"]

transforms = [
    "faucet-core/transforms",
    "faucet-source-rest?/transforms",
    "transform-filter",
    "transform-explode",
    "transform-cdc-unwrap",
]

# Per-transform feature flags forwarded to faucet-core. Lets the CLI's
# `#[cfg(feature = "transform-X")]` gates compile in just the rows the user
# asked for. The `transforms` aggregate above enables every transform at once.
transform-filter = ["faucet-core/transform-filter"]
transform-explode = ["faucet-core/transform-explode"]
transform-cdc-unwrap = ["faucet-core/transform-cdc-unwrap"]
transform-sql = ["dep:faucet-transform-sql"]

observability = [
    "faucet-core/observability-install",
    "dep:metrics-exporter-prometheus",
    "dep:tracing-subscriber",
]

# OTLP (OpenTelemetry) export of traces + metrics (#201). Forwards to faucet-core.
otel = ["observability", "faucet-core/otel"]

# Built-in cron scheduler (`faucet schedule` + the `schedule:` config block).
# Pulls a cron parser + IANA timezone db; opt out of it for slim builds.
schedule = ["dep:croner"]

# HTTP control plane (`faucet serve`). Pulls axum + an in-process run registry.
# Enables `observability` (serve renders /metrics on its own port). Gates the
# tokio net/time features here (rather than the base dep) so slim non-serve
# builds don't link the TCP stack.
serve = ["observability", "tokio/net", "tokio/time", "dep:axum", "dep:tower-http", "dep:subtle", "dep:dashmap", "dep:sha2", "dep:async-stream"]
serve-history-postgres = ["serve", "dep:sqlx"]
serve-history-sqlite = ["serve", "dep:sqlx"]
# Embedded web console (single-page UI) served by `faucet serve` at `/`.
# Vanilla assets embedded via rust-embed; no JS build step. Implies `serve`.
serve-ui = ["serve", "dep:rust-embed"]

# Event-driven pipeline triggers for `faucet serve` (#196). Base = framework +
# webhook trigger (reuses serve's axum). Per-backend sub-features add the polling
# watchers + their client deps. Implies `serve`. CLI-only.
triggers = ["serve"]
triggers-object-store = ["triggers", "dep:object_store"]
triggers-redis = ["triggers", "dep:redis"]
triggers-kafka = ["triggers", "dep:rdkafka"]

[dependencies]
faucet-core.workspace = true
faucet-auth.workspace = true
faucet-source-rest = { workspace = true, optional = true }
faucet-source-graphql = { workspace = true, optional = true }
faucet-source-xml = { workspace = true, optional = true }
faucet-source-grpc = { workspace = true, optional = true }
faucet-source-postgres = { workspace = true, optional = true }
faucet-source-postgres-cdc = { workspace = true, optional = true }
faucet-source-mysql = { workspace = true, optional = true }
faucet-source-mssql = { workspace = true, optional = true }
faucet-source-sqlite = { workspace = true, optional = true }
faucet-source-s3 = { workspace = true, optional = true }
faucet-source-mongodb = { workspace = true, optional = true }
faucet-source-mongodb-cdc = { workspace = true, optional = true }
faucet-source-mysql-cdc = { workspace = true, optional = true }
faucet-source-redis = { workspace = true, optional = true }
faucet-source-webhook = { workspace = true, optional = true }
faucet-source-websocket = { workspace = true, optional = true }
faucet-source-csv = { workspace = true, optional = true }
faucet-source-elasticsearch = { workspace = true, optional = true }
faucet-source-kafka = { workspace = true, optional = true }
faucet-source-kinesis = { workspace = true, optional = true }
faucet-source-parquet = { workspace = true, optional = true }
faucet-source-delta = { workspace = true, optional = true }
faucet-source-databricks = { workspace = true, optional = true }
faucet-source-gcs = { workspace = true, optional = true }
faucet-source-bigquery = { workspace = true, optional = true }
faucet-source-snowflake = { workspace = true, optional = true }
faucet-source-spanner = { workspace = true, optional = true }
faucet-source-singer = { workspace = true, optional = true }
faucet-sink-bigquery = { workspace = true, optional = true }
faucet-sink-iceberg = { workspace = true, optional = true }
faucet-sink-postgres = { workspace = true, optional = true }
faucet-sink-jsonl = { workspace = true, optional = true }
faucet-sink-snowflake = { workspace = true, optional = true }
faucet-sink-mysql = { workspace = true, optional = true }
faucet-sink-mssql = { workspace = true, optional = true }
faucet-sink-sqlite = { workspace = true, optional = true }
faucet-sink-s3 = { workspace = true, optional = true }
faucet-sink-mongodb = { workspace = true, optional = true }
faucet-sink-redis = { workspace = true, optional = true }
faucet-sink-csv = { workspace = true, optional = true }
faucet-sink-elasticsearch = { workspace = true, optional = true }
faucet-sink-http = { workspace = true, optional = true }
faucet-sink-kafka = { workspace = true, optional = true }
faucet-sink-kinesis = { workspace = true, optional = true }
faucet-sink-spanner = { workspace = true, optional = true }
faucet-sink-stdout = { workspace = true, optional = true }
faucet-sink-parquet = { workspace = true, optional = true }
faucet-sink-delta = { workspace = true, optional = true }
faucet-sink-gcs = { workspace = true, optional = true }
faucet-state-redis = { workspace = true, optional = true }
faucet-state-postgres = { workspace = true, optional = true }
faucet-lineage = { workspace = true, optional = true }
faucet-transform-sql = { workspace = true, optional = true }

clap = { version = "4", features = ["derive", "env"] }
dotenvy = "0.15"
schemars.workspace = true
serde.workspace = true
serde_json.workspace = true
serde_yaml = "0.9"
thiserror.workspace = true
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "signal", "fs", "sync", "time"] }
tracing.workspace = true
tracing-subscriber = { workspace = true, optional = true }
async-trait.workspace = true
metrics.workspace = true
metrics-exporter-prometheus = { workspace = true, optional = true }
uuid.workspace = true
inquire = { version = "0.9", optional = true, default-features = false, features = ["crossterm"] }
# `faucet dev` filesystem watch (#283). Renamed key so it doesn't collide with
# the `notify` *feature* (the notification/incident-routing layer).
notify-fs = { package = "notify", version = "8", optional = true }
ratatui = { workspace = true, optional = true }
futures = { workspace = true }
glob = { workspace = true }
reqwest = { workspace = true, optional = true }
aws-config = { workspace = true, optional = true }
aws-sdk-secretsmanager = { workspace = true, optional = true }
google-cloud-auth = { workspace = true, optional = true }
base64 = { workspace = true, optional = true }
azure_security_keyvault_secrets = { workspace = true, optional = true }
azure_identity = { workspace = true, optional = true }
azure_core = { workspace = true, optional = true }
chrono = { workspace = true }
# Base dep (not optional): `faucet backfill` (#282) computes window boundaries
# timezone-correctly in every build; `faucet schedule` reuses it.
chrono-tz = { workspace = true }
croner = { workspace = true, optional = true }
axum = { workspace = true, optional = true }
tower-http = { workspace = true, optional = true }
# `CancellationToken` for cooperative run cancellation in the executor + serve
# runner (flush-on-cancel, #146 H16). Non-optional: the executor is used by
# `faucet run`/`schedule`, not just `serve`.
tokio-util = { workspace = true }
subtle = { workspace = true, optional = true }
# faucet serve run registry (Phase 2+); only compiled with the `serve` feature.
dashmap = { workspace = true, optional = true }
# faucet serve idempotency-key hashing (Phase 2+; `serve`) + notify webhook
# HMAC signing (#280; `notify`).
sha2 = { workspace = true, optional = true }
# notify webhook HMAC-SHA256 signature (#280); only compiled with `notify`.
hmac = { workspace = true, optional = true }
# faucet serve SSE log streaming (Phase 4); only compiled with `serve`.
async-stream = { workspace = true, optional = true }
sqlx = { workspace = true, optional = true }
# faucet serve embedded web console (serve-ui feature): compile-time asset embedding.
rust-embed = { version = "8", optional = true, features = ["mime-guess"] }
# faucet serve event-driven triggers (#196). object_store lists S3/GCS prefixes
# for the object_arrival trigger; redis/rdkafka poll queue depth. All optional,
# pulled only by the matching `triggers-*` feature.
object_store = { workspace = true, optional = true, features = ["aws", "gcp"] }
redis = { workspace = true, optional = true }
rdkafka = { workspace = true, optional = true }

[dev-dependencies]
tokio = { workspace = true, features = ["test-util"] }
tempfile = "3"
# `faucet schema config` composed-schema tests (#213): validate the shipped
# example configs against the emitted top-level JSON Schema.
jsonschema = { workspace = true }
assert_cmd = "2"
predicates = "3"
wiremock = "0.6"
serial_test = "3"
metrics-util.workspace = true
serde_yaml = "0.9"
async-stream.workspace = true
futures-core.workspace = true
reqwest = { workspace = true, features = ["json"] }
tower = { version = "0.5", features = ["util"] }
# faucet replicate snapshot→CDC integration tests (#189): real Postgres via
# testcontainers; versions match crates/source/postgres-cdc/Cargo.toml.
testcontainers = "0.27"
testcontainers-modules = { version = "0.15", features = ["postgres"] }
# faucet backfill end-to-end tests (#282): seed/inspect the SQLite source and
# destination DBs directly (offline — no Docker).
sqlx = { workspace = true, features = ["runtime-tokio", "sqlite"] }
tokio-postgres = "0.7"

[package.metadata.docs.rs]
all-features = true
rustdoc-args = ["--cfg", "docsrs"]