faucet_core/
lib.rs

1#![cfg_attr(docsrs, feature(doc_cfg))]
2
3//! # faucet-core
4//!
5//! Shared types, traits, and utilities for the faucet-stream ecosystem.
6//!
7//! This crate provides the common foundation used by all faucet source and
8//! sink connectors:
9//!
10//! - [`FaucetError`] — unified error type
11//! - [`Source`] / [`Sink`] — async traits for data connectors
12//! - [`RecordTransform`] — record transformation pipeline
13//! - [`ReplicationMethod`] — incremental replication support
14//! - [`schema::infer_schema`] — JSON Schema inference from record samples
15
16pub mod adaptive;
17pub mod auth;
18pub mod check;
19pub mod config;
20#[cfg(feature = "contract")]
21pub mod contract;
22pub mod dlq;
23pub mod drift;
24pub mod error;
25pub mod idempotency;
26#[cfg(feature = "masking")]
27pub mod masking;
28pub mod observability;
29pub mod pipeline;
30#[cfg(feature = "quality")]
31pub mod quality;
32pub mod replication;
33pub mod resilience;
34pub mod retry;
35pub mod schema;
36pub mod shard;
37pub mod stage;
38pub mod state;
39pub mod traits;
40pub mod transform;
41pub mod transforming_source;
42pub mod util;
43pub mod verify;
44pub mod write_mode;
45
46#[cfg(feature = "compression")]
47pub mod compression;
48
49pub use adaptive::{
50    AdaptiveBatchConfig, AdjustDirection, AdjustReason, Adjustment, AimdController, Observation,
51};
52pub use auth::{AuthProvider, AuthReference, AuthSpec, Credential, SharedAuthProvider};
53pub use check::{CheckContext, CheckReport, Probe, ProbeStatus};
54pub use dlq::{
55    DlqConfig, DlqReason, DlqStats, EnvelopeError, OnBatchError, UnwrappedEnvelope, build_envelope,
56    unwrap_envelope,
57};
58pub use drift::{
59    ColumnChange, OnDrift, OnIncompatible, SchemaDiff, SchemaDriftPolicy, SchemaDriftSpec,
60    SchemaEvolution, SqlBaseType, adds_null, base_widened, json_schema_base_type,
61};
62pub use error::FaucetError;
63pub use idempotency::{DeliveryMode, format_token, parse_token, unwrap_state, wrap_state};
64#[cfg(feature = "contract")]
65pub use observability::instrumented_apply_contract;
66#[cfg(feature = "masking")]
67pub use observability::instrumented_apply_masking;
68#[cfg(feature = "quality")]
69pub use observability::instrumented_apply_quality;
70pub use observability::otel::{OtelConfig, OtelProtocol, OtelSignal, shutdown_otel};
71pub use observability::{
72    DurationGuard, InstallError, InstallReport, InstrumentedSink, InstrumentedSource,
73    InstrumentedStateStore, Labels, ObservabilityConfig, PrometheusConfig, RunStreamOptions,
74    TracingConfig, install_observability, instrumented_apply_stages, register_build_info,
75    update_bookmark_lag,
76};
77pub use pipeline::{
78    DEFAULT_BATCH_SIZE, MAX_BATCH_SIZE, Pipeline, PipelineResult, StreamPage, run_stream,
79    validate_batch_size,
80};
81pub use replication::ReplicationMethod;
82pub use resilience::{
83    BackoffKind, CircuitBreaker, CircuitBreakerConfig, PoisonAction, PoisonPolicy,
84    ResiliencePolicy, RetryClass, RetryClassSet, RetryMetrics, RetryPolicy, classify,
85    execute_with_policy, execute_with_policy_metered,
86};
87pub use retry::execute_with_retry;
88pub use shard::ShardSpec;
89#[cfg(feature = "transform-cdc-unwrap")]
90pub use stage::CdcUnwrapSpec;
91#[cfg(feature = "transform-explode")]
92pub use stage::{ExplodeSpec, OnMissing};
93#[cfg(feature = "transform-filter")]
94pub use stage::{FilterOp, FilterSpec};
95pub use stage::{TransformStage, compile_stage};
96pub use state::{FileStateStore, MemoryStateStore, StateStore};
97pub use traits::{RowOutcome, Sink, Source};
98#[cfg(feature = "transform-keys-case")]
99pub use transform::KeyCaseMode;
100pub use transform::RecordTransform;
101#[cfg(feature = "transform-value-case")]
102pub use transform::ValueCaseMode;
103#[cfg(feature = "transform-cast")]
104pub use transform::{CastOnError, CastType};
105pub use transforming_source::TransformingSource;
106pub use util::redact_uri_credentials;
107pub use verify::{IntegrityCheck, LengthCheck, VerifyingReader};
108pub use write_mode::{
109    DeleteMarker, KeyTuple, WriteMode, WritePlan, WriteSpec, key_to_doc_id, key_to_filter,
110    plan_writes,
111};
112
113// Re-export dependencies that connector authors need, so they only depend on
114// `faucet-core` instead of adding `async-trait` and `serde_json` themselves.
115pub use async_stream;
116pub use async_trait::async_trait;
117pub use futures_core::{self, Stream};
118pub use schemars::{self, JsonSchema, schema_for};
119pub use serde_json::{self, Value, json};
120/// Re-exported so callers of [`Pipeline::with_cancel`](pipeline::Pipeline::with_cancel)
121/// / [`RunStreamOptions::with_cancel`] can name the token type without adding
122/// `tokio-util` themselves.
123pub use tokio_util::sync::CancellationToken;
124
125#[cfg(feature = "compression")]
126pub use compression::{Compression, CompressionConfig, compress_buf, warn_mismatch};
127
128#[cfg(feature = "quality")]
129pub use quality::{
130    BatchCheck, CheckTally, CompareOp, CompiledQuality, JsonType, OnFailure, QualityOutcome,
131    QualitySpec, QuarantinedRecord, RecordCheck, apply_quality,
132};
133
134#[cfg(feature = "contract")]
135pub use contract::{
136    CompiledContract, ContractFieldType, ContractOutcome, ContractSpec, ContractViolation,
137    FieldContract, OnBreach, ViolatingRecord, apply_contract, to_json_schema, to_openlineage_facet,
138};
139
140#[cfg(feature = "masking")]
141pub use masking::{
142    CompiledMasking, Detector, MaskAction, MaskHit, MaskRule, MaskingOutcome, MaskingSpec,
143    MatchSpec, apply_masking,
144};
faucet_core/lib.rs

faucet_core/
lib.rs
