faucet_core/
lib.rs

1#![cfg_attr(docsrs, feature(doc_cfg))]
2
3//! # faucet-core
4//!
5//! Shared types, traits, and utilities for the faucet-stream ecosystem.
6//!
7//! This crate provides the common foundation used by all faucet source and
8//! sink connectors:
9//!
10//! - [`FaucetError`] — unified error type
11//! - [`Source`] / [`Sink`] — async traits for data connectors
12//! - [`RecordTransform`] — record transformation pipeline
13//! - [`ReplicationMethod`] — incremental replication support
14//! - [`schema::infer_schema`] — JSON Schema inference from record samples
15
16pub mod adaptive;
17pub mod auth;
18pub mod check;
19pub mod config;
20#[cfg(feature = "contract")]
21pub mod contract;
22pub mod discover;
23pub mod dlq;
24pub mod drift;
25#[cfg(feature = "encryption")]
26pub mod encryption;
27pub mod error;
28pub mod idempotency;
29#[cfg(feature = "masking")]
30pub mod masking;
31pub mod observability;
32pub mod pipeline;
33#[cfg(feature = "quality")]
34pub mod quality;
35pub mod replication;
36pub mod resilience;
37pub mod retry;
38pub mod schema;
39pub mod shard;
40pub mod stage;
41pub mod state;
42pub mod traits;
43pub mod transform;
44pub mod transforming_source;
45pub mod util;
46pub mod verify;
47pub mod write_mode;
48
49#[cfg(feature = "compression")]
50pub mod compression;
51
52pub use adaptive::{
53    AdaptiveBatchConfig, AdjustDirection, AdjustReason, Adjustment, AimdController, Observation,
54};
55pub use auth::{AuthProvider, AuthReference, AuthSpec, Credential, SharedAuthProvider};
56pub use check::{CheckContext, CheckReport, Probe, ProbeStatus};
57pub use discover::{DatasetDescriptor, columns_to_schema, nullable_type, sql_type_to_json_schema};
58pub use dlq::{
59    DlqConfig, DlqReason, DlqStats, EnvelopeError, OnBatchError, UnwrappedEnvelope, build_envelope,
60    unwrap_envelope,
61};
62pub use drift::{
63    ColumnChange, OnDrift, OnIncompatible, SchemaDiff, SchemaDriftPolicy, SchemaDriftSpec,
64    SchemaEvolution, SqlBaseType, adds_null, base_widened, json_schema_base_type,
65};
66#[cfg(feature = "encryption")]
67pub use encryption::{CompiledEncryption, EncryptionAlgorithm, EncryptionSpec};
68pub use error::FaucetError;
69pub use idempotency::{
70    DeliveryGuarantee, DeliveryMode, EffectivelyOnceMechanism, GuaranteeInputs, ReplayGuarantee,
71    SinkGuarantee, derive_delivery_guarantee, format_token, format_token_with_bookmark,
72    parse_token, parse_token_parts, unwrap_state, wrap_state,
73};
74#[cfg(feature = "contract")]
75pub use observability::instrumented_apply_contract;
76#[cfg(feature = "masking")]
77pub use observability::instrumented_apply_masking;
78#[cfg(feature = "quality")]
79pub use observability::instrumented_apply_quality;
80pub use observability::otel::{OtelConfig, OtelProtocol, OtelSignal, shutdown_otel};
81pub use observability::{
82    DurationGuard, InstallError, InstallReport, InstrumentedSink, InstrumentedSource,
83    InstrumentedStateStore, Labels, ObservabilityConfig, PrometheusConfig, RunStreamOptions,
84    TracingConfig, install_observability, instrumented_apply_stages, register_build_info,
85    update_bookmark_lag,
86};
87pub use pipeline::{
88    DEFAULT_BATCH_SIZE, MAX_BATCH_SIZE, Pipeline, PipelineResult, StreamPage, run_stream,
89    validate_batch_size,
90};
91pub use replication::{ReplicationMethod, json_gt};
92pub use resilience::{
93    BackoffKind, CircuitBreaker, CircuitBreakerConfig, PoisonAction, PoisonPolicy,
94    ResiliencePolicy, RetryClass, RetryClassSet, RetryMetrics, RetryPolicy, classify,
95    execute_with_policy, execute_with_policy_metered,
96};
97pub use retry::execute_with_retry;
98pub use shard::ShardSpec;
99#[cfg(feature = "transform-cdc-unwrap")]
100pub use stage::CdcUnwrapSpec;
101#[cfg(feature = "transform-explode")]
102pub use stage::{ExplodeSpec, OnMissing};
103#[cfg(feature = "transform-filter")]
104pub use stage::{FilterOp, FilterSpec};
105pub use stage::{TransformStage, compile_stage};
106pub use state::{FileStateStore, MemoryStateStore, StateStore};
107pub use traits::{RowOutcome, Sink, Source};
108#[cfg(feature = "transform-keys-case")]
109pub use transform::KeyCaseMode;
110pub use transform::RecordTransform;
111#[cfg(feature = "transform-value-case")]
112pub use transform::ValueCaseMode;
113#[cfg(feature = "transform-cast")]
114pub use transform::{CastOnError, CastType};
115pub use transforming_source::TransformingSource;
116pub use util::redact_uri_credentials;
117pub use verify::{IntegrityCheck, LengthCheck, VerifyingReader};
118pub use write_mode::{
119    DeleteMarker, KeyTuple, WriteMode, WritePlan, WriteSpec, key_to_doc_id, key_to_filter,
120    plan_writes,
121};
122
123// Re-export dependencies that connector authors need, so they only depend on
124// `faucet-core` instead of adding `async-trait` and `serde_json` themselves.
125pub use async_stream;
126pub use async_trait::async_trait;
127pub use futures_core::{self, Stream};
128pub use schemars::{self, JsonSchema, schema_for};
129pub use serde_json::{self, Value, json};
130/// Re-exported so callers of [`Pipeline::with_cancel`](pipeline::Pipeline::with_cancel)
131/// / [`RunStreamOptions::with_cancel`] can name the token type without adding
132/// `tokio-util` themselves.
133pub use tokio_util::sync::CancellationToken;
134
135#[cfg(feature = "compression")]
136pub use compression::{Compression, CompressionConfig, compress_buf, warn_mismatch};
137
138#[cfg(feature = "quality")]
139pub use quality::{
140    BatchCheck, CheckTally, CompareOp, CompiledQuality, JsonType, OnFailure, QualityOutcome,
141    QualitySpec, QuarantinedRecord, RecordCheck, apply_quality,
142};
143
144#[cfg(feature = "contract")]
145pub use contract::{
146    CompiledContract, ContractFieldType, ContractOutcome, ContractSpec, ContractViolation,
147    FieldContract, OnBreach, ViolatingRecord, apply_contract, to_json_schema, to_openlineage_facet,
148};
149
150#[cfg(feature = "masking")]
151pub use masking::{
152    CompiledMasking, Detector, MaskAction, MaskHit, MaskRule, MaskingOutcome, MaskingSpec,
153    MatchSpec, apply_masking,
154};
faucet_core/lib.rs

faucet_core/
lib.rs
