faucet_core/
lib.rs

1#![cfg_attr(docsrs, feature(doc_cfg))]
2
3//! # faucet-core
4//!
5//! Shared types, traits, and utilities for the faucet-stream ecosystem.
6//!
7//! This crate provides the common foundation used by all faucet source and
8//! sink connectors:
9//!
10//! - [`FaucetError`] — unified error type
11//! - [`Source`] / [`Sink`] — async traits for data connectors
12//! - [`RecordTransform`] — record transformation pipeline
13//! - [`ReplicationMethod`] — incremental replication support
14//! - [`schema::infer_schema`] — JSON Schema inference from record samples
15
16pub mod adaptive;
17pub mod anomaly;
18pub mod auth;
19pub mod budget;
20pub mod check;
21pub mod cleanup;
22#[cfg(feature = "arrow")]
23pub mod columnar;
24#[cfg(feature = "arrow")]
25pub mod columnar_transform;
26pub mod config;
27#[cfg(feature = "contract")]
28pub mod contract;
29pub mod create_table;
30#[cfg(feature = "transform-cross-join")]
31pub mod cross_join;
32pub mod diff;
33pub mod discover;
34pub mod dlq;
35pub mod drift;
36#[cfg(feature = "encryption")]
37pub mod encryption;
38pub mod error;
39pub mod file_format;
40pub mod idempotency;
41pub mod join;
42pub mod json_numbers;
43pub mod lag;
44pub mod lease;
45pub mod local_outputs;
46#[cfg(feature = "masking")]
47pub mod masking;
48pub mod metadata;
49pub mod native;
50pub mod object_rollover;
51pub mod observability;
52pub mod pipeline;
53#[cfg(feature = "policy")]
54pub mod policy;
55pub mod profiling;
56#[cfg(feature = "quality")]
57pub mod quality;
58pub mod redact;
59pub mod replication;
60pub mod resilience;
61pub mod retry;
62pub mod rollback;
63pub mod schema;
64pub mod shard;
65#[cfg(test)]
66mod sink_forwarding;
67pub mod stage;
68pub mod staging;
69pub mod state;
70pub mod state_version;
71pub mod tls;
72pub mod topology;
73pub mod traits;
74pub mod transform;
75pub mod transforming_source;
76#[cfg(feature = "transform-tree-flatten")]
77pub mod tree;
78pub mod usage;
79pub mod util;
80pub mod verify;
81pub mod window;
82pub mod write_mode;
83#[cfg(feature = "transform-zip-columns")]
84pub mod zip_columns;
85
86#[cfg(feature = "compression")]
87pub mod compression;
88
89pub use adaptive::{
90    AdaptiveBatchConfig, AdjustDirection, AdjustReason, Adjustment, AimdController, Observation,
91};
92pub use anomaly::AnomalyMethod;
93pub use auth::{
94    AuthProvider, AuthReference, AuthSpec, Credential, CredentialPlacement, RequestAuth,
95    SharedAuthProvider, rejects_credential, send_with_reauth,
96};
97pub use budget::{BudgetKind, BudgetSink, BudgetSpec, BudgetState, BudgetTimer, BudgetVerdict};
98pub use check::{CheckContext, CheckReport, Probe, ProbeStatus};
99pub use cleanup::{CleanupMode, CleanupPolicy, DEFAULT_MAX_KEYS, SeenKeys};
100#[cfg(feature = "arrow")]
101pub use columnar::{
102    ColumnarPage, check_integral, infer_arrow_schema, record_batch_to_values,
103    values_to_record_batch, values_to_record_batch_inferred,
104};
105pub use create_table::{
106    PlannedColumn, missing_target_error, plan_columns, plan_keyed_columns, render_column_defs,
107    render_columns, render_primary_key,
108};
109#[cfg(feature = "transform-cross-join")]
110pub use cross_join::{CompiledCrossJoin, CrossJoinSpec, OnEmpty as CrossJoinOnEmpty};
111pub use diff::{
112    ContentDigest, Difference, DifferenceKind, DigestAccumulator, KeyRange, Normalizer,
113    ServerDigest, VerifyReport, diff_rows, plan_ranges, row_hash,
114};
115pub use discover::{
116    DatasetDescriptor, attach_primary_keys, columns_to_schema, nullable_type,
117    sql_type_to_json_schema,
118};
119pub use dlq::{
120    BatchAtomicity, BatchOutcome, BatchOutcomeCounters, BatchOutcomes, DlqConfig, DlqReason,
121    DlqStats, EnvelopeError, OnBatchError, UnwrappedEnvelope, build_envelope, check_dlq_all_policy,
122    dlq_all_is_safe, dlq_all_refusal, unwrap_envelope,
123};
124pub use drift::{
125    ColumnChange, DRIFT_ALSO_ACCEPTS, OnDrift, OnIncompatible, SchemaDiff, SchemaDriftPolicy,
126    SchemaDriftSpec, SchemaEvolution, SqlBaseType, adds_null, base_widened, json_schema_base_type,
127};
128#[cfg(feature = "encryption")]
129pub use encryption::{CompiledEncryption, EncryptionAlgorithm, EncryptionSpec, SealedLine};
130pub use error::FaucetError;
131pub use file_format::parquet_io::ParquetReadOptions;
132pub use file_format::{
133    AvroCodec, AvroOptions, ContainerDecoder, CsvOptions, CsvUnknownField, ExcelOptions,
134    FileFormat, FileInput, FormatOptions, OrcOptions, XmlOptions,
135};
136pub use idempotency::{
137    DeliveryGuarantee, DeliveryMode, EffectivelyOnceMechanism, GuaranteeInputs, ReplayGuarantee,
138    SinkGuarantee, derive_delivery_guarantee, format_token, format_token_with_bookmark,
139    parse_token, parse_token_parts, unwrap_state, wrap_state,
140};
141pub use join::{
142    HashJoin, JoinConfig, JoinMode, JoinStats, KeyNormalize, OnCollision, OnDuplicate, Projection,
143};
144pub use json_numbers::{JsonBigNumbers, parse_json_exact};
145pub use lag::{LagObserver, SourceLag};
146pub use local_outputs::{LocalOutput, LocalOutputLog, probe_pre_existing};
147pub use metadata::{
148    CompiledMetadata, MetadataColumn, MetadataColumnsSpec, MetadataContext, MetadataSink,
149};
150pub use native::{
151    CsvDialect, NativeBatch, NativeFormat, NativeLoadCapability, NativeLoadContext, NativePayload,
152    NativePlan, NativePlanInputs, plan_native_transfer,
153};
154pub use object_rollover::{CompletedObject, ObjectAccumulator, PageAccumulator};
155#[cfg(feature = "contract")]
156pub use observability::instrumented_apply_contract;
157#[cfg(feature = "masking")]
158pub use observability::instrumented_apply_masking;
159#[cfg(feature = "quality")]
160pub use observability::instrumented_apply_quality;
161pub use observability::otel::{OtelConfig, OtelProtocol, OtelSignal, shutdown_otel};
162pub use observability::{
163    DurationGuard, InstallError, InstallReport, InstrumentedSink, InstrumentedSource,
164    InstrumentedStateStore, Labels, ObservabilityConfig, PrometheusConfig, RunStreamOptions,
165    TracingConfig, build_version, install_observability, instrumented_apply_stages,
166    register_build_info, set_build_version, update_bookmark_lag,
167};
168pub use pipeline::{
169    DEFAULT_BATCH_SIZE, MAX_BATCH_SIZE, Pipeline, PipelineResult, StreamPage, run_stream,
170    validate_batch_size,
171};
172#[cfg(feature = "policy")]
173pub use policy::{
174    ColumnFacts, CompiledPolicy, PolicyRule, PolicyScope, PolicySink, PolicySpec, RuntimeAction,
175    SinkFacts, Violation, ViolationKind,
176};
177pub use profiling::{
178    ColumnProfile, DriftMetric, OnProfileDrift, ProfileDrift, Profiler, ProfilingSink,
179    ProfilingSpec, RunProfile, detect_drift,
180};
181pub use replication::{
182    BindFormat, BindTarget, BindValueType, IncrementalFilter, OnMissingKey, ReplicationBind,
183    ReplicationKey, ReplicationMethod, format_bookmark, format_instant, json_gt, parse_instant,
184    set_body_pointer,
185};
186pub use resilience::{
187    BackoffFrom, BackoffKind, CircuitBreaker, CircuitBreakerConfig, DEFAULT_MAX_WAIT_SECS,
188    PoisonAction, PoisonPolicy, ResiliencePolicy, RetryClass, RetryClassSet, RetryMatcher,
189    RetryMetrics, RetryPolicy, WaitUnit, classify, execute_with_policy,
190    execute_with_policy_metered, execute_with_policy_recorded,
191};
192pub use retry::execute_with_retry;
193pub use rollback::{
194    DEFAULT_RUN_ID_COLUMN, PREVIOUS_TABLE_SUFFIX, RUN_JOURNAL_TABLE, RollbackMode, RollbackOptions,
195    RollbackOutcome, RollbackWriteSpec,
196};
197pub use shard::ShardSpec;
198#[cfg(feature = "transform-cdc-unwrap")]
199pub use stage::{CdcMissingImage, CdcUnwrapSpec};
200#[cfg(feature = "transform-unpivot")]
201pub use stage::{CompiledUnpivot, UnpivotSpec};
202#[cfg(feature = "transform-explode")]
203pub use stage::{ExplodeSpec, OnMissing};
204#[cfg(feature = "transform-filter")]
205pub use stage::{FilterOp, FilterSpec};
206pub use stage::{TransformStage, apply_stages, compile_stage};
207pub use staging::{
208    StagedFile, StagingCleanup, StagingCompression, StagingFormat, StagingLocation, StagingScheme,
209    StagingSpec, serialize_records,
210};
211pub use state::{FileStateStore, MemoryStateStore, StateExport, StateStore};
212pub use state_version::{
213    ResolvedState, STATE_FORMAT, StateCompat, StoredState, check_compat, peel_versioned,
214    resolve_for_source, wrap_versioned,
215};
216pub use tls::TlsClientConfig;
217pub use topology::{
218    Edge, JoinNode, Node, NodeKind, Topology, TopologyBuilder, TopologyOnError, TopologyOptions,
219    TopologyResult,
220};
221pub use traits::{RowOutcome, Sink, Source};
222#[cfg(feature = "transform-json-parse")]
223pub use transform::JsonParseOnError;
224#[cfg(feature = "transform-lookup")]
225pub use transform::LookupOnMissing;
226pub use transform::RecordTransform;
227#[cfg(feature = "transform-value-case")]
228pub use transform::ValueCaseMode;
229#[cfg(feature = "transform-cast")]
230pub use transform::{CastOnError, CastType};
231#[cfg(feature = "transform-hash")]
232pub use transform::{HashAlgorithm, HashEncoding};
233#[cfg(feature = "transform-keys-case")]
234pub use transform::{KeyCaseMode, KeyCollision};
235pub use transforming_source::TransformingSource;
236#[cfg(feature = "transform-tree-flatten")]
237pub use tree::{AncestorsSpec, ColumnsSpec, CompiledTreeFlatten, TreeFlattenSpec};
238pub use usage::{CostSignal, UsageMeter, UsageSide, UsageSnapshot, estimate_json_bytes};
239pub use util::redact_uri_credentials;
240pub use verify::{IntegrityCheck, LengthCheck, VerifyingReader};
241pub use window::{
242    WINDOW_END_PLACEHOLDER, WINDOW_PLACEHOLDER, WINDOW_START_PLACEHOLDER, Window, WindowBind,
243    WindowSpec, enumerate_windows, parse_step,
244};
245pub use write_mode::{
246    DeleteMarker, KeyTuple, OverwriteScope, WriteMode, WritePlan, WriteSpec, key_to_doc_id,
247    key_to_filter, plan_writes, sql_literal,
248};
249#[cfg(feature = "transform-zip-columns")]
250pub use zip_columns::{ColumnGroupSpec, CompiledZipColumns, ZipColumnsSpec};
251
252// Re-export dependencies that connector authors need, so they only depend on
253// `faucet-core` instead of adding `async-trait` and `serde_json` themselves.
254pub use async_stream;
255pub use async_trait::async_trait;
256pub use futures_core::{self, Stream};
257pub use schemars::{self, JsonSchema, schema_for};
258pub use serde_json::{self, Value, json};
259/// Re-exported so callers of [`Pipeline::with_cancel`](pipeline::Pipeline::with_cancel)
260/// / [`RunStreamOptions::with_cancel`] can name the token type without adding
261/// `tokio-util` themselves.
262pub use tokio_util::sync::CancellationToken;
263
264#[cfg(feature = "compression")]
265pub use compression::{Compression, CompressionConfig, compress_buf, warn_mismatch};
266
267#[cfg(feature = "quality")]
268pub use quality::{
269    BatchCheck, CheckTally, CompareOp, CompiledQuality, JsonType, OnFailure, QualityOutcome,
270    QualitySpec, QuarantinedRecord, RecordCheck, apply_quality,
271};
272
273#[cfg(feature = "contract")]
274pub use contract::{
275    CompiledContract, ContractFieldType, ContractOutcome, ContractSpec, ContractViolation,
276    FieldContract, OnBreach, ViolatingRecord, apply_contract, to_json_schema, to_openlineage_facet,
277};
278
279#[cfg(feature = "masking")]
280pub use masking::{
281    CompiledMasking, Detector, MaskAction, MaskHit, MaskRule, MaskingOutcome, MaskingSpec,
282    MatchSpec, apply_masking,
283};
faucet_core/lib.rs

faucet_core/
lib.rs
