faucet_core/
lib.rs

1#![cfg_attr(docsrs, feature(doc_cfg))]
2
3//! # faucet-core
4//!
5//! Shared types, traits, and utilities for the faucet-stream ecosystem.
6//!
7//! This crate provides the common foundation used by all faucet source and
8//! sink connectors:
9//!
10//! - [`FaucetError`] — unified error type
11//! - [`Source`] / [`Sink`] — async traits for data connectors
12//! - [`RecordTransform`] — record transformation pipeline
13//! - [`ReplicationMethod`] — incremental replication support
14//! - [`schema::infer_schema`] — JSON Schema inference from record samples
15
16pub mod accounting;
17pub mod adaptive;
18pub mod anomaly;
19pub mod auth;
20pub mod batching;
21pub mod budget;
22pub mod capabilities;
23pub mod check;
24pub mod cleanup;
25#[cfg(feature = "arrow")]
26pub mod columnar;
27#[cfg(feature = "arrow")]
28pub mod columnar_transform;
29pub mod compat;
30pub mod config;
31#[cfg(feature = "contract")]
32pub mod contract;
33pub mod create_table;
34#[cfg(feature = "transform-cross-join")]
35pub mod cross_join;
36pub mod diff;
37pub mod discover;
38pub mod dlq;
39pub mod drift;
40pub mod driver;
41#[cfg(feature = "encryption")]
42pub mod encryption;
43pub mod error;
44pub mod fidelity;
45pub mod file_format;
46pub mod forward;
47pub mod governance;
48pub mod http;
49// The conformance crate's forwarding harness, compiled into core's own unit
50// tests under the `faucet_core::` paths it is written against.
51#[cfg(test)]
52extern crate self as faucet_core;
53#[cfg(test)]
54#[path = "../../conformance/src/forwarding.rs"]
55#[allow(dead_code)]
56mod forwarding;
57pub mod idempotency;
58pub mod io;
59pub mod join;
60pub mod json_numbers;
61pub mod lag;
62pub mod lease;
63pub mod local_outputs;
64#[cfg(feature = "masking")]
65pub mod masking;
66pub mod metadata;
67pub mod native;
68pub mod object_rollover;
69pub mod observability;
70pub mod pipeline;
71#[cfg(feature = "policy")]
72pub mod policy;
73pub mod profiling;
74#[cfg(feature = "quality")]
75pub mod quality;
76pub mod redact;
77pub mod replication;
78pub mod resilience;
79pub mod resolve;
80pub mod retry;
81pub mod rollback;
82pub mod schema;
83pub mod shard;
84pub mod stage;
85pub mod staging;
86pub mod state;
87pub mod state_key;
88pub mod state_version;
89pub mod tls;
90pub mod topology;
91pub mod traits;
92pub mod transform;
93pub mod transforming_source;
94#[cfg(feature = "transform-tree-flatten")]
95pub mod tree;
96pub mod usage;
97pub mod util;
98pub mod verify;
99pub mod window;
100pub mod write_mode;
101#[cfg(feature = "transform-zip-columns")]
102pub mod zip_columns;
103
104#[cfg(feature = "compression")]
105pub mod compression;
106
107pub use accounting::{RowAccounts, RowAccountsSnapshot, WriteReport};
108pub use adaptive::{
109    AdaptiveBatchConfig, AdjustDirection, AdjustReason, Adjustment, AimdController, Observation,
110};
111pub use anomaly::AnomalyMethod;
112pub use auth::{
113    AuthProvider, AuthReference, AuthSpec, Credential, CredentialPlacement, RequestAuth,
114    SharedAuthProvider, rejects_credential, send_with_reauth,
115};
116pub use budget::{BudgetKind, BudgetSink, BudgetSpec, BudgetState, BudgetTimer, BudgetVerdict};
117pub use capabilities::{Capabilities, CapabilitiesBuilder, ConnectorDescriptor, SchemaReport};
118pub use check::{CheckContext, CheckReport, Probe, ProbeStatus};
119pub use cleanup::{CleanupMode, CleanupPolicy, DEFAULT_MAX_KEYS, SeenKeys};
120#[cfg(feature = "arrow")]
121pub use columnar::{
122    ColumnarPage, check_integral, infer_arrow_schema, record_batch_to_values,
123    values_to_record_batch, values_to_record_batch_inferred,
124};
125pub use create_table::{
126    PlannedColumn, missing_target_error, plan_columns, plan_keyed_columns, render_column_defs,
127    render_columns, render_primary_key,
128};
129#[cfg(feature = "transform-cross-join")]
130pub use cross_join::{CompiledCrossJoin, CrossJoinSpec, OnEmpty as CrossJoinOnEmpty};
131pub use diff::{
132    ContentDigest, Difference, DifferenceKind, DigestAccumulator, KeyRange, Normalizer,
133    ServerDigest, VerifyReport, diff_rows, plan_ranges, row_hash,
134};
135pub use discover::{
136    DatasetDescriptor, attach_primary_keys, columns_to_schema, nullable_type,
137    sql_type_to_json_schema,
138};
139pub use dlq::{
140    BatchAtomicity, BatchOutcome, BatchOutcomeCounters, BatchOutcomes, DlqConfig, DlqReason,
141    DlqStats, EnvelopeError, OnBatchError, UnwrappedEnvelope, build_envelope, check_dlq_all_policy,
142    dlq_all_is_safe, dlq_all_refusal, unwrap_envelope,
143};
144pub use drift::{
145    ColumnChange, DRIFT_ALSO_ACCEPTS, OnDrift, OnIncompatible, SchemaDiff, SchemaDriftPolicy,
146    SchemaDriftSpec, SchemaEvolution, SqlBaseType, adds_null, base_widened, json_schema_base_type,
147};
148#[cfg(feature = "encryption")]
149pub use encryption::{CompiledEncryption, EncryptionAlgorithm, EncryptionSpec, SealedLine};
150pub use error::FaucetError;
151pub use fidelity::{FidelityError, NonFiniteCounter, TimePrecision};
152pub use file_format::parquet_io::ParquetReadOptions;
153pub use file_format::{
154    AvroCodec, AvroOptions, ContainerDecoder, CsvOptions, CsvUnknownField, ExcelOptions,
155    FileFormat, FileInput, FormatOptions, OrcOptions, XmlOptions,
156};
157#[cfg(feature = "policy")]
158pub use governance::PolicyGovernance;
159pub use governance::{
160    DlqGovernance, Governance, GovernedSink, GovernedSinkBuilder, OptOut, Pass, SINK_LAYERS,
161    SinkLayer,
162};
163pub use idempotency::{
164    DeliveryGuarantee, DeliveryMode, EffectivelyOnceMechanism, GuaranteeInputs, ReplayGuarantee,
165    SinkGuarantee, derive_delivery_guarantee, format_token, format_token_with_bookmark,
166    parse_token, parse_token_parts, unwrap_state, wrap_state,
167};
168pub use io::{CallContext, CallExt, CallReader, CallSlot, DEFAULT_CALL_TIMEOUT};
169pub use join::{
170    HashJoin, JoinConfig, JoinMode, JoinStats, KeyNormalize, OnCollision, OnDuplicate, Projection,
171};
172pub use json_numbers::{JsonBigNumbers, parse_json_exact};
173pub use lag::{LagObserver, SourceLag};
174pub use local_outputs::{LocalOutput, LocalOutputLog, probe_pre_existing};
175pub use metadata::{
176    CompiledMetadata, MetadataColumn, MetadataColumnsSpec, MetadataContext, MetadataSink,
177};
178pub use native::{
179    CsvDialect, NativeBatch, NativeFormat, NativeLoadCapability, NativeLoadContext, NativePayload,
180    NativePlan, NativePlanInputs, plan_native_transfer,
181};
182pub use object_rollover::{CompletedObject, ObjectAccumulator, PageAccumulator};
183#[cfg(feature = "contract")]
184pub use observability::instrumented_apply_contract;
185#[cfg(feature = "masking")]
186pub use observability::instrumented_apply_masking;
187#[cfg(feature = "quality")]
188pub use observability::instrumented_apply_quality;
189pub use observability::otel::{OtelConfig, OtelProtocol, OtelSignal, shutdown_otel};
190pub use observability::{
191    DurationGuard, InstallError, InstallReport, InstrumentedSink, InstrumentedSource,
192    InstrumentedStateStore, Labels, ObservabilityConfig, PrometheusConfig, RunStreamOptions,
193    TracingConfig, build_version, install_observability, instrumented_apply_stages,
194    register_build_info, set_build_version, update_bookmark_lag,
195};
196pub use pipeline::{
197    DEFAULT_BATCH_SIZE, MAX_BATCH_SIZE, Pipeline, PipelineResult, StreamPage, run_stream,
198    validate_batch_size,
199};
200#[cfg(feature = "policy")]
201pub use policy::{
202    ColumnFacts, CompiledPolicy, PolicyRule, PolicyScope, PolicySink, PolicySpec, RuntimeAction,
203    SinkFacts, Violation, ViolationKind,
204};
205pub use profiling::{
206    ColumnProfile, DriftMetric, OnProfileDrift, ProfileDrift, Profiler, ProfilingSink,
207    ProfilingSpec, RunProfile, detect_drift,
208};
209pub use replication::{
210    BindFormat, BindTarget, BindValueType, IncrementalFilter, OnMissingKey, ReplicationBind,
211    ReplicationKey, ReplicationMethod, format_bookmark, format_instant, json_gt, parse_instant,
212    set_body_pointer,
213};
214pub use resilience::{
215    BackoffFrom, BackoffKind, CircuitBreaker, CircuitBreakerConfig, DEFAULT_MAX_WAIT_SECS,
216    PoisonAction, PoisonPolicy, ResiliencePolicy, RetryClass, RetryClassSet, RetryMatcher,
217    RetryMetrics, RetryPolicy, WaitUnit, classify, execute_with_policy,
218    execute_with_policy_metered, execute_with_policy_recorded,
219};
220pub use retry::execute_with_retry;
221pub use rollback::{
222    DEFAULT_RUN_ID_COLUMN, PREVIOUS_TABLE_SUFFIX, RUN_JOURNAL_TABLE, RollbackMode, RollbackOptions,
223    RollbackOutcome, RollbackWriteSpec,
224};
225pub use shard::ShardSpec;
226#[cfg(feature = "transform-cdc-unwrap")]
227pub use stage::{CdcMissingImage, CdcUnwrapSpec};
228#[cfg(feature = "transform-unpivot")]
229pub use stage::{CompiledUnpivot, UnpivotSpec};
230#[cfg(feature = "transform-explode")]
231pub use stage::{ExplodeSpec, OnMissing};
232#[cfg(feature = "transform-filter")]
233pub use stage::{FilterOp, FilterSpec};
234pub use stage::{TransformStage, apply_stages, compile_stage};
235pub use staging::{
236    StagedFile, StagingCleanup, StagingCompression, StagingFormat, StagingLocation, StagingScheme,
237    StagingSpec, serialize_records,
238};
239pub use state::{FileStateStore, MemoryStateStore, StateExport, StateStore};
240pub use state_key::{KEY_SCHEME_VERSION, KeyMigration, KeySuffix, StateKey, migrate_state_key};
241pub use state_version::{
242    ResolvedState, STATE_FORMAT, StateCompat, StoredState, check_compat, peel_versioned,
243    resolve_for_source, wrap_versioned,
244};
245pub use tls::TlsClientConfig;
246pub use topology::{
247    Edge, JoinNode, Node, NodeKind, Topology, TopologyBuilder, TopologyOnError, TopologyOptions,
248    TopologyResult,
249};
250pub use traits::{RowOutcome, Sink, Source};
251#[cfg(feature = "transform-json-parse")]
252pub use transform::JsonParseOnError;
253#[cfg(feature = "transform-lookup")]
254pub use transform::LookupOnMissing;
255pub use transform::RecordTransform;
256#[cfg(feature = "transform-value-case")]
257pub use transform::ValueCaseMode;
258#[cfg(feature = "transform-cast")]
259pub use transform::{CastOnError, CastType};
260#[cfg(feature = "transform-hash")]
261pub use transform::{HashAlgorithm, HashEncoding};
262#[cfg(feature = "transform-keys-case")]
263pub use transform::{KeyCaseMode, KeyCollision};
264pub use transforming_source::TransformingSource;
265#[cfg(feature = "transform-tree-flatten")]
266pub use tree::{AncestorsSpec, ColumnsSpec, CompiledTreeFlatten, TreeFlattenSpec};
267pub use usage::{CostSignal, UsageMeter, UsageSide, UsageSnapshot, estimate_json_bytes};
268pub use util::{redact_uri_credentials, redacted_uri_authority};
269pub use verify::{IntegrityCheck, LengthCheck, VerifyingReader};
270pub use window::{
271    WINDOW_END_PLACEHOLDER, WINDOW_PLACEHOLDER, WINDOW_START_PLACEHOLDER, Window, WindowBind,
272    WindowSpec, enumerate_windows, parse_step,
273};
274pub use write_mode::{
275    DeleteMarker, KeyTuple, OverwriteScope, WriteMode, WritePlan, WriteSpec, key_to_doc_id,
276    key_to_filter, plan_writes, plan_writes_exact, plan_writes_ordered, sql_literal,
277};
278#[cfg(feature = "transform-zip-columns")]
279pub use zip_columns::{ColumnGroupSpec, CompiledZipColumns, ZipColumnsSpec};
280
281// Re-export dependencies that connector authors need, so they only depend on
282// `faucet-core` instead of adding `async-trait` and `serde_json` themselves.
283pub use async_stream;
284pub use async_trait::async_trait;
285pub use futures_core::{self, Stream};
286pub use schemars::{self, JsonSchema, schema_for};
287pub use serde_json::{self, Value, json};
288/// Re-exported so callers of [`Pipeline::with_cancel`](pipeline::Pipeline::with_cancel)
289/// / [`RunStreamOptions::with_cancel`] can name the token type without adding
290/// `tokio-util` themselves.
291pub use tokio_util::sync::CancellationToken;
292
293#[cfg(feature = "compression")]
294pub use compression::{Compression, CompressionConfig, compress_buf, warn_mismatch};
295
296#[cfg(feature = "quality")]
297pub use quality::{
298    BatchCheck, CheckTally, CompareOp, CompiledQuality, JsonType, OnFailure, QualityOutcome,
299    QualitySpec, QuarantinedRecord, RecordCheck, apply_quality,
300};
301
302#[cfg(feature = "contract")]
303pub use contract::{
304    CompiledContract, ContractFieldType, ContractOutcome, ContractSpec, ContractViolation,
305    FieldContract, OnBreach, ViolatingRecord, apply_contract, to_json_schema, to_openlineage_facet,
306};
307
308#[cfg(feature = "masking")]
309pub use masking::{
310    CompiledMasking, Detector, MaskAction, MaskHit, MaskRule, MaskingOutcome, MaskingSpec,
311    MatchSpec, apply_masking,
312};
faucet_core/lib.rs

faucet_core/
lib.rs
