faucet_core/
lib.rs

1#![cfg_attr(docsrs, feature(doc_cfg))]
2
3//! # faucet-core
4//!
5//! Shared types, traits, and utilities for the faucet-stream ecosystem.
6//!
7//! This crate provides the common foundation used by all faucet source and
8//! sink connectors:
9//!
10//! - [`FaucetError`] — unified error type
11//! - [`Source`] / [`Sink`] — async traits for data connectors
12//! - [`RecordTransform`] — record transformation pipeline
13//! - [`ReplicationMethod`] — incremental replication support
14//! - [`schema::infer_schema`] — JSON Schema inference from record samples
15
16pub mod accounting;
17pub mod adaptive;
18pub mod anomaly;
19pub mod auth;
20pub mod batching;
21pub mod budget;
22pub mod capabilities;
23pub mod check;
24pub mod cleanup;
25#[cfg(feature = "arrow")]
26pub mod columnar;
27#[cfg(feature = "arrow")]
28pub mod columnar_transform;
29pub mod compat;
30pub mod config;
31#[cfg(feature = "contract")]
32pub mod contract;
33pub mod create_table;
34#[cfg(feature = "transform-cross-join")]
35pub mod cross_join;
36pub mod diff;
37pub mod discover;
38pub mod dlq;
39pub mod drift;
40pub mod driver;
41#[cfg(feature = "encryption")]
42pub mod encryption;
43pub mod error;
44pub mod fidelity;
45pub mod file_format;
46pub mod forward;
47pub mod governance;
48pub mod http;
49// The conformance crate's forwarding harness, compiled into core's own unit
50// tests under the `faucet_core::` paths it is written against.
51#[cfg(test)]
52extern crate self as faucet_core;
53#[cfg(test)]
54#[path = "../../conformance/src/forwarding.rs"]
55#[allow(dead_code)]
56mod forwarding;
57pub mod idempotency;
58pub mod io;
59pub mod join;
60pub mod json_numbers;
61pub mod lag;
62pub mod lease;
63pub mod local_outputs;
64#[cfg(feature = "masking")]
65pub mod masking;
66pub mod metadata;
67pub mod native;
68pub mod object_rollover;
69pub mod observability;
70pub mod pipeline;
71#[cfg(feature = "policy")]
72pub mod policy;
73pub mod profiling;
74#[cfg(feature = "quality")]
75pub mod quality;
76pub mod redact;
77pub mod replication;
78pub mod resilience;
79pub mod resolve;
80pub mod retry;
81pub mod rollback;
82pub mod schema;
83pub mod shard;
84pub mod stage;
85pub mod staging;
86pub mod state;
87pub mod state_key;
88pub mod state_version;
89pub mod tls;
90pub mod topology;
91pub mod traits;
92pub mod transform;
93pub mod transforming_source;
94#[cfg(feature = "transform-tree-flatten")]
95pub mod tree;
96pub mod usage;
97pub mod util;
98pub mod verify;
99pub mod window;
100pub mod write_mode;
101#[cfg(feature = "transform-zip-columns")]
102pub mod zip_columns;
103
104#[cfg(feature = "compression")]
105pub mod compression;
106
107pub use accounting::{RowAccounts, RowAccountsSnapshot, WriteReport};
108pub use adaptive::{
109    AdaptiveBatchConfig, AdjustDirection, AdjustReason, Adjustment, AimdController, Observation,
110};
111pub use anomaly::AnomalyMethod;
112pub use auth::{
113    AuthProvider, AuthReference, AuthSpec, Credential, CredentialPlacement, RequestAuth,
114    SharedAuthProvider, rejects_credential, send_with_reauth,
115};
116pub use budget::{BudgetKind, BudgetSink, BudgetSpec, BudgetState, BudgetTimer, BudgetVerdict};
117pub use capabilities::{Capabilities, CapabilitiesBuilder, ConnectorDescriptor, SchemaReport};
118pub use check::{CheckContext, CheckReport, Probe, ProbeStatus};
119pub use cleanup::{
120    CleanupMode, CleanupPolicy, CleanupTrigger, DEFAULT_COMPLETE_READ_MAX_KEYS, DEFAULT_MAX_KEYS,
121    ReadCompleteness, ReadState, SeenKeys,
122};
123#[cfg(feature = "arrow")]
124pub use columnar::{
125    ColumnarPage, check_integral, infer_arrow_schema, record_batch_to_values,
126    values_to_record_batch, values_to_record_batch_inferred,
127};
128pub use create_table::{
129    PlannedColumn, missing_target_error, plan_columns, plan_keyed_columns, render_column_defs,
130    render_columns, render_primary_key,
131};
132#[cfg(feature = "transform-cross-join")]
133pub use cross_join::{CompiledCrossJoin, CrossJoinSpec, OnEmpty as CrossJoinOnEmpty};
134pub use diff::{
135    ContentDigest, Difference, DifferenceKind, DigestAccumulator, KeyRange, Normalizer,
136    ServerDigest, VerifyReport, diff_rows, plan_ranges, row_hash,
137};
138pub use discover::{
139    DatasetDescriptor, attach_primary_keys, columns_to_schema, nullable_type,
140    sql_type_to_json_schema,
141};
142pub use dlq::{
143    BatchAtomicity, BatchOutcome, BatchOutcomeCounters, BatchOutcomes, DlqConfig, DlqReason,
144    DlqStats, EnvelopeError, OnBatchError, UnwrappedEnvelope, build_envelope, check_dlq_all_policy,
145    dlq_all_is_safe, dlq_all_refusal, unwrap_envelope,
146};
147pub use drift::{
148    ColumnChange, DRIFT_ALSO_ACCEPTS, OnDrift, OnIncompatible, SchemaDiff, SchemaDriftPolicy,
149    SchemaDriftSpec, SchemaEvolution, SqlBaseType, adds_null, base_widened, json_schema_base_type,
150};
151#[cfg(feature = "encryption")]
152pub use encryption::{CompiledEncryption, EncryptionAlgorithm, EncryptionSpec, SealedLine};
153pub use error::FaucetError;
154pub use fidelity::{FidelityError, NonFiniteCounter, TimePrecision};
155pub use file_format::parquet_io::ParquetReadOptions;
156pub use file_format::{
157    AvroCodec, AvroOptions, ContainerDecoder, CsvOptions, CsvUnknownField, ExcelOptions,
158    FileFormat, FileInput, FormatOptions, OrcOptions, XmlOptions,
159};
160#[cfg(feature = "policy")]
161pub use governance::PolicyGovernance;
162pub use governance::{
163    DlqGovernance, Governance, GovernedSink, GovernedSinkBuilder, OptOut, Pass, SINK_LAYERS,
164    SinkLayer,
165};
166pub use idempotency::{
167    DeliveryGuarantee, DeliveryMode, EffectivelyOnceMechanism, GuaranteeInputs, ReplayGuarantee,
168    SinkGuarantee, derive_delivery_guarantee, format_token, format_token_with_bookmark,
169    parse_token, parse_token_parts, unwrap_state, wrap_state,
170};
171pub use io::{CallContext, CallExt, CallReader, CallSlot, DEFAULT_CALL_TIMEOUT};
172pub use join::{
173    HashJoin, JoinConfig, JoinMode, JoinStats, KeyNormalize, OnCollision, OnDuplicate, Projection,
174};
175pub use json_numbers::{JsonBigNumbers, parse_json_exact};
176pub use lag::{LagObserver, SourceLag};
177pub use local_outputs::{LocalOutput, LocalOutputLog, probe_pre_existing};
178pub use metadata::{
179    CompiledMetadata, MetadataColumn, MetadataColumnsSpec, MetadataContext, MetadataSink,
180};
181pub use native::{
182    CsvDialect, NativeBatch, NativeFormat, NativeLoadCapability, NativeLoadContext, NativePayload,
183    NativePlan, NativePlanInputs, plan_native_transfer,
184};
185pub use object_rollover::{CompletedObject, ObjectAccumulator, PageAccumulator};
186#[cfg(feature = "contract")]
187pub use observability::instrumented_apply_contract;
188#[cfg(feature = "masking")]
189pub use observability::instrumented_apply_masking;
190#[cfg(feature = "quality")]
191pub use observability::instrumented_apply_quality;
192pub use observability::otel::{OtelConfig, OtelProtocol, OtelSignal, shutdown_otel};
193pub use observability::{
194    DurationGuard, InstallError, InstallReport, InstrumentedSink, InstrumentedSource,
195    InstrumentedStateStore, Labels, ObservabilityConfig, PrometheusConfig, RunStreamOptions,
196    TracingConfig, build_version, install_observability, instrumented_apply_stages,
197    register_build_info, set_build_version, update_bookmark_lag,
198};
199pub use pipeline::{
200    DEFAULT_BATCH_SIZE, MAX_BATCH_SIZE, Pipeline, PipelineResult, StreamPage, run_stream,
201    validate_batch_size,
202};
203#[cfg(feature = "policy")]
204pub use policy::{
205    ColumnFacts, CompiledPolicy, PolicyRule, PolicyScope, PolicySink, PolicySpec, RuntimeAction,
206    SinkFacts, Violation, ViolationKind,
207};
208pub use profiling::{
209    ColumnProfile, DriftMetric, OnProfileDrift, ProfileDrift, Profiler, ProfilingSink,
210    ProfilingSpec, RunProfile, detect_drift,
211};
212pub use replication::{
213    BindFormat, BindTarget, BindValueType, IncrementalFilter, OnMissingKey, ReplicationBind,
214    ReplicationKey, ReplicationMethod, format_bookmark, format_instant, json_gt, parse_instant,
215    set_body_pointer,
216};
217pub use resilience::{
218    BackoffFrom, BackoffKind, CircuitBreaker, CircuitBreakerConfig, DEFAULT_MAX_WAIT_SECS,
219    PoisonAction, PoisonPolicy, ResiliencePolicy, RetryClass, RetryClassSet, RetryMatcher,
220    RetryMetrics, RetryPolicy, WaitUnit, classify, execute_with_policy,
221    execute_with_policy_metered, execute_with_policy_recorded,
222};
223pub use retry::execute_with_retry;
224pub use rollback::{
225    DEFAULT_RUN_ID_COLUMN, PREVIOUS_TABLE_SUFFIX, RUN_JOURNAL_TABLE, RollbackMode, RollbackOptions,
226    RollbackOutcome, RollbackWriteSpec,
227};
228pub use shard::ShardSpec;
229#[cfg(feature = "transform-cdc-unwrap")]
230pub use stage::{CdcMissingImage, CdcUnwrapSpec};
231#[cfg(feature = "transform-unpivot")]
232pub use stage::{CompiledUnpivot, UnpivotSpec};
233#[cfg(feature = "transform-explode")]
234pub use stage::{ExplodeSpec, OnMissing};
235#[cfg(feature = "transform-filter")]
236pub use stage::{FilterOp, FilterSpec};
237pub use stage::{TransformStage, apply_stages, compile_stage};
238pub use staging::{
239    StagedFile, StagingCleanup, StagingCompression, StagingFormat, StagingLocation, StagingScheme,
240    StagingSpec, serialize_records,
241};
242pub use state::{FileStateStore, MemoryStateStore, StateExport, StateStore};
243pub use state_key::{KEY_SCHEME_VERSION, KeyMigration, KeySuffix, StateKey, migrate_state_key};
244pub use state_version::{
245    ResolvedState, STATE_FORMAT, StateCompat, StoredState, check_compat, peel_versioned,
246    resolve_for_source, wrap_versioned,
247};
248pub use tls::TlsClientConfig;
249pub use topology::{
250    Edge, JoinNode, Node, NodeKind, Topology, TopologyBuilder, TopologyOnError, TopologyOptions,
251    TopologyResult,
252};
253pub use traits::{RowOutcome, Sink, Source};
254#[cfg(feature = "transform-json-parse")]
255pub use transform::JsonParseOnError;
256#[cfg(feature = "transform-lookup")]
257pub use transform::LookupOnMissing;
258pub use transform::RecordTransform;
259#[cfg(feature = "transform-value-case")]
260pub use transform::ValueCaseMode;
261#[cfg(feature = "transform-cast")]
262pub use transform::{CastOnError, CastType};
263#[cfg(feature = "transform-hash")]
264pub use transform::{HashAlgorithm, HashEncoding};
265#[cfg(feature = "transform-keys-case")]
266pub use transform::{KeyCaseMode, KeyCollision};
267pub use transforming_source::TransformingSource;
268#[cfg(feature = "transform-tree-flatten")]
269pub use tree::{AncestorsSpec, ColumnsSpec, CompiledTreeFlatten, TreeFlattenSpec};
270pub use usage::{CostSignal, UsageMeter, UsageSide, UsageSnapshot, estimate_json_bytes};
271pub use util::{redact_uri_credentials, redacted_uri_authority};
272pub use verify::{IntegrityCheck, LengthCheck, VerifyingReader};
273pub use window::{
274    WINDOW_END_PLACEHOLDER, WINDOW_PLACEHOLDER, WINDOW_START_PLACEHOLDER, Window, WindowBind,
275    WindowSpec, enumerate_windows, parse_step,
276};
277pub use write_mode::{
278    DeleteMarker, KeyTuple, OverwriteScope, WriteMode, WritePlan, WriteSpec, key_to_doc_id,
279    key_to_filter, plan_writes, plan_writes_exact, plan_writes_ordered, sql_literal,
280};
281#[cfg(feature = "transform-zip-columns")]
282pub use zip_columns::{ColumnGroupSpec, CompiledZipColumns, ZipColumnsSpec};
283
284// Re-export dependencies that connector authors need, so they only depend on
285// `faucet-core` instead of adding `async-trait` and `serde_json` themselves.
286pub use async_stream;
287pub use async_trait::async_trait;
288pub use futures_core::{self, Stream};
289pub use schemars::{self, JsonSchema, schema_for};
290pub use serde_json::{self, Value, json};
291/// Re-exported so callers of [`Pipeline::with_cancel`](pipeline::Pipeline::with_cancel)
292/// / [`RunStreamOptions::with_cancel`] can name the token type without adding
293/// `tokio-util` themselves.
294pub use tokio_util::sync::CancellationToken;
295
296#[cfg(feature = "compression")]
297pub use compression::{Compression, CompressionConfig, compress_buf, warn_mismatch};
298
299#[cfg(feature = "quality")]
300pub use quality::{
301    BatchCheck, CheckTally, CompareOp, CompiledQuality, JsonType, OnFailure, QualityOutcome,
302    QualitySpec, QuarantinedRecord, RecordCheck, apply_quality,
303};
304
305#[cfg(feature = "contract")]
306pub use contract::{
307    CompiledContract, ContractFieldType, ContractOutcome, ContractSpec, ContractViolation,
308    FieldContract, OnBreach, ViolatingRecord, apply_contract, to_json_schema, to_openlineage_facet,
309};
310
311#[cfg(feature = "masking")]
312pub use masking::{
313    CompiledMasking, Detector, MaskAction, MaskHit, MaskRule, MaskingOutcome, MaskingSpec,
314    MatchSpec, apply_masking,
315};
faucet_core/lib.rs

faucet_core/
lib.rs
