faucet_core/
lib.rs

1#![cfg_attr(docsrs, feature(doc_cfg))]
2
3//! # faucet-core
4//!
5//! Shared types, traits, and utilities for the faucet-stream ecosystem.
6//!
7//! This crate provides the common foundation used by all faucet source and
8//! sink connectors:
9//!
10//! - [`FaucetError`] — unified error type
11//! - [`Source`] / [`Sink`] — async traits for data connectors
12//! - [`RecordTransform`] — record transformation pipeline
13//! - [`ReplicationMethod`] — incremental replication support
14//! - [`schema::infer_schema`] — JSON Schema inference from record samples
15
16pub mod adaptive;
17pub mod auth;
18pub mod check;
19pub mod config;
20pub mod dlq;
21pub mod drift;
22pub mod error;
23pub mod idempotency;
24pub mod observability;
25pub mod pipeline;
26#[cfg(feature = "quality")]
27pub mod quality;
28pub mod replication;
29pub mod resilience;
30pub mod retry;
31pub mod schema;
32pub mod shard;
33pub mod stage;
34pub mod state;
35pub mod traits;
36pub mod transform;
37pub mod transforming_source;
38pub mod util;
39pub mod verify;
40pub mod write_mode;
41
42#[cfg(feature = "compression")]
43pub mod compression;
44
45pub use adaptive::{
46    AdaptiveBatchConfig, AdjustDirection, AdjustReason, Adjustment, AimdController, Observation,
47};
48pub use auth::{AuthProvider, AuthReference, AuthSpec, Credential, SharedAuthProvider};
49pub use check::{CheckContext, CheckReport, Probe, ProbeStatus};
50pub use dlq::{DlqConfig, DlqReason, DlqStats, OnBatchError, build_envelope};
51pub use drift::{
52    ColumnChange, OnDrift, OnIncompatible, SchemaDiff, SchemaDriftPolicy, SchemaDriftSpec,
53    SchemaEvolution, SqlBaseType, adds_null, base_widened, json_schema_base_type,
54};
55pub use error::FaucetError;
56pub use idempotency::{DeliveryMode, format_token, parse_token, unwrap_state, wrap_state};
57#[cfg(feature = "quality")]
58pub use observability::instrumented_apply_quality;
59pub use observability::otel::{OtelConfig, OtelProtocol, OtelSignal, shutdown_otel};
60pub use observability::{
61    DurationGuard, InstallError, InstallReport, InstrumentedSink, InstrumentedSource,
62    InstrumentedStateStore, Labels, ObservabilityConfig, PrometheusConfig, RunStreamOptions,
63    TracingConfig, install_observability, instrumented_apply_stages, register_build_info,
64    update_bookmark_lag,
65};
66pub use pipeline::{
67    DEFAULT_BATCH_SIZE, MAX_BATCH_SIZE, Pipeline, PipelineResult, StreamPage, run_stream,
68    validate_batch_size,
69};
70pub use replication::ReplicationMethod;
71pub use resilience::{
72    BackoffKind, CircuitBreaker, CircuitBreakerConfig, PoisonAction, PoisonPolicy,
73    ResiliencePolicy, RetryClass, RetryClassSet, RetryMetrics, RetryPolicy, classify,
74    execute_with_policy, execute_with_policy_metered,
75};
76pub use retry::execute_with_retry;
77pub use shard::ShardSpec;
78#[cfg(feature = "transform-cdc-unwrap")]
79pub use stage::CdcUnwrapSpec;
80#[cfg(feature = "transform-explode")]
81pub use stage::{ExplodeSpec, OnMissing};
82#[cfg(feature = "transform-filter")]
83pub use stage::{FilterOp, FilterSpec};
84pub use stage::{TransformStage, compile_stage};
85pub use state::{FileStateStore, MemoryStateStore, StateStore};
86pub use traits::{RowOutcome, Sink, Source};
87#[cfg(feature = "transform-keys-case")]
88pub use transform::KeyCaseMode;
89pub use transform::RecordTransform;
90#[cfg(feature = "transform-value-case")]
91pub use transform::ValueCaseMode;
92#[cfg(feature = "transform-cast")]
93pub use transform::{CastOnError, CastType};
94pub use transforming_source::TransformingSource;
95pub use util::redact_uri_credentials;
96pub use verify::{IntegrityCheck, LengthCheck, VerifyingReader};
97pub use write_mode::{
98    DeleteMarker, KeyTuple, WriteMode, WritePlan, WriteSpec, key_to_doc_id, key_to_filter,
99    plan_writes,
100};
101
102// Re-export dependencies that connector authors need, so they only depend on
103// `faucet-core` instead of adding `async-trait` and `serde_json` themselves.
104pub use async_stream;
105pub use async_trait::async_trait;
106pub use futures_core::{self, Stream};
107pub use schemars::{self, JsonSchema, schema_for};
108pub use serde_json::{self, Value, json};
109/// Re-exported so callers of [`Pipeline::with_cancel`](pipeline::Pipeline::with_cancel)
110/// / [`RunStreamOptions::with_cancel`] can name the token type without adding
111/// `tokio-util` themselves.
112pub use tokio_util::sync::CancellationToken;
113
114#[cfg(feature = "compression")]
115pub use compression::{Compression, CompressionConfig, compress_buf, warn_mismatch};
116
117#[cfg(feature = "quality")]
118pub use quality::{
119    BatchCheck, CheckTally, CompareOp, CompiledQuality, JsonType, OnFailure, QualityOutcome,
120    QualitySpec, QuarantinedRecord, RecordCheck, apply_quality,
121};
faucet_core/lib.rs

faucet_core/
lib.rs
