faucet_core/
lib.rs

1#![cfg_attr(docsrs, feature(doc_cfg))]
2
3//! # faucet-core
4//!
5//! Shared types, traits, and utilities for the faucet-stream ecosystem.
6//!
7//! This crate provides the common foundation used by all faucet source and
8//! sink connectors:
9//!
10//! - [`FaucetError`] — unified error type
11//! - [`Source`] / [`Sink`] — async traits for data connectors
12//! - [`RecordTransform`] — record transformation pipeline
13//! - [`ReplicationMethod`] — incremental replication support
14//! - [`schema::infer_schema`] — JSON Schema inference from record samples
15
16pub mod adaptive;
17pub mod auth;
18pub mod check;
19pub mod config;
20pub mod dlq;
21pub mod error;
22pub mod observability;
23pub mod pipeline;
24#[cfg(feature = "quality")]
25pub mod quality;
26pub mod replication;
27pub mod retry;
28pub mod schema;
29pub mod stage;
30pub mod state;
31pub mod traits;
32pub mod transform;
33pub mod transforming_source;
34pub mod util;
35
36#[cfg(feature = "compression")]
37pub mod compression;
38
39pub use adaptive::{
40    AdaptiveBatchConfig, AdjustDirection, AdjustReason, Adjustment, AimdController, Observation,
41};
42pub use auth::{AuthProvider, AuthReference, AuthSpec, Credential, SharedAuthProvider};
43pub use check::{CheckContext, CheckReport, Probe, ProbeStatus};
44pub use dlq::{DlqConfig, DlqReason, DlqStats, OnBatchError, build_envelope};
45pub use error::FaucetError;
46#[cfg(feature = "quality")]
47pub use observability::instrumented_apply_quality;
48pub use observability::{
49    DurationGuard, InstallError, InstallReport, InstrumentedSink, InstrumentedSource,
50    InstrumentedStateStore, Labels, ObservabilityConfig, PrometheusConfig, RunStreamOptions,
51    TracingConfig, install_observability, instrumented_apply_stages, register_build_info,
52    update_bookmark_lag,
53};
54pub use pipeline::{
55    DEFAULT_BATCH_SIZE, MAX_BATCH_SIZE, Pipeline, PipelineResult, StreamPage, run_stream,
56    validate_batch_size,
57};
58pub use replication::ReplicationMethod;
59pub use retry::execute_with_retry;
60#[cfg(feature = "transform-explode")]
61pub use stage::{ExplodeSpec, OnMissing};
62#[cfg(feature = "transform-filter")]
63pub use stage::{FilterOp, FilterSpec};
64pub use stage::{TransformStage, compile_stage};
65pub use state::{FileStateStore, MemoryStateStore, StateStore};
66pub use traits::{RowOutcome, Sink, Source};
67#[cfg(feature = "transform-keys-case")]
68pub use transform::KeyCaseMode;
69pub use transform::RecordTransform;
70#[cfg(feature = "transform-value-case")]
71pub use transform::ValueCaseMode;
72#[cfg(feature = "transform-cast")]
73pub use transform::{CastOnError, CastType};
74pub use transforming_source::TransformingSource;
75
76// Re-export dependencies that connector authors need, so they only depend on
77// `faucet-core` instead of adding `async-trait` and `serde_json` themselves.
78pub use async_stream;
79pub use async_trait::async_trait;
80pub use futures_core::{self, Stream};
81pub use schemars::{self, JsonSchema, schema_for};
82pub use serde_json::{self, Value, json};
83/// Re-exported so callers of [`Pipeline::with_cancel`](pipeline::Pipeline::with_cancel)
84/// / [`RunStreamOptions::with_cancel`] can name the token type without adding
85/// `tokio-util` themselves.
86pub use tokio_util::sync::CancellationToken;
87
88#[cfg(feature = "compression")]
89pub use compression::{Compression, CompressionConfig, compress_buf, warn_mismatch};
90
91#[cfg(feature = "quality")]
92pub use quality::{
93    BatchCheck, CheckTally, CompareOp, CompiledQuality, JsonType, OnFailure, QualityOutcome,
94    QualitySpec, QuarantinedRecord, RecordCheck, apply_quality,
95};
faucet_core/lib.rs

faucet_core/
lib.rs
