faucet_source_file/
lib.rs

1#![cfg_attr(docsrs, feature(doc_cfg))]
2//! Local file source for faucet-stream: JSON Lines, JSON, CSV, Excel, XML,
3//! Parquet, Avro and ORC from a path, a directory, a glob, or one
4//! `http(s)://` URL — the local counterpart of the object-store sources.
5//!
6//! Formats resolve per file from the extension (through a `.gz` / `.zst`
7//! suffix) unless one is set; incremental mode reads only files that are new
8//! since the previous run, by modification time or by name. A directory or
9//! glob skips a file sink's unfinished output: its scratch files and the swap
10//! area of an overwrite run that has not committed.
11//!
12//! **Experimental** (PRINCIPLES.md §3): this block's shape may change in a
13//! minor release; any change is called out in the changelog.
14
15mod descriptor;
16pub use descriptor::DESCRIPTOR;
17
18pub mod config;
19#[cfg(feature = "encryption")]
20mod decrypt;
21pub mod http;
22#[cfg(feature = "file-format-parquet")]
23mod parquet;
24pub mod plan;
25mod stream;
26
27pub use config::{
28    FileSourceConfig, FileSourceFormat, IncrementalBy, IncrementalSpec, ParquetReadOptions,
29};
30pub use stream::FileSource;
31
32#[cfg(not(feature = "file-format-parquet"))]
33pub(crate) fn parquet_missing() -> faucet_core::FaucetError {
34    faucet_core::FaucetError::Config(
35        "file source: Parquet files need the `file-format-parquet` build feature".into(),
36    )
37}
38
39/// Read every record at `path` with every default — format resolved per file
40/// from its extension, compression from its suffix. The one-call read-back a
41/// test or a script wants; build a [`FileSource`] for anything more.
42pub async fn read_records(
43    path: impl Into<String>,
44) -> Result<Vec<faucet_core::Value>, faucet_core::FaucetError> {
45    use faucet_core::Source;
46    FileSource::new(FileSourceConfig::new(path))?
47        .fetch_all()
48        .await
49}
faucet_source_file/lib.rs

faucet_source_file/
lib.rs
