spark_connect/
lib.rs

1//! Pure-Rust Spark Connect DataFrame client mirroring the PySpark API surface.
2//!
3//! This crate provides the core data types and operations for interacting with
4//! Apache Spark via the Spark Connect protocol.
5
6#![allow(clippy::result_large_err)] // SparkError is large; changing the type would break the API
7
8pub mod catalog;
9pub mod column;
10pub mod conf;
11pub mod dataframe;
12pub mod datasource;
13pub mod expression;
14pub mod functions;
15pub mod group;
16pub mod merge;
17pub mod ml;
18pub mod observation;
19pub mod pipelines;
20pub mod plan;
21pub mod profiler;
22pub mod readwriter;
23pub mod resource;
24pub mod row;
25pub mod session;
26pub mod storage;
27pub mod streaming;
28pub mod table_arg;
29pub mod tvf;
30pub mod types;
31pub mod udf;
32/// Run Rust UDFs on Spark via WebAssembly. Requires the `wasm-udf` feature.
33#[cfg(feature = "wasm-udf")]
34pub mod wasm_udf;
35pub mod window;
36
37// Re-export commonly used types
38pub use column::{col, lit, lit_boolean, lit_double, lit_string, Column};
39pub use dataframe::{DataFrame, LocalRowIterator};
40pub use datasource::{CommonInlineUserDefinedDataSourceExpression, PythonDataSourcePayload};
41pub use expression::{
42    Alias, CaseWhen, Cast, ColumnReference, Expression, LiteralExpression, SortOrder,
43    UnresolvedFunction,
44};
45pub use group::{CoGroupedData, GroupedData};
46pub use merge::MergeIntoWriter;
47pub use observation::Observation;
48pub use profiler::ProfilerCollector;
49pub use readwriter::{
50    DataFrameReader, DataFrameWriter, DataFrameWriterV2, ReadType, SaveMode, TableSaveMethod,
51};
52pub use resource::{
53    ExecutorResourceRequests, ResourceProfile, ResourceProfileBuilder, TaskResourceRequests,
54};
55pub use session::{SparkSession, SparkSessionBuilder};
56/// Re-exported so fallible APIs (e.g. the `#[spark_wasm_udf]`-generated UDF
57/// constructors) can name the client's `Result`/error types.
58pub use spark_connect_core::error::{Result, SparkError};
59/// Re-exported so `persist(...)` / `storage_level()` have a public storage-level type.
60pub use spark_connect_proto::StorageLevel;
61/// Named `StorageLevel` presets (`MEMORY_AND_DISK`, `DISK_ONLY`, ...) mirroring `pyspark.StorageLevel`.
62pub use storage::StorageLevelExt;
63pub use streaming::{
64    DataStreamReader, DataStreamWriter, StreamingQuery, StreamingQueryManager, Trigger,
65};
66pub use types::{DataType, StructField};
67pub use window::{FrameBound, FrameType, Window, WindowSpec};
spark_connect/lib.rs

spark_connect/
lib.rs
