apache-spark-connect 4.2.0

Pure-Rust Spark Connect DataFrame client mirroring the PySpark API surface
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67

//! Pure-Rust Spark Connect DataFrame client mirroring the PySpark API surface.
//!
//! This crate provides the core data types and operations for interacting with
//! Apache Spark via the Spark Connect protocol.

#![allow(clippy::result_large_err)] // SparkError is large; changing the type would break the API

pub mod catalog;
pub mod column;
pub mod conf;
pub mod dataframe;
pub mod datasource;
pub mod expression;
pub mod functions;
pub mod group;
pub mod merge;
pub mod ml;
pub mod observation;
pub mod pipelines;
pub mod plan;
pub mod profiler;
pub mod readwriter;
pub mod resource;
pub mod row;
pub mod session;
pub mod storage;
pub mod streaming;
pub mod table_arg;
pub mod tvf;
pub mod types;
pub mod udf;
/// Run Rust UDFs on Spark via WebAssembly. Requires the `wasm-udf` feature.
#[cfg(feature = "wasm-udf")]
pub mod wasm_udf;
pub mod window;

// Re-export commonly used types
pub use column::{col, lit, lit_boolean, lit_double, lit_string, Column};
pub use dataframe::{DataFrame, LocalRowIterator};
pub use datasource::{CommonInlineUserDefinedDataSourceExpression, PythonDataSourcePayload};
pub use expression::{
    Alias, CaseWhen, Cast, ColumnReference, Expression, LiteralExpression, SortOrder,
    UnresolvedFunction,
};
pub use group::{CoGroupedData, GroupedData};
pub use merge::MergeIntoWriter;
pub use observation::Observation;
pub use profiler::ProfilerCollector;
pub use readwriter::{
    DataFrameReader, DataFrameWriter, DataFrameWriterV2, ReadType, SaveMode, TableSaveMethod,
};
pub use resource::{
    ExecutorResourceRequests, ResourceProfile, ResourceProfileBuilder, TaskResourceRequests,
};
pub use session::{SparkSession, SparkSessionBuilder};
/// Re-exported so fallible APIs (e.g. the `#[spark_wasm_udf]`-generated UDF
/// constructors) can name the client's `Result`/error types.
pub use spark_connect_core::error::{Result, SparkError};
/// Re-exported so `persist(...)` / `storage_level()` have a public storage-level type.
pub use spark_connect_proto::StorageLevel;
/// Named `StorageLevel` presets (`MEMORY_AND_DISK`, `DISK_ONLY`, ...) mirroring `pyspark.StorageLevel`.
pub use storage::StorageLevelExt;
pub use streaming::{
    DataStreamReader, DataStreamWriter, StreamingQuery, StreamingQueryManager, Trigger,
};
pub use types::{DataType, StructField};
pub use window::{FrameBound, FrameType, Window, WindowSpec};