spark_connect_core/lib.rs
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136
//! Spark Connection Client for Rust
//!
//! Currently, the Spark Connect client for Rust is **highly experimental** and **should
//! not be used in any production setting**. This is currently a "proof of concept" to identify the methods
//! of interacting with Spark cluster from rust.
//!
//! # Quickstart
//!
//! Create a Spark Session and create a [DataFrame] from a [arrow::array::RecordBatch].
//!
//! ```rust
//! use spark_connect_rs::{SparkSession, SparkSessionBuilder};
//! use spark_connect_rs::functions::{col, lit}
//!
//! #[tokio::main]
//! async fn main() -> Result<(), Box<dyn std::error::Error>> {
//!
//! let spark: SparkSession = SparkSessionBuilder::remote("sc://127.0.0.1:15002/;user_id=example_rs")
//! .build()
//! .await?;
//!
//! let name: ArrayRef = Arc::new(StringArray::from(vec!["Tom", "Alice", "Bob"]));
//! let age: ArrayRef = Arc::new(Int64Array::from(vec![14, 23, 16]));
//!
//! let data = RecordBatch::try_from_iter(vec![("name", name), ("age", age)])?
//!
//! let df = spark.create_dataframe(&data).await?
//!
//! // 2 records total
//! let records = df.select("*")
//! .with_column("age_plus", col("age") + lit(4))
//! .filter(col("name").contains("o"))
//! .count()
//! .await?;
//!
//! Ok(())
//! };
//!```
//!
//! Create a Spark Session and create a DataFrame from a SQL statement:
//!
//! ```rust
//! use spark_connect_rs::{SparkSession, SparkSessionBuilder};
//!
//! #[tokio::main]
//! async fn main() -> Result<(), Box<dyn std::error::Error>> {
//!
//! let spark: SparkSession = SparkSessionBuilder::remote("sc://127.0.0.1:15002/;user_id=example_rs")
//! .build()
//! .await?;
//!
//! let df = spark.sql("SELECT * FROM json.`/datasets/employees.json`").await?;
//!
//! // Show the first 5 records
//! df.filter("salary > 3000").show(Some(5), None, None).await?;
//!
//! Ok(())
//! };
//!```
//!
//! Create a Spark Session, read a CSV file into a DataFrame, apply function transformations, and write the results:
//!
//! ```rust
//! use spark_connect_rs::{SparkSession, SparkSessionBuilder};
//!
//! use spark_connect_rs::functions as F;
//!
//! #[tokio::main]
//! async fn main() -> Result<(), Box<dyn std::error::Error>> {
//!
//! let spark: SparkSession = SparkSessionBuilder::remote("sc://127.0.0.1:15002/;user_id=example_rs")
//! .build()
//! .await?;
//!
//! let paths = ["/datasets/people.csv"];
//!
//! let df = spark
//! .read()
//! .format("csv")
//! .option("header", "True")
//! .option("delimiter", ";")
//! .load(paths)?;
//!
//! let df = df
//! .filter("age > 30")
//! .select([
//! F::col("name"),
//! F::col("age").cast("int")
//! ]);
//!
//! df.write()
//! .format("csv")
//! .option("header", "true")
//! .save("/opt/spark/examples/src/main/rust/people/")
//! .await?;
//!
//! Ok(())
//! };
//!```
//!
//! ## Databricks Connection
//!
//! Spark Connect is enabled for Databricks Runtime 13.3 LTS and above, and requires the feature
//! flag `feature = "tls"`. The connection string for the remote session must contain the following
//! values in the string;
//!
//! ```rust
//! "sc://<workspace id>:443/;token=<personal access token>;x-databricks-cluster-id=<cluster-id>"
//! ```
//!
//!
/// Spark Connect gRPC protobuf translated using [tonic]
pub mod spark {
tonic::include_proto!("spark.connect");
}
pub mod catalog;
pub mod client;
pub mod column;
pub mod conf;
pub mod dataframe;
pub mod errors;
pub mod expressions;
pub mod functions;
pub mod group;
pub mod plan;
pub mod readwriter;
pub mod session;
pub mod storage;
pub mod streaming;
pub mod types;
pub mod window;
pub use dataframe::{DataFrame, DataFrameReader, DataFrameWriter};
pub use session::{SparkSession, SparkSessionBuilder};