spark-connect 0.3.0

Rust client for Apache Spark Connect.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33

use spark_connect::SparkSession;

#[tokio::main]
async fn main() {
    // Retrieve Databricks connection parameters from environment variables.
    let host = std::env::var("DATABRICKS_HOST")
        .expect("DATABRICKS_HOST env var not set");
    let cluster_id = std::env::var("DATABRICKS_CLUSTER_ID")
        .expect("DATABRICKS_CLUSTER_ID env var not set");
    let token = std::env::var("DATABRICKS_TOKEN")
        .expect("DATABRICKS_TOKEN env var not set");

    // Build the Spark session.
    let session = SparkSession::builder()
        .remote(&format!(
            "sc://{host}:443/;\
            use_ssl=true;\
            x-databricks-cluster-id={cluster_id};\
            token={token};"
        ))
        .get_or_create()
        .await
        .expect("Failed to create Spark session");

    // Execute a simple query to verify the connection.
    let batches = session
        .query("SELECT COUNT(*) FROM raw.raw_balance")
        .execute()
        .await
        .unwrap();
    
    println!("Retrieved: {:?}", batches);
}