Struct SparkSession

pub struct SparkSession { /* private fields */ }

The entry point to connecting to a Spark Cluster using the Spark Connection gRPC protocol.

Implementations

impl SparkSession

pub fn new( client: SparkConnectClient<InterceptedService<Channel, MetadataInterceptor>>, ) -> Self

pub fn session(&self) -> SparkSession

pub fn range( &self, start: Option<i64>, end: i64, step: i64, num_partitions: Option<i32>, ) -> DataFrame

Create a DataFrame with a spingle column named id, containing elements in a range from start (default 0) to end (exclusive) with a step value step, and control the number of partitions with num_partitions

pub fn read(&self) -> DataFrameReader

Returns a DataFrameReader that can be used to read datra in as a DataFrame

pub fn read_stream(&self) -> DataStreamReader

Returns a DataFrameReader that can be used to read datra in as a DataFrame

pub fn table(&self, name: &str) -> Result<DataFrame, SparkError>

pub fn catalog(&self) -> Catalog

Interface through which the user may create, drop, alter or query underlying databases, tables, functions, etc.

pub async fn sql(&self, sql_query: &str) -> Result<DataFrame, SparkError>

Returns a DataFrame representing the result of the given query

pub fn create_dataframe( &self, data: &RecordBatch, ) -> Result<DataFrame, SparkError>

pub fn session_id(&self) -> &str

Return the session ID

pub fn client( self, ) -> SparkConnectClient<InterceptedService<Channel, MetadataInterceptor>>

Spark Connection gRPC client interface

pub async fn interrupt_all(&self) -> Result<Vec<String>, SparkError>

Interrupt all operations of this session currently running on the connected server.

pub async fn interrupt_tag(&self, tag: &str) -> Result<Vec<String>, SparkError>

Interrupt all operations of this session with the given operation tag.

pub async fn interrupt_operation( &self, op_id: &str, ) -> Result<Vec<String>, SparkError>

Interrupt an operation of this session with the given operationId.

pub fn add_tag(&mut self, tag: &str) -> Result<(), SparkError>

Add a tag to be assigned to all the operations started by this thread in this session.

pub fn remove_tag(&mut self, tag: &str) -> Result<(), SparkError>

Remove a tag previously added to be assigned to all the operations started by this thread in this session.

pub fn get_tags(&mut self) -> &Vec<String>

Get the tags that are currently set to be assigned to all the operations started by this thread.

pub fn clear_tags(&mut self)

Clear the current thread’s operation tags.

pub async fn version(&self) -> Result<String, SparkError>

The version of Spark on which this application is running.

pub fn conf(&self) -> RunTimeConfig

RunTimeConfig configuration interface for Spark.

Trait Implementations

impl Clone for SparkSession

fn clone(&self) -> SparkSession

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for SparkSession

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.