Struct spark_connect_rs::column::Column

pub struct Column {
    pub expression: Expression,
}

Column

A column holds a specific spark::Expression which will be resolved once an action is called. The columns are resolved by the Spark Connect server of the remote session.

A column instance can be created by in a similar way as to the Spark API. A column with created with col("*") or col("name.*") is created as an unresolved star attribute which will select all columns or references in the specified column.

use spark_connect_rs::{SparkSession, SparkSessionBuilder};

let spark: SparkSession = SparkSessionBuilder::remote("sc://127.0.0.1:15002/;user_id=example_rs".to_string())
        .build()
        .await?;

// As a &str representing an unresolved column in the dataframe
spark.range(None, 1, 1, Some(1)).select("id");

// By using the `col` function
spark.range(None, 1, 1, Some(1)).select(col("id"));

// By using the `lit` function to return a literal value
spark.range(None, 1, 1, Some(1)).select(lit(4.0).alias("num_col"));

Fields

expression: Expression

a spark::Expression containing any unresolved value to be leveraged in a spark::Plan

Implementations

impl Column

pub fn alias(self, value: &str) -> Column

Returns the column with a new name

Example:

let cols = [
    col("name").alias("new_name"),
    col("age").alias("new_age")
];

df.select(cols);

Examples found in repository

examples/reader.rs (line 24)

async fn main() -> Result<(), Box<dyn std::error::Error>> {
    let spark: SparkSession = SparkSessionBuilder::default().build().await?;

    let path = ["/opt/spark/examples/src/main/resources/people.csv"];

    let df = spark
        .read()
        .format("csv")
        .option("header", "True")
        .option("delimiter", ";")
        .load(path)?;

    df.select([
        F::col("name"),
        F::col("age").cast("int").alias("age_int"),
        (F::lit(3.0) + F::col("age").cast("int")).alias("addition"),
    ])
    .sort(vec![F::col("name").desc()])
    .show(Some(5), None, None)
    .await?;

    // print results
    // +--------------------------+
    // | show_string              |
    // +--------------------------+
    // | +-----+-------+--------+ |
    // | |name |age_int|addition| |
    // | +-----+-------+--------+ |
    // | |Jorge|30     |33.0    | |
    // | |Bob  |32     |35.0    | |
    // | +-----+-------+--------+ |
    // |                          |
    // +--------------------------+

    Ok(())
}

examples/writer.rs (line 21)

async fn main() -> Result<(), Box<dyn std::error::Error>> {
    let spark: SparkSession = SparkSessionBuilder::remote("sc://127.0.0.1:15002/")
        .build()
        .await?;

    let df = spark
        .clone()
        .range(None, 1000, 1, Some(16))
        .select(col("id").alias("range_id"));

    let path = "/opt/spark/examples/src/main/rust/employees/";

    df.write()
        .format("csv")
        .mode(SaveMode::Overwrite)
        .option("header", "true")
        .save(path)
        .await?;

    let df = spark
        .clone()
        .read()
        .format("csv")
        .option("header", "true")
        .load([path])?;

    df.show(Some(10), None, None).await?;

    // print results may slighty vary but should be close to the below
    // +--------------------------+
    // | show_string              |
    // +--------------------------+
    // | +--------+               |
    // | |range_id|               |
    // | +--------+               |
    // | |312     |               |
    // | |313     |               |
    // | |314     |               |
    // | |315     |               |
    // | |316     |               |
    // | |317     |               |
    // | |318     |               |
    // | |319     |               |
    // | |320     |               |
    // | |321     |               |
    // | +--------+               |
    // | only showing top 10 rows |
    // |                          |
    // +--------------------------+

    Ok(())
}

pub fn name(self, value: &str) -> Column

An alias for the function alias

pub fn asc(self) -> Column

Returns a sorted expression based on the ascending order of the column

Example:

let df: DataFrame = df.sort(col("id").asc());

let df: DataFrame = df.sort(asc(col("id")));

pub fn asc_nulls_first(self) -> Column

pub fn asc_nulls_last(self) -> Column

pub fn desc(self) -> Column

Returns a sorted expression based on the ascending order of the column

Example:

let df: DataFrame = df.sort(col("id").desc());

let df: DataFrame = df.sort(desc(col("id")));

Examples found in repository

examples/reader.rs (line 27)

async fn main() -> Result<(), Box<dyn std::error::Error>> {
    let spark: SparkSession = SparkSessionBuilder::default().build().await?;

    let path = ["/opt/spark/examples/src/main/resources/people.csv"];

    let df = spark
        .read()
        .format("csv")
        .option("header", "True")
        .option("delimiter", ";")
        .load(path)?;

    df.select([
        F::col("name"),
        F::col("age").cast("int").alias("age_int"),
        (F::lit(3.0) + F::col("age").cast("int")).alias("addition"),
    ])
    .sort(vec![F::col("name").desc()])
    .show(Some(5), None, None)
    .await?;

    // print results
    // +--------------------------+
    // | show_string              |
    // +--------------------------+
    // | +-----+-------+--------+ |
    // | |name |age_int|addition| |
    // | +-----+-------+--------+ |
    // | |Jorge|30     |33.0    | |
    // | |Bob  |32     |35.0    | |
    // | +-----+-------+--------+ |
    // |                          |
    // +--------------------------+

    Ok(())
}

pub fn desc_nulls_first(self) -> Column

pub fn desc_nulls_last(self) -> Column

pub fn cast(self, to_type: &str) -> Column

Casts the column into the Spark type represented as a &str

Arguments:

• to_type is the string representation of the datatype

Example:

let df = df.select([
      col("age").cast("int"),
      col("name").cast("string")
    ])

Examples found in repository

examples/reader.rs (line 24)

async fn main() -> Result<(), Box<dyn std::error::Error>> {
    let spark: SparkSession = SparkSessionBuilder::default().build().await?;

    let path = ["/opt/spark/examples/src/main/resources/people.csv"];

    let df = spark
        .read()
        .format("csv")
        .option("header", "True")
        .option("delimiter", ";")
        .load(path)?;

    df.select([
        F::col("name"),
        F::col("age").cast("int").alias("age_int"),
        (F::lit(3.0) + F::col("age").cast("int")).alias("addition"),
    ])
    .sort(vec![F::col("name").desc()])
    .show(Some(5), None, None)
    .await?;

    // print results
    // +--------------------------+
    // | show_string              |
    // +--------------------------+
    // | +-----+-------+--------+ |
    // | |name |age_int|addition| |
    // | +-----+-------+--------+ |
    // | |Jorge|30     |33.0    | |
    // | |Bob  |32     |35.0    | |
    // | +-----+-------+--------+ |
    // |                          |
    // +--------------------------+

    Ok(())
}

pub fn isin<T>(self, cols: Vec<T>) -> Column

where T: ToLiteralExpr,

A boolean expression that is evaluated to true if the value of the expression is contained by the evaluated values of the arguments

Arguments:

• cols a value that implements the ToLiteralExpr trait

Example:

df.filter(col("name").isin(["Jorge", "Bob"]));

pub fn contains<T>(self, other: T) -> Column

where T: ToLiteralExpr,

A boolean expression that is evaluated to true if the value is in the Column

Arguments:

• cols: a col reference that is translated into an spark::Expression

Example:

df.filter(col("name").contains("ge"));

pub fn startswith<T>(self, other: T) -> Column

where T: ToLiteralExpr,

A filter expression that evaluates if the column startswith a string literal

pub fn endswith<T>(self, other: T) -> Column

where T: ToLiteralExpr,

A filter expression that evaluates if the column endswith a string literal

pub fn like<T>(self, other: T) -> Column

where T: ToLiteralExpr,

A SQL LIKE filter expression that evaluates the column based on a case sensitive match

pub fn ilike<T>(self, other: T) -> Column

where T: ToLiteralExpr,

A SQL ILIKE filter expression that evaluates the column based on a case insensitive match

pub fn rlike<T>(self, other: T) -> Column

where T: ToLiteralExpr,

A SQL RLIKE filter expression that evaluates the column based on a regex match

pub fn eq<T>(self, other: T) -> Column

where T: ToExpr,

Equality comparion. Cannot overload the ‘==’ and return something other than a bool

pub fn and<T>(self, other: T) -> Column

where T: ToExpr,

Logical AND comparion. Cannot overload the ‘&&’ and return something other than a bool

pub fn or<T>(self, other: T) -> Column

where T: ToExpr,

Logical OR comparion.

pub fn isNull(self) -> Column

A filter expression that evaluates to true is the expression is null

pub fn isNotNull(self) -> Column

A filter expression that evaluates to true is the expression is NOT null

pub fn isNaN(self) -> Column

pub fn over(self, window: WindowSpec) -> Column

Defines a windowing column

Arguments:

• window: a WindowSpec

Example

let window = Window::new()
    .partitionBy(col("name"))
    .orderBy([col("age")])
    .rangeBetween(Window::unboundedPreceding(), Window::currentRow());

let df = df.withColumn("rank", rank().over(window.clone()))
    .withColumn("min", min("age").over(window));

Trait Implementations

impl Add for Column

type Output = Column

The resulting type after applying the + operator.

fn add(self, other: Column) -> Column

Performs the + operation.

impl BitAnd for Column

type Output = Column

The resulting type after applying the & operator.

fn bitand(self, other: Column) -> Column

Performs the & operation.

impl BitOr for Column

type Output = Column

The resulting type after applying the | operator.

fn bitor(self, other: Column) -> Column

Performs the | operation.

impl BitXor for Column

type Output = Column

The resulting type after applying the ^ operator.

fn bitxor(self, other: Column) -> Column

Performs the ^ operation.

impl Clone for Column

fn clone(&self) -> Column

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for Column

fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error>

Formats the value using the given formatter.

impl Div for Column

type Output = Column

The resulting type after applying the / operator.

fn div(self, other: Column) -> Column

Performs the / operation.

impl From<&str> for Column

fn from(value: &str) -> Column

&str values containing a * will be created as an unresolved star expression Otherwise, the value is created as an unresolved attribute

impl From<Expression> for Column

fn from(expression: Expression) -> Column

Used for creating columns from a spark::Expression

impl Mul for Column

type Output = Column

The resulting type after applying the * operator.

fn mul(self, other: Column) -> Column

Performs the * operation.

impl Neg for Column

type Output = Column

The resulting type after applying the - operator.

fn neg(self) -> Column

Performs the unary - operation.

impl Not for Column

type Output = Column

The resulting type after applying the ! operator.

fn not(self) -> <Column as Not>::Output

Performs the unary ! operation.

impl Rem for Column

type Output = Column

The resulting type after applying the % operator.

fn rem(self, other: Column) -> Column

Performs the % operation.

impl Sub for Column

type Output = Column

The resulting type after applying the - operator.

fn sub(self, other: Column) -> Column

Performs the - operation.

impl ToExpr for Column

fn to_expr(&self) -> Expression

impl ToFilterExpr for Column

fn to_filter_expr(&self) -> Option<Expression>

impl ToLiteralExpr for Column

fn to_literal_expr(&self) -> Expression