Struct spark_connect_rs::column::Column

pub struct Column {
    pub expression: Expression,
}

A Column is composed of a expressions which is used as input into a Plan object

Fields

expression: Expression

An expression is an unresolved value to be leveraged in a Spark Plan

Implementations

impl Column

pub fn alias(&mut self, value: &str) -> Column

Returns the column with a new name

Example:

let cols = vec![
    col("name").alias("new_name"),
    col("age").alias("new_age")
];

df.select(cols);

Examples found in repository

examples/reader.rs (line 23)

async fn main() -> Result<(), Box<dyn std::error::Error>> {
    let spark: SparkSession = SparkSessionBuilder::default().build().await?;

    let path = ["/opt/spark/examples/src/main/resources/people.csv"];

    let mut df = spark
        .read()
        .format("csv")
        .option("header", "True")
        .option("delimiter", ";")
        .load(path);

    df.select([
        F::col("name"),
        F::col("age").cast("int").alias("age_int"),
        (F::lit(3.0) + F::col("age").cast("int")).alias("addition"),
    ])
    .sort(vec![F::col("name").desc()])
    .show(Some(5), None, None)
    .await?;

    // print results
    // +--------------------------+
    // | show_string              |
    // +--------------------------+
    // | +-----+-------+--------+ |
    // | |name |age_int|addition| |
    // | +-----+-------+--------+ |
    // | |Jorge|30     |33.0    | |
    // | |Bob  |32     |35.0    | |
    // | +-----+-------+--------+ |
    // |                          |
    // +--------------------------+

    Ok(())
}

examples/writer.rs (line 19)

async fn main() -> Result<(), Box<dyn std::error::Error>> {
    let spark: SparkSession = SparkSessionBuilder::default().build().await?;

    let df = spark
        .clone()
        .range(None, 1000, 1, Some(16))
        .select(col("id").alias("range_id"));

    let path = "/opt/spark/examples/src/main/rust/employees/";

    df.write()
        .format("csv")
        .mode(SaveMode::Overwrite)
        .option("header", "true")
        .save(path)
        .await?;

    let mut df = spark
        .clone()
        .read()
        .format("csv")
        .option("header", "true")
        .load([path]);

    df.show(Some(10), None, None).await?;

    // print results may slighty vary but should be close to the below
    // +--------------------------+
    // | show_string              |
    // +--------------------------+
    // | +--------+               |
    // | |range_id|               |
    // | +--------+               |
    // | |312     |               |
    // | |313     |               |
    // | |314     |               |
    // | |315     |               |
    // | |316     |               |
    // | |317     |               |
    // | |318     |               |
    // | |319     |               |
    // | |320     |               |
    // | |321     |               |
    // | +--------+               |
    // | only showing top 10 rows |
    // |                          |
    // +--------------------------+

    Ok(())
}

pub fn name(&mut self, value: &str) -> Column

An alias for the function alias

pub fn asc(&mut self) -> Column

Returns a sorted expression based on the ascending order of the column

Example:

let mut df: DataFrame = df.sort(col("id").asc());

let mut df: DataFrame = df.sort(asc(col("id")));

pub fn asc_nulls_first(&mut self) -> Column

pub fn asc_nulls_last(&mut self) -> Column

pub fn desc(&mut self) -> Column

Returns a sorted expression based on the ascending order of the column

Example:

let mut df: DataFrame = df.sort(col("id").desc());

let mut df: DataFrame = df.sort(desc(col("id")));

Examples found in repository

examples/reader.rs (line 26)

async fn main() -> Result<(), Box<dyn std::error::Error>> {
    let spark: SparkSession = SparkSessionBuilder::default().build().await?;

    let path = ["/opt/spark/examples/src/main/resources/people.csv"];

    let mut df = spark
        .read()
        .format("csv")
        .option("header", "True")
        .option("delimiter", ";")
        .load(path);

    df.select([
        F::col("name"),
        F::col("age").cast("int").alias("age_int"),
        (F::lit(3.0) + F::col("age").cast("int")).alias("addition"),
    ])
    .sort(vec![F::col("name").desc()])
    .show(Some(5), None, None)
    .await?;

    // print results
    // +--------------------------+
    // | show_string              |
    // +--------------------------+
    // | +-----+-------+--------+ |
    // | |name |age_int|addition| |
    // | +-----+-------+--------+ |
    // | |Jorge|30     |33.0    | |
    // | |Bob  |32     |35.0    | |
    // | +-----+-------+--------+ |
    // |                          |
    // +--------------------------+

    Ok(())
}

pub fn desc_nulls_first(&mut self) -> Column

pub fn desc_nulls_last(&mut self) -> Column

pub fn cast(&mut self, to_type: &str) -> Column

Casts the column into the Spark type represented as a &str

Arguments:

• to_type is the string representation of the datatype

Example:

let mut df = df.select(vec![
      col("age").cast("int"),
      col("name").cast("string")
    ])
    .collect()
    .await?;

Examples found in repository

examples/reader.rs (line 23)

async fn main() -> Result<(), Box<dyn std::error::Error>> {
    let spark: SparkSession = SparkSessionBuilder::default().build().await?;

    let path = ["/opt/spark/examples/src/main/resources/people.csv"];

    let mut df = spark
        .read()
        .format("csv")
        .option("header", "True")
        .option("delimiter", ";")
        .load(path);

    df.select([
        F::col("name"),
        F::col("age").cast("int").alias("age_int"),
        (F::lit(3.0) + F::col("age").cast("int")).alias("addition"),
    ])
    .sort(vec![F::col("name").desc()])
    .show(Some(5), None, None)
    .await?;

    // print results
    // +--------------------------+
    // | show_string              |
    // +--------------------------+
    // | +-----+-------+--------+ |
    // | |name |age_int|addition| |
    // | +-----+-------+--------+ |
    // | |Jorge|30     |33.0    | |
    // | |Bob  |32     |35.0    | |
    // | +-----+-------+--------+ |
    // |                          |
    // +--------------------------+

    Ok(())
}

pub fn isin<T: ToLiteralExpr>(&self, cols: Vec<T>) -> Column

A boolean expression that is evaluated to true if the value of the expression is contained by the evaluated values of the arguments

Arguments:

• cols a value that implements the [ToLiteralExpr] trait

Example:

let paths = vec!["/opt/spark/examples/src/main/resources/people.csv".to_string()];
let mut df = spark
    .read()
    .format("csv")
    .option("header", "True")
    .option("delimiter", ";")
    .load(paths);

let row = df
    .filter(col("name").isin(vec!["Jorge", "Bob"]))
    .select("name");

pub fn contains<T: ToLiteralExpr>(&self, other: T) -> Column

A boolean expression that is evaluated to true if the value is in the Column

Arguments:

• cols: a col reference that is translated into an spark::Expression

Example:

let paths = vec!["/opt/spark/examples/src/main/resources/people.csv".to_string()];
let mut df = spark
    .read()
    .format("csv")
    .option("header", "True")
    .option("delimiter", ";")
    .load(paths);

let row = df
    .filter(col("name").contains("ge"))
    .select("name");

pub fn startswith<T: ToLiteralExpr>(&self, other: T) -> Column

A filter expression that evaluates if the column startswith a string literal

pub fn endswith<T: ToLiteralExpr>(&self, other: T) -> Column

A filter expression that evaluates if the column endswith a string literal

pub fn like<T: ToLiteralExpr>(&self, other: T) -> Column

A SQL LIKE filter expression that evaluates the column based on a case sensitive match

pub fn ilike<T: ToLiteralExpr>(&self, other: T) -> Column

A SQL ILIKE filter expression that evaluates the column based on a case insensitive match

pub fn rlike<T: ToLiteralExpr>(&self, other: T) -> Column

A SQL RLIKE filter expression that evaluates the column based on a regex match

pub fn eq<T: ToLiteralExpr>(&self, other: T) -> Column

Equality comparion. Cannot overload the ‘==’ and return something other than a bool

pub fn isNull(&self) -> Column

A filter expression that evaluates to true is the expression is null

pub fn isNotNull(&self) -> Column

A filter expression that evaluates to true is the expression is NOT null

pub fn isNaN(&self) -> Column

Trait Implementations

impl Add for Column

type Output = Column

The resulting type after applying the + operator.

fn add(self, other: Self) -> Self

Performs the + operation.

impl BitAnd for Column

type Output = Column

The resulting type after applying the & operator.

fn bitand(self, other: Self) -> Self

Performs the & operation.

impl BitOr for Column

type Output = Column

The resulting type after applying the | operator.

fn bitor(self, other: Self) -> Self

Performs the | operation.

impl BitXor for Column

type Output = Column

The resulting type after applying the ^ operator.

fn bitxor(self, other: Self) -> Self

Performs the ^ operation.

impl Clone for Column

fn clone(&self) -> Column

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for Column

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Div for Column

type Output = Column

The resulting type after applying the / operator.

fn div(self, other: Self) -> Self

Performs the / operation.

impl From<&str> for Column

fn from(value: &str) -> Self

Converts to this type from the input type.

impl From<Expression> for Column

fn from(expression: Expression) -> Self

Converts to this type from the input type.

impl Mul for Column

type Output = Column

The resulting type after applying the * operator.

fn mul(self, other: Self) -> Self

Performs the * operation.

impl Neg for Column

type Output = Column

The resulting type after applying the - operator.

fn neg(self) -> Self

Performs the unary - operation.

impl Rem for Column

type Output = Column

The resulting type after applying the % operator.

fn rem(self, other: Self) -> Self

Performs the % operation.

impl Sub for Column

type Output = Column

The resulting type after applying the - operator.

fn sub(self, other: Self) -> Self

Performs the - operation.