Struct Column

pub struct Column {
    pub expression: Expression,
}

A column in a DataFrame.

A column holds a specific spark::Expression which will be resolved once an action is called. The columns are resolved by the Spark Connect server of the remote session.

A column instance can be created by in a similar way as to the Spark API. A column with created with col("*") or col("name.*") is created as an unresolved star attribute which will select all columns or references in the specified column.

use spark_connect_rs::{SparkSession, SparkSessionBuilder};

let spark: SparkSession = SparkSessionBuilder::remote("sc://127.0.0.1:15002/;user_id=example_rs".to_string())
        .build()
        .await?;

// As a &str representing an unresolved column in the dataframe
spark.range(None, 1, 1, Some(1)).select(["id"]);

// By using the `col` function
spark.range(None, 1, 1, Some(1)).select([col("id")]);

// By using the `lit` function to return a literal value
spark.range(None, 1, 1, Some(1)).select([lit(4.0).alias("num_col")]);

Fields

expression: Expression

a spark::Expression containing any unresolved value to be leveraged in a spark::Plan

Implementations

impl Column

pub fn from_str(s: &str) -> Self

pub fn from_string(s: String) -> Self

pub fn alias(self, value: &str) -> Column

Returns the column with a new name

Example:

let cols = [
    col("name").alias("new_name"),
    col("age").alias("new_age")
];

df.select(cols);

pub fn name(self, value: &str) -> Column

An alias for the function alias

pub fn asc(self) -> Column

Returns a sorted expression based on the ascending order of the column

Example:

let df: DataFrame = df.sort([col("id").asc()]);

let df: DataFrame = df.sort([asc(col("id"))]);

pub fn asc_nulls_first(self) -> Column

pub fn asc_nulls_last(self) -> Column

pub fn desc(self) -> Column

Returns a sorted expression based on the ascending order of the column

Example:

let df: DataFrame = df.sort(col("id").desc());

let df: DataFrame = df.sort(desc(col("id")));

pub fn desc_nulls_first(self) -> Column

pub fn desc_nulls_last(self) -> Column

pub fn drop_fields<I>(self, field_names: I) -> Column

where I: IntoIterator<Item: AsRef<str>>,

pub fn with_field(self, field_name: &str, col: impl Into<Column>) -> Column

pub fn substr( self, start_pos: impl Into<Column>, length: impl Into<Column>, ) -> Column

pub fn cast(self, to_type: impl Into<CastToType>) -> Column

Casts the column into the Spark DataType

Arguments:

• to_type is a string or crate::types::DataType of the target type

Example:

use crate::types::DataType;

let df = df.select([
      col("age").cast("int"),
      col("name").cast("string")
    ])

// Using DataTypes
let df = df.select([
      col("age").cast(DataType::Integer),
      col("name").cast(DataType::String)
    ])

pub fn isin(self, cols: Vec<Column>) -> Column

A boolean expression that is evaluated to true if the value of the expression is contained by the evaluated values of the arguments

Arguments:

• cols a vector of Columns

Example:

df.filter(col("name").isin([lit("Jorge"), lit("Bob")]));

pub fn contains(self, other: impl Into<Column>) -> Column

A boolean expression that is evaluated to true if the value is in the Column

Arguments:

• cols: a col reference that is translated into an spark::Expression

Example:

df.filter(col("name").contains("ge"));

pub fn startswith(self, other: impl Into<Column>) -> Column

A filter expression that evaluates if the column startswith a string literal

pub fn endswith(self, other: impl Into<Column>) -> Column

A filter expression that evaluates if the column endswith a string literal

pub fn like(self, other: impl Into<Column>) -> Column

A SQL LIKE filter expression that evaluates the column based on a case sensitive match

pub fn ilike(self, other: impl Into<Column>) -> Column

A SQL ILIKE filter expression that evaluates the column based on a case insensitive match

pub fn rlike(self, other: impl Into<Column>) -> Column

A SQL RLIKE filter expression that evaluates the column based on a regex match

pub fn eq(self, other: impl Into<Column>) -> Column

Equality comparion. Cannot overload the ‘==’ and return something other than a bool

pub fn and(self, other: impl Into<Column>) -> Column

Logical AND comparion. Cannot overload the ‘&&’ and return something other than a bool

pub fn or(self, other: impl Into<Column>) -> Column

Logical OR comparion.

pub fn is_null(self) -> Column

A filter expression that evaluates to true is the expression is null

pub fn is_not_null(self) -> Column

A filter expression that evaluates to true is the expression is NOT null

pub fn is_nan(self) -> Column

pub fn over(self, window: WindowSpec) -> Column

Defines a windowing column

Arguments:

• window: a WindowSpec

Example

let window = Window::new()
    .partition_by([col("name")])
    .order_by([col("age")])
    .range_between(Window::unbounded_preceding(), Window::current_row());

let df = df.with_column("rank", rank().over(window.clone()))
    .with_column("min", min("age").over(window));

Trait Implementations

impl Add for Column

type Output = Column

The resulting type after applying the + operator.

fn add(self, other: Self) -> Self

Performs the + operation.

impl BitAnd for Column

type Output = Column

The resulting type after applying the & operator.

fn bitand(self, other: Self) -> Self

Performs the & operation.

impl BitOr for Column

type Output = Column

The resulting type after applying the | operator.

fn bitor(self, other: Self) -> Self

Performs the | operation.

impl BitXor for Column

type Output = Column

The resulting type after applying the ^ operator.

fn bitxor(self, other: Self) -> Self

Performs the ^ operation.

impl Clone for Column

fn clone(&self) -> Column

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for Column

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Div for Column

type Output = Column

The resulting type after applying the / operator.

fn div(self, other: Self) -> Self

Performs the / operation.

impl From<&String> for Column

fn from(value: &String) -> Self

Converts to this type from the input type.

impl From<&str> for Column

fn from(value: &str) -> Self

&str values containing a * will be created as an unresolved star expression Otherwise, the value is created as an unresolved attribute

impl From<Column> for Expression

fn from(value: Column) -> Self

Converts to this type from the input type.

impl From<Expression> for Column

fn from(expression: Expression) -> Self

Used for creating columns from a spark::Expression

impl From<Literal> for Column

fn from(expression: Literal) -> Self

Used for creating columns from a spark::Expression

impl From<String> for Column

fn from(value: String) -> Self

Converts to this type from the input type.

impl Mul for Column

type Output = Column

The resulting type after applying the * operator.

fn mul(self, other: Self) -> Self

Performs the * operation.

impl Neg for Column

type Output = Column

The resulting type after applying the - operator.

fn neg(self) -> Self

Performs the unary - operation.

impl Not for Column

type Output = Column

The resulting type after applying the ! operator.

fn not(self) -> Self::Output

Performs the unary ! operation.

impl Rem for Column

type Output = Column

The resulting type after applying the % operator.

fn rem(self, other: Self) -> Self

Performs the % operation.

impl Sub for Column

type Output = Column

The resulting type after applying the - operator.

fn sub(self, other: Self) -> Self

Performs the - operation.

impl ToFilterExpr for Column

fn to_filter_expr(&self) -> Option<Expression>