Struct Column

Source
pub struct Column {
    pub expression: Expression,
}
Expand description

§Column

A column holds a specific spark::Expression which will be resolved once an action is called. The columns are resolved by the Spark Connect server of the remote session.

A column instance can be created by in a similar way as to the Spark API. A column with created with col("*") or col("name.*") is created as an unresolved star attribute which will select all columns or references in the specified column.

use spark_connect_rs::{SparkSession, SparkSessionBuilder};

let spark: SparkSession = SparkSessionBuilder::remote("sc://127.0.0.1:15002/;user_id=example_rs".to_string())
        .build()
        .await?;

// As a &str representing an unresolved column in the dataframe
spark.range(None, 1, 1, Some(1)).select("id");

// By using the `col` function
spark.range(None, 1, 1, Some(1)).select(col("id"));

// By using the `lit` function to return a literal value
spark.range(None, 1, 1, Some(1)).select(lit(4.0).alias("num_col"));

Fields§

§expression: Expression

a spark::Expression containing any unresolved value to be leveraged in a spark::Plan

Implementations§

Source§

impl Column

Source

pub fn alias(self, value: &str) -> Column

Returns the column with a new name

§Example:
let cols = [
    col("name").alias("new_name"),
    col("age").alias("new_age")
];

df.select(cols);
Source

pub fn name(self, value: &str) -> Column

An alias for the function alias

Source

pub fn asc(self) -> Column

Returns a sorted expression based on the ascending order of the column

§Example:
let df: DataFrame = df.sort(col("id").asc());

let df: DataFrame = df.sort(asc(col("id")));
Source

pub fn asc_nulls_first(self) -> Column

Source

pub fn asc_nulls_last(self) -> Column

Source

pub fn desc(self) -> Column

Returns a sorted expression based on the ascending order of the column

§Example:
let df: DataFrame = df.sort(col("id").desc());

let df: DataFrame = df.sort(desc(col("id")));
Source

pub fn desc_nulls_first(self) -> Column

Source

pub fn desc_nulls_last(self) -> Column

Source

pub fn drop_fields<'a, I>(self, field_names: I) -> Column
where I: IntoIterator<Item = &'a str>,

Source

pub fn with_field(self, field_name: &str, col: Column) -> Column

Source

pub fn substr<T: ToExpr>(self, start_pos: T, length: T) -> Column

Source

pub fn cast<T: CastToDataType>(self, to_type: T) -> Column

Casts the column into the Spark DataType

§Arguments:
  • to_type is a string or DataType of the target type
§Example:
use crate::types::DataType;

let df = df.select([
      col("age").cast("int"),
      col("name").cast("string")
    ])

// Using DataTypes
let df = df.select([
      col("age").cast(DataType::Integer),
      col("name").cast(DataType::String)
    ])
Source

pub fn isin<T: ToLiteralExpr>(self, cols: Vec<T>) -> Column

A boolean expression that is evaluated to true if the value of the expression is contained by the evaluated values of the arguments

§Arguments:
§Example:
df.filter(col("name").isin(["Jorge", "Bob"]));
Source

pub fn contains<T: ToLiteralExpr>(self, other: T) -> Column

A boolean expression that is evaluated to true if the value is in the Column

§Arguments:
§Example:
df.filter(col("name").contains("ge"));
Source

pub fn startswith<T: ToLiteralExpr>(self, other: T) -> Column

A filter expression that evaluates if the column startswith a string literal

Source

pub fn endswith<T: ToLiteralExpr>(self, other: T) -> Column

A filter expression that evaluates if the column endswith a string literal

Source

pub fn like<T: ToLiteralExpr>(self, other: T) -> Column

A SQL LIKE filter expression that evaluates the column based on a case sensitive match

Source

pub fn ilike<T: ToLiteralExpr>(self, other: T) -> Column

A SQL ILIKE filter expression that evaluates the column based on a case insensitive match

Source

pub fn rlike<T: ToLiteralExpr>(self, other: T) -> Column

A SQL RLIKE filter expression that evaluates the column based on a regex match

Source

pub fn eq<T: ToExpr>(self, other: T) -> Column

Equality comparion. Cannot overload the ‘==’ and return something other than a bool

Source

pub fn and<T: ToExpr>(self, other: T) -> Column

Logical AND comparion. Cannot overload the ‘&&’ and return something other than a bool

Source

pub fn or<T: ToExpr>(self, other: T) -> Column

Logical OR comparion.

Source

pub fn is_null(self) -> Column

A filter expression that evaluates to true is the expression is null

Source

pub fn is_not_null(self) -> Column

A filter expression that evaluates to true is the expression is NOT null

Source

pub fn is_nan(self) -> Column

Source

pub fn over(self, window: WindowSpec) -> Column

Defines a windowing column

§Arguments:
§Example
let window = Window::new()
    .partition_by(col("name"))
    .order_by([col("age")])
    .range_between(Window::unbounded_preceding(), Window::current_row());

let df = df.with_column("rank", rank().over(window.clone()))
    .with_column("min", min("age").over(window));

Trait Implementations§

Source§

impl Add for Column

Source§

type Output = Column

The resulting type after applying the + operator.
Source§

fn add(self, other: Self) -> Self

Performs the + operation. Read more
Source§

impl BitAnd for Column

Source§

type Output = Column

The resulting type after applying the & operator.
Source§

fn bitand(self, other: Self) -> Self

Performs the & operation. Read more
Source§

impl BitOr for Column

Source§

type Output = Column

The resulting type after applying the | operator.
Source§

fn bitor(self, other: Self) -> Self

Performs the | operation. Read more
Source§

impl BitXor for Column

Source§

type Output = Column

The resulting type after applying the ^ operator.
Source§

fn bitxor(self, other: Self) -> Self

Performs the ^ operation. Read more
Source§

impl Clone for Column

Source§

fn clone(&self) -> Column

Returns a copy of the value. Read more
1.0.0 · Source§

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source. Read more
Source§

impl Debug for Column

Source§

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter. Read more
Source§

impl Div for Column

Source§

type Output = Column

The resulting type after applying the / operator.
Source§

fn div(self, other: Self) -> Self

Performs the / operation. Read more
Source§

impl From<&str> for Column

Source§

fn from(value: &str) -> Self

&str values containing a * will be created as an unresolved star expression Otherwise, the value is created as an unresolved attribute

Source§

impl From<Expression> for Column

Source§

fn from(expression: Expression) -> Self

Used for creating columns from a spark::Expression

Source§

impl Mul for Column

Source§

type Output = Column

The resulting type after applying the * operator.
Source§

fn mul(self, other: Self) -> Self

Performs the * operation. Read more
Source§

impl Neg for Column

Source§

type Output = Column

The resulting type after applying the - operator.
Source§

fn neg(self) -> Self

Performs the unary - operation. Read more
Source§

impl Not for Column

Source§

type Output = Column

The resulting type after applying the ! operator.
Source§

fn not(self) -> Self::Output

Performs the unary ! operation. Read more
Source§

impl Rem for Column

Source§

type Output = Column

The resulting type after applying the % operator.
Source§

fn rem(self, other: Self) -> Self

Performs the % operation. Read more
Source§

impl Sub for Column

Source§

type Output = Column

The resulting type after applying the - operator.
Source§

fn sub(self, other: Self) -> Self

Performs the - operation. Read more
Source§

impl ToExpr for Column

Source§

impl ToFilterExpr for Column

Source§

impl ToLiteralExpr for Column

Auto Trait Implementations§

§

impl Freeze for Column

§

impl RefUnwindSafe for Column

§

impl Send for Column

§

impl Sync for Column

§

impl Unpin for Column

§

impl UnwindSafe for Column

Blanket Implementations§

Source§

impl<T> Any for T
where T: 'static + ?Sized,

Source§

fn type_id(&self) -> TypeId

Gets the TypeId of self. Read more
Source§

impl<T> Borrow<T> for T
where T: ?Sized,

Source§

fn borrow(&self) -> &T

Immutably borrows from an owned value. Read more
Source§

impl<T> BorrowMut<T> for T
where T: ?Sized,

Source§

fn borrow_mut(&mut self) -> &mut T

Mutably borrows from an owned value. Read more
Source§

impl<T> CloneToUninit for T
where T: Clone,

Source§

unsafe fn clone_to_uninit(&self, dst: *mut u8)

🔬This is a nightly-only experimental API. (clone_to_uninit)
Performs copy-assignment from self to dst. Read more
Source§

impl<T> From<T> for T

Source§

fn from(t: T) -> T

Returns the argument unchanged.

Source§

impl<T> FromRef<T> for T
where T: Clone,

Source§

fn from_ref(input: &T) -> T

Converts to this type from a reference to the input type.
Source§

impl<T> Instrument for T

Source§

fn instrument(self, span: Span) -> Instrumented<Self>

Instruments this type with the provided Span, returning an Instrumented wrapper. Read more
Source§

fn in_current_span(self) -> Instrumented<Self>

Instruments this type with the current Span, returning an Instrumented wrapper. Read more
Source§

impl<T, U> Into<U> for T
where U: From<T>,

Source§

fn into(self) -> U

Calls U::from(self).

That is, this conversion is whatever the implementation of From<T> for U chooses to do.

Source§

impl<T> IntoRequest<T> for T

Source§

fn into_request(self) -> Request<T>

Wrap the input message T in a tonic::Request
Source§

impl<T> ToOwned for T
where T: Clone,

Source§

type Owned = T

The resulting type after obtaining ownership.
Source§

fn to_owned(&self) -> T

Creates owned data from borrowed data, usually by cloning. Read more
Source§

fn clone_into(&self, target: &mut T)

Uses borrowed data to replace owned data, usually by cloning. Read more
Source§

impl<T> ToVecExpr for T
where T: ToExpr,

Source§

impl<T, U> TryFrom<U> for T
where U: Into<T>,

Source§

type Error = Infallible

The type returned in the event of a conversion error.
Source§

fn try_from(value: U) -> Result<T, <T as TryFrom<U>>::Error>

Performs the conversion.
Source§

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,

Source§

type Error = <U as TryFrom<T>>::Error

The type returned in the event of a conversion error.
Source§

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

Performs the conversion.
Source§

impl<V, T> VZip<V> for T
where V: MultiLane<T>,

Source§

fn vzip(self) -> V

Source§

impl<T> WithSubscriber for T

Source§

fn with_subscriber<S>(self, subscriber: S) -> WithDispatch<Self>
where S: Into<Dispatch>,

Attaches the provided Subscriber to this type, returning a WithDispatch wrapper. Read more
Source§

fn with_current_subscriber(self) -> WithDispatch<Self>

Attaches the current default Subscriber to this type, returning a WithDispatch wrapper. Read more
Source§

impl<T> Allocation for T
where T: RefUnwindSafe + Send + Sync,

Source§

impl<T> ErasedDestructor for T
where T: 'static,

Source§

impl<T> MaybeSendSync for T

Source§

impl<T, Rhs, Output> NumOps<Rhs, Output> for T
where T: Sub<Rhs, Output = Output> + Mul<Rhs, Output = Output> + Div<Rhs, Output = Output> + Add<Rhs, Output = Output> + Rem<Rhs, Output = Output>,