pub struct Column {
pub expression: Expression,
}
Expand description
§Column
A column holds a specific spark::Expression which will be resolved once an action is called. The columns are resolved by the Spark Connect server of the remote session.
A column instance can be created by in a similar way as to the Spark API. A column with created
with col("*")
or col("name.*")
is created as an unresolved star attribute which will select
all columns or references in the specified column.
use spark_connect_rs::{SparkSession, SparkSessionBuilder};
let spark: SparkSession = SparkSessionBuilder::remote("sc://127.0.0.1:15002/;user_id=example_rs".to_string())
.build()
.await?;
// As a &str representing an unresolved column in the dataframe
spark.range(None, 1, 1, Some(1)).select("id");
// By using the `col` function
spark.range(None, 1, 1, Some(1)).select(col("id"));
// By using the `lit` function to return a literal value
spark.range(None, 1, 1, Some(1)).select(lit(4.0).alias("num_col"));
Fields§
§expression: Expression
a spark::Expression containing any unresolved value to be leveraged in a spark::Plan
Implementations§
Source§impl Column
impl Column
Sourcepub fn alias(self, value: &str) -> Column
pub fn alias(self, value: &str) -> Column
Returns the column with a new name
§Example:
let cols = [
col("name").alias("new_name"),
col("age").alias("new_age")
];
df.select(cols);
Sourcepub fn asc(self) -> Column
pub fn asc(self) -> Column
Returns a sorted expression based on the ascending order of the column
§Example:
let df: DataFrame = df.sort(col("id").asc());
let df: DataFrame = df.sort(asc(col("id")));
pub fn asc_nulls_first(self) -> Column
pub fn asc_nulls_last(self) -> Column
Sourcepub fn desc(self) -> Column
pub fn desc(self) -> Column
Returns a sorted expression based on the ascending order of the column
§Example:
let df: DataFrame = df.sort(col("id").desc());
let df: DataFrame = df.sort(desc(col("id")));
pub fn desc_nulls_first(self) -> Column
pub fn desc_nulls_last(self) -> Column
pub fn drop_fields<'a, I>(self, field_names: I) -> Columnwhere
I: IntoIterator<Item = &'a str>,
pub fn with_field(self, field_name: &str, col: Column) -> Column
pub fn substr<T: ToExpr>(self, start_pos: T, length: T) -> Column
Sourcepub fn cast<T: CastToDataType>(self, to_type: T) -> Column
pub fn cast<T: CastToDataType>(self, to_type: T) -> Column
Casts the column into the Spark DataType
§Arguments:
to_type
is a string or DataType of the target type
§Example:
use crate::types::DataType;
let df = df.select([
col("age").cast("int"),
col("name").cast("string")
])
// Using DataTypes
let df = df.select([
col("age").cast(DataType::Integer),
col("name").cast(DataType::String)
])
Sourcepub fn isin<T: ToLiteralExpr>(self, cols: Vec<T>) -> Column
pub fn isin<T: ToLiteralExpr>(self, cols: Vec<T>) -> Column
A boolean expression that is evaluated to true
if the value of the expression is
contained by the evaluated values of the arguments
§Arguments:
cols
a value that implements the ToLiteralExpr trait
§Example:
df.filter(col("name").isin(["Jorge", "Bob"]));
Sourcepub fn contains<T: ToLiteralExpr>(self, other: T) -> Column
pub fn contains<T: ToLiteralExpr>(self, other: T) -> Column
A boolean expression that is evaluated to true
if the value is in the Column
§Arguments:
cols
: a col reference that is translated into an spark::Expression
§Example:
df.filter(col("name").contains("ge"));
Sourcepub fn startswith<T: ToLiteralExpr>(self, other: T) -> Column
pub fn startswith<T: ToLiteralExpr>(self, other: T) -> Column
A filter expression that evaluates if the column startswith a string literal
Sourcepub fn endswith<T: ToLiteralExpr>(self, other: T) -> Column
pub fn endswith<T: ToLiteralExpr>(self, other: T) -> Column
A filter expression that evaluates if the column endswith a string literal
Sourcepub fn like<T: ToLiteralExpr>(self, other: T) -> Column
pub fn like<T: ToLiteralExpr>(self, other: T) -> Column
A SQL LIKE filter expression that evaluates the column based on a case sensitive match
Sourcepub fn ilike<T: ToLiteralExpr>(self, other: T) -> Column
pub fn ilike<T: ToLiteralExpr>(self, other: T) -> Column
A SQL ILIKE filter expression that evaluates the column based on a case insensitive match
Sourcepub fn rlike<T: ToLiteralExpr>(self, other: T) -> Column
pub fn rlike<T: ToLiteralExpr>(self, other: T) -> Column
A SQL RLIKE filter expression that evaluates the column based on a regex match
Sourcepub fn eq<T: ToExpr>(self, other: T) -> Column
pub fn eq<T: ToExpr>(self, other: T) -> Column
Equality comparion. Cannot overload the ‘==’ and return something other than a bool
Sourcepub fn and<T: ToExpr>(self, other: T) -> Column
pub fn and<T: ToExpr>(self, other: T) -> Column
Logical AND comparion. Cannot overload the ‘&&’ and return something other than a bool
Sourcepub fn is_null(self) -> Column
pub fn is_null(self) -> Column
A filter expression that evaluates to true is the expression is null
Sourcepub fn is_not_null(self) -> Column
pub fn is_not_null(self) -> Column
A filter expression that evaluates to true is the expression is NOT null
pub fn is_nan(self) -> Column
Sourcepub fn over(self, window: WindowSpec) -> Column
pub fn over(self, window: WindowSpec) -> Column
Defines a windowing column
§Arguments:
window
: a WindowSpec
§Example
let window = Window::new()
.partition_by(col("name"))
.order_by([col("age")])
.range_between(Window::unbounded_preceding(), Window::current_row());
let df = df.with_column("rank", rank().over(window.clone()))
.with_column("min", min("age").over(window));
Trait Implementations§
Source§impl From<Expression> for Column
impl From<Expression> for Column
Source§fn from(expression: Expression) -> Self
fn from(expression: Expression) -> Self
Used for creating columns from a spark::Expression
Source§impl ToFilterExpr for Column
impl ToFilterExpr for Column
fn to_filter_expr(&self) -> Option<Expression>
Source§impl ToLiteralExpr for Column
impl ToLiteralExpr for Column
fn to_literal_expr(&self) -> Expression
Auto Trait Implementations§
impl Freeze for Column
impl RefUnwindSafe for Column
impl Send for Column
impl Sync for Column
impl Unpin for Column
impl UnwindSafe for Column
Blanket Implementations§
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Source§impl<T> CloneToUninit for Twhere
T: Clone,
impl<T> CloneToUninit for Twhere
T: Clone,
Source§impl<T> Instrument for T
impl<T> Instrument for T
Source§fn instrument(self, span: Span) -> Instrumented<Self>
fn instrument(self, span: Span) -> Instrumented<Self>
Source§fn in_current_span(self) -> Instrumented<Self>
fn in_current_span(self) -> Instrumented<Self>
Source§impl<T> IntoRequest<T> for T
impl<T> IntoRequest<T> for T
Source§fn into_request(self) -> Request<T>
fn into_request(self) -> Request<T>
T
in a tonic::Request